From c94de0ab606e5bbe7331a58456204a5775df779b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 16:55:01 +0100 Subject: [PATCH 1/3] Add WHERE clause support to index creation --- synapse/storage/background_updates.py | 22 ++++++++++++++-------- synapse/storage/event_push_actions.py | 3 ++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 003f5ba20..94b2bcc54 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -219,7 +219,7 @@ class BackgroundUpdateStore(SQLBaseStore): self._background_update_handlers[update_name] = update_handler def register_background_index_update(self, update_name, index_name, - table, columns): + table, columns, where_clause=None): """Helper for store classes to do a background index addition To use: @@ -243,14 +243,20 @@ class BackgroundUpdateStore(SQLBaseStore): conc = True else: conc = False + # We don't use partial indices on SQLite as it wasn't introduced + # until 3.8, and wheezy has 3.7 + where_clause = None - sql = "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" \ - % { - "conc": "CONCURRENTLY" if conc else "", - "name": index_name, - "table": table, - "columns": ", ".join(columns), - } + sql = ( + "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" + " %(where_clause)s" + ) % { + "conc": "CONCURRENTLY" if conc else "", + "name": index_name, + "table": table, + "columns": ", ".join(columns), + "where_clause": "WHERE " + where_clause if where_clause else "" + } def create_index_concurrently(conn): conn.rollback() diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 51b13e949..efa1db3b9 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,8 @@ class EventPushActionsStore(SQLBaseStore): self.EPA_HIGHLIGHT_INDEX, index_name="event_push_actions_u_highlight", table="event_push_actions", - columns=["user_id", "stream_ordering", "highlight"], + columns=["user_id", "stream_ordering"], + where_clause="highlight = 1", ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): From 0b32bb20bbf025b6e69bae23a6d4a96903f41885 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 16:57:05 +0100 Subject: [PATCH 2/3] Index contains_url for file search queries --- synapse/storage/events.py | 8 ++++++++ .../storage/schema/delta/35/contains_url.sql | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 synapse/storage/schema/delta/35/contains_url.sql diff --git a/synapse/storage/events.py b/synapse/storage/events.py index ed182c8d1..6dc46fa50 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -189,6 +189,14 @@ class EventsStore(SQLBaseStore): self._background_reindex_fields_sender, ) + self.register_background_index_update( + "event_contains_url_index", + index_name="event_contains_url_index", + table="events", + columns=["room_id", "topological_ordering", "stream_ordering"], + where_clause="contains_url = true AND outlier = false", + ) + self._event_persist_queue = _EventPeristenceQueue() def persist_events(self, events_and_contexts, backfilled=False): diff --git a/synapse/storage/schema/delta/35/contains_url.sql b/synapse/storage/schema/delta/35/contains_url.sql new file mode 100644 index 000000000..6cd123027 --- /dev/null +++ b/synapse/storage/schema/delta/35/contains_url.sql @@ -0,0 +1,17 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + INSERT into background_updates (update_name, progress_json) + VALUES ('event_contains_url_index', '{}'); From b17af156c71b9859edbd27516eacd2c55f488e31 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 17:05:54 +0100 Subject: [PATCH 3/3] Remove where clause --- synapse/storage/event_push_actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index efa1db3b9..9cd923eb9 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -37,7 +37,6 @@ class EventPushActionsStore(SQLBaseStore): index_name="event_push_actions_u_highlight", table="event_push_actions", columns=["user_id", "stream_ordering"], - where_clause="highlight = 1", ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):