From 15ca0c6a4d4b7ddbff8e3348ba949177e1562108 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:36:36 +0100 Subject: [PATCH 1/5] Make reindex happen in bg --- synapse/storage/event_push_actions.py | 30 +++++++++++++++++++ .../delta/35/event_push_actions_index.sql | 5 ++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index a87d90741..40bfe754b 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore from twisted.internet import defer from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.types import RoomStreamToken +from synapse.storage.engines import PostgresEngine from .stream import lower_bound import logging @@ -26,10 +27,17 @@ logger = logging.getLogger(__name__) class EventPushActionsStore(SQLBaseStore): + EPA_HIGHLIGHT_INDEX = "epa_highlight_index" + def __init__(self, hs): self.stream_ordering_month_ago = None super(EventPushActionsStore, self).__init__(hs) + self.register_background_update_handler( + self.EPA_HIGHLIGHT_INDEX, + self._background_index_epa_highlight, + ) + def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ Args: @@ -500,6 +508,28 @@ class EventPushActionsStore(SQLBaseStore): return range_end + @defer.inlineCallbacks + def _background_index_epa_highlight(self, progress, batch_size): + def reindex_txn(txn): + if isinstance(self.database_engine, PostgresEngine): + txn.execute( + "CREATE INDEX CONCURRENTLY event_push_actions_u_highlight" + " on event_push_actions(user_id, highlight, stream_ordering)" + ) + else: + txn.execute( + "CREATE INDEX event_push_actions_u_highlight" + " on event_push_actions(user_id, highlight, stream_ordering)" + ) + + yield self.runInteraction( + self.EPA_HIGHLIGHT_INDEX, reindex_txn + ) + + yield self._end_background_update(self.EPA_HIGHLIGHT_INDEX) + + defer.returnValue(1) + def _action_has_highlight(actions): for action in actions: diff --git a/synapse/storage/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/delta/35/event_push_actions_index.sql index 4fc32c351..2e836d8e9 100644 --- a/synapse/storage/schema/delta/35/event_push_actions_index.sql +++ b/synapse/storage/schema/delta/35/event_push_actions_index.sql @@ -13,6 +13,5 @@ * limitations under the License. */ - CREATE INDEX event_push_actions_user_id_highlight_stream_ordering on event_push_actions( - user_id, highlight, stream_ordering - ); + INSERT into background_updates (update_name, progress_json) + VALUES ('epa_highlight_index', '{}'); From 0294c14ec43bed0b116c7ff531482539fb713443 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:43:56 +0100 Subject: [PATCH 2/5] Add back in query change --- synapse/storage/event_push_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 8632b2f93..40bfe754b 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -361,12 +361,14 @@ class EventPushActionsStore(SQLBaseStore): before_clause += " " before_clause += "AND epa.highlight = 1" + # NB. This assumes event_ids are globally unique since + # it makes the query easier to index sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," " epa.actions, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" - " WHERE epa.room_id = e.room_id AND epa.event_id = e.event_id" + " WHERE epa.event_id = e.event_id" " AND epa.user_id = ? %s" " ORDER BY epa.stream_ordering DESC" " LIMIT ?" From 7cd6edb9470ed949b2c63317624dbc9e38950c95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:54:38 +0100 Subject: [PATCH 3/5] Use register_background_index_update --- synapse/storage/event_push_actions.py | 28 ++++----------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 40bfe754b..7974a108a 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -33,9 +33,11 @@ class EventPushActionsStore(SQLBaseStore): self.stream_ordering_month_ago = None super(EventPushActionsStore, self).__init__(hs) - self.register_background_update_handler( + self.register_background_index_update( self.EPA_HIGHLIGHT_INDEX, - self._background_index_epa_highlight, + index_name="event_push_actions_u_highlight", + table="event_push_actions", + columns=["user_id", "highlight", "stream_ordering"], ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): @@ -508,28 +510,6 @@ class EventPushActionsStore(SQLBaseStore): return range_end - @defer.inlineCallbacks - def _background_index_epa_highlight(self, progress, batch_size): - def reindex_txn(txn): - if isinstance(self.database_engine, PostgresEngine): - txn.execute( - "CREATE INDEX CONCURRENTLY event_push_actions_u_highlight" - " on event_push_actions(user_id, highlight, stream_ordering)" - ) - else: - txn.execute( - "CREATE INDEX event_push_actions_u_highlight" - " on event_push_actions(user_id, highlight, stream_ordering)" - ) - - yield self.runInteraction( - self.EPA_HIGHLIGHT_INDEX, reindex_txn - ) - - yield self._end_background_update(self.EPA_HIGHLIGHT_INDEX) - - defer.returnValue(1) - def _action_has_highlight(actions): for action in actions: From 5ef5435529778d6bf0bf19f36b7fb6febc26a718 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 13:32:58 +0100 Subject: [PATCH 4/5] Remove unused import --- synapse/storage/event_push_actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 7974a108a..e02afdb1d 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -17,7 +17,6 @@ from ._base import SQLBaseStore from twisted.internet import defer from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.types import RoomStreamToken -from synapse.storage.engines import PostgresEngine from .stream import lower_bound import logging From fa20c9ce9410d71d7144932c33156b9dacd554f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 14:04:08 +0100 Subject: [PATCH 5/5] Change the index to be stream_ordering, highlight --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index e02afdb1d..51b13e949 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,7 @@ class EventPushActionsStore(SQLBaseStore): self.EPA_HIGHLIGHT_INDEX, index_name="event_push_actions_u_highlight", table="event_push_actions", - columns=["user_id", "highlight", "stream_ordering"], + columns=["user_id", "stream_ordering", "highlight"], ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):