Move the event storage into a single transaction

synapse/storage/__init__.py

@@ -57,19 +57,21 @@ class DataStore(RoomMemberStore, RoomStore,
     @defer.inlineCallbacks
     @log_function
     def persist_event(self, event, backfilled=False):
-        if event.type == RoomMemberEvent.TYPE:
+        # FIXME (erikj): This should be removed when we start amalgamating
-            yield self._store_room_member(event)
+        # event and pdu storage
-        elif event.type == FeedbackEvent.TYPE:
+        yield self.hs.get_federation().fill_out_prev_events(event)
-            yield self._store_feedback(event)
-#        elif event.type == RoomConfigEvent.TYPE:
-#            yield self._store_room_config(event)
-        elif event.type == RoomNameEvent.TYPE:
-            yield self._store_room_name(event)
-        elif event.type == RoomTopicEvent.TYPE:
-            yield self._store_room_topic(event)
 
-        ret = yield self._store_event(event, backfilled)
+        stream_ordering = None
-        defer.returnValue(ret)
+        if backfilled:
+            if not self.min_token_deferred.called:
+                yield self.min_token_deferred
+            self.min_token -= 1
+            stream_ordering = self.min_token
+
+        latest = yield self._db_pool.runInteraction(
+            _persist_event_txn, event, backfilled, stream_ordering
+        )
+        defer.returnValue(latest)
 
     @defer.inlineCallbacks
     def get_event(self, event_id):
@@ -89,12 +91,18 @@ class DataStore(RoomMemberStore, RoomStore,
         event = self._parse_event_from_row(events_dict)
         defer.returnValue(event)
 
-    @defer.inlineCallbacks
     @log_function
-    def _store_event(self, event, backfilled):
+    def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None):
-        # FIXME (erikj): This should be removed when we start amalgamating
+        if event.type == RoomMemberEvent.TYPE:
-        # event and pdu storage
+            self._store_room_member_txn(txn, event)
-        yield self.hs.get_federation().fill_out_prev_events(event)
+        elif event.type == FeedbackEvent.TYPE:
+            self._store_feedback_txn(txn,event)
+#        elif event.type == RoomConfigEvent.TYPE:
+#            self._store_room_config_txn(txn, event)
+        elif event.type == RoomNameEvent.TYPE:
+            self._store_room_name_txn(txn, event)
+        elif event.type == RoomTopicEvent.TYPE:
+            self._store_room_topic_txn(txn, event)
 
         vals = {
             "topological_ordering": event.depth,
@@ -105,11 +113,8 @@ class DataStore(RoomMemberStore, RoomStore,
             "processed": True,
         }
 
-        if backfilled:
+        if stream_ordering is not None:
-            if not self.min_token_deferred.called:
+            vals["stream_ordering"] = stream_ordering
-                yield self.min_token_deferred
-            self.min_token -= 1
-            vals["stream_ordering"] = self.min_token
 
         unrec = {
             k: v
@@ -119,7 +124,7 @@ class DataStore(RoomMemberStore, RoomStore,
         vals["unrecognized_keys"] = json.dumps(unrec)
 
         try:
-            yield self._simple_insert("events", vals)
+            self._simple_insert_txn(txn, "events", vals)
         except:
             logger.exception(
                 "Failed to persist, probably duplicate: %s",
@@ -138,9 +143,10 @@ class DataStore(RoomMemberStore, RoomStore,
             if hasattr(event, "prev_state"):
                 vals["prev_state"] = event.prev_state
 
-            yield self._simple_insert("state_events", vals)
+            self._simple_insert_txn(txn, "state_events", vals)
 
-            yield self._simple_insert(
+            self._simple_insert_txn(
+                txn
                 "current_state_events",
                 {
                     "event_id": event.event_id,
@@ -150,8 +156,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 }
             )
 
-        latest = yield self.get_room_events_max_id()
+        return self._get_room_events_max_id_(txn)
-        defer.returnValue(latest)
 
     @defer.inlineCallbacks
     def get_current_state(self, room_id, event_type=None, state_key=""):

synapse/storage/_base.py

@@ -86,16 +86,18 @@ class SQLBaseStore(object):
             table : string giving the table name
             values : dict of new column names and values for them
         """
+        return self._db_pool.runInteraction(
+            self._simple_insert_txn, table, values,
+        )
+
+    def _simple_insert_txn(self, txn, table, values):
         sql = "INSERT INTO %s (%s) VALUES(%s)" % (
             table,
             ", ".join(k for k in values),
             ", ".join("?" for k in values)
         )
-
-        def func(txn):
         txn.execute(sql, values.values())
         return txn.lastrowid
-        return self._db_pool.runInteraction(func)
 
     def _simple_select_one(self, table, keyvalues, retcols,
                            allow_none=False):

synapse/storage/feedback.py

@@ -24,8 +24,8 @@ import json
 
 class FeedbackStore(SQLBaseStore):
 
-    def _store_feedback(self, event):
+    def _store_feedback_txn(self, txn, event):
-        return self._simple_insert("feedback", {
+        self._simple_insert_txn(txn, "feedback", {
             "event_id": event.event_id,
             "feedback_type": event.feedback_type,
             "room_id": event.room_id,

synapse/storage/room.py

@@ -131,8 +131,9 @@ class RoomStore(SQLBaseStore):
 
         defer.returnValue(ret)
 
-    def _store_room_topic(self, event):
+    def _store_room_topic_txn(self, txn, event):
-        return self._simple_insert(
+        self._simple_insert_txn(
+            txn,
             "topics",
             {
                 "event_id": event.event_id,
@@ -141,8 +142,9 @@ class RoomStore(SQLBaseStore):
             }
         )
 
-    def _store_room_name(self, event):
+    def _store_room_name_txn(self, txn, event):
-        return self._simple_insert(
+        self._simple_insert_txn(
+            txn,
             "room_names",
             {
                 "event_id": event.event_id,

synapse/storage/roommember.py

@@ -31,13 +31,13 @@ logger = logging.getLogger(__name__)
 
 class RoomMemberStore(SQLBaseStore):
 
-    @defer.inlineCallbacks
+    def _store_room_member_txn(self, txn, event):
-    def _store_room_member(self, event):
         """Store a room member in the database.
         """
         domain = self.hs.parse_userid(event.target_user_id).domain
 
-        yield self._simple_insert(
+        self._simple_insert_txn(
+            txn,
             "room_memberships",
             {
                 "event_id": event.event_id,
@@ -54,13 +54,13 @@ class RoomMemberStore(SQLBaseStore):
                 "INSERT OR IGNORE INTO room_hosts (room_id, host) "
                 "VALUES (?, ?)"
             )
-            yield self._execute(None, sql, event.room_id, domain)
+            txn.execute(sql, event.room_id, domain)
         else:
             sql = (
                 "DELETE FROM room_hosts WHERE room_id = ? AND host = ?"
             )
 
-            yield self._execute(None, sql, event.room_id, domain)
+            txn.execute(sql, event.room_id, domain)
 
     @defer.inlineCallbacks
     def get_room_member(self, user_id, room_id):

synapse/storage/stream.py

@@ -283,17 +283,20 @@ class StreamStore(SQLBaseStore):
             )
         )
 
-    @defer.inlineCallbacks
     def get_room_events_max_id(self):
-        res = yield self._execute_and_decode(
+        return self._db_pool.runInteraction(self._get_room_events_max_id_txn)
+
+    def _get_room_events_max_id_txn(self, txn):
+        txn.execute(
             "SELECT MAX(stream_ordering) as m FROM events"
         )
 
+        res = self.cursor_to_dict(txn)
+
         logger.debug("get_room_events_max_id: %s", res)
 
         if not res or not res[0] or not res[0]["m"]:
-            defer.returnValue("s1")
+            return "s1"
-            return
 
         key = res[0]["m"] + 1
-        defer.returnValue("s%d" % (key,))
+        return "s%d" % (key,)