Move the event storage into a single transaction

This commit is contained in:
Mark Haines 2014-08-26 14:31:48 +01:00
parent 1379dcae6f
commit 4b2ad549d5
6 changed files with 60 additions and 48 deletions

View File

@ -57,19 +57,21 @@ class DataStore(RoomMemberStore, RoomStore,
@defer.inlineCallbacks @defer.inlineCallbacks
@log_function @log_function
def persist_event(self, event, backfilled=False): def persist_event(self, event, backfilled=False):
if event.type == RoomMemberEvent.TYPE: # FIXME (erikj): This should be removed when we start amalgamating
yield self._store_room_member(event) # event and pdu storage
elif event.type == FeedbackEvent.TYPE: yield self.hs.get_federation().fill_out_prev_events(event)
yield self._store_feedback(event)
# elif event.type == RoomConfigEvent.TYPE:
# yield self._store_room_config(event)
elif event.type == RoomNameEvent.TYPE:
yield self._store_room_name(event)
elif event.type == RoomTopicEvent.TYPE:
yield self._store_room_topic(event)
ret = yield self._store_event(event, backfilled) stream_ordering = None
defer.returnValue(ret) if backfilled:
if not self.min_token_deferred.called:
yield self.min_token_deferred
self.min_token -= 1
stream_ordering = self.min_token
latest = yield self._db_pool.runInteraction(
_persist_event_txn, event, backfilled, stream_ordering
)
defer.returnValue(latest)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_event(self, event_id): def get_event(self, event_id):
@ -89,12 +91,18 @@ class DataStore(RoomMemberStore, RoomStore,
event = self._parse_event_from_row(events_dict) event = self._parse_event_from_row(events_dict)
defer.returnValue(event) defer.returnValue(event)
@defer.inlineCallbacks
@log_function @log_function
def _store_event(self, event, backfilled): def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None):
# FIXME (erikj): This should be removed when we start amalgamating if event.type == RoomMemberEvent.TYPE:
# event and pdu storage self._store_room_member_txn(txn, event)
yield self.hs.get_federation().fill_out_prev_events(event) elif event.type == FeedbackEvent.TYPE:
self._store_feedback_txn(txn,event)
# elif event.type == RoomConfigEvent.TYPE:
# self._store_room_config_txn(txn, event)
elif event.type == RoomNameEvent.TYPE:
self._store_room_name_txn(txn, event)
elif event.type == RoomTopicEvent.TYPE:
self._store_room_topic_txn(txn, event)
vals = { vals = {
"topological_ordering": event.depth, "topological_ordering": event.depth,
@ -105,11 +113,8 @@ class DataStore(RoomMemberStore, RoomStore,
"processed": True, "processed": True,
} }
if backfilled: if stream_ordering is not None:
if not self.min_token_deferred.called: vals["stream_ordering"] = stream_ordering
yield self.min_token_deferred
self.min_token -= 1
vals["stream_ordering"] = self.min_token
unrec = { unrec = {
k: v k: v
@ -119,7 +124,7 @@ class DataStore(RoomMemberStore, RoomStore,
vals["unrecognized_keys"] = json.dumps(unrec) vals["unrecognized_keys"] = json.dumps(unrec)
try: try:
yield self._simple_insert("events", vals) self._simple_insert_txn(txn, "events", vals)
except: except:
logger.exception( logger.exception(
"Failed to persist, probably duplicate: %s", "Failed to persist, probably duplicate: %s",
@ -138,9 +143,10 @@ class DataStore(RoomMemberStore, RoomStore,
if hasattr(event, "prev_state"): if hasattr(event, "prev_state"):
vals["prev_state"] = event.prev_state vals["prev_state"] = event.prev_state
yield self._simple_insert("state_events", vals) self._simple_insert_txn(txn, "state_events", vals)
yield self._simple_insert( self._simple_insert_txn(
txn
"current_state_events", "current_state_events",
{ {
"event_id": event.event_id, "event_id": event.event_id,
@ -150,8 +156,7 @@ class DataStore(RoomMemberStore, RoomStore,
} }
) )
latest = yield self.get_room_events_max_id() return self._get_room_events_max_id_(txn)
defer.returnValue(latest)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_current_state(self, room_id, event_type=None, state_key=""): def get_current_state(self, room_id, event_type=None, state_key=""):

View File

@ -86,16 +86,18 @@ class SQLBaseStore(object):
table : string giving the table name table : string giving the table name
values : dict of new column names and values for them values : dict of new column names and values for them
""" """
return self._db_pool.runInteraction(
self._simple_insert_txn, table, values,
)
def _simple_insert_txn(self, txn, table, values):
sql = "INSERT INTO %s (%s) VALUES(%s)" % ( sql = "INSERT INTO %s (%s) VALUES(%s)" % (
table, table,
", ".join(k for k in values), ", ".join(k for k in values),
", ".join("?" for k in values) ", ".join("?" for k in values)
) )
def func(txn):
txn.execute(sql, values.values()) txn.execute(sql, values.values())
return txn.lastrowid return txn.lastrowid
return self._db_pool.runInteraction(func)
def _simple_select_one(self, table, keyvalues, retcols, def _simple_select_one(self, table, keyvalues, retcols,
allow_none=False): allow_none=False):

View File

@ -24,8 +24,8 @@ import json
class FeedbackStore(SQLBaseStore): class FeedbackStore(SQLBaseStore):
def _store_feedback(self, event): def _store_feedback_txn(self, txn, event):
return self._simple_insert("feedback", { self._simple_insert_txn(txn, "feedback", {
"event_id": event.event_id, "event_id": event.event_id,
"feedback_type": event.feedback_type, "feedback_type": event.feedback_type,
"room_id": event.room_id, "room_id": event.room_id,

View File

@ -131,8 +131,9 @@ class RoomStore(SQLBaseStore):
defer.returnValue(ret) defer.returnValue(ret)
def _store_room_topic(self, event): def _store_room_topic_txn(self, txn, event):
return self._simple_insert( self._simple_insert_txn(
txn,
"topics", "topics",
{ {
"event_id": event.event_id, "event_id": event.event_id,
@ -141,8 +142,9 @@ class RoomStore(SQLBaseStore):
} }
) )
def _store_room_name(self, event): def _store_room_name_txn(self, txn, event):
return self._simple_insert( self._simple_insert_txn(
txn,
"room_names", "room_names",
{ {
"event_id": event.event_id, "event_id": event.event_id,

View File

@ -31,13 +31,13 @@ logger = logging.getLogger(__name__)
class RoomMemberStore(SQLBaseStore): class RoomMemberStore(SQLBaseStore):
@defer.inlineCallbacks def _store_room_member_txn(self, txn, event):
def _store_room_member(self, event):
"""Store a room member in the database. """Store a room member in the database.
""" """
domain = self.hs.parse_userid(event.target_user_id).domain domain = self.hs.parse_userid(event.target_user_id).domain
yield self._simple_insert( self._simple_insert_txn(
txn,
"room_memberships", "room_memberships",
{ {
"event_id": event.event_id, "event_id": event.event_id,
@ -54,13 +54,13 @@ class RoomMemberStore(SQLBaseStore):
"INSERT OR IGNORE INTO room_hosts (room_id, host) " "INSERT OR IGNORE INTO room_hosts (room_id, host) "
"VALUES (?, ?)" "VALUES (?, ?)"
) )
yield self._execute(None, sql, event.room_id, domain) txn.execute(sql, event.room_id, domain)
else: else:
sql = ( sql = (
"DELETE FROM room_hosts WHERE room_id = ? AND host = ?" "DELETE FROM room_hosts WHERE room_id = ? AND host = ?"
) )
yield self._execute(None, sql, event.room_id, domain) txn.execute(sql, event.room_id, domain)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_room_member(self, user_id, room_id): def get_room_member(self, user_id, room_id):

View File

@ -283,17 +283,20 @@ class StreamStore(SQLBaseStore):
) )
) )
@defer.inlineCallbacks
def get_room_events_max_id(self): def get_room_events_max_id(self):
res = yield self._execute_and_decode( return self._db_pool.runInteraction(self._get_room_events_max_id_txn)
def _get_room_events_max_id_txn(self, txn):
txn.execute(
"SELECT MAX(stream_ordering) as m FROM events" "SELECT MAX(stream_ordering) as m FROM events"
) )
res = self.cursor_to_dict(txn)
logger.debug("get_room_events_max_id: %s", res) logger.debug("get_room_events_max_id: %s", res)
if not res or not res[0] or not res[0]["m"]: if not res or not res[0] or not res[0]["m"]:
defer.returnValue("s1") return "s1"
return
key = res[0]["m"] + 1 key = res[0]["m"] + 1
defer.returnValue("s%d" % (key,)) return "s%d" % (key,)