mirror of
https://mau.dev/maunium/synapse.git
synced 2024-09-05 00:52:20 +00:00
Batch process handling state groups
This commit is contained in:
parent
67a1e315cc
commit
47a9da28ca
@ -37,6 +37,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process
|
|||||||
from synapse.storage.background_updates import BackgroundUpdateStore
|
from synapse.storage.background_updates import BackgroundUpdateStore
|
||||||
from synapse.storage.event_federation import EventFederationStore
|
from synapse.storage.event_federation import EventFederationStore
|
||||||
from synapse.storage.events_worker import EventsWorkerStore
|
from synapse.storage.events_worker import EventsWorkerStore
|
||||||
|
from synapse.storage.state import StateGroupWorkerStore
|
||||||
from synapse.types import RoomStreamToken, get_domain_from_id
|
from synapse.types import RoomStreamToken, get_domain_from_id
|
||||||
from synapse.util.async_helpers import ObservableDeferred
|
from synapse.util.async_helpers import ObservableDeferred
|
||||||
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
|
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
|
||||||
@ -203,7 +204,8 @@ def _retry_on_integrity_error(func):
|
|||||||
|
|
||||||
# inherits from EventFederationStore so that we can call _update_backward_extremities
|
# inherits from EventFederationStore so that we can call _update_backward_extremities
|
||||||
# and _handle_mult_prev_events (though arguably those could both be moved in here)
|
# and _handle_mult_prev_events (though arguably those could both be moved in here)
|
||||||
class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore):
|
class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore,
|
||||||
|
BackgroundUpdateStore):
|
||||||
EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
|
EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
|
||||||
EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
|
EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
|
||||||
|
|
||||||
@ -1995,70 +1997,29 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
|
|||||||
|
|
||||||
logger.info("[purge] finding redundant state groups")
|
logger.info("[purge] finding redundant state groups")
|
||||||
|
|
||||||
# Get all state groups that are only referenced by events that are
|
# Get all state groups that are referenced by events that are to be
|
||||||
# to be deleted.
|
# deleted. We then go and check if they are referenced by other events
|
||||||
# This works by first getting state groups that we may want to delete,
|
# or state groups, and if not we delete them.
|
||||||
# joining against event_to_state_groups to get events that use that
|
|
||||||
# state group, then left joining against events_to_purge again. Any
|
|
||||||
# state group where the left join produce *no nulls* are referenced
|
|
||||||
# only by events that are going to be purged.
|
|
||||||
txn.execute("""
|
txn.execute("""
|
||||||
SELECT state_group FROM
|
|
||||||
(
|
|
||||||
SELECT DISTINCT state_group FROM events_to_purge
|
SELECT DISTINCT state_group FROM events_to_purge
|
||||||
INNER JOIN event_to_state_groups USING (event_id)
|
INNER JOIN event_to_state_groups USING (event_id)
|
||||||
) AS sp
|
|
||||||
INNER JOIN event_to_state_groups USING (state_group)
|
|
||||||
LEFT JOIN events_to_purge AS ep USING (event_id)
|
|
||||||
GROUP BY state_group
|
|
||||||
HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0
|
|
||||||
""")
|
""")
|
||||||
|
|
||||||
state_rows = txn.fetchall()
|
referenced_state_groups = set(sg for sg, in txn)
|
||||||
logger.info("[purge] found %i redundant state groups", len(state_rows))
|
logger.info(
|
||||||
|
"[purge] found %i referenced state groups",
|
||||||
# make a set of the redundant state groups, so that we can look them up
|
len(referenced_state_groups),
|
||||||
# efficiently
|
|
||||||
state_groups_to_delete = set([sg for sg, in state_rows])
|
|
||||||
|
|
||||||
# Now we get all the state groups that rely on these state groups
|
|
||||||
logger.info("[purge] finding state groups which depend on redundant"
|
|
||||||
" state groups")
|
|
||||||
remaining_state_groups = []
|
|
||||||
unreferenced_state_groups = 0
|
|
||||||
for i in range(0, len(state_rows), 100):
|
|
||||||
chunk = [sg for sg, in state_rows[i:i + 100]]
|
|
||||||
# look for state groups whose prev_state_group is one we are about
|
|
||||||
# to delete
|
|
||||||
rows = self._simple_select_many_txn(
|
|
||||||
txn,
|
|
||||||
table="state_group_edges",
|
|
||||||
column="prev_state_group",
|
|
||||||
iterable=chunk,
|
|
||||||
retcols=["state_group"],
|
|
||||||
keyvalues={},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for row in rows:
|
logger.info("[purge] finding state groups that can be deleted")
|
||||||
sg = row["state_group"]
|
|
||||||
|
|
||||||
if sg in state_groups_to_delete:
|
state_groups_to_delete, remaining_state_groups = self._find_unreferenced_groups(
|
||||||
# exclude state groups we are about to delete: no point in
|
txn, referenced_state_groups,
|
||||||
# updating them
|
)
|
||||||
continue
|
|
||||||
|
|
||||||
if not self._is_state_group_referenced(txn, sg):
|
|
||||||
# Let's also delete unreferenced state groups while we're
|
|
||||||
# here, since otherwise we'd need to de-delta them
|
|
||||||
state_groups_to_delete.add(sg)
|
|
||||||
unreferenced_state_groups += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
remaining_state_groups.append(sg)
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"[purge] found %i extra unreferenced state groups to delete",
|
"[purge] found %i state groups to delete",
|
||||||
unreferenced_state_groups,
|
len(state_groups_to_delete),
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
@ -2109,11 +2070,11 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore
|
|||||||
logger.info("[purge] removing redundant state groups")
|
logger.info("[purge] removing redundant state groups")
|
||||||
txn.executemany(
|
txn.executemany(
|
||||||
"DELETE FROM state_groups_state WHERE state_group = ?",
|
"DELETE FROM state_groups_state WHERE state_group = ?",
|
||||||
state_rows
|
((sg,) for sg in state_groups_to_delete),
|
||||||
)
|
)
|
||||||
txn.executemany(
|
txn.executemany(
|
||||||
"DELETE FROM state_groups WHERE id = ?",
|
"DELETE FROM state_groups WHERE id = ?",
|
||||||
state_rows
|
((sg,) for sg in state_groups_to_delete),
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("[purge] removing events from event_to_state_groups")
|
logger.info("[purge] removing events from event_to_state_groups")
|
||||||
|
@ -1041,55 +1041,85 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
|
|||||||
|
|
||||||
return count
|
return count
|
||||||
|
|
||||||
def _is_state_group_referenced(self, txn, state_group):
|
def _find_unreferenced_groups(self, txn, state_groups):
|
||||||
"""Checks if a given state group is referenced, or is safe to delete.
|
"""Used when purging history to figure out which state groups can be
|
||||||
|
deleted and which need to be de-delta'ed (due to one of its prev groups
|
||||||
|
being scheduled for deletion).
|
||||||
|
|
||||||
A state group is referenced if it or any of its descendants are
|
Args:
|
||||||
pointed at by an event. (A descendant is a state_group whose chain of
|
txn
|
||||||
prev_groups includes the given state_group.)
|
state_groups (set[int]): Set of state groups referenced by events
|
||||||
|
that are going to be deleted.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple[set[int], set[int]]: The set of state groups that can be
|
||||||
|
deleted and the set of state groups that need to be de-delta'ed
|
||||||
"""
|
"""
|
||||||
|
# Graph of state group -> previous group
|
||||||
|
graph = {}
|
||||||
|
|
||||||
# We check this by doing a depth first search to look for any
|
# Set of events that we have found to be referenced by events
|
||||||
# descendant referenced by `event_to_state_groups`.
|
referenced_groups = set()
|
||||||
|
|
||||||
# State groups we need to check, contains state groups that are
|
# Set of state groups we've already seen
|
||||||
# descendants of `state_group`
|
state_groups_seen = set(state_groups)
|
||||||
state_groups_to_search = [state_group]
|
|
||||||
|
|
||||||
# Set of state groups we've already checked
|
# Set of state groups to handle next.
|
||||||
state_groups_searched = set()
|
next_to_search = set(state_groups)
|
||||||
|
while next_to_search:
|
||||||
|
# We bound size of groups we're looking up at once, to stop the
|
||||||
|
# SQL query getting too big
|
||||||
|
if len(next_to_search) < 100:
|
||||||
|
current_search = next_to_search
|
||||||
|
next_to_search = set()
|
||||||
|
else:
|
||||||
|
lst = list(next_to_search)
|
||||||
|
current_search = set(lst[:100])
|
||||||
|
next_to_search = set(lst[100:])
|
||||||
|
|
||||||
while state_groups_to_search:
|
# Check if state groups are referenced
|
||||||
state_group = state_groups_to_search.pop() # Next state group to check
|
sql = """
|
||||||
|
SELECT state_group, count(*) FROM event_to_state_groups
|
||||||
|
LEFT JOIN events_to_purge AS ep USING (event_id)
|
||||||
|
WHERE state_group IN (%s) AND ep.event_id IS NULL
|
||||||
|
GROUP BY state_group
|
||||||
|
""" % (",".join("?" for _ in current_search),)
|
||||||
|
txn.execute(sql, list(current_search))
|
||||||
|
|
||||||
is_referenced = self._simple_select_one_onecol_txn(
|
referenced = set(sg for sg, cnt in txn if cnt > 0)
|
||||||
txn,
|
referenced_groups |= referenced
|
||||||
table="event_to_state_groups",
|
|
||||||
keyvalues={"state_group": state_group},
|
|
||||||
retcol="event_id",
|
|
||||||
allow_none=True,
|
|
||||||
)
|
|
||||||
if is_referenced:
|
|
||||||
# A descendant is referenced by event_to_state_groups, so
|
|
||||||
# original state group is referenced.
|
|
||||||
return True
|
|
||||||
|
|
||||||
state_groups_searched.add(state_group)
|
# We don't continue iterating up the state group graphs for state
|
||||||
|
# groups that are referenced.
|
||||||
|
current_search -= referenced
|
||||||
|
|
||||||
# Find all children of current state group and add to search
|
rows = self._simple_select_many_txn(
|
||||||
references = self._simple_select_onecol_txn(
|
|
||||||
txn,
|
txn,
|
||||||
table="state_group_edges",
|
table="state_group_edges",
|
||||||
keyvalues={"prev_state_group": state_group},
|
column="prev_state_group",
|
||||||
retcol="state_group",
|
iterable=current_search,
|
||||||
|
keyvalues={},
|
||||||
|
retcols=("prev_state_group", "state_group",),
|
||||||
)
|
)
|
||||||
state_groups_to_search.extend(references)
|
|
||||||
|
|
||||||
# Lets be paranoid and check for cycles
|
next_to_search.update(row["state_group"] for row in rows)
|
||||||
if state_groups_searched.intersection(references):
|
# We don't bother re-handling groups we've already seen
|
||||||
raise Exception("State group %s has cyclic dependency", state_group)
|
next_to_search -= state_groups_seen
|
||||||
|
state_groups_seen |= next_to_search
|
||||||
|
|
||||||
return False
|
for row in rows:
|
||||||
|
# Note: Each state group can have at most one prev group
|
||||||
|
graph[row["state_group"]] = row["prev_state_group"]
|
||||||
|
|
||||||
|
to_delete = state_groups_seen - referenced_groups
|
||||||
|
|
||||||
|
to_dedelta = set()
|
||||||
|
for sg in referenced_groups:
|
||||||
|
prev_sg = graph.get(sg)
|
||||||
|
if prev_sg and prev_sg in to_delete:
|
||||||
|
to_dedelta.add(sg)
|
||||||
|
|
||||||
|
return to_delete, to_dedelta
|
||||||
|
|
||||||
|
|
||||||
class StateStore(StateGroupWorkerStore, BackgroundUpdateStore):
|
class StateStore(StateGroupWorkerStore, BackgroundUpdateStore):
|
||||||
|
Loading…
Reference in New Issue
Block a user