mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2025-05-13 04:42:11 -04:00
Move additional tasks to the background worker, part 3 (#8489)
This commit is contained in:
Patrick Cloke
GitHub
parent
c9c0ad5e20
commit
fe0f4a3591
8 changed files with 276 additions and 282 deletions
|
|
@ -13,15 +13,14 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import attr
|
||||
|
||||
from synapse.metrics.background_process_metrics import run_as_background_process
|
||||
from synapse.metrics.background_process_metrics import wrap_as_background_process
|
||||
from synapse.storage._base import SQLBaseStore, db_to_json
|
||||
from synapse.storage.database import DatabasePool
|
||||
from synapse.storage.database import DatabasePool, LoggingTransaction
|
||||
from synapse.util import json_encoder
|
||||
from synapse.util.caches.descriptors import cached
|
||||
|
||||
|
|
@ -81,8 +80,14 @@ class EventPushActionsWorkerStore(SQLBaseStore):
|
|||
self.find_stream_orderings_looping_call = self._clock.looping_call(
|
||||
self._find_stream_orderings_for_times, 10 * 60 * 1000
|
||||
)
|
||||
|
||||
self._rotate_delay = 3
|
||||
self._rotate_count = 10000
|
||||
self._doing_notif_rotation = False
|
||||
if hs.config.run_background_tasks:
|
||||
self._rotate_notif_loop = self._clock.looping_call(
|
||||
self._rotate_notifs, 30 * 60 * 1000
|
||||
)
|
||||
|
||||
@cached(num_args=3, tree=True, max_entries=5000)
|
||||
async def get_unread_event_push_actions_by_room_for_user(
|
||||
|
|
@ -514,15 +519,14 @@ class EventPushActionsWorkerStore(SQLBaseStore):
|
|||
"Error removing push actions after event persistence failure"
|
||||
)
|
||||
|
||||
def _find_stream_orderings_for_times(self):
|
||||
return run_as_background_process(
|
||||
"event_push_action_stream_orderings",
|
||||
self.db_pool.runInteraction,
|
||||
@wrap_as_background_process("event_push_action_stream_orderings")
|
||||
async def _find_stream_orderings_for_times(self) -> None:
|
||||
await self.db_pool.runInteraction(
|
||||
"_find_stream_orderings_for_times",
|
||||
self._find_stream_orderings_for_times_txn,
|
||||
)
|
||||
|
||||
def _find_stream_orderings_for_times_txn(self, txn):
|
||||
def _find_stream_orderings_for_times_txn(self, txn: LoggingTransaction) -> None:
|
||||
logger.info("Searching for stream ordering 1 month ago")
|
||||
self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn(
|
||||
txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000
|
||||
|
|
@ -652,129 +656,7 @@ class EventPushActionsWorkerStore(SQLBaseStore):
|
|||
)
|
||||
return result[0] if result else None
|
||||
|
||||
|
||||
class EventPushActionsStore(EventPushActionsWorkerStore):
|
||||
EPA_HIGHLIGHT_INDEX = "epa_highlight_index"
|
||||
|
||||
def __init__(self, database: DatabasePool, db_conn, hs):
|
||||
super().__init__(database, db_conn, hs)
|
||||
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
self.EPA_HIGHLIGHT_INDEX,
|
||||
index_name="event_push_actions_u_highlight",
|
||||
table="event_push_actions",
|
||||
columns=["user_id", "stream_ordering"],
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"event_push_actions_highlights_index",
|
||||
index_name="event_push_actions_highlights_index",
|
||||
table="event_push_actions",
|
||||
columns=["user_id", "room_id", "topological_ordering", "stream_ordering"],
|
||||
where_clause="highlight=1",
|
||||
)
|
||||
|
||||
self._doing_notif_rotation = False
|
||||
self._rotate_notif_loop = self._clock.looping_call(
|
||||
self._start_rotate_notifs, 30 * 60 * 1000
|
||||
)
|
||||
|
||||
async def get_push_actions_for_user(
|
||||
self, user_id, before=None, limit=50, only_highlight=False
|
||||
):
|
||||
def f(txn):
|
||||
before_clause = ""
|
||||
if before:
|
||||
before_clause = "AND epa.stream_ordering < ?"
|
||||
args = [user_id, before, limit]
|
||||
else:
|
||||
args = [user_id, limit]
|
||||
|
||||
if only_highlight:
|
||||
if len(before_clause) > 0:
|
||||
before_clause += " "
|
||||
before_clause += "AND epa.highlight = 1"
|
||||
|
||||
# NB. This assumes event_ids are globally unique since
|
||||
# it makes the query easier to index
|
||||
sql = (
|
||||
"SELECT epa.event_id, epa.room_id,"
|
||||
" epa.stream_ordering, epa.topological_ordering,"
|
||||
" epa.actions, epa.highlight, epa.profile_tag, e.received_ts"
|
||||
" FROM event_push_actions epa, events e"
|
||||
" WHERE epa.event_id = e.event_id"
|
||||
" AND epa.user_id = ? %s"
|
||||
" AND epa.notif = 1"
|
||||
" ORDER BY epa.stream_ordering DESC"
|
||||
" LIMIT ?" % (before_clause,)
|
||||
)
|
||||
txn.execute(sql, args)
|
||||
return self.db_pool.cursor_to_dict(txn)
|
||||
|
||||
push_actions = await self.db_pool.runInteraction("get_push_actions_for_user", f)
|
||||
for pa in push_actions:
|
||||
pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"])
|
||||
return push_actions
|
||||
|
||||
async def get_latest_push_action_stream_ordering(self):
|
||||
def f(txn):
|
||||
txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
|
||||
return txn.fetchone()
|
||||
|
||||
result = await self.db_pool.runInteraction(
|
||||
"get_latest_push_action_stream_ordering", f
|
||||
)
|
||||
return result[0] or 0
|
||||
|
||||
def _remove_old_push_actions_before_txn(
|
||||
self, txn, room_id, user_id, stream_ordering
|
||||
):
|
||||
"""
|
||||
Purges old push actions for a user and room before a given
|
||||
stream_ordering.
|
||||
|
||||
We however keep a months worth of highlighted notifications, so that
|
||||
users can still get a list of recent highlights.
|
||||
|
||||
Args:
|
||||
txn: The transcation
|
||||
room_id: Room ID to delete from
|
||||
user_id: user ID to delete for
|
||||
stream_ordering: The lowest stream ordering which will
|
||||
not be deleted.
|
||||
"""
|
||||
txn.call_after(
|
||||
self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
|
||||
(room_id, user_id),
|
||||
)
|
||||
|
||||
# We need to join on the events table to get the received_ts for
|
||||
# event_push_actions and sqlite won't let us use a join in a delete so
|
||||
# we can't just delete where received_ts < x. Furthermore we can
|
||||
# only identify event_push_actions by a tuple of room_id, event_id
|
||||
# we we can't use a subquery.
|
||||
# Instead, we look up the stream ordering for the last event in that
|
||||
# room received before the threshold time and delete event_push_actions
|
||||
# in the room with a stream_odering before that.
|
||||
txn.execute(
|
||||
"DELETE FROM event_push_actions "
|
||||
" WHERE user_id = ? AND room_id = ? AND "
|
||||
" stream_ordering <= ?"
|
||||
" AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)",
|
||||
(user_id, room_id, stream_ordering, self.stream_ordering_month_ago),
|
||||
)
|
||||
|
||||
txn.execute(
|
||||
"""
|
||||
DELETE FROM event_push_summary
|
||||
WHERE room_id = ? AND user_id = ? AND stream_ordering <= ?
|
||||
""",
|
||||
(room_id, user_id, stream_ordering),
|
||||
)
|
||||
|
||||
def _start_rotate_notifs(self):
|
||||
return run_as_background_process("rotate_notifs", self._rotate_notifs)
|
||||
|
||||
@wrap_as_background_process("rotate_notifs")
|
||||
async def _rotate_notifs(self):
|
||||
if self._doing_notif_rotation or self.stream_ordering_day_ago is None:
|
||||
return
|
||||
|
|
@ -954,6 +836,121 @@ class EventPushActionsStore(EventPushActionsWorkerStore):
|
|||
)
|
||||
|
||||
|
||||
class EventPushActionsStore(EventPushActionsWorkerStore):
|
||||
EPA_HIGHLIGHT_INDEX = "epa_highlight_index"
|
||||
|
||||
def __init__(self, database: DatabasePool, db_conn, hs):
|
||||
super().__init__(database, db_conn, hs)
|
||||
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
self.EPA_HIGHLIGHT_INDEX,
|
||||
index_name="event_push_actions_u_highlight",
|
||||
table="event_push_actions",
|
||||
columns=["user_id", "stream_ordering"],
|
||||
)
|
||||
|
||||
self.db_pool.updates.register_background_index_update(
|
||||
"event_push_actions_highlights_index",
|
||||
index_name="event_push_actions_highlights_index",
|
||||
table="event_push_actions",
|
||||
columns=["user_id", "room_id", "topological_ordering", "stream_ordering"],
|
||||
where_clause="highlight=1",
|
||||
)
|
||||
|
||||
async def get_push_actions_for_user(
|
||||
self, user_id, before=None, limit=50, only_highlight=False
|
||||
):
|
||||
def f(txn):
|
||||
before_clause = ""
|
||||
if before:
|
||||
before_clause = "AND epa.stream_ordering < ?"
|
||||
args = [user_id, before, limit]
|
||||
else:
|
||||
args = [user_id, limit]
|
||||
|
||||
if only_highlight:
|
||||
if len(before_clause) > 0:
|
||||
before_clause += " "
|
||||
before_clause += "AND epa.highlight = 1"
|
||||
|
||||
# NB. This assumes event_ids are globally unique since
|
||||
# it makes the query easier to index
|
||||
sql = (
|
||||
"SELECT epa.event_id, epa.room_id,"
|
||||
" epa.stream_ordering, epa.topological_ordering,"
|
||||
" epa.actions, epa.highlight, epa.profile_tag, e.received_ts"
|
||||
" FROM event_push_actions epa, events e"
|
||||
" WHERE epa.event_id = e.event_id"
|
||||
" AND epa.user_id = ? %s"
|
||||
" AND epa.notif = 1"
|
||||
" ORDER BY epa.stream_ordering DESC"
|
||||
" LIMIT ?" % (before_clause,)
|
||||
)
|
||||
txn.execute(sql, args)
|
||||
return self.db_pool.cursor_to_dict(txn)
|
||||
|
||||
push_actions = await self.db_pool.runInteraction("get_push_actions_for_user", f)
|
||||
for pa in push_actions:
|
||||
pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"])
|
||||
return push_actions
|
||||
|
||||
async def get_latest_push_action_stream_ordering(self):
|
||||
def f(txn):
|
||||
txn.execute("SELECT MAX(stream_ordering) FROM event_push_actions")
|
||||
return txn.fetchone()
|
||||
|
||||
result = await self.db_pool.runInteraction(
|
||||
"get_latest_push_action_stream_ordering", f
|
||||
)
|
||||
return result[0] or 0
|
||||
|
||||
def _remove_old_push_actions_before_txn(
|
||||
self, txn, room_id, user_id, stream_ordering
|
||||
):
|
||||
"""
|
||||
Purges old push actions for a user and room before a given
|
||||
stream_ordering.
|
||||
|
||||
We however keep a months worth of highlighted notifications, so that
|
||||
users can still get a list of recent highlights.
|
||||
|
||||
Args:
|
||||
txn: The transcation
|
||||
room_id: Room ID to delete from
|
||||
user_id: user ID to delete for
|
||||
stream_ordering: The lowest stream ordering which will
|
||||
not be deleted.
|
||||
"""
|
||||
txn.call_after(
|
||||
self.get_unread_event_push_actions_by_room_for_user.invalidate_many,
|
||||
(room_id, user_id),
|
||||
)
|
||||
|
||||
# We need to join on the events table to get the received_ts for
|
||||
# event_push_actions and sqlite won't let us use a join in a delete so
|
||||
# we can't just delete where received_ts < x. Furthermore we can
|
||||
# only identify event_push_actions by a tuple of room_id, event_id
|
||||
# we we can't use a subquery.
|
||||
# Instead, we look up the stream ordering for the last event in that
|
||||
# room received before the threshold time and delete event_push_actions
|
||||
# in the room with a stream_odering before that.
|
||||
txn.execute(
|
||||
"DELETE FROM event_push_actions "
|
||||
" WHERE user_id = ? AND room_id = ? AND "
|
||||
" stream_ordering <= ?"
|
||||
" AND ((stream_ordering < ? AND highlight = 1) or highlight = 0)",
|
||||
(user_id, room_id, stream_ordering, self.stream_ordering_month_ago),
|
||||
)
|
||||
|
||||
txn.execute(
|
||||
"""
|
||||
DELETE FROM event_push_summary
|
||||
WHERE room_id = ? AND user_id = ? AND stream_ordering <= ?
|
||||
""",
|
||||
(room_id, user_id, stream_ordering),
|
||||
)
|
||||
|
||||
|
||||
def _action_has_highlight(actions):
|
||||
for action in actions:
|
||||
try:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue