Matrix-Mirrors/synapse-product
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse-product.git synced 2024-09-16 20:41:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert pusher databases to async/await. (#8075)

Browse Source
This commit is contained in:
Patrick Cloke 2020-08-14 10:30:16 -04:00 committed by GitHub
parent e8861957d9
commit b069b78bb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 90 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/8075.misc Normal file
View File

@ -0,0 +1 @@
Convert various parts of the codebase to async/await.

9
synapse/rest/client/v1/push_rule.py
View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from synapse.api.errors import ( from synapse.api.errors import (
NotFoundError, NotFoundError,
StoreError, StoreError,
@ -163,7 +162,7 @@ class PushRuleRestServlet(RestServlet):
stream_id, _ = self.store.get_push_rules_stream_token() stream_id, _ = self.store.get_push_rules_stream_token()
self.notifier.on_new_event("push_rules_key", stream_id, users=[user_id]) self.notifier.on_new_event("push_rules_key", stream_id, users=[user_id])
def set_rule_attr(self, user_id, spec, val): async def set_rule_attr(self, user_id, spec, val):
if spec["attr"] == "enabled": if spec["attr"] == "enabled":
if isinstance(val, dict) and "enabled" in val: if isinstance(val, dict) and "enabled" in val:
val = val["enabled"] val = val["enabled"]
@ -173,7 +172,9 @@ class PushRuleRestServlet(RestServlet):
# bools directly, so let's not break them. # bools directly, so let's not break them.
raise SynapseError(400, "Value for 'enabled' must be boolean") raise SynapseError(400, "Value for 'enabled' must be boolean")
namespaced_rule_id = _namespaced_rule_id_from_spec(spec) namespaced_rule_id = _namespaced_rule_id_from_spec(spec)
return self.store.set_push_rule_enabled(user_id, namespaced_rule_id, val) return await self.store.set_push_rule_enabled(
user_id, namespaced_rule_id, val
)
elif spec["attr"] == "actions": elif spec["attr"] == "actions":
actions = val.get("actions") actions = val.get("actions")
_check_actions(actions) _check_actions(actions)
@ -188,7 +189,7 @@ class PushRuleRestServlet(RestServlet):
if namespaced_rule_id not in rule_ids: if namespaced_rule_id not in rule_ids:
raise SynapseError(404, "Unknown rule %r" % (namespaced_rule_id,)) raise SynapseError(404, "Unknown rule %r" % (namespaced_rule_id,))
return self.store.set_push_rule_actions( return await self.store.set_push_rule_actions(
user_id, namespaced_rule_id, actions, is_default_rule user_id, namespaced_rule_id, actions, is_default_rule
) )
else: else:

80
synapse/storage/databases/main/push_rule.py
View File

@ -32,7 +32,7 @@ from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
from synapse.storage.util.id_generators import ChainedIdGenerator from synapse.storage.util.id_generators import ChainedIdGenerator
from synapse.util import json_encoder from synapse.util import json_encoder
from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.caches.stream_change_cache import StreamChangeCache
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -115,9 +115,9 @@ class PushRulesWorkerStore(
""" """
raise NotImplementedError() raise NotImplementedError()
@cachedInlineCallbacks(max_entries=5000) @cached(max_entries=5000)
def get_push_rules_for_user(self, user_id): async def get_push_rules_for_user(self, user_id):
rows = yield self.db_pool.simple_select_list( rows = await self.db_pool.simple_select_list(
table="push_rules", table="push_rules",
keyvalues={"user_name": user_id}, keyvalues={"user_name": user_id},
retcols=( retcols=(
@ -133,17 +133,15 @@ class PushRulesWorkerStore(
rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"]))) rows.sort(key=lambda row: (-int(row["priority_class"]), -int(row["priority"])))
enabled_map = yield self.get_push_rules_enabled_for_user(user_id) enabled_map = await self.get_push_rules_enabled_for_user(user_id)
use_new_defaults = user_id in self._users_new_default_push_rules use_new_defaults = user_id in self._users_new_default_push_rules
rules = _load_rules(rows, enabled_map, use_new_defaults) return _load_rules(rows, enabled_map, use_new_defaults)
return rules @cached(max_entries=5000)
async def get_push_rules_enabled_for_user(self, user_id):
@cachedInlineCallbacks(max_entries=5000) results = await self.db_pool.simple_select_list(
def get_push_rules_enabled_for_user(self, user_id):
results = yield self.db_pool.simple_select_list(
table="push_rules_enable", table="push_rules_enable",
keyvalues={"user_name": user_id}, keyvalues={"user_name": user_id},
retcols=("user_name", "rule_id", "enabled"), retcols=("user_name", "rule_id", "enabled"),
@ -202,14 +200,15 @@ class PushRulesWorkerStore(
return results return results
@defer.inlineCallbacks async def copy_push_rule_from_room_to_room(
def copy_push_rule_from_room_to_room(self, new_room_id, user_id, rule): self, new_room_id: str, user_id: str, rule: dict
) -> None:
"""Copy a single push rule from one room to another for a specific user. """Copy a single push rule from one room to another for a specific user.
Args: Args:
new_room_id (str): ID of the new room. new_room_id: ID of the new room.
user_id (str): ID of user the push rule belongs to. user_id : ID of user the push rule belongs to.
rule (Dict): A push rule. rule: A push rule.
""" """
# Create new rule id # Create new rule id
rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1]) rule_id_scope = "/".join(rule["rule_id"].split("/")[:-1])
@ -221,7 +220,7 @@ class PushRulesWorkerStore(
condition["pattern"] = new_room_id condition["pattern"] = new_room_id
# Add the rule for the new room # Add the rule for the new room
yield self.add_push_rule( await self.add_push_rule(
user_id=user_id, user_id=user_id,
rule_id=new_rule_id, rule_id=new_rule_id,
priority_class=rule["priority_class"], priority_class=rule["priority_class"],
@ -229,20 +228,19 @@ class PushRulesWorkerStore(
actions=rule["actions"], actions=rule["actions"],
) )
@defer.inlineCallbacks async def copy_push_rules_from_room_to_room_for_user(
def copy_push_rules_from_room_to_room_for_user( self, old_room_id: str, new_room_id: str, user_id: str
self, old_room_id, new_room_id, user_id ) -> None:
):
"""Copy all of the push rules from one room to another for a specific """Copy all of the push rules from one room to another for a specific
user. user.
Args: Args:
old_room_id (str): ID of the old room. old_room_id: ID of the old room.
new_room_id (str): ID of the new room. new_room_id: ID of the new room.
user_id (str): ID of user to copy push rules for. user_id: ID of user to copy push rules for.
""" """
# Retrieve push rules for this user # Retrieve push rules for this user
user_push_rules = yield self.get_push_rules_for_user(user_id) user_push_rules = await self.get_push_rules_for_user(user_id)
# Get rules relating to the old room and copy them to the new room # Get rules relating to the old room and copy them to the new room
for rule in user_push_rules: for rule in user_push_rules:
@ -251,7 +249,7 @@ class PushRulesWorkerStore(
(c.get("key") == "room_id" and c.get("pattern") == old_room_id) (c.get("key") == "room_id" and c.get("pattern") == old_room_id)
for c in conditions for c in conditions
): ):
yield self.copy_push_rule_from_room_to_room(new_room_id, user_id, rule) await self.copy_push_rule_from_room_to_room(new_room_id, user_id, rule)
@cachedList( @cachedList(
cached_method_name="get_push_rules_enabled_for_user", cached_method_name="get_push_rules_enabled_for_user",
@ -328,8 +326,7 @@ class PushRulesWorkerStore(
class PushRuleStore(PushRulesWorkerStore): class PushRuleStore(PushRulesWorkerStore):
@defer.inlineCallbacks async def add_push_rule(
def add_push_rule(
self, self,
user_id, user_id,
rule_id, rule_id,
@ -338,13 +335,13 @@ class PushRuleStore(PushRulesWorkerStore):
actions, actions,
before=None, before=None,
after=None, after=None,
): ) -> None:
conditions_json = json_encoder.encode(conditions) conditions_json = json_encoder.encode(conditions)
actions_json = json_encoder.encode(actions) actions_json = json_encoder.encode(actions)
with self._push_rules_stream_id_gen.get_next() as ids: with self._push_rules_stream_id_gen.get_next() as ids:
stream_id, event_stream_ordering = ids stream_id, event_stream_ordering = ids
if before or after: if before or after:
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"_add_push_rule_relative_txn", "_add_push_rule_relative_txn",
self._add_push_rule_relative_txn, self._add_push_rule_relative_txn,
stream_id, stream_id,
@ -358,7 +355,7 @@ class PushRuleStore(PushRulesWorkerStore):
after, after,
) )
else: else:
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"_add_push_rule_highest_priority_txn", "_add_push_rule_highest_priority_txn",
self._add_push_rule_highest_priority_txn, self._add_push_rule_highest_priority_txn,
stream_id, stream_id,
@ -542,16 +539,15 @@ class PushRuleStore(PushRulesWorkerStore):
}, },
) )
@defer.inlineCallbacks async def delete_push_rule(self, user_id: str, rule_id: str) -> None:
def delete_push_rule(self, user_id, rule_id):
""" """
Delete a push rule. Args specify the row to be deleted and can be Delete a push rule. Args specify the row to be deleted and can be
any of the columns in the push_rule table, but below are the any of the columns in the push_rule table, but below are the
standard ones standard ones
Args: Args:
user_id (str): The matrix ID of the push rule owner user_id: The matrix ID of the push rule owner
rule_id (str): The rule_id of the rule to be deleted rule_id: The rule_id of the rule to be deleted
""" """
def delete_push_rule_txn(txn, stream_id, event_stream_ordering): def delete_push_rule_txn(txn, stream_id, event_stream_ordering):
@ -565,18 +561,17 @@ class PushRuleStore(PushRulesWorkerStore):
with self._push_rules_stream_id_gen.get_next() as ids: with self._push_rules_stream_id_gen.get_next() as ids:
stream_id, event_stream_ordering = ids stream_id, event_stream_ordering = ids
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"delete_push_rule", "delete_push_rule",
delete_push_rule_txn, delete_push_rule_txn,
stream_id, stream_id,
event_stream_ordering, event_stream_ordering,
) )
@defer.inlineCallbacks async def set_push_rule_enabled(self, user_id, rule_id, enabled) -> None:
def set_push_rule_enabled(self, user_id, rule_id, enabled):
with self._push_rules_stream_id_gen.get_next() as ids: with self._push_rules_stream_id_gen.get_next() as ids:
stream_id, event_stream_ordering = ids stream_id, event_stream_ordering = ids
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"_set_push_rule_enabled_txn", "_set_push_rule_enabled_txn",
self._set_push_rule_enabled_txn, self._set_push_rule_enabled_txn,
stream_id, stream_id,
@ -607,8 +602,9 @@ class PushRuleStore(PushRulesWorkerStore):
op="ENABLE" if enabled else "DISABLE", op="ENABLE" if enabled else "DISABLE",
) )
@defer.inlineCallbacks async def set_push_rule_actions(
def set_push_rule_actions(self, user_id, rule_id, actions, is_default_rule): self, user_id, rule_id, actions, is_default_rule
) -> None:
actions_json = json_encoder.encode(actions) actions_json = json_encoder.encode(actions)
def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering): def set_push_rule_actions_txn(txn, stream_id, event_stream_ordering):
@ -649,7 +645,7 @@ class PushRuleStore(PushRulesWorkerStore):
with self._push_rules_stream_id_gen.get_next() as ids: with self._push_rules_stream_id_gen.get_next() as ids:
stream_id, event_stream_ordering = ids stream_id, event_stream_ordering = ids
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"set_push_rule_actions", "set_push_rule_actions",
set_push_rule_actions_txn, set_push_rule_actions_txn,
stream_id, stream_id,

95
synapse/storage/databases/main/pusher.py
View File

@ -19,10 +19,8 @@ from typing import Iterable, Iterator, List, Tuple
from canonicaljson import encode_canonical_json from canonicaljson import encode_canonical_json
from twisted.internet import defer
from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage._base import SQLBaseStore, db_to_json
from synapse.util.caches.descriptors import cachedInlineCallbacks, cachedList from synapse.util.caches.descriptors import cached, cachedList
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -34,23 +32,22 @@ class PusherWorkerStore(SQLBaseStore):
Drops any rows whose data cannot be decoded Drops any rows whose data cannot be decoded
""" """
for r in rows: for r in rows:
dataJson = r["data"] data_json = r["data"]
try: try:
r["data"] = db_to_json(dataJson) r["data"] = db_to_json(data_json)
except Exception as e: except Exception as e:
logger.warning( logger.warning(
"Invalid JSON in data for pusher %d: %s, %s", "Invalid JSON in data for pusher %d: %s, %s",
r["id"], r["id"],
dataJson, data_json,
e.args[0], e.args[0],
) )
continue continue
yield r yield r
@defer.inlineCallbacks async def user_has_pusher(self, user_id):
def user_has_pusher(self, user_id): ret = await self.db_pool.simple_select_one_onecol(
ret = yield self.db_pool.simple_select_one_onecol(
"pushers", {"user_name": user_id}, "id", allow_none=True "pushers", {"user_name": user_id}, "id", allow_none=True
) )
return ret is not None return ret is not None
@ -61,9 +58,8 @@ class PusherWorkerStore(SQLBaseStore):
def get_pushers_by_user_id(self, user_id): def get_pushers_by_user_id(self, user_id):
return self.get_pushers_by({"user_name": user_id}) return self.get_pushers_by({"user_name": user_id})
@defer.inlineCallbacks async def get_pushers_by(self, keyvalues):
def get_pushers_by(self, keyvalues): ret = await self.db_pool.simple_select_list(
ret = yield self.db_pool.simple_select_list(
"pushers", "pushers",
keyvalues, keyvalues,
[ [
@ -87,16 +83,14 @@ class PusherWorkerStore(SQLBaseStore):
) )
return self._decode_pushers_rows(ret) return self._decode_pushers_rows(ret)
@defer.inlineCallbacks async def get_all_pushers(self):
def get_all_pushers(self):
def get_pushers(txn): def get_pushers(txn):
txn.execute("SELECT * FROM pushers") txn.execute("SELECT * FROM pushers")
rows = self.db_pool.cursor_to_dict(txn) rows = self.db_pool.cursor_to_dict(txn)
return self._decode_pushers_rows(rows) return self._decode_pushers_rows(rows)
rows = yield self.db_pool.runInteraction("get_all_pushers", get_pushers) return await self.db_pool.runInteraction("get_all_pushers", get_pushers)
return rows
async def get_all_updated_pushers_rows( async def get_all_updated_pushers_rows(
self, instance_name: str, last_id: int, current_id: int, limit: int self, instance_name: str, last_id: int, current_id: int, limit: int
@ -164,8 +158,8 @@ class PusherWorkerStore(SQLBaseStore):
"get_all_updated_pushers_rows", get_all_updated_pushers_rows_txn "get_all_updated_pushers_rows", get_all_updated_pushers_rows_txn
) )
@cachedInlineCallbacks(num_args=1, max_entries=15000) @cached(num_args=1, max_entries=15000)
def get_if_user_has_pusher(self, user_id): async def get_if_user_has_pusher(self, user_id):
# This only exists for the cachedList decorator # This only exists for the cachedList decorator
raise NotImplementedError() raise NotImplementedError()
@ -186,34 +180,38 @@ class PusherWorkerStore(SQLBaseStore):
return result return result
@defer.inlineCallbacks async def update_pusher_last_stream_ordering(
def update_pusher_last_stream_ordering(
self, app_id, pushkey, user_id, last_stream_ordering self, app_id, pushkey, user_id, last_stream_ordering
): ) -> None:
yield self.db_pool.simple_update_one( await self.db_pool.simple_update_one(
"pushers", "pushers",
{"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, {"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
{"last_stream_ordering": last_stream_ordering}, {"last_stream_ordering": last_stream_ordering},
desc="update_pusher_last_stream_ordering", desc="update_pusher_last_stream_ordering",
) )
@defer.inlineCallbacks async def update_pusher_last_stream_ordering_and_success(
def update_pusher_last_stream_ordering_and_success( self,
self, app_id, pushkey, user_id, last_stream_ordering, last_success app_id: str,
): pushkey: str,
user_id: str,
last_stream_ordering: int,
last_success: int,
) -> bool:
"""Update the last stream ordering position we've processed up to for """Update the last stream ordering position we've processed up to for
the given pusher. the given pusher.
Args: Args:
app_id (str) app_id
pushkey (str) pushkey
last_stream_ordering (int) user_id
last_success (int) last_stream_ordering
last_success
Returns: Returns:
Deferred[bool]: True if the pusher still exists; False if it has been deleted. True if the pusher still exists; False if it has been deleted.
""" """
updated = yield self.db_pool.simple_update( updated = await self.db_pool.simple_update(
table="pushers", table="pushers",
keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
updatevalues={ updatevalues={
@ -225,18 +223,18 @@ class PusherWorkerStore(SQLBaseStore):
return bool(updated) return bool(updated)
@defer.inlineCallbacks async def update_pusher_failing_since(
def update_pusher_failing_since(self, app_id, pushkey, user_id, failing_since): self, app_id, pushkey, user_id, failing_since
yield self.db_pool.simple_update( ) -> None:
await self.db_pool.simple_update(
table="pushers", table="pushers",
keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
updatevalues={"failing_since": failing_since}, updatevalues={"failing_since": failing_since},
desc="update_pusher_failing_since", desc="update_pusher_failing_since",
) )
@defer.inlineCallbacks async def get_throttle_params_by_room(self, pusher_id):
def get_throttle_params_by_room(self, pusher_id): res = await self.db_pool.simple_select_list(
res = yield self.db_pool.simple_select_list(
"pusher_throttle", "pusher_throttle",
{"pusher": pusher_id}, {"pusher": pusher_id},
["room_id", "last_sent_ts", "throttle_ms"], ["room_id", "last_sent_ts", "throttle_ms"],
@ -252,11 +250,10 @@ class PusherWorkerStore(SQLBaseStore):
return params_by_room return params_by_room
@defer.inlineCallbacks async def set_throttle_params(self, pusher_id, room_id, params) -> None:
def set_throttle_params(self, pusher_id, room_id, params):
# no need to lock because `pusher_throttle` has a primary key on # no need to lock because `pusher_throttle` has a primary key on
# (pusher, room_id) so simple_upsert will retry # (pusher, room_id) so simple_upsert will retry
yield self.db_pool.simple_upsert( await self.db_pool.simple_upsert(
"pusher_throttle", "pusher_throttle",
{"pusher": pusher_id, "room_id": room_id}, {"pusher": pusher_id, "room_id": room_id},
params, params,
@ -269,8 +266,7 @@ class PusherStore(PusherWorkerStore):
def get_pushers_stream_token(self): def get_pushers_stream_token(self):
return self._pushers_id_gen.get_current_token() return self._pushers_id_gen.get_current_token()
@defer.inlineCallbacks async def add_pusher(
def add_pusher(
self, self,
user_id, user_id,
access_token, access_token,
@ -284,11 +280,11 @@ class PusherStore(PusherWorkerStore):
data, data,
last_stream_ordering, last_stream_ordering,
profile_tag="", profile_tag="",
): ) -> None:
with self._pushers_id_gen.get_next() as stream_id: with self._pushers_id_gen.get_next() as stream_id:
# no need to lock because `pushers` has a unique key on # no need to lock because `pushers` has a unique key on
# (app_id, pushkey, user_name) so simple_upsert will retry # (app_id, pushkey, user_name) so simple_upsert will retry
yield self.db_pool.simple_upsert( await self.db_pool.simple_upsert(
table="pushers", table="pushers",
keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id}, keyvalues={"app_id": app_id, "pushkey": pushkey, "user_name": user_id},
values={ values={
@ -313,15 +309,16 @@ class PusherStore(PusherWorkerStore):
if user_has_pusher is not True: if user_has_pusher is not True:
# invalidate, since we the user might not have had a pusher before # invalidate, since we the user might not have had a pusher before
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"add_pusher", "add_pusher",
self._invalidate_cache_and_stream, self._invalidate_cache_and_stream,
self.get_if_user_has_pusher, self.get_if_user_has_pusher,
(user_id,), (user_id,),
) )
@defer.inlineCallbacks async def delete_pusher_by_app_id_pushkey_user_id(
def delete_pusher_by_app_id_pushkey_user_id(self, app_id, pushkey, user_id): self, app_id, pushkey, user_id
) -> None:
def delete_pusher_txn(txn, stream_id): def delete_pusher_txn(txn, stream_id):
self._invalidate_cache_and_stream( self._invalidate_cache_and_stream(
txn, self.get_if_user_has_pusher, (user_id,) txn, self.get_if_user_has_pusher, (user_id,)
@ -348,6 +345,6 @@ class PusherStore(PusherWorkerStore):
) )
with self._pushers_id_gen.get_next() as stream_id: with self._pushers_id_gen.get_next() as stream_id:
yield self.db_pool.runInteraction( await self.db_pool.runInteraction(
"delete_pusher", delete_pusher_txn, stream_id "delete_pusher", delete_pusher_txn, stream_id
) )