Add bulk push rule evaluator which actually still evaluates rules one by one, but does far fewer db queries to fetch the rules

synapse/push/action_generator.py

@@ -15,9 +15,7 @@
 
 from twisted.internet import defer
 
-from synapse.types import UserID
+import bulk_push_rule_evaluator
-
-import push_rule_evaluator
 
 import logging
 
@@ -39,17 +37,13 @@ class ActionGenerator:
     def handle_event(self, event):
         users = yield self.store.get_users_in_room(event['room_id'])
 
-        for uid in users:
+        bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id(
-            if not self.hs.is_mine(UserID.from_string(uid)):
+            event['room_id'], self.hs, self.store
-                continue
+        )
 
-            evaluator = yield push_rule_evaluator.\
+        actions_by_user = bulk_evaluator.action_for_event_by_user(event)
-                evaluator_for_user_name_and_profile_tag(
+
-                    uid, None, event['room_id'], self.store
+        for uid,actions in actions_by_user.items():
-                )
+            self.store.set_actions_for_event(
-            actions = yield evaluator.actions_for_event(event)
+                event, uid, None, actions
-            logger.info("actions for user %s: %s", uid, actions)
+            )
-            if len(actions):
-                self.store.set_actions_for_event(
-                    event, uid, None, actions
-                )

synapse/push/bulk_push_rule_evaluator.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import simplejson as json
+
+from twisted.internet import defer
+
+from synapse.types import UserID
+
+import baserules
+from push_rule_evaluator import PushRuleEvaluator
+
+logger = logging.getLogger(__name__)
+
+
+def decode_rule_json(rule):
+    rule['conditions'] = json.loads(rule['conditions'])
+    rule['actions'] = json.loads(rule['actions'])
+    return rule
+
+
+@defer.inlineCallbacks
+def evaluator_for_room_id(room_id, hs, store):
+    users = yield store.get_users_in_room(room_id)
+    rules_by_user = yield store.bulk_get_push_rules(users)
+    rules_by_user = {
+        uid: baserules.list_with_base_rules(
+            [decode_rule_json(rule_list) for rule_list in rules_by_user[uid]]
+            if uid in rules_by_user else [],
+            UserID.from_string(uid)
+        )
+        for uid in users
+    }
+    member_events = yield store.get_current_state(
+        room_id=room_id,
+        event_type='m.room.member',
+    )
+    display_names = {}
+    for ev in member_events:
+        if ev.content.get("displayname"):
+            display_names[ev.state_key] = ev.content.get("displayname")
+
+    defer.returnValue(BulkPushRuleEvaluator(
+            room_id, rules_by_user, display_names, users
+    ))
+
+
+class BulkPushRuleEvaluator:
+    def __init__(self, room_id, rules_by_user, display_names, users_in_room):
+        self.room_id = room_id
+        self.rules_by_user = rules_by_user
+        self.display_names = display_names
+        self.users_in_room = users_in_room
+
+    def action_for_event_by_user(self, event):
+        actions_by_user = {}
+
+        for uid, rules in self.rules_by_user.items():
+            display_name = None
+            if uid in self.display_names:
+                display_name = self.display_names[uid]
+
+            for rule in rules:
+                if 'enabled' in rule and not rule['enabled']:
+                    continue
+
+                # XXX: profile tags
+                if BulkPushRuleEvaluator.event_matches_rule(
+                    event, rule,
+                    display_name, len(self.users_in_room), None
+                ):
+                    actions = [x for x in rule['actions'] if x != 'dont_notify']
+                    if len(actions) > 0:
+                        actions_by_user[uid] = actions
+                    break
+        return actions_by_user
+
+    @staticmethod
+    def event_matches_rule(event, rule,
+                           display_name, room_member_count, profile_tag):
+        matches = True
+        for cond in rule['conditions']:
+            matches &= PushRuleEvaluator._event_fulfills_condition(
+                event, cond, display_name, room_member_count, profile_tag
+            )
+        return matches

synapse/push/push_rule_evaluator.py

@@ -113,7 +113,8 @@ class PushRuleEvaluator:
             for c in conditions:
                 matches &= self._event_fulfills_condition(
                     ev, c, display_name=my_display_name,
-                    room_member_count=room_member_count
+                    room_member_count=room_member_count,
+                    profile_tag=self.profile_tag
                 )
             logger.debug(
                 "Rule %s %s",
@@ -156,16 +157,18 @@ class PushRuleEvaluator:
                                           re.sub(r'\\\-', '-', x.group(2)))), r)
         return r
 
-    def _event_fulfills_condition(self, ev, condition, display_name, room_member_count):
+    @staticmethod
+    def _event_fulfills_condition(ev, condition,
+                                  display_name, room_member_count, profile_tag):
         if condition['kind'] == 'event_match':
             if 'pattern' not in condition:
                 logger.warn("event_match condition with no pattern")
                 return False
             # XXX: optimisation: cache our pattern regexps
             if condition['key'] == 'content.body':
-                r = r'\b%s\b' % self._glob_to_regexp(condition['pattern'])
+                r = r'\b%s\b' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
             else:
-                r = r'^%s$' % self._glob_to_regexp(condition['pattern'])
+                r = r'^%s$' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
             val = _value_for_dotted_key(condition['key'], ev)
             if val is None:
                 return False
@@ -174,7 +177,7 @@ class PushRuleEvaluator:
         elif condition['kind'] == 'device':
             if 'profile_tag' not in condition:
                 return True
-            return condition['profile_tag'] == self.profile_tag
+            return condition['profile_tag'] == profile_tag
 
         elif condition['kind'] == 'contains_display_name':
             # This is special because display names can be different

synapse/storage/push_rule.py

@@ -55,6 +55,47 @@ class PushRuleStore(SQLBaseStore):
             r['rule_id']: False if r['enabled'] == 0 else True for r in results
         })
 
+    @defer.inlineCallbacks
+    def bulk_get_push_rules(self, user_ids):
+        batch_size = 100
+
+        def f(txn, user_ids_to_fetch):
+            sql = (
+                "SELECT " +
+                ",".join(map(lambda x: "pr."+x, PushRuleTable.fields)) +
+                " FROM " + PushRuleTable.table_name + " pr " +
+                " LEFT JOIN " + PushRuleEnableTable.table_name + " pre " +
+                " ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " +
+                " WHERE pr.user_name " +
+                " IN (" + ",".join(["?" for _ in user_ids_to_fetch]) + ")"
+                " AND (pre.enabled is null or pre.enabled = 1)"
+                " ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC"
+            )
+            txn.execute(sql, user_ids_to_fetch)
+            return txn.fetchall()
+
+        results = {}
+
+        batch_start = 0
+        while batch_start < len(user_ids):
+            batch_end = max(len(user_ids), batch_size)
+            batch_user_ids = user_ids[batch_start:batch_end]
+            batch_start = batch_end
+
+            rows = yield self.runInteraction(
+                "bulk_get_push_rules", f, batch_user_ids
+            )
+
+            for r in rows:
+                rawdict = {
+                    PushRuleTable.fields[i]: r[i] for i in range(len(r))
+                }
+
+                if rawdict['user_name'] not in results:
+                    results[rawdict['user_name']] = []
+                results[rawdict['user_name']].append(rawdict)
+        defer.returnValue(results)
+
     @defer.inlineCallbacks
     def add_push_rule(self, before, after, **kwargs):
         vals = kwargs