Merge pull request #6340 from matrix-org/babolivier/pagination_query

Fix the SQL SELECT query in _paginate_room_events_txn

changelog.d/6340.feature

@@ -0,0 +1 @@
+Implement label-based filtering on `/sync` and `/messages` ([MSC2326](https://github.com/matrix-org/matrix-doc/pull/2326)).

synapse/storage/data_stores/main/stream.py

@@ -871,14 +871,38 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         args.append(int(limit))
 
-        sql = (
+        select_keywords = "SELECT"
-            "SELECT DISTINCT event_id, topological_ordering, stream_ordering"
+        join_clause = ""
-            " FROM events"
+        if event_filter and event_filter.labels:
-            " LEFT JOIN event_labels USING (event_id, room_id, topological_ordering)"
+            # If we're not filtering on a label, then joining on event_labels will
-            " WHERE outlier = ? AND room_id = ? AND %(bounds)s"
+            # return as many row for a single event as the number of labels it has. To
-            " ORDER BY topological_ordering %(order)s,"
+            # avoid this, only join if we're filtering on at least one label.
-            " stream_ordering %(order)s LIMIT ?"
+            join_clause = """
-        ) % {"bounds": bounds, "order": order}
+                LEFT JOIN event_labels
+                USING (event_id, room_id, topological_ordering)
+            """
+            if len(event_filter.labels) > 1:
+                # Using DISTINCT in this SELECT query is quite expensive, because it
+                # requires the engine to sort on the entire (not limited) result set,
+                # i.e. the entire events table. We only need to use it when we're
+                # filtering on more than two labels, because that's the only scenario
+                # in which we can possibly to get multiple times the same event ID in
+                # the results.
+                select_keywords += "DISTINCT"
+
+        sql = """
+            %(select_keywords)s event_id, topological_ordering, stream_ordering
+            FROM events
+            %(join_clause)s
+            WHERE outlier = ? AND room_id = ? AND %(bounds)s
+            ORDER BY topological_ordering %(order)s,
+            stream_ordering %(order)s LIMIT ?
+        """ % {
+            "select_keywords": select_keywords,
+            "join_clause": join_clause,
+            "bounds": bounds,
+            "order": order,
+        }
 
         txn.execute(sql, args)