mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2024-09-09 13:31:55 +00:00
Land improved room list based on room stats (#6019)
Use room_stats and room_state for room directory search
This commit is contained in:
parent
864f144543
commit
a5166e4d5f
1
changelog.d/6019.misc
Normal file
1
changelog.d/6019.misc
Normal file
@ -0,0 +1 @@
|
||||
Improve performance of the public room list directory.
|
@ -765,6 +765,10 @@ class PublicRoomList(BaseFederationServlet):
|
||||
else:
|
||||
network_tuple = ThirdPartyInstanceID(None, None)
|
||||
|
||||
if limit == 0:
|
||||
# zero is a special value which corresponds to no limit.
|
||||
limit = None
|
||||
|
||||
data = await maybeDeferred(
|
||||
self.handler.get_local_public_room_list,
|
||||
limit,
|
||||
@ -800,6 +804,10 @@ class PublicRoomList(BaseFederationServlet):
|
||||
if search_filter is None:
|
||||
logger.warning("Nonefilter")
|
||||
|
||||
if limit == 0:
|
||||
# zero is a special value which corresponds to no limit.
|
||||
limit = None
|
||||
|
||||
data = await self.handler.get_local_public_room_list(
|
||||
limit=limit,
|
||||
since_token=since_token,
|
||||
|
@ -16,8 +16,7 @@
|
||||
import logging
|
||||
from collections import namedtuple
|
||||
|
||||
from six import PY3, iteritems
|
||||
from six.moves import range
|
||||
from six import iteritems
|
||||
|
||||
import msgpack
|
||||
from unpaddedbase64 import decode_base64, encode_base64
|
||||
@ -27,7 +26,6 @@ from twisted.internet import defer
|
||||
from synapse.api.constants import EventTypes, JoinRules
|
||||
from synapse.api.errors import Codes, HttpResponseException
|
||||
from synapse.types import ThirdPartyInstanceID
|
||||
from synapse.util.async_helpers import concurrently_execute
|
||||
from synapse.util.caches.descriptors import cachedInlineCallbacks
|
||||
from synapse.util.caches.response_cache import ResponseCache
|
||||
|
||||
@ -37,7 +35,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
REMOTE_ROOM_LIST_POLL_INTERVAL = 60 * 1000
|
||||
|
||||
|
||||
# This is used to indicate we should only return rooms published to the main list.
|
||||
EMPTY_THIRD_PARTY_ID = ThirdPartyInstanceID(None, None)
|
||||
|
||||
@ -72,6 +69,8 @@ class RoomListHandler(BaseHandler):
|
||||
This can be (None, None) to indicate the main list, or a particular
|
||||
appservice and network id to use an appservice specific one.
|
||||
Setting to None returns all public rooms across all lists.
|
||||
from_federation (bool): true iff the request comes from the federation
|
||||
API
|
||||
"""
|
||||
if not self.enable_room_list_search:
|
||||
return defer.succeed({"chunk": [], "total_room_count_estimate": 0})
|
||||
@ -133,239 +132,109 @@ class RoomListHandler(BaseHandler):
|
||||
from_federation (bool): Whether this request originated from a
|
||||
federating server or a client. Used for room filtering.
|
||||
timeout (int|None): Amount of seconds to wait for a response before
|
||||
timing out.
|
||||
timing out. TODO
|
||||
"""
|
||||
if since_token and since_token != "END":
|
||||
since_token = RoomListNextBatch.from_token(since_token)
|
||||
else:
|
||||
since_token = None
|
||||
|
||||
rooms_to_order_value = {}
|
||||
rooms_to_num_joined = {}
|
||||
# Pagination tokens work by storing the room ID sent in the last batch,
|
||||
# plus the direction (forwards or backwards). Next batch tokens always
|
||||
# go forwards, prev batch tokens always go backwards.
|
||||
|
||||
newly_visible = []
|
||||
newly_unpublished = []
|
||||
if since_token:
|
||||
stream_token = since_token.stream_ordering
|
||||
current_public_id = yield self.store.get_current_public_room_stream_id()
|
||||
public_room_stream_id = since_token.public_room_stream_id
|
||||
newly_visible, newly_unpublished = yield self.store.get_public_room_changes(
|
||||
public_room_stream_id, current_public_id, network_tuple=network_tuple
|
||||
)
|
||||
else:
|
||||
stream_token = yield self.store.get_room_max_stream_ordering()
|
||||
public_room_stream_id = yield self.store.get_current_public_room_stream_id()
|
||||
batch_token = RoomListNextBatch.from_token(since_token)
|
||||
|
||||
room_ids = yield self.store.get_public_room_ids_at_stream_id(
|
||||
public_room_stream_id, network_tuple=network_tuple
|
||||
last_room_id = batch_token.last_room_id
|
||||
forwards = batch_token.direction_is_forward
|
||||
else:
|
||||
batch_token = None
|
||||
|
||||
last_room_id = None
|
||||
forwards = True
|
||||
|
||||
# we request one more than wanted to see if there are more pages to come
|
||||
probing_limit = limit + 1 if limit is not None else None
|
||||
|
||||
results = yield self.store.get_largest_public_rooms(
|
||||
network_tuple,
|
||||
search_filter,
|
||||
probing_limit,
|
||||
last_room_id=last_room_id,
|
||||
forwards=forwards,
|
||||
ignore_non_federatable=from_federation,
|
||||
)
|
||||
|
||||
# We want to return rooms in a particular order: the number of joined
|
||||
# users. We then arbitrarily use the room_id as a tie breaker.
|
||||
def build_room_entry(room):
|
||||
entry = {
|
||||
"room_id": room["room_id"],
|
||||
"name": room["name"],
|
||||
"topic": room["topic"],
|
||||
"canonical_alias": room["canonical_alias"],
|
||||
"num_joined_members": room["joined_members"],
|
||||
"avatar_url": room["avatar"],
|
||||
"world_readable": room["history_visibility"] == "world_readable",
|
||||
"guest_can_join": room["guest_access"] == "can_join",
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_order_for_room(room_id):
|
||||
# Most of the rooms won't have changed between the since token and
|
||||
# now (especially if the since token is "now"). So, we can ask what
|
||||
# the current users are in a room (that will hit a cache) and then
|
||||
# check if the room has changed since the since token. (We have to
|
||||
# do it in that order to avoid races).
|
||||
# If things have changed then fall back to getting the current state
|
||||
# at the since token.
|
||||
joined_users = yield self.store.get_users_in_room(room_id)
|
||||
if self.store.has_room_changed_since(room_id, stream_token):
|
||||
latest_event_ids = yield self.store.get_forward_extremeties_for_room(
|
||||
room_id, stream_token
|
||||
)
|
||||
# Filter out Nones – rather omit the field altogether
|
||||
return {k: v for k, v in entry.items() if v is not None}
|
||||
|
||||
if not latest_event_ids:
|
||||
return
|
||||
results = [build_room_entry(r) for r in results]
|
||||
|
||||
joined_users = yield self.state_handler.get_current_users_in_room(
|
||||
room_id, latest_event_ids
|
||||
)
|
||||
response = {}
|
||||
num_results = len(results)
|
||||
if limit is not None:
|
||||
more_to_come = num_results == probing_limit
|
||||
|
||||
num_joined_users = len(joined_users)
|
||||
rooms_to_num_joined[room_id] = num_joined_users
|
||||
# Depending on direction we trim either the front or back.
|
||||
if forwards:
|
||||
results = results[:limit]
|
||||
else:
|
||||
results = results[-limit:]
|
||||
else:
|
||||
more_to_come = False
|
||||
|
||||
if num_joined_users == 0:
|
||||
return
|
||||
if num_results > 0:
|
||||
final_room_id = results[-1]["room_id"]
|
||||
initial_room_id = results[0]["room_id"]
|
||||
|
||||
# We want larger rooms to be first, hence negating num_joined_users
|
||||
rooms_to_order_value[room_id] = (-num_joined_users, room_id)
|
||||
if forwards:
|
||||
if batch_token:
|
||||
# If there was a token given then we assume that there
|
||||
# must be previous results.
|
||||
response["prev_batch"] = RoomListNextBatch(
|
||||
last_room_id=initial_room_id, direction_is_forward=False
|
||||
).to_token()
|
||||
|
||||
logger.info(
|
||||
"Getting ordering for %i rooms since %s", len(room_ids), stream_token
|
||||
if more_to_come:
|
||||
response["next_batch"] = RoomListNextBatch(
|
||||
last_room_id=final_room_id, direction_is_forward=True
|
||||
).to_token()
|
||||
else:
|
||||
if batch_token:
|
||||
response["next_batch"] = RoomListNextBatch(
|
||||
last_room_id=final_room_id, direction_is_forward=True
|
||||
).to_token()
|
||||
|
||||
if more_to_come:
|
||||
response["prev_batch"] = RoomListNextBatch(
|
||||
last_room_id=initial_room_id, direction_is_forward=False
|
||||
).to_token()
|
||||
|
||||
for room in results:
|
||||
# populate search result entries with additional fields, namely
|
||||
# 'aliases'
|
||||
room_id = room["room_id"]
|
||||
|
||||
aliases = yield self.store.get_aliases_for_room(room_id)
|
||||
if aliases:
|
||||
room["aliases"] = aliases
|
||||
|
||||
response["chunk"] = results
|
||||
|
||||
response["total_room_count_estimate"] = yield self.store.count_public_rooms(
|
||||
network_tuple, ignore_non_federatable=from_federation
|
||||
)
|
||||
yield concurrently_execute(get_order_for_room, room_ids, 10)
|
||||
|
||||
sorted_entries = sorted(rooms_to_order_value.items(), key=lambda e: e[1])
|
||||
sorted_rooms = [room_id for room_id, _ in sorted_entries]
|
||||
|
||||
# `sorted_rooms` should now be a list of all public room ids that is
|
||||
# stable across pagination. Therefore, we can use indices into this
|
||||
# list as our pagination tokens.
|
||||
|
||||
# Filter out rooms that we don't want to return
|
||||
rooms_to_scan = [
|
||||
r
|
||||
for r in sorted_rooms
|
||||
if r not in newly_unpublished and rooms_to_num_joined[r] > 0
|
||||
]
|
||||
|
||||
total_room_count = len(rooms_to_scan)
|
||||
|
||||
if since_token:
|
||||
# Filter out rooms we've already returned previously
|
||||
# `since_token.current_limit` is the index of the last room we
|
||||
# sent down, so we exclude it and everything before/after it.
|
||||
if since_token.direction_is_forward:
|
||||
rooms_to_scan = rooms_to_scan[since_token.current_limit + 1 :]
|
||||
else:
|
||||
rooms_to_scan = rooms_to_scan[: since_token.current_limit]
|
||||
rooms_to_scan.reverse()
|
||||
|
||||
logger.info("After sorting and filtering, %i rooms remain", len(rooms_to_scan))
|
||||
|
||||
# _append_room_entry_to_chunk will append to chunk but will stop if
|
||||
# len(chunk) > limit
|
||||
#
|
||||
# Normally we will generate enough results on the first iteration here,
|
||||
# but if there is a search filter, _append_room_entry_to_chunk may
|
||||
# filter some results out, in which case we loop again.
|
||||
#
|
||||
# We don't want to scan over the entire range either as that
|
||||
# would potentially waste a lot of work.
|
||||
#
|
||||
# XXX if there is no limit, we may end up DoSing the server with
|
||||
# calls to get_current_state_ids for every single room on the
|
||||
# server. Surely we should cap this somehow?
|
||||
#
|
||||
if limit:
|
||||
step = limit + 1
|
||||
else:
|
||||
# step cannot be zero
|
||||
step = len(rooms_to_scan) if len(rooms_to_scan) != 0 else 1
|
||||
|
||||
chunk = []
|
||||
for i in range(0, len(rooms_to_scan), step):
|
||||
if timeout and self.clock.time() > timeout:
|
||||
raise Exception("Timed out searching room directory")
|
||||
|
||||
batch = rooms_to_scan[i : i + step]
|
||||
logger.info("Processing %i rooms for result", len(batch))
|
||||
yield concurrently_execute(
|
||||
lambda r: self._append_room_entry_to_chunk(
|
||||
r,
|
||||
rooms_to_num_joined[r],
|
||||
chunk,
|
||||
limit,
|
||||
search_filter,
|
||||
from_federation=from_federation,
|
||||
),
|
||||
batch,
|
||||
5,
|
||||
)
|
||||
logger.info("Now %i rooms in result", len(chunk))
|
||||
if len(chunk) >= limit + 1:
|
||||
break
|
||||
|
||||
chunk.sort(key=lambda e: (-e["num_joined_members"], e["room_id"]))
|
||||
|
||||
# Work out the new limit of the batch for pagination, or None if we
|
||||
# know there are no more results that would be returned.
|
||||
# i.e., [since_token.current_limit..new_limit] is the batch of rooms
|
||||
# we've returned (or the reverse if we paginated backwards)
|
||||
# We tried to pull out limit + 1 rooms above, so if we have <= limit
|
||||
# then we know there are no more results to return
|
||||
new_limit = None
|
||||
if chunk and (not limit or len(chunk) > limit):
|
||||
|
||||
if not since_token or since_token.direction_is_forward:
|
||||
if limit:
|
||||
chunk = chunk[:limit]
|
||||
last_room_id = chunk[-1]["room_id"]
|
||||
else:
|
||||
if limit:
|
||||
chunk = chunk[-limit:]
|
||||
last_room_id = chunk[0]["room_id"]
|
||||
|
||||
new_limit = sorted_rooms.index(last_room_id)
|
||||
|
||||
results = {"chunk": chunk, "total_room_count_estimate": total_room_count}
|
||||
|
||||
if since_token:
|
||||
results["new_rooms"] = bool(newly_visible)
|
||||
|
||||
if not since_token or since_token.direction_is_forward:
|
||||
if new_limit is not None:
|
||||
results["next_batch"] = RoomListNextBatch(
|
||||
stream_ordering=stream_token,
|
||||
public_room_stream_id=public_room_stream_id,
|
||||
current_limit=new_limit,
|
||||
direction_is_forward=True,
|
||||
).to_token()
|
||||
|
||||
if since_token:
|
||||
results["prev_batch"] = since_token.copy_and_replace(
|
||||
direction_is_forward=False,
|
||||
current_limit=since_token.current_limit + 1,
|
||||
).to_token()
|
||||
else:
|
||||
if new_limit is not None:
|
||||
results["prev_batch"] = RoomListNextBatch(
|
||||
stream_ordering=stream_token,
|
||||
public_room_stream_id=public_room_stream_id,
|
||||
current_limit=new_limit,
|
||||
direction_is_forward=False,
|
||||
).to_token()
|
||||
|
||||
if since_token:
|
||||
results["next_batch"] = since_token.copy_and_replace(
|
||||
direction_is_forward=True,
|
||||
current_limit=since_token.current_limit - 1,
|
||||
).to_token()
|
||||
|
||||
return results
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _append_room_entry_to_chunk(
|
||||
self,
|
||||
room_id,
|
||||
num_joined_users,
|
||||
chunk,
|
||||
limit,
|
||||
search_filter,
|
||||
from_federation=False,
|
||||
):
|
||||
"""Generate the entry for a room in the public room list and append it
|
||||
to the `chunk` if it matches the search filter
|
||||
|
||||
Args:
|
||||
room_id (str): The ID of the room.
|
||||
num_joined_users (int): The number of joined users in the room.
|
||||
chunk (list)
|
||||
limit (int|None): Maximum amount of rooms to display. Function will
|
||||
return if length of chunk is greater than limit + 1.
|
||||
search_filter (dict|None)
|
||||
from_federation (bool): Whether this request originated from a
|
||||
federating server or a client. Used for room filtering.
|
||||
"""
|
||||
if limit and len(chunk) > limit + 1:
|
||||
# We've already got enough, so lets just drop it.
|
||||
return
|
||||
|
||||
result = yield self.generate_room_entry(room_id, num_joined_users)
|
||||
if not result:
|
||||
return
|
||||
|
||||
if from_federation and not result.get("m.federate", True):
|
||||
# This is a room that other servers cannot join. Do not show them
|
||||
# this room.
|
||||
return
|
||||
|
||||
if _matches_room_entry(result, search_filter):
|
||||
chunk.append(result)
|
||||
return response
|
||||
|
||||
@cachedInlineCallbacks(num_args=1, cache_context=True)
|
||||
def generate_room_entry(
|
||||
@ -580,32 +449,18 @@ class RoomListNextBatch(
|
||||
namedtuple(
|
||||
"RoomListNextBatch",
|
||||
(
|
||||
"stream_ordering", # stream_ordering of the first public room list
|
||||
"public_room_stream_id", # public room stream id for first public room list
|
||||
"current_limit", # The number of previous rooms returned
|
||||
"last_room_id", # The room_id to get rooms after/before
|
||||
"direction_is_forward", # Bool if this is a next_batch, false if prev_batch
|
||||
),
|
||||
)
|
||||
):
|
||||
|
||||
KEY_DICT = {
|
||||
"stream_ordering": "s",
|
||||
"public_room_stream_id": "p",
|
||||
"current_limit": "n",
|
||||
"direction_is_forward": "d",
|
||||
}
|
||||
KEY_DICT = {"last_room_id": "r", "direction_is_forward": "d"}
|
||||
|
||||
REVERSE_KEY_DICT = {v: k for k, v in KEY_DICT.items()}
|
||||
|
||||
@classmethod
|
||||
def from_token(cls, token):
|
||||
if PY3:
|
||||
# The argument raw=False is only available on new versions of
|
||||
# msgpack, and only really needed on Python 3. Gate it behind
|
||||
# a PY3 check to avoid causing issues on Debian-packaged versions.
|
||||
decoded = msgpack.loads(decode_base64(token), raw=False)
|
||||
else:
|
||||
decoded = msgpack.loads(decode_base64(token))
|
||||
decoded = msgpack.loads(decode_base64(token), raw=False)
|
||||
return RoomListNextBatch(
|
||||
**{cls.REVERSE_KEY_DICT[key]: val for key, val in decoded.items()}
|
||||
)
|
||||
|
@ -361,6 +361,10 @@ class PublicRoomListRestServlet(TransactionRestServlet):
|
||||
limit = parse_integer(request, "limit", 0)
|
||||
since_token = parse_string(request, "since", None)
|
||||
|
||||
if limit == 0:
|
||||
# zero is a special value which corresponds to no limit.
|
||||
limit = None
|
||||
|
||||
handler = self.hs.get_room_list_handler()
|
||||
if server:
|
||||
data = yield handler.get_remote_public_room_list(
|
||||
@ -398,6 +402,10 @@ class PublicRoomListRestServlet(TransactionRestServlet):
|
||||
else:
|
||||
network_tuple = ThirdPartyInstanceID.from_string(third_party_instance_id)
|
||||
|
||||
if limit == 0:
|
||||
# zero is a special value which corresponds to no limit.
|
||||
limit = None
|
||||
|
||||
handler = self.hs.get_room_list_handler()
|
||||
if server:
|
||||
data = yield handler.get_remote_public_room_list(
|
||||
|
@ -1,5 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2014-2016 OpenMarket Ltd
|
||||
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@ -63,103 +64,176 @@ class RoomWorkerStore(SQLBaseStore):
|
||||
desc="get_public_room_ids",
|
||||
)
|
||||
|
||||
@cached(num_args=2, max_entries=100)
|
||||
def get_public_room_ids_at_stream_id(self, stream_id, network_tuple):
|
||||
"""Get pulbic rooms for a particular list, or across all lists.
|
||||
def count_public_rooms(self, network_tuple, ignore_non_federatable):
|
||||
"""Counts the number of public rooms as tracked in the room_stats_current
|
||||
and room_stats_state table.
|
||||
|
||||
Args:
|
||||
stream_id (int)
|
||||
network_tuple (ThirdPartyInstanceID): The list to use (None, None)
|
||||
means the main list, None means all lsits.
|
||||
network_tuple (ThirdPartyInstanceID|None)
|
||||
ignore_non_federatable (bool): If true filters out non-federatable rooms
|
||||
"""
|
||||
return self.runInteraction(
|
||||
"get_public_room_ids_at_stream_id",
|
||||
self.get_public_room_ids_at_stream_id_txn,
|
||||
stream_id,
|
||||
network_tuple=network_tuple,
|
||||
)
|
||||
|
||||
def get_public_room_ids_at_stream_id_txn(self, txn, stream_id, network_tuple):
|
||||
return {
|
||||
rm
|
||||
for rm, vis in self.get_published_at_stream_id_txn(
|
||||
txn, stream_id, network_tuple=network_tuple
|
||||
).items()
|
||||
if vis
|
||||
def _count_public_rooms_txn(txn):
|
||||
query_args = []
|
||||
|
||||
if network_tuple:
|
||||
if network_tuple.appservice_id:
|
||||
published_sql = """
|
||||
SELECT room_id from appservice_room_list
|
||||
WHERE appservice_id = ? AND network_id = ?
|
||||
"""
|
||||
query_args.append(network_tuple.appservice_id)
|
||||
query_args.append(network_tuple.network_id)
|
||||
else:
|
||||
published_sql = """
|
||||
SELECT room_id FROM rooms WHERE is_public
|
||||
"""
|
||||
else:
|
||||
published_sql = """
|
||||
SELECT room_id FROM rooms WHERE is_public
|
||||
UNION SELECT room_id from appservice_room_list
|
||||
"""
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
COALESCE(COUNT(*), 0)
|
||||
FROM (
|
||||
%(published_sql)s
|
||||
) published
|
||||
INNER JOIN room_stats_state USING (room_id)
|
||||
INNER JOIN room_stats_current USING (room_id)
|
||||
WHERE
|
||||
(
|
||||
join_rules = 'public' OR history_visibility = 'world_readable'
|
||||
)
|
||||
AND joined_members > 0
|
||||
""" % {
|
||||
"published_sql": published_sql
|
||||
}
|
||||
|
||||
txn.execute(sql, query_args)
|
||||
return txn.fetchone()[0]
|
||||
|
||||
return self.runInteraction("count_public_rooms", _count_public_rooms_txn)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_largest_public_rooms(
|
||||
self,
|
||||
network_tuple,
|
||||
search_filter,
|
||||
limit,
|
||||
last_room_id,
|
||||
forwards,
|
||||
ignore_non_federatable=False,
|
||||
):
|
||||
"""Gets the largest public rooms (where largest is in terms of joined
|
||||
members, as tracked in the statistics table).
|
||||
|
||||
Args:
|
||||
network_tuple (ThirdPartyInstanceID|None):
|
||||
search_filter (dict|None):
|
||||
limit (int|None): Maxmimum number of rows to return, unlimited otherwise.
|
||||
last_room_id (str|None): if present, a room ID which bounds the
|
||||
result set, and is always *excluded* from the result set.
|
||||
forwards (bool): true iff going forwards, going backwards otherwise
|
||||
ignore_non_federatable (bool): If true filters out non-federatable rooms.
|
||||
|
||||
Returns:
|
||||
Rooms in order: biggest number of joined users first.
|
||||
We then arbitrarily use the room_id as a tie breaker.
|
||||
|
||||
"""
|
||||
|
||||
where_clauses = []
|
||||
query_args = []
|
||||
|
||||
if last_room_id:
|
||||
if forwards:
|
||||
where_clauses.append("room_id < ?")
|
||||
else:
|
||||
where_clauses.append("? < room_id")
|
||||
|
||||
query_args += [last_room_id]
|
||||
|
||||
if search_filter and search_filter.get("generic_search_term", None):
|
||||
search_term = "%" + search_filter["generic_search_term"] + "%"
|
||||
|
||||
where_clauses.append(
|
||||
"""
|
||||
(
|
||||
name LIKE ?
|
||||
OR topic LIKE ?
|
||||
OR canonical_alias LIKE ?
|
||||
)
|
||||
"""
|
||||
)
|
||||
query_args += [search_term, search_term, search_term]
|
||||
|
||||
if network_tuple:
|
||||
if network_tuple.appservice_id:
|
||||
published_sql = """
|
||||
SELECT room_id from appservice_room_list
|
||||
WHERE appservice_id = ? AND network_id = ?
|
||||
"""
|
||||
query_args.append(network_tuple.appservice_id)
|
||||
query_args.append(network_tuple.network_id)
|
||||
else:
|
||||
published_sql = """
|
||||
SELECT room_id FROM rooms WHERE is_public
|
||||
"""
|
||||
else:
|
||||
published_sql = """
|
||||
SELECT room_id FROM rooms WHERE is_public
|
||||
UNION SELECT room_id from appservice_room_list
|
||||
"""
|
||||
|
||||
where_clause = ""
|
||||
if where_clauses:
|
||||
where_clause = " AND " + " AND ".join(where_clauses)
|
||||
|
||||
sql = """
|
||||
SELECT
|
||||
room_id, name, topic, canonical_alias, joined_members,
|
||||
avatar, history_visibility, joined_members, guest_access
|
||||
FROM (
|
||||
%(published_sql)s
|
||||
) published
|
||||
INNER JOIN room_stats_state USING (room_id)
|
||||
INNER JOIN room_stats_current USING (room_id)
|
||||
WHERE
|
||||
(
|
||||
join_rules = 'public' OR history_visibility = 'world_readable'
|
||||
)
|
||||
AND joined_members > 0
|
||||
%(where_clause)s
|
||||
ORDER BY joined_members %(dir)s, room_id %(dir)s
|
||||
""" % {
|
||||
"published_sql": published_sql,
|
||||
"where_clause": where_clause,
|
||||
"dir": "DESC" if forwards else "ASC",
|
||||
}
|
||||
|
||||
def get_published_at_stream_id_txn(self, txn, stream_id, network_tuple):
|
||||
if network_tuple:
|
||||
# We want to get from a particular list. No aggregation required.
|
||||
if limit is not None:
|
||||
query_args.append(limit)
|
||||
|
||||
sql = """
|
||||
SELECT room_id, visibility FROM public_room_list_stream
|
||||
INNER JOIN (
|
||||
SELECT room_id, max(stream_id) AS stream_id
|
||||
FROM public_room_list_stream
|
||||
WHERE stream_id <= ? %s
|
||||
GROUP BY room_id
|
||||
) grouped USING (room_id, stream_id)
|
||||
sql += """
|
||||
LIMIT ?
|
||||
"""
|
||||
|
||||
if network_tuple.appservice_id is not None:
|
||||
txn.execute(
|
||||
sql % ("AND appservice_id = ? AND network_id = ?",),
|
||||
(stream_id, network_tuple.appservice_id, network_tuple.network_id),
|
||||
)
|
||||
else:
|
||||
txn.execute(sql % ("AND appservice_id IS NULL",), (stream_id,))
|
||||
return dict(txn)
|
||||
else:
|
||||
# We want to get from all lists, so we need to aggregate the results
|
||||
def _get_largest_public_rooms_txn(txn):
|
||||
txn.execute(sql, query_args)
|
||||
|
||||
logger.info("Executing full list")
|
||||
results = self.cursor_to_dict(txn)
|
||||
|
||||
sql = """
|
||||
SELECT room_id, visibility
|
||||
FROM public_room_list_stream
|
||||
INNER JOIN (
|
||||
SELECT
|
||||
room_id, max(stream_id) AS stream_id, appservice_id,
|
||||
network_id
|
||||
FROM public_room_list_stream
|
||||
WHERE stream_id <= ?
|
||||
GROUP BY room_id, appservice_id, network_id
|
||||
) grouped USING (room_id, stream_id)
|
||||
"""
|
||||
|
||||
txn.execute(sql, (stream_id,))
|
||||
|
||||
results = {}
|
||||
# A room is visible if its visible on any list.
|
||||
for room_id, visibility in txn:
|
||||
results[room_id] = bool(visibility) or results.get(room_id, False)
|
||||
if not forwards:
|
||||
results.reverse()
|
||||
|
||||
return results
|
||||
|
||||
def get_public_room_changes(self, prev_stream_id, new_stream_id, network_tuple):
|
||||
def get_public_room_changes_txn(txn):
|
||||
then_rooms = self.get_public_room_ids_at_stream_id_txn(
|
||||
txn, prev_stream_id, network_tuple
|
||||
)
|
||||
|
||||
now_rooms_dict = self.get_published_at_stream_id_txn(
|
||||
txn, new_stream_id, network_tuple
|
||||
)
|
||||
|
||||
now_rooms_visible = set(rm for rm, vis in now_rooms_dict.items() if vis)
|
||||
now_rooms_not_visible = set(
|
||||
rm for rm, vis in now_rooms_dict.items() if not vis
|
||||
)
|
||||
|
||||
newly_visible = now_rooms_visible - then_rooms
|
||||
newly_unpublished = now_rooms_not_visible & then_rooms
|
||||
|
||||
return newly_visible, newly_unpublished
|
||||
|
||||
return self.runInteraction(
|
||||
"get_public_room_changes", get_public_room_changes_txn
|
||||
ret_val = yield self.runInteraction(
|
||||
"get_largest_public_rooms", _get_largest_public_rooms_txn
|
||||
)
|
||||
defer.returnValue(ret_val)
|
||||
|
||||
@cached(max_entries=10000)
|
||||
def is_room_blocked(self, room_id):
|
||||
|
16
synapse/storage/schema/delta/56/public_room_list_idx.sql
Normal file
16
synapse/storage/schema/delta/56/public_room_list_idx.sql
Normal file
@ -0,0 +1,16 @@
|
||||
/* Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
CREATE INDEX public_room_list_stream_network ON public_room_list_stream (appservice_id, network_id, room_id);
|
@ -1,39 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2018 New Vector Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from synapse.handlers.room_list import RoomListNextBatch
|
||||
|
||||
import tests.unittest
|
||||
import tests.utils
|
||||
|
||||
|
||||
class RoomListTestCase(tests.unittest.TestCase):
|
||||
""" Tests RoomList's RoomListNextBatch. """
|
||||
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def test_check_read_batch_tokens(self):
|
||||
batch_token = RoomListNextBatch(
|
||||
stream_ordering="abcdef",
|
||||
public_room_stream_id="123",
|
||||
current_limit=20,
|
||||
direction_is_forward=True,
|
||||
).to_token()
|
||||
next_batch = RoomListNextBatch.from_token(batch_token)
|
||||
self.assertEquals(next_batch.stream_ordering, "abcdef")
|
||||
self.assertEquals(next_batch.public_room_stream_id, "123")
|
||||
self.assertEquals(next_batch.current_limit, 20)
|
||||
self.assertEquals(next_batch.direction_is_forward, True)
|
Loading…
Reference in New Issue
Block a user