mirror of
https://mau.dev/maunium/synapse.git
synced 2024-08-25 03:35:28 +00:00
Merge pull request #808 from matrix-org/dbkr/room_list_spider
Add secondary_directory_servers option to fetch room list from other servers
This commit is contained in:
commit
00c487a8db
@ -29,6 +29,7 @@ class ServerConfig(Config):
|
|||||||
self.user_agent_suffix = config.get("user_agent_suffix")
|
self.user_agent_suffix = config.get("user_agent_suffix")
|
||||||
self.use_frozen_dicts = config.get("use_frozen_dicts", True)
|
self.use_frozen_dicts = config.get("use_frozen_dicts", True)
|
||||||
self.public_baseurl = config.get("public_baseurl")
|
self.public_baseurl = config.get("public_baseurl")
|
||||||
|
self.secondary_directory_servers = config.get("secondary_directory_servers", [])
|
||||||
|
|
||||||
if self.public_baseurl is not None:
|
if self.public_baseurl is not None:
|
||||||
if self.public_baseurl[-1] != '/':
|
if self.public_baseurl[-1] != '/':
|
||||||
@ -156,6 +157,15 @@ class ServerConfig(Config):
|
|||||||
# hard limit.
|
# hard limit.
|
||||||
soft_file_limit: 0
|
soft_file_limit: 0
|
||||||
|
|
||||||
|
# A list of other Home Servers to fetch the public room directory from
|
||||||
|
# and include in the public room directory of this home server
|
||||||
|
# This is a temporary stopgap solution to populate new server with a
|
||||||
|
# list of rooms until there exists a good solution of a decentralized
|
||||||
|
# room directory.
|
||||||
|
# secondary_directory_servers:
|
||||||
|
# - matrix.org
|
||||||
|
# - vector.im
|
||||||
|
|
||||||
# List of ports that Synapse should listen on, their purpose and their
|
# List of ports that Synapse should listen on, their purpose and their
|
||||||
# configuration.
|
# configuration.
|
||||||
listeners:
|
listeners:
|
||||||
|
@ -24,6 +24,7 @@ from synapse.api.errors import (
|
|||||||
CodeMessageException, HttpResponseException, SynapseError,
|
CodeMessageException, HttpResponseException, SynapseError,
|
||||||
)
|
)
|
||||||
from synapse.util import unwrapFirstError
|
from synapse.util import unwrapFirstError
|
||||||
|
from synapse.util.async import concurrently_execute
|
||||||
from synapse.util.caches.expiringcache import ExpiringCache
|
from synapse.util.caches.expiringcache import ExpiringCache
|
||||||
from synapse.util.logutils import log_function
|
from synapse.util.logutils import log_function
|
||||||
from synapse.events import FrozenEvent
|
from synapse.events import FrozenEvent
|
||||||
@ -550,6 +551,25 @@ class FederationClient(FederationBase):
|
|||||||
|
|
||||||
raise RuntimeError("Failed to send to any server.")
|
raise RuntimeError("Failed to send to any server.")
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def get_public_rooms(self, destinations):
|
||||||
|
results_by_server = {}
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _get_result(s):
|
||||||
|
if s == self.server_name:
|
||||||
|
defer.returnValue()
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = yield self.transport_layer.get_public_rooms(s)
|
||||||
|
results_by_server[s] = result
|
||||||
|
except:
|
||||||
|
logger.exception("Error getting room list from server %r", s)
|
||||||
|
|
||||||
|
yield concurrently_execute(_get_result, destinations, 3)
|
||||||
|
|
||||||
|
defer.returnValue(results_by_server)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def query_auth(self, destination, room_id, event_id, local_auth):
|
def query_auth(self, destination, room_id, event_id, local_auth):
|
||||||
"""
|
"""
|
||||||
|
@ -224,6 +224,18 @@ class TransportLayerClient(object):
|
|||||||
|
|
||||||
defer.returnValue(response)
|
defer.returnValue(response)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
@log_function
|
||||||
|
def get_public_rooms(self, remote_server):
|
||||||
|
path = PREFIX + "/publicRooms"
|
||||||
|
|
||||||
|
response = yield self.client.get_json(
|
||||||
|
destination=remote_server,
|
||||||
|
path=path,
|
||||||
|
)
|
||||||
|
|
||||||
|
defer.returnValue(response)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
@log_function
|
@log_function
|
||||||
def exchange_third_party_invite(self, destination, room_id, event_dict):
|
def exchange_third_party_invite(self, destination, room_id, event_dict):
|
||||||
|
@ -134,10 +134,12 @@ class Authenticator(object):
|
|||||||
|
|
||||||
|
|
||||||
class BaseFederationServlet(object):
|
class BaseFederationServlet(object):
|
||||||
def __init__(self, handler, authenticator, ratelimiter, server_name):
|
def __init__(self, handler, authenticator, ratelimiter, server_name,
|
||||||
|
room_list_handler):
|
||||||
self.handler = handler
|
self.handler = handler
|
||||||
self.authenticator = authenticator
|
self.authenticator = authenticator
|
||||||
self.ratelimiter = ratelimiter
|
self.ratelimiter = ratelimiter
|
||||||
|
self.room_list_handler = room_list_handler
|
||||||
|
|
||||||
def _wrap(self, code):
|
def _wrap(self, code):
|
||||||
authenticator = self.authenticator
|
authenticator = self.authenticator
|
||||||
@ -492,6 +494,50 @@ class OpenIdUserInfo(BaseFederationServlet):
|
|||||||
return code
|
return code
|
||||||
|
|
||||||
|
|
||||||
|
class PublicRoomList(BaseFederationServlet):
|
||||||
|
"""
|
||||||
|
Fetch the public room list for this server.
|
||||||
|
|
||||||
|
This API returns information in the same format as /publicRooms on the
|
||||||
|
client API, but will only ever include local public rooms and hence is
|
||||||
|
intended for consumption by other home servers.
|
||||||
|
|
||||||
|
GET /publicRooms HTTP/1.1
|
||||||
|
|
||||||
|
HTTP/1.1 200 OK
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"chunk": [
|
||||||
|
{
|
||||||
|
"aliases": [
|
||||||
|
"#test:localhost"
|
||||||
|
],
|
||||||
|
"guest_can_join": false,
|
||||||
|
"name": "test room",
|
||||||
|
"num_joined_members": 3,
|
||||||
|
"room_id": "!whkydVegtvatLfXmPN:localhost",
|
||||||
|
"world_readable": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"end": "END",
|
||||||
|
"start": "START"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
PATH = "/publicRooms"
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def on_GET(self, request):
|
||||||
|
data = yield self.room_list_handler.get_local_public_room_list()
|
||||||
|
defer.returnValue((200, data))
|
||||||
|
|
||||||
|
# Avoid doing remote HS authorization checks which are done by default by
|
||||||
|
# BaseFederationServlet.
|
||||||
|
def _wrap(self, code):
|
||||||
|
return code
|
||||||
|
|
||||||
|
|
||||||
SERVLET_CLASSES = (
|
SERVLET_CLASSES = (
|
||||||
FederationSendServlet,
|
FederationSendServlet,
|
||||||
FederationPullServlet,
|
FederationPullServlet,
|
||||||
@ -513,6 +559,7 @@ SERVLET_CLASSES = (
|
|||||||
FederationThirdPartyInviteExchangeServlet,
|
FederationThirdPartyInviteExchangeServlet,
|
||||||
On3pidBindServlet,
|
On3pidBindServlet,
|
||||||
OpenIdUserInfo,
|
OpenIdUserInfo,
|
||||||
|
PublicRoomList,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -523,4 +570,5 @@ def register_servlets(hs, resource, authenticator, ratelimiter):
|
|||||||
authenticator=authenticator,
|
authenticator=authenticator,
|
||||||
ratelimiter=ratelimiter,
|
ratelimiter=ratelimiter,
|
||||||
server_name=hs.hostname,
|
server_name=hs.hostname,
|
||||||
|
room_list_handler=hs.get_room_list_handler(),
|
||||||
).register(resource)
|
).register(resource)
|
||||||
|
@ -36,6 +36,8 @@ import string
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
REMOTE_ROOM_LIST_POLL_INTERVAL = 60 * 1000
|
||||||
|
|
||||||
id_server_scheme = "https://"
|
id_server_scheme = "https://"
|
||||||
|
|
||||||
|
|
||||||
@ -344,8 +346,14 @@ class RoomListHandler(BaseHandler):
|
|||||||
def __init__(self, hs):
|
def __init__(self, hs):
|
||||||
super(RoomListHandler, self).__init__(hs)
|
super(RoomListHandler, self).__init__(hs)
|
||||||
self.response_cache = ResponseCache()
|
self.response_cache = ResponseCache()
|
||||||
|
self.remote_list_request_cache = ResponseCache()
|
||||||
|
self.remote_list_cache = {}
|
||||||
|
self.fetch_looping_call = hs.get_clock().looping_call(
|
||||||
|
self.fetch_all_remote_lists, REMOTE_ROOM_LIST_POLL_INTERVAL
|
||||||
|
)
|
||||||
|
self.fetch_all_remote_lists()
|
||||||
|
|
||||||
def get_public_room_list(self):
|
def get_local_public_room_list(self):
|
||||||
result = self.response_cache.get(())
|
result = self.response_cache.get(())
|
||||||
if not result:
|
if not result:
|
||||||
result = self.response_cache.set((), self._get_public_room_list())
|
result = self.response_cache.set((), self._get_public_room_list())
|
||||||
@ -427,6 +435,55 @@ class RoomListHandler(BaseHandler):
|
|||||||
# FIXME (erikj): START is no longer a valid value
|
# FIXME (erikj): START is no longer a valid value
|
||||||
defer.returnValue({"start": "START", "end": "END", "chunk": results})
|
defer.returnValue({"start": "START", "end": "END", "chunk": results})
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def fetch_all_remote_lists(self):
|
||||||
|
deferred = self.hs.get_replication_layer().get_public_rooms(
|
||||||
|
self.hs.config.secondary_directory_servers
|
||||||
|
)
|
||||||
|
self.remote_list_request_cache.set((), deferred)
|
||||||
|
yield deferred
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def get_aggregated_public_room_list(self):
|
||||||
|
"""
|
||||||
|
Get the public room list from this server and the servers
|
||||||
|
specified in the secondary_directory_servers config option.
|
||||||
|
XXX: Pagination...
|
||||||
|
"""
|
||||||
|
# We return the results from out cache which is updated by a looping call,
|
||||||
|
# unless we're missing a cache entry, in which case wait for the result
|
||||||
|
# of the fetch if there's one in progress. If not, omit that server.
|
||||||
|
wait = False
|
||||||
|
for s in self.hs.config.secondary_directory_servers:
|
||||||
|
if s not in self.remote_list_cache:
|
||||||
|
logger.warn("No cached room list from %s: waiting for fetch", s)
|
||||||
|
wait = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if wait and self.remote_list_request_cache.get(()):
|
||||||
|
yield self.remote_list_request_cache.get(())
|
||||||
|
|
||||||
|
public_rooms = yield self.get_local_public_room_list()
|
||||||
|
|
||||||
|
# keep track of which room IDs we've seen so we can de-dup
|
||||||
|
room_ids = set()
|
||||||
|
|
||||||
|
# tag all the ones in our list with our server name.
|
||||||
|
# Also add the them to the de-deping set
|
||||||
|
for room in public_rooms['chunk']:
|
||||||
|
room["server_name"] = self.hs.hostname
|
||||||
|
room_ids.add(room["room_id"])
|
||||||
|
|
||||||
|
# Now add the results from federation
|
||||||
|
for server_name, server_result in self.remote_list_cache.items():
|
||||||
|
for room in server_result["chunk"]:
|
||||||
|
if room["room_id"] not in room_ids:
|
||||||
|
room["server_name"] = server_name
|
||||||
|
public_rooms["chunk"].append(room)
|
||||||
|
room_ids.add(room["room_id"])
|
||||||
|
|
||||||
|
defer.returnValue(public_rooms)
|
||||||
|
|
||||||
|
|
||||||
class RoomContextHandler(BaseHandler):
|
class RoomContextHandler(BaseHandler):
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
|
@ -280,7 +280,8 @@ class PublicRoomListRestServlet(ClientV1RestServlet):
|
|||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def on_GET(self, request):
|
def on_GET(self, request):
|
||||||
handler = self.hs.get_room_list_handler()
|
handler = self.hs.get_room_list_handler()
|
||||||
data = yield handler.get_public_room_list()
|
data = yield handler.get_aggregated_public_room_list()
|
||||||
|
|
||||||
defer.returnValue((200, data))
|
defer.returnValue((200, data))
|
||||||
|
|
||||||
|
|
||||||
|
@ -67,6 +67,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
|
|||||||
version_string="Synapse/tests",
|
version_string="Synapse/tests",
|
||||||
database_engine=create_engine(config.database_config),
|
database_engine=create_engine(config.database_config),
|
||||||
get_db_conn=db_pool.get_db_conn,
|
get_db_conn=db_pool.get_db_conn,
|
||||||
|
room_list_handler=object(),
|
||||||
**kargs
|
**kargs
|
||||||
)
|
)
|
||||||
hs.setup()
|
hs.setup()
|
||||||
@ -75,6 +76,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
|
|||||||
name, db_pool=None, datastore=datastore, config=config,
|
name, db_pool=None, datastore=datastore, config=config,
|
||||||
version_string="Synapse/tests",
|
version_string="Synapse/tests",
|
||||||
database_engine=create_engine(config.database_config),
|
database_engine=create_engine(config.database_config),
|
||||||
|
room_list_handler=object(),
|
||||||
**kargs
|
**kargs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user