Matrix-Mirrors/synapse-product
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse-product.git synced 2024-10-01 08:25:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

Batching in the user directory import (#4900)

Browse Source
This commit is contained in:
Amber Brown 2019-03-21 03:06:36 +11:00 committed by GitHub
parent cdb8036161
commit 4d53017432
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/4900.feature Normal file
View File

@ -0,0 +1 @@
The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

13
synapse/storage/user_directory.py
View File

@ -32,6 +32,11 @@ TEMP_TABLE = "_temp_populate_user_directory"
class UserDirectoryStore(BackgroundUpdateStore): class UserDirectoryStore(BackgroundUpdateStore):
# How many records do we calculate before sending it to
# add_users_who_share_private_rooms?
SHARE_PRIVATE_WORKING_SET = 500
def __init__(self, db_conn, hs): def __init__(self, db_conn, hs):
super(UserDirectoryStore, self).__init__(db_conn, hs) super(UserDirectoryStore, self).__init__(db_conn, hs)
@ -218,6 +223,14 @@ class UserDirectoryStore(BackgroundUpdateStore):
user_set = (user_id, other_user_id) user_set = (user_id, other_user_id)
to_insert.add(user_set) to_insert.add(user_set)
# If it gets too big, stop and write to the database
# to prevent storing too much in RAM.
if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET:
yield self.add_users_who_share_private_room(
room_id, to_insert
)
to_insert.clear()
if to_insert: if to_insert:
yield self.add_users_who_share_private_room(room_id, to_insert) yield self.add_users_who_share_private_room(room_id, to_insert)
to_insert.clear() to_insert.clear()