From b932b4ea256fcbfa112f006ccb2e1c10ef195c17 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 14:36:39 +0200 Subject: [PATCH 01/46] use repr, not str Signed-off-by: Adrian Tschira --- synapse/http/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 202a99050..75e32924e 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -56,7 +56,7 @@ class SynapseRequest(Request): def __repr__(self): # We overwrite this so that we don't log ``access_token`` - return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( + return '<%s at 0x%x method=%r uri=%r clientproto=%r site=%r>' % ( self.__class__.__name__, id(self), self.method, From aafb0f6b0d7db313ac54a8e5e933970feae4bff3 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 10:44:31 +0200 Subject: [PATCH 02/46] py3-ize url preview --- synapse/rest/media/v1/preview_url_resource.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 2839207ab..565cef2b8 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -24,7 +24,9 @@ import shutil import sys import traceback import simplejson as json -import urlparse + +from six.moves import urllib_parse as urlparse +from six import string_types from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -590,8 +592,8 @@ def _iterate_over_text(tree, *tags_to_ignore): # to be returned. elements = iter([tree]) while True: - el = elements.next() - if isinstance(el, basestring): + el = next(elements) + if isinstance(el, string_types): yield el elif el is not None and el.tag not in tags_to_ignore: # el.text is the text before the first child, so we can immediately From 73cbdef5f773ddd091713d71ac9c63639d7c0825 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:55:46 +0200 Subject: [PATCH 03/46] fix py3 intern and remove unnecessary py3 encode Signed-off-by: Adrian Tschira --- synapse/util/caches/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 4adae9668..329ccbb86 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -16,6 +16,9 @@ import synapse.metrics import os +from six.moves import intern +import six + CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5)) metrics = synapse.metrics.get_metrics_for("synapse.util.caches") @@ -66,7 +69,9 @@ def intern_string(string): return None try: - string = string.encode("ascii") + if six.PY2: + string = string.encode("ascii") + return intern(string) except UnicodeEncodeError: return string From dcc235b47d694d9ef7181b379d4e38bed2677028 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Mon, 30 Apr 2018 16:38:23 +0200 Subject: [PATCH 04/46] use stand-in value if maxint is not available Signed-off-by: Adrian Tschira --- synapse/storage/_base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2262776ab..962cc270b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -30,6 +30,12 @@ import threading logger = logging.getLogger(__name__) +try: + MAX_TXN_ID = sys.maxint - 1 +except AttributeError: + # python 3 does not have a maximum int value + MAX_TXN_ID = 2**63 - 1 + sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") perf_logger = logging.getLogger("synapse.storage.TIME") @@ -222,7 +228,7 @@ class SQLBaseStore(object): # We don't really need these to be unique, so lets stop it from # growing really large. - self._TXN_ID = (self._TXN_ID + 1) % (sys.maxint - 1) + self._TXN_ID = (self._TXN_ID + 1) % (MAX_TXN_ID) name = "%s-%x" % (desc, txn_id, ) From 45b55e23d34371986ee0da5be784a1f2134fd58a Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 29 Apr 2018 13:54:38 +0200 Subject: [PATCH 05/46] Add batch_iter to utils There's a frequent idiom I noticed where an iterable is split up into a number of chunks/batches. Unfortunately that method does not work with iterators like dict.keys() in python3. This implementation works with iterators. Signed-off-by: Adrian Tschira --- synapse/util/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 814a7bf71..fc11e2662 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -20,6 +20,8 @@ from twisted.internet import defer, reactor, task import time import logging +from itertools import islice + logger = logging.getLogger(__name__) @@ -79,3 +81,19 @@ class Clock(object): except Exception: if not ignore_errs: raise + + +def batch_iter(iterable, size): + """batch an iterable up into tuples with a maximum size + + Args: + iterable (iterable): the iterable to slice + size (int): the maximum batch size + + Returns: + an iterator over the chunks + """ + # make sure we can deal with iterables like lists too + sourceiter = iter(iterable) + # call islice until it returns an empty tuple + return iter(lambda: tuple(islice(sourceiter, size)), ()) From d9fe2b2d9dbb539da32f969b8bd752159fd5eb6f Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sun, 15 Apr 2018 21:43:35 +0200 Subject: [PATCH 06/46] Replace some more comparisons with six plus a bonus b"" string I missed last time Signed-off-by: Adrian Tschira --- synapse/events/utils.py | 4 +++- synapse/events/validator.py | 6 ++++-- synapse/groups/groups_server.py | 4 +++- synapse/http/matrixfederationclient.py | 8 +++++--- synapse/push/push_rule_evaluator.py | 4 +++- synapse/rest/client/v1/presence.py | 8 +++++--- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 824f4a42e..29ae08678 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -20,6 +20,8 @@ from frozendict import frozendict import re +from six import string_types + # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' # (? Date: Sat, 28 Apr 2018 13:19:12 +0200 Subject: [PATCH 07/46] replace some iteritems with six Signed-off-by: Adrian Tschira --- synapse/handlers/device.py | 14 ++++++++------ synapse/handlers/e2e_keys.py | 13 +++++++------ synapse/handlers/groups_local.py | 3 ++- synapse/handlers/presence.py | 15 ++++++++------- synapse/handlers/sync.py | 14 ++++++++------ synapse/handlers/user_directory.py | 3 ++- synapse/metrics/process_collector.py | 3 ++- synapse/push/bulk_push_rule_evaluator.py | 13 +++++++------ synapse/replication/tcp/protocol.py | 9 +++++---- synapse/replication/tcp/resource.py | 3 ++- synapse/rest/media/v1/media_repository.py | 3 ++- synapse/storage/client_ips.py | 6 ++++-- synapse/storage/devices.py | 9 +++++---- synapse/storage/end_to_end_keys.py | 6 ++++-- synapse/storage/event_push_actions.py | 4 +++- 15 files changed, 69 insertions(+), 49 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index f7457a708..31bd0e60c 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -26,6 +26,8 @@ from ._base import BaseHandler import logging +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -318,7 +320,7 @@ class DeviceHandler(BaseHandler): # The user may have left the room # TODO: Check if they actually did or if we were just invited. if room_id not in room_ids: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -338,7 +340,7 @@ class DeviceHandler(BaseHandler): # special-case for an empty prev state: include all members # in the changed list if not event_ids: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -354,10 +356,10 @@ class DeviceHandler(BaseHandler): # Check if we've joined the room? If so we just blindly add all the users to # the "possibly changed" users. - for state_dict in prev_state_ids.itervalues(): + for state_dict in itervalues(prev_state_ids): member_event = state_dict.get((EventTypes.Member, user_id), None) if not member_event or member_event != current_member_id: - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue @@ -367,14 +369,14 @@ class DeviceHandler(BaseHandler): # If there has been any change in membership, include them in the # possibly changed list. We'll check if they are joined below, # and we're not toooo worried about spuriously adding users. - for key, event_id in current_state_ids.iteritems(): + for key, event_id in iteritems(current_state_ids): etype, state_key = key if etype != EventTypes.Member: continue # check if this member has changed since any of the extremities # at the stream_ordering, and add them to the list if so. - for state_dict in prev_state_ids.itervalues(): + for state_dict in itervalues(prev_state_ids): prev_event_id = state_dict.get(key, None) if not prev_event_id or prev_event_id != event_id: if state_key != user_id: diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 25aec624a..8a2d17753 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -19,6 +19,7 @@ import logging from canonicaljson import encode_canonical_json from twisted.internet import defer +from six import iteritems from synapse.api.errors import ( SynapseError, CodeMessageException, FederationDeniedError, @@ -92,7 +93,7 @@ class E2eKeysHandler(object): remote_queries_not_in_cache = {} if remote_queries: query_list = [] - for user_id, device_ids in remote_queries.iteritems(): + for user_id, device_ids in iteritems(remote_queries): if device_ids: query_list.extend((user_id, device_id) for device_id in device_ids) else: @@ -103,9 +104,9 @@ class E2eKeysHandler(object): query_list ) ) - for user_id, devices in remote_results.iteritems(): + for user_id, devices in iteritems(remote_results): user_devices = results.setdefault(user_id, {}) - for device_id, device in devices.iteritems(): + for device_id, device in iteritems(devices): keys = device.get("keys", None) device_display_name = device.get("device_display_name", None) if keys: @@ -250,9 +251,9 @@ class E2eKeysHandler(object): "Claimed one-time-keys: %s", ",".join(( "%s for %s:%s" % (key_id, user_id, device_id) - for user_id, user_keys in json_result.iteritems() - for device_id, device_keys in user_keys.iteritems() - for key_id, _ in device_keys.iteritems() + for user_id, user_keys in iteritems(json_result) + for device_id, device_keys in iteritems(user_keys) + for key_id, _ in iteritems(device_keys) )), ) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 977993e7d..dcae08373 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -15,6 +15,7 @@ # limitations under the License. from twisted.internet import defer +from six import iteritems from synapse.api.errors import SynapseError from synapse.types import get_domain_from_id @@ -449,7 +450,7 @@ class GroupsLocalHandler(object): results = {} failed_results = [] - for destination, dest_user_ids in destinations.iteritems(): + for destination, dest_user_ids in iteritems(destinations): try: r = yield self.transport_client.bulk_get_publicised_groups( destination, list(dest_user_ids), diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 91218e40e..b51f92522 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -25,6 +25,8 @@ The methods that define policy are: from twisted.internet import defer, reactor from contextlib import contextmanager +from six import itervalues, iteritems + from synapse.api.errors import SynapseError from synapse.api.constants import PresenceState from synapse.storage.presence import UserPresenceState @@ -40,7 +42,6 @@ import synapse.metrics import logging - logger = logging.getLogger(__name__) metrics = synapse.metrics.get_metrics_for(__name__) @@ -526,7 +527,7 @@ class PresenceHandler(object): prev_state.copy_and_replace( last_user_sync_ts=time_now_ms, ) - for prev_state in prev_states.itervalues() + for prev_state in itervalues(prev_states) ]) self.external_process_last_updated_ms.pop(process_id, None) @@ -549,14 +550,14 @@ class PresenceHandler(object): for user_id in user_ids } - missing = [user_id for user_id, state in states.iteritems() if not state] + missing = [user_id for user_id, state in iteritems(states) if not state] if missing: # There are things not in our in memory cache. Lets pull them out of # the database. res = yield self.store.get_presence_for_users(missing) states.update(res) - missing = [user_id for user_id, state in states.iteritems() if not state] + missing = [user_id for user_id, state in iteritems(states) if not state] if missing: new = { user_id: UserPresenceState.default(user_id) @@ -1044,7 +1045,7 @@ class PresenceEventSource(object): defer.returnValue((updates.values(), max_token)) else: defer.returnValue(([ - s for s in updates.itervalues() + s for s in itervalues(updates) if s.state != PresenceState.OFFLINE ], max_token)) @@ -1301,11 +1302,11 @@ def get_interested_remotes(store, states, state_handler): # hosts in those rooms. room_ids_to_states, users_to_states = yield get_interested_parties(store, states) - for room_id, states in room_ids_to_states.iteritems(): + for room_id, states in iteritems(room_ids_to_states): hosts = yield state_handler.get_current_hosts_in_room(room_id) hosts_and_states.append((hosts, states)) - for user_id, states in users_to_states.iteritems(): + for user_id, states in iteritems(users_to_states): host = get_domain_from_id(user_id) hosts_and_states.append(([host], states)) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 263e42dde..d0c99c35e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -28,6 +28,8 @@ import collections import logging import itertools +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -275,7 +277,7 @@ class SyncHandler(object): # result returned by the event source is poor form (it might cache # the object) room_id = event["room_id"] - event_copy = {k: v for (k, v) in event.iteritems() + event_copy = {k: v for (k, v) in iteritems(event) if k != "room_id"} ephemeral_by_room.setdefault(room_id, []).append(event_copy) @@ -294,7 +296,7 @@ class SyncHandler(object): for event in receipts: room_id = event["room_id"] # exclude room id, as above - event_copy = {k: v for (k, v) in event.iteritems() + event_copy = {k: v for (k, v) in iteritems(event) if k != "room_id"} ephemeral_by_room.setdefault(room_id, []).append(event_copy) @@ -325,7 +327,7 @@ class SyncHandler(object): current_state_ids = frozenset() if any(e.is_state() for e in recents): current_state_ids = yield self.state.get_current_state_ids(room_id) - current_state_ids = frozenset(current_state_ids.itervalues()) + current_state_ids = frozenset(itervalues(current_state_ids)) recents = yield filter_events_for_client( self.store, @@ -382,7 +384,7 @@ class SyncHandler(object): current_state_ids = frozenset() if any(e.is_state() for e in loaded_recents): current_state_ids = yield self.state.get_current_state_ids(room_id) - current_state_ids = frozenset(current_state_ids.itervalues()) + current_state_ids = frozenset(itervalues(current_state_ids)) loaded_recents = yield filter_events_for_client( self.store, @@ -984,7 +986,7 @@ class SyncHandler(object): if since_token: for joined_sync in sync_result_builder.joined: it = itertools.chain( - joined_sync.timeline.events, joined_sync.state.itervalues() + joined_sync.timeline.events, itervalues(joined_sync.state) ) for event in it: if event.type == EventTypes.Member: @@ -1062,7 +1064,7 @@ class SyncHandler(object): newly_left_rooms = [] room_entries = [] invited = [] - for room_id, events in mem_change_events_by_room_id.iteritems(): + for room_id, events in iteritems(mem_change_events_by_room_id): non_joins = [e for e in events if e.membership != Membership.JOIN] has_join = len(non_joins) != len(events) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 714f0195c..cd0b7290f 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -22,6 +22,7 @@ from synapse.util.metrics import Measure from synapse.util.async import sleep from synapse.types import get_localpart_from_id +from six import iteritems logger = logging.getLogger(__name__) @@ -403,7 +404,7 @@ class UserDirectoryHandler(object): if change: users_with_profile = yield self.state.get_current_user_in_room(room_id) - for user_id, profile in users_with_profile.iteritems(): + for user_id, profile in iteritems(users_with_profile): yield self._handle_new_user(room_id, user_id, profile) else: users = yield self.store.get_users_in_public_due_to_room(room_id) diff --git a/synapse/metrics/process_collector.py b/synapse/metrics/process_collector.py index 6fec3de39..50e5b48a2 100644 --- a/synapse/metrics/process_collector.py +++ b/synapse/metrics/process_collector.py @@ -15,6 +15,7 @@ import os +from six import iteritems TICKS_PER_SEC = 100 BYTES_PER_PAGE = 4096 @@ -55,7 +56,7 @@ def update_resource_metrics(): # line is PID (command) more stats go here ... raw_stats = line.split(") ", 1)[1].split(" ") - for (name, index) in STAT_FIELDS.iteritems(): + for (name, index) in iteritems(STAT_FIELDS): # subtract 3 from the index, because proc(5) is 1-based, and # we've lost the first two fields in PID and COMMAND above stats[name] = int(raw_stats[index - 3]) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 7c680659b..2f7e77f5f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -30,6 +30,7 @@ from synapse.state import POWER_KEY from collections import namedtuple +from six import itervalues, iteritems logger = logging.getLogger(__name__) @@ -126,7 +127,7 @@ class BulkPushRuleEvaluator(object): ) auth_events = yield self.store.get_events(auth_events_ids) auth_events = { - (e.type, e.state_key): e for e in auth_events.itervalues() + (e.type, e.state_key): e for e in itervalues(auth_events) } sender_level = get_user_power_level(event.sender, auth_events) @@ -160,7 +161,7 @@ class BulkPushRuleEvaluator(object): condition_cache = {} - for uid, rules in rules_by_user.iteritems(): + for uid, rules in iteritems(rules_by_user): if event.sender == uid: continue @@ -406,7 +407,7 @@ class RulesForRoom(object): # If the event is a join event then it will be in current state evnts # map but not in the DB, so we have to explicitly insert it. if event.type == EventTypes.Member: - for event_id in member_event_ids.itervalues(): + for event_id in itervalues(member_event_ids): if event_id == event.event_id: members[event_id] = (event.state_key, event.membership) @@ -414,7 +415,7 @@ class RulesForRoom(object): logger.debug("Found members %r: %r", self.room_id, members.values()) interested_in_user_ids = set( - user_id for user_id, membership in members.itervalues() + user_id for user_id, membership in itervalues(members) if membership == Membership.JOIN ) @@ -426,7 +427,7 @@ class RulesForRoom(object): ) user_ids = set( - uid for uid, have_pusher in if_users_with_pushers.iteritems() if have_pusher + uid for uid, have_pusher in iteritems(if_users_with_pushers) if have_pusher ) logger.debug("With pushers: %r", user_ids) @@ -447,7 +448,7 @@ class RulesForRoom(object): ) ret_rules_by_user.update( - item for item in rules_by_user.iteritems() if item[0] is not None + item for item in iteritems(rules_by_user) if item[0] is not None ) self.update_cache(sequence, members, ret_rules_by_user, state_group) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index d7d38464b..7ca1588f6 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -68,6 +68,7 @@ import synapse.metrics import struct import fcntl +from six import iterkeys, iteritems metrics = synapse.metrics.get_metrics_for(__name__) @@ -392,7 +393,7 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): if stream_name == "ALL": # Subscribe to all streams we're publishing to. - for stream in self.streamer.streams_by_name.iterkeys(): + for stream in iterkeys(self.streamer.streams_by_name): self.subscribe_to_stream(stream, token) else: self.subscribe_to_stream(stream_name, token) @@ -498,7 +499,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol): BaseReplicationStreamProtocol.connectionMade(self) # Once we've connected subscribe to the necessary streams - for stream_name, token in self.handler.get_streams_to_replicate().iteritems(): + for stream_name, token in iteritems(self.handler.get_streams_to_replicate()): self.replicate(stream_name, token) # Tell the server if we have any users currently syncing (should only @@ -633,7 +634,7 @@ metrics.register_callback( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.inbound_commands_counter.counts.iteritems() + for k, count in iteritems(p.inbound_commands_counter.counts) }, labels=["command", "name", "conn_id"], ) @@ -643,7 +644,7 @@ metrics.register_callback( lambda: { (k[0], p.name, p.conn_id): count for p in connected_connections - for k, count in p.outbound_commands_counter.counts.iteritems() + for k, count in iteritems(p.outbound_commands_counter.counts) }, labels=["command", "name", "conn_id"], ) diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index a41af4fd6..d1c291e17 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -26,6 +26,7 @@ from synapse.util.metrics import Measure, measure_func import logging import synapse.metrics +from six import itervalues metrics = synapse.metrics.get_metrics_for(__name__) stream_updates_counter = metrics.register_counter( @@ -79,7 +80,7 @@ class ReplicationStreamer(object): # We only support federation stream if federation sending hase been # disabled on the master. self.streams = [ - stream(hs) for stream in STREAMS_MAP.itervalues() + stream(hs) for stream in itervalues(STREAMS_MAP) if stream != FederationStream or not hs.config.send_federation ] diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 9800ce758..2ac767d2d 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -48,6 +48,7 @@ import shutil import cgi import logging from six.moves.urllib import parse as urlparse +from six import iteritems logger = logging.getLogger(__name__) @@ -603,7 +604,7 @@ class MediaRepository(object): thumbnails[(t_width, t_height, r_type)] = r_method # Now we generate the thumbnails for each dimension, store it - for (t_width, t_height, t_type), t_method in thumbnails.iteritems(): + for (t_width, t_height, t_type), t_method in iteritems(thumbnails): # Generate the thumbnail if t_method == "crop": t_byte_source = yield make_deferred_yieldable(threads.deferToThread( diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index ba4690773..ce338514e 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -22,6 +22,8 @@ from . import background_updates from synapse.util.caches import CACHE_SIZE_FACTOR +from six import iteritems + logger = logging.getLogger(__name__) @@ -99,7 +101,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): def _update_client_ips_batch_txn(self, txn, to_update): self.database_engine.lock_table(txn, "user_ips") - for entry in to_update.iteritems(): + for entry in iteritems(to_update): (user_id, access_token, ip), (user_agent, device_id, last_seen) = entry self._simple_upsert_txn( @@ -231,5 +233,5 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): "user_agent": user_agent, "last_seen": last_seen, } - for (access_token, ip), (user_agent, last_seen) in results.iteritems() + for (access_token, ip), (user_agent, last_seen) in iteritems(results) )) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 712106b83..d149d8392 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -21,6 +21,7 @@ from synapse.api.errors import StoreError from ._base import SQLBaseStore, Cache from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks +from six import itervalues, iteritems logger = logging.getLogger(__name__) @@ -360,7 +361,7 @@ class DeviceStore(SQLBaseStore): return (now_stream_id, []) if len(query_map) >= 20: - now_stream_id = max(stream_id for stream_id in query_map.itervalues()) + now_stream_id = max(stream_id for stream_id in itervalues(query_map)) devices = self._get_e2e_device_keys_txn( txn, query_map.keys(), include_all_devices=True @@ -373,13 +374,13 @@ class DeviceStore(SQLBaseStore): """ results = [] - for user_id, user_devices in devices.iteritems(): + for user_id, user_devices in iteritems(devices): # The prev_id for the first row is always the last row before # `from_stream_id` txn.execute(prev_sent_id_sql, (destination, user_id, from_stream_id)) rows = txn.fetchall() prev_id = rows[0][0] - for device_id, device in user_devices.iteritems(): + for device_id, device in iteritems(user_devices): stream_id = query_map[(user_id, device_id)] result = { "user_id": user_id, @@ -483,7 +484,7 @@ class DeviceStore(SQLBaseStore): if devices: user_devices = devices[user_id] results = [] - for device_id, device in user_devices.iteritems(): + for device_id, device in iteritems(user_devices): result = { "device_id": device_id, } diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index ff8538ddf..b14648794 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -21,6 +21,8 @@ import simplejson as json from ._base import SQLBaseStore +from six import iteritems + class EndToEndKeyStore(SQLBaseStore): def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys): @@ -81,8 +83,8 @@ class EndToEndKeyStore(SQLBaseStore): query_list, include_all_devices, ) - for user_id, device_keys in results.iteritems(): - for device_id, device_info in device_keys.iteritems(): + for user_id, device_keys in iteritems(results): + for device_id, device_info in iteritems(device_keys): device_info["keys"] = json.loads(device_info.pop("key_json")) defer.returnValue(results) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index f084a5f54..d0350ee5f 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,6 +22,8 @@ from synapse.util.caches.descriptors import cachedInlineCallbacks import logging import simplejson as json +from six import iteritems + logger = logging.getLogger(__name__) @@ -420,7 +422,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): txn.executemany(sql, ( _gen_entry(user_id, actions) - for user_id, actions in user_id_actions.iteritems() + for user_id, actions in iteritems(user_id_actions) )) return self.runInteraction( From cb2a2ad791155bd1e1ef58caa2ef2894a74fab08 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 16:23:39 +0100 Subject: [PATCH 08/46] get_domains_from_state returns list of tuples --- synapse/handlers/federation.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba3ede802..c85d19451 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -751,6 +751,16 @@ class FederationHandler(BaseHandler): curr_state = yield self.state_handler.get_current_state(room_id) def get_domains_from_state(state): + """Get joined domains from state + + Args: + state (dict[tuple, FrozenEvent]): State map from type/state + key to event. + + Returns: + list[tuple[str, int]]: Returns a list of servers with the + lowest depth of their joins. Sorted by lowest depth first. + """ joined_users = [ (state_key, int(event.depth)) for (e_type, state_key), event in state.items() @@ -861,13 +871,13 @@ class FederationHandler(BaseHandler): likely_domains = get_domains_from_state(states[e_id]) success = yield try_backfill([ - dom for dom in likely_domains + dom for dom, _ in likely_domains if dom not in tried_domains ]) if success: defer.returnValue(True) - tried_domains.update(likely_domains) + tried_domains.update(dom for dom, _ in likely_domains) defer.returnValue(False) From 0a078026eae694337cfdf8c9460d1b231b1bd04a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 22 May 2018 17:14:06 +0100 Subject: [PATCH 09/46] comment typo --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 4eb18775e..d58ea6c65 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -61,7 +61,7 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) - # Add the user to a table of users penpding deactivation (ie. + # Add the user to a table of users pending deactivation (ie. # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) From 08bfc48abf8c138b540ede6c816338c044130dec Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 22 May 2018 17:27:03 +0100 Subject: [PATCH 10/46] custom error code for not leaving server notices room --- synapse/api/errors.py | 1 + synapse/handlers/room_member.py | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index aca804280..e6ad3768f 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -53,6 +53,7 @@ class Codes(object): INVALID_USERNAME = "M_INVALID_USERNAME" SERVER_NOT_TRUSTED = "M_SERVER_NOT_TRUSTED" CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" + CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5263f09aa..82adfc8fd 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -306,6 +306,7 @@ class RoomMemberHandler(object): raise SynapseError( http_client.FORBIDDEN, "You cannot leave this room", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, ) if effective_membership_state == Membership.INVITE: From ad2823ee27d7bfe766c8b84efbd0bcde02962048 Mon Sep 17 00:00:00 2001 From: hera Date: Tue, 22 May 2018 16:38:46 +0000 Subject: [PATCH 11/46] fix synchrotron --- synapse/server_notices/worker_server_notices_sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server_notices/worker_server_notices_sender.py b/synapse/server_notices/worker_server_notices_sender.py index 25abb4ccf..4a133026c 100644 --- a/synapse/server_notices/worker_server_notices_sender.py +++ b/synapse/server_notices/worker_server_notices_sender.py @@ -32,7 +32,7 @@ class WorkerServerNoticesSender(object): Returns: Deferred """ - return defer.succeed() + return defer.succeed(None) def on_user_ip(self, user_id): """Called on the master when a worker process saw a client request. From 5494c1d71e47245b9750fc260462fe4c2ca3e273 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 18:15:21 +0100 Subject: [PATCH 12/46] Don't support limitless pagination The pagination storage function supported not specifiying a limit on the number of events returned. This was triggered when using the search or context API with a limit of zero, which the storage function took to mean not being limited. --- synapse/storage/stream.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index ea24710ad..1701d0386 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -684,8 +684,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): results to only those before direction(char): Either 'b' or 'f' to indicate whether we are paginating forwards or backwards from `from_key`. - limit (int): The maximum number of events to return. Zero or less - means no limit. + limit (int): The maximum number of events to return. event_filter (Filter|None): If provided filters the events to those that match the filter. @@ -694,6 +693,9 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): as a list of _EventDictReturn and a token that points to the end of the result set. """ + + assert int(limit) >= 0 + # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -723,18 +725,14 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): bounds += " AND " + filter_clause args.extend(filter_args) - if int(limit) > 0: - args.append(int(limit)) - limit_str = " LIMIT ?" - else: - limit_str = "" + args.append(int(limit)) sql = ( "SELECT event_id, topological_ordering, stream_ordering" " FROM events" " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," - " stream_ordering %(order)s %(limit)s" + " stream_ordering %(order)s LIMIT ?" ) % { "bounds": bounds, "order": order, From a17e901f4dd1825d4bf21e55488939d95e1d7d80 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 18:24:32 +0100 Subject: [PATCH 13/46] Remove unused string formatting param --- synapse/storage/stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 1701d0386..fb463c525 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -736,7 +736,6 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ) % { "bounds": bounds, "order": order, - "limit": limit_str } txn.execute(sql, args) From 586b66b1977e2c678e23a3b2d25a5b699b637a8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 19:00:48 +0100 Subject: [PATCH 14/46] Fix that states is a dict of dicts --- synapse/handlers/federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c85d19451..6a381f002 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -853,10 +853,13 @@ class FederationHandler(BaseHandler): [resolve(room_id, [e]) for e in event_ids], consumeErrors=True, )) + + # dict[str, dict[tuple, str]], a map from event_id to state map of + # event_ids. states = dict(zip(event_ids, [s.state for s in states])) state_map = yield self.store.get_events( - [e_id for ids in states.values() for e_id in ids], + [e_id for ids in states.values() for e_id in ids.itervalues()], get_prev_content=False ) states = { From e85b5a0ff717e0388179ad9664a8fdfca4b61d9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 22 May 2018 19:02:48 +0100 Subject: [PATCH 15/46] Use iter* methods --- synapse/handlers/federation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6a381f002..786c20037 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -763,7 +763,7 @@ class FederationHandler(BaseHandler): """ joined_users = [ (state_key, int(event.depth)) - for (e_type, state_key), event in state.items() + for (e_type, state_key), event in state.iteritems() if e_type == EventTypes.Member and event.membership == Membership.JOIN ] @@ -780,7 +780,7 @@ class FederationHandler(BaseHandler): except Exception: pass - return sorted(joined_domains.items(), key=lambda d: d[1]) + return sorted(joined_domains.iteritems(), key=lambda d: d[1]) curr_domains = get_domains_from_state(curr_state) @@ -797,7 +797,7 @@ class FederationHandler(BaseHandler): yield self.backfill( dom, room_id, limit=100, - extremities=[e for e in extremities.keys()] + extremities=extremities, ) # If this succeeded then we probably already have the # appropriate stuff. @@ -843,7 +843,7 @@ class FederationHandler(BaseHandler): tried_domains = set(likely_domains) tried_domains.add(self.server_name) - event_ids = list(extremities.keys()) + event_ids = list(extremities.iterkeys()) logger.debug("calling resolve_state_groups in _maybe_backfill") resolve = logcontext.preserve_fn( @@ -859,15 +859,15 @@ class FederationHandler(BaseHandler): states = dict(zip(event_ids, [s.state for s in states])) state_map = yield self.store.get_events( - [e_id for ids in states.values() for e_id in ids.itervalues()], + [e_id for ids in states.itervalues() for e_id in ids.itervalues()], get_prev_content=False ) states = { key: { k: state_map[e_id] - for k, e_id in state_dict.items() + for k, e_id in state_dict.iteritems() if e_id in state_map - } for key, state_dict in states.items() + } for key, state_dict in states.iteritems() } for e_id, _ in sorted_extremeties_tuple: From 0a4bca413425fee20aed2103d6886a7701ac6b09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 May 2018 10:04:23 +0100 Subject: [PATCH 16/46] Use iter* methods for _filter_events_for_server --- synapse/handlers/federation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba3ede802..c2dc1e269 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -479,18 +479,18 @@ class FederationHandler(BaseHandler): # to get all state ids that we're interested in. event_map = yield self.store.get_events([ e_id - for key_to_eid in event_to_state_ids.values() - for key, e_id in key_to_eid.items() + for key_to_eid in event_to_state_ids.itervalues() + for key, e_id in key_to_eid.iteritems() if key[0] != EventTypes.Member or check_match(key[1]) ]) event_to_state = { e_id: { key: event_map[inner_e_id] - for key, inner_e_id in key_to_eid.items() + for key, inner_e_id in key_to_eid.iteritems() if inner_e_id in event_map } - for e_id, key_to_eid in event_to_state_ids.items() + for e_id, key_to_eid in event_to_state_ids.iteritems() } def redact_disallowed(event, state): From 5aaa3189d5a1ef0edc4ed09d7cc61e1a7452efed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 May 2018 10:13:05 +0100 Subject: [PATCH 17/46] s/values/itervalues/ --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c2dc1e269..c15c8c1ed 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -505,7 +505,7 @@ class FederationHandler(BaseHandler): # membership states for the requesting server to determine # if the server is either in the room or has been invited # into the room. - for ev in state.values(): + for ev in state.itervalues(): if ev.type != EventTypes.Member: continue try: From e7598b666ba6c91f69f312d367017caca8f44a8f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 11:14:23 +0100 Subject: [PATCH 18/46] Some docs about server notices --- docs/server_notices.md | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/server_notices.md diff --git a/docs/server_notices.md b/docs/server_notices.md new file mode 100644 index 000000000..a37aed679 --- /dev/null +++ b/docs/server_notices.md @@ -0,0 +1,53 @@ +Server Notices +============== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of the communication of Privacy Polices (see +[privacy_policy.md]), however the intention is that they may also find a use +for features such as "Message of the day". + +This is a feature specific to Synapse, but it uses standard Matrix +communication mechanisms, so should work with any Matrix client. + +User experience +--------------- + +When the user is first sent a server notice, they will get an invitation to a +room (typically called 'Server Notices', though this is configurable in +`homeserver.yaml`). They will be **unable to reject** this invitation - +attempts to do so will receive an error. + +Once they accept the invitation, they will see the notice message in the room +history; it will appear to have come from the 'server notices user' (see +below). + +The user is prevented from sending any messages in this room by the power +levels. They also cannot leave it. + +Synapse configuration +--------------------- + +Server notices come from a specific user id on the server. Server +administrators are free to choose the user id - something like `server` is +suggested, meaning the notices will come from +`@server:`. Once the server notices user is configured, that +user id becomes a special, privileged user, so administrators should ensure +that **it is not already allocated**. + +In order to support server notices, it is necessary to add some configuration +to the `homeserver.yaml` file. In particular, you should add a `server_notices` +section, which should look like this: + +```yaml +server_notices: + system_mxid_localpart: server + system_mxid_display_name: "Server Notices" + room_name: "Server Notices" +``` + +The only compulsory setting is `system_mxid_localpart`, which defines the user +id of the server notices user, as above. `system_mxid_display_name` and +`room_name` define the displayname of the system notices user, and of +the notices room, respectively. From 3601a240aaec9af7c95c11821d851fd5fa03b0c8 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 23 May 2018 12:03:23 +0100 Subject: [PATCH 19/46] bump version and changelog --- CHANGES.rst | 36 ++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 851cb9b2d..ee418abff 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,39 @@ +Changes in synapse v0.30.0-rc1 (2018-05-23) +========================================== + +This version includes support for GDPR consent collection. Users signal their +consent through responding to a private message from a bot named +@server:matrix.org For more information see XXXXXX + +GDPR Support: + +* ConsentResource to gather policy consent from users (PR #3213) +* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) +* Infrastructure for a server notices room (PR #3232) +* Send users a server notice about consent (PR #3236) +* Reject attempts to send event before privacy consent is given (PR #3257) +* Add a 'has_consented' template var to consent forms (PR #3262) +* Fix dependency on jinja2 (PR #3263) + +Features: +* Cohort analytics (PR #3163, #3241, #3251) +* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! +* Add in flight request metrics (PR #3252) + +Changes: + +* Remove unused `update_external_syncs` (PR #3233) +* Use stream rather depth ordering for push actions (PR #3212) +* Make purge_history operate on tokens (PR #3221) +* Don't support limitless pagination (PR #3265) + +Bug Fixes: + +* Fix logcontext resource usage tracking (PR #3258) +* Fix error in handling receipts (PR #3235) +* Stop the transaction cache caching failures (PR #3255) + + Changes in synapse v0.29.1 (2018-05-17) ========================================== Changes: diff --git a/synapse/__init__.py b/synapse/__init__.py index b11a1d3ea..353387f15 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.1" +__version__ = "0.30.0-rc1" From 9e8ab0a4f44a3ec9e4b049f5571c14e333e8f0fa Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 23 May 2018 12:05:58 +0100 Subject: [PATCH 20/46] style --- CHANGES.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ee418abff..096fa4794 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,9 @@ Changes in synapse v0.30.0-rc1 (2018-05-23) ========================================== -This version includes support for GDPR consent collection. Users signal their -consent through responding to a private message from a bot named -@server:matrix.org For more information see XXXXXX +This version includes support for Privacy Notice agreement collection. More +details live at docs/privacy_policy_templates/README.md + GDPR Support: @@ -16,6 +16,7 @@ GDPR Support: * Fix dependency on jinja2 (PR #3263) Features: + * Cohort analytics (PR #3163, #3241, #3251) * Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! * Add in flight request metrics (PR #3252) From 833db2d92245a0b7cab9e16f9badc135346f0750 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 12:32:38 +0100 Subject: [PATCH 21/46] consent tracking docs --- docs/consent_tracking.md | 152 ++++++++++++++++++++++++ docs/privacy_policy_templates/README.md | 23 ---- docs/server_notices.md | 4 +- 3 files changed, 154 insertions(+), 25 deletions(-) create mode 100644 docs/consent_tracking.md delete mode 100644 docs/privacy_policy_templates/README.md diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md new file mode 100644 index 000000000..eb34749b5 --- /dev/null +++ b/docs/consent_tracking.md @@ -0,0 +1,152 @@ +Support in Synapse for tracking agreement to server terms and conditions +======================================================================== + +Synapse 0.30 introduces support for tracking whether users have agreed to the +terms and conditions set by the administrator of a server - and blocking access +to the server until they have. + +There are several parts to this functionality; each requires some specific +configuration in `homeserver.yaml` to be enabled. + +Note that various parts of the configuation and this document refer to the +"privacy policy": agreement with a privacy policy is one particular use of this +feature, but of course adminstrators can specify other terms and conditions +unrelated to "privacy" per se. + +Collecting policy agreement from a user +--------------------------------------- + +Synapse can be configured to serve the user a simple policy form with an +"accept" button. Clicking "Accept" records the user's acceptance in the +database and shows a success page. + +To enable this, first create templates for the policy and success pages. +These should be stored on the local filesystem. + +These templates use the [Jinja2](http://jinja.pocoo.org) templating language, +and the `privacy_policy_templates` subdirectory of this `docs` directory gives +examples of the sort of thing that can be done. + +Note that the templates must be stored under a name giving the language of the +template - currently this must always be `en` (for "English"); +internationalisation support is intended for the future. + +The template for the policy itself should be versioned - for example +`1.0.html`. The version of the policy which the user has agreed to is stored in +the database. + +Once the templates are in place, make the following changes to `homeserver.yaml`: + + 1. Add a `user_consent` section, which should look like: + + ```yaml + user_consent: + template_dir: privacy_policy_templates + version: 1.0 + ``` + + `template_dir` points to the directory containing the policy + templates. `version` defines the version of the policy which will be served + to the user. In the example above, Synapse will serve + `privacy_policy_templates/en/1.0.html`. + + + 2. Add a `form_secret` setting at the top level: + + + ```yaml + form_secret: "" + ``` + + This should be set to an arbitrary secret string (try `pwgen -y 30` to + generate suitable secrets). + + More on what this is used for below. + + 3. Add `consent` wherever the `client` resource is currently enabled in the + `listeners` configuration. For example: + + ``` + listeners: + - port: 8008 + resources: + - names: + - client + - consent + ``` + + +Finally, ensure that `jinja2` is installed. If you are using a virtualenv, this +should be a matter of `pip install Jinja2`. On debian, try `apt-get install +python-jinja2`. + +Once this is complete, and the server has been restarted, try visiting +`https:///_matrix/consent`. If configuration has been done correctly, +this should give an error "Missing string query parameter 'u'". It is now +possible to manually construct URIs where users can give their consent. + +Constructing the consent URI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It may be useful to manually construct the "consent URI" for a given user - for +instance, in order to send them an email asking them to consent. To do this, +take the base `https:///_matrix/consent` URL and add the following +query parameters: + + * `u`: the user id of the user. This can either be a full MXID + (`@user:server.com`) or just the localpart (`user`). + + * `h`: hex-encoded HMAC-SHA256 of `u` using the `form_secret` as a key. It is + possible to calculate this on the commandline with something like: + + ```bash + echo -n '' | openssl sha256 -hmac '' + ``` + + This should result in a URI which looks something like: + `https:///_matrix/consent?u=&h=68a152465a4d...`. + + +Sending users a server notice asking them to agree to the policy +---------------------------------------------------------------- + +It is possible to configure Synapse to send a [server +notice](server_notices.md) to anybody who has not yet agreed to the current +version of the policy. To do so: + + * ensure that the consent resource is configured, as in the previous section + + * ensure that server notices are configured, as in [server_notices.md]. + + * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For + example: + + ```yaml + user_consent: + server_notice_content: + msgtype: m.text + body: >- + Please give your consent to the privacy policy at %(consent_uri)s. + ``` + + Synapse automatically replaces the placeholder `%(consent_uri)s` with the + consent uri for that user. + +Blocking users from using the server until they agree to the policy +------------------------------------------------------------------- + +Synapse can be configured to block any attempts to join rooms or send messages +until the user has given their agreement to the policy. (Joining the server +notices room is exempted from this). + +To enable this, add `block_events_error` under `user_consent`. For example: + +``` +user_consent: + block_events_error: >- + You can't send any messages until you consent to the privacy policy at + %(consent_uri)s. +``` + +Synapse automatically replaces the placeholder `%(consent_uri)s` with the +consent uri for that user. diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md deleted file mode 100644 index a3e6fc098..000000000 --- a/docs/privacy_policy_templates/README.md +++ /dev/null @@ -1,23 +0,0 @@ -If enabling the 'consent' resource in synapse, you will need some templates -for the HTML to be served to the user. This directory contains very simple -examples of the sort of thing that can be done. - -You'll need to add this sort of thing to your homeserver.yaml: - -``` -form_secret: - -user_consent: - template_dir: docs/privacy_policy_templates - version: 1.0 -``` - -You should then be able to enable the `consent` resource under a `listener` -entry. For example: - -``` -listeners: - - port: 8008 - resources: - - names: [client, consent] -``` diff --git a/docs/server_notices.md b/docs/server_notices.md index a37aed679..5d7857d15 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -4,8 +4,8 @@ Server Notices 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of the communication of Privacy Polices (see -[privacy_policy.md]), however the intention is that they may also find a use +They are used as part of the communication of the server polices (see +[consent_tracking.md]), however the intention is that they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix From 2574ea3dc83c58d1cb596606ac760d91022f336f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 12:34:34 +0100 Subject: [PATCH 22/46] server_notices.md: fix link --- docs/server_notices.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 5d7857d15..22bca332d 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -5,8 +5,8 @@ Server Notices channel whereby server administrators can send messages to users on the server. They are used as part of the communication of the server polices (see -[consent_tracking.md]), however the intention is that they may also find a use -for features such as "Message of the day". +[consent_tracking.md](consent_tracking.md)), however the intention is that +they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix communication mechanisms, so should work with any Matrix client. From 563606b8f2216f6a8ba64c4b55bf389f67c99e9e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 12:37:39 +0100 Subject: [PATCH 23/46] consent_tracking: formatting etc --- docs/consent_tracking.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index eb34749b5..61e41468d 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -31,9 +31,9 @@ Note that the templates must be stored under a name giving the language of the template - currently this must always be `en` (for "English"); internationalisation support is intended for the future. -The template for the policy itself should be versioned - for example -`1.0.html`. The version of the policy which the user has agreed to is stored in -the database. +The template for the policy itself should be versioned and named according to +the version: for example `1.0.html`. The version of the policy which the user +has agreed to is stored in the database. Once the templates are in place, make the following changes to `homeserver.yaml`: @@ -66,7 +66,7 @@ Once the templates are in place, make the following changes to `homeserver.yaml` 3. Add `consent` wherever the `client` resource is currently enabled in the `listeners` configuration. For example: - ``` + ```yaml listeners: - port: 8008 resources: @@ -85,8 +85,7 @@ Once this is complete, and the server has been restarted, try visiting this should give an error "Missing string query parameter 'u'". It is now possible to manually construct URIs where users can give their consent. -Constructing the consent URI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Constructing the consent URI It may be useful to manually construct the "consent URI" for a given user - for instance, in order to send them an email asking them to consent. To do this, @@ -116,7 +115,7 @@ version of the policy. To do so: * ensure that the consent resource is configured, as in the previous section - * ensure that server notices are configured, as in [server_notices.md]. + * ensure that server notices are configured, as in [server_notices.md](server_notices.md). * Add `server_notice_content` under `user_consent` in `homeserver.yaml`. For example: @@ -141,7 +140,7 @@ notices room is exempted from this). To enable this, add `block_events_error` under `user_consent`. For example: -``` +```yaml user_consent: block_events_error: >- You can't send any messages until you consent to the privacy policy at From 5ad1149f384e1a826ad73a791896d63c910d874c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:45:17 +0100 Subject: [PATCH 24/46] Notes on the manhole --- docs/manhole.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 docs/manhole.md diff --git a/docs/manhole.md b/docs/manhole.md new file mode 100644 index 000000000..7375f5ad4 --- /dev/null +++ b/docs/manhole.md @@ -0,0 +1,43 @@ +Using the synapse manhole +========================= + +The "manhole" allows server administrators to access a Python shell on a running +Synapse installation. This is a very powerful mechanism for administration and +debugging. + +To enable it, first uncomment the `manhole` listener configuration in +`homeserver.yaml`: + +```yaml +listeners: + - port: 9000 + bind_addresses: ['::1', '127.0.0.1'] + type: manhole +``` + +(`bind_addresses` in the above is important: it ensures that access to the +manhole is only possible for local users). + +Note that this will give administrative access to synapse to **all users** with +shell access to the server. It should therefore **not** be enabled in +environments where untrusted users have shell access. + +Then restart synapse, and point an ssh client at port 9000 on localhost, using +the username `matrix`: + +```bash +ssh -p9000 matrix@localhost +``` + +The password is `rabbithole`. + +This gives a Python REPL in which `hs` gives access to the +`synapse.server.HomeServer` object - which in turn gives access to many other +parts of the process. + +As a simple example, retrieving an event from the database: + +``` +>>> hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org') +> +``` From 052d08a6a579f4f9a98a76a3cb39c05d8d5aaa90 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:55:39 +0100 Subject: [PATCH 25/46] Using the manhole to send server notices --- docs/server_notices.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/server_notices.md b/docs/server_notices.md index 22bca332d..a96a5b88a 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -51,3 +51,16 @@ The only compulsory setting is `system_mxid_localpart`, which defines the user id of the server notices user, as above. `system_mxid_display_name` and `room_name` define the displayname of the system notices user, and of the notices room, respectively. + +Sending notices +--------------- + +As of the current version of synapse, there is no convenient interface for +sending notices (other than the automated ones sent as part of consent +tracking). + +In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like: + +``` +>>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) +``` From 1cbb8e5a33326441ec311ce212bb62fcd6704669 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 13:58:28 +0100 Subject: [PATCH 26/46] fix wrapping --- docs/server_notices.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index a96a5b88a..989688138 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -59,7 +59,9 @@ As of the current version of synapse, there is no convenient interface for sending notices (other than the automated ones sent as part of consent tracking). -In the meantime, it is possible to test this feature using the manhole. Having gone into the manhole as described in [manhole.md](manhole.md), a notice can be sent with something like: +In the meantime, it is possible to test this feature using the manhole. Having +gone into the manhole as described in [manhole.md](manhole.md), a notice can be +sent with something like: ``` >>> hs.get_server_notices_manager().send_notice('@user:server.com', {'msgtype':'m.text', 'body':'foo'}) From 321f02d26347fb56e1eafbba60d7faef95ae9c3f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 14:30:47 +0100 Subject: [PATCH 27/46] Block attempts to send server notices to remote users --- synapse/server_notices/server_notices_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index f535b9c9d..9f76c7554 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ class ServerNoticesManager(object): self._config = hs.config self._room_creation_handler = hs.get_room_creation_handler() self._event_creation_handler = hs.get_event_creation_handler() + self._is_mine_id = hs.is_mine_id def is_enabled(self): """Checks if server notices are enabled on this server. @@ -89,6 +90,9 @@ class ServerNoticesManager(object): if not self.is_enabled(): raise Exception("Server notices not enabled") + assert self._is_mine_id(user_id), \ + "Cannot send server notices to remote users" + rooms = yield self._store.get_rooms_for_user_where_membership_is( user_id, [Membership.INVITE, Membership.JOIN], ) From cd8ab9a0d8d3e0ee305ab56788f76459596075f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 14:43:09 +0100 Subject: [PATCH 28/46] mention public_baseurl --- docs/consent_tracking.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 61e41468d..66beb9263 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -131,6 +131,11 @@ version of the policy. To do so: Synapse automatically replaces the placeholder `%(consent_uri)s` with the consent uri for that user. + * ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base + URI that clients use to connect to the server. (It is used to construct + `consent_uri` in the server notice.) + + Blocking users from using the server until they agree to the policy ------------------------------------------------------------------- @@ -149,3 +154,7 @@ user_consent: Synapse automatically replaces the placeholder `%(consent_uri)s` with the consent uri for that user. + +ensure that `public_baseurl` is set in `homeserver.yaml`, and gives the base +URI that clients use to connect to the server. (It is used to construct +`consent_uri` in the error.) From 82191b08f6a45c693551983a48b49a9fc30184c4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:24:31 +0100 Subject: [PATCH 29/46] Support for putting %(consent_uri)s in messages Make it possible to put the URI in the error message and the server notice that get sent by the server --- synapse/config/consent_config.py | 11 +++-- synapse/handlers/message.py | 5 ++- .../server_notices/consent_server_notices.py | 41 ++++++++++++++++++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 44c4711e6..ddcd305f4 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -43,10 +43,13 @@ DEFAULT_CONFIG = """\ # version: 1.0 # server_notice_content: # msgtype: m.text -# body: | -# Pls do consent kthx -# block_events_error: | -# You can't send any messages until you consent to the privacy policy. +# body: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# block_events_error: >- +# To continue using this homeserver you must review and agree to the +# terms and conditions at %(consent_uri)s +# """ diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c3adbc6c9..088660702 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -575,8 +575,11 @@ class EventCreationHandler(object): return consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + msg = self.config.block_events_without_consent_error % { + 'consent_uri': consent_uri, + } raise ConsentNotGivenError( - msg=self.config.block_events_without_consent_error, + msg=msg, consent_uri=consent_uri, ) diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index 440f6b1cd..a70980285 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -14,10 +14,13 @@ # limitations under the License. import logging +from six import (iteritems, string_types) from twisted.internet import defer from synapse.api.errors import SynapseError +from synapse.api.urls import ConsentURIBuilder from synapse.config import ConfigError +from synapse.types import get_localpart_from_id logger = logging.getLogger(__name__) @@ -52,6 +55,8 @@ class ConsentServerNotices(object): "key.", ) + self._consent_uri_builder = ConsentURIBuilder(hs.config) + @defer.inlineCallbacks def maybe_send_server_notice_to_user(self, user_id): """Check if we need to send a notice to this user, and does so if so @@ -81,10 +86,18 @@ class ConsentServerNotices(object): # we've already sent a notice to the user return - # need to send a message + # need to send a message. try: + consent_uri = self._consent_uri_builder.build_user_consent_uri( + get_localpart_from_id(user_id), + ) + content = copy_with_str_subst( + self._server_notice_content, { + 'consent_uri': consent_uri, + }, + ) yield self._server_notices_manager.send_notice( - user_id, self._server_notice_content, + user_id, content, ) yield self._store.user_set_consent_server_notice_sent( user_id, self._current_consent_version, @@ -93,3 +106,27 @@ class ConsentServerNotices(object): logger.error("Error sending server notice about user consent: %s", e) finally: self._users_in_progress.remove(user_id) + + +def copy_with_str_subst(x, substitutions): + """Deep-copy a structure, carrying out string substitions on any strings + + Args: + x (object): structure to be copied + substitutions (object): substitutions to be made - passed into the + string '%' operator + + Returns: + copy of x + """ + if isinstance(x, string_types): + return x % substitutions + if isinstance(x, dict): + return { + k: copy_with_str_subst(v, substitutions) for (k, v) in iteritems(x) + } + if isinstance(x, (list, tuple)): + return [copy_with_str_subst(y) for y in x] + + # assume it's uninterested and can be shallow-copied. + return x From 3ff6f50eac86b063706f68387d3cf3e96a79a541 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:28:23 +0100 Subject: [PATCH 30/46] Use the localpart in the consent uri ... because it's shorter. --- synapse/handlers/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c3adbc6c9..7936a60af 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -574,7 +574,9 @@ class EventCreationHandler(object): if u["consent_version"] == self.config.user_consent_version: return - consent_uri = self._consent_uri_builder.build_user_consent_uri(user_id) + consent_uri = self._consent_uri_builder.build_user_consent_uri( + requester.user.localpart, + ) raise ConsentNotGivenError( msg=self.config.block_events_without_consent_error, consent_uri=consent_uri, From dda40fb55d0aa7bde6387e5e32ffed872fddea4e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:29:58 +0100 Subject: [PATCH 31/46] fix typo --- synapse/server_notices/server_notices_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 9f76c7554..7e4055c84 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -56,7 +56,7 @@ class ServerNoticesManager(object): event_content (dict): content of event to send Returns: - Deferrred[None] + Deferred[None] """ room_id = yield self.get_notice_room_for_user(user_id) From 2df8c3139ab7be10af4aaa750bbdc4ad39209583 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 15:39:52 +0100 Subject: [PATCH 32/46] minor post-review tweaks --- docs/consent_tracking.md | 6 +++--- docs/server_notices.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 66beb9263..9dc009388 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -81,9 +81,9 @@ should be a matter of `pip install Jinja2`. On debian, try `apt-get install python-jinja2`. Once this is complete, and the server has been restarted, try visiting -`https:///_matrix/consent`. If configuration has been done correctly, -this should give an error "Missing string query parameter 'u'". It is now -possible to manually construct URIs where users can give their consent. +`https:///_matrix/consent`. If correctly configured, this should give +an error "Missing string query parameter 'u'". It is now possible to manually +construct URIs where users can give their consent. ### Constructing the consent URI diff --git a/docs/server_notices.md b/docs/server_notices.md index 989688138..8e18e3d95 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -4,7 +4,7 @@ Server Notices 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of the communication of the server polices (see +They are used as part of communication of the server polices(see [consent_tracking.md](consent_tracking.md)), however the intention is that they may also find a use for features such as "Message of the day". @@ -32,7 +32,7 @@ Synapse configuration Server notices come from a specific user id on the server. Server administrators are free to choose the user id - something like `server` is suggested, meaning the notices will come from -`@server:`. Once the server notices user is configured, that +`@server:`. Once the Server Notices user is configured, that user id becomes a special, privileged user, so administrators should ensure that **it is not already allocated**. From e206b2c9ace8a202bf47ec419581e19f0baa39fb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 23 May 2018 15:57:10 +0100 Subject: [PATCH 33/46] consent_tracking.md: clarify link --- docs/consent_tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 9dc009388..064eae82f 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -24,7 +24,7 @@ To enable this, first create templates for the policy and success pages. These should be stored on the local filesystem. These templates use the [Jinja2](http://jinja.pocoo.org) templating language, -and the `privacy_policy_templates` subdirectory of this `docs` directory gives +and [docs/privacy_policy_templates](privacy_policy_templates) gives examples of the sort of thing that can be done. Note that the templates must be stored under a name giving the language of the From 9bf4b2bda343dcbdba7b0e9d752bc560f8e344fd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 May 2018 17:43:30 +0100 Subject: [PATCH 34/46] Allow overriding the server_notices user's avatar probably should have done this in the first place, like @turt2live suggested. --- docs/server_notices.md | 9 ++++++--- synapse/config/server_notices_config.py | 15 ++++++++++++--- .../server_notices/server_notices_manager.py | 17 ++++++++++++++--- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 8e18e3d95..221553b24 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -44,13 +44,16 @@ section, which should look like this: server_notices: system_mxid_localpart: server system_mxid_display_name: "Server Notices" + system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" room_name: "Server Notices" ``` The only compulsory setting is `system_mxid_localpart`, which defines the user -id of the server notices user, as above. `system_mxid_display_name` and -`room_name` define the displayname of the system notices user, and of -the notices room, respectively. +id of the Server Notices user, as above. `room_name` defines the name of the +room which will be created. + +`system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the +displayname and avatar of the Server Notices user. Sending notices --------------- diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py index ccef8d2ec..be1d1f762 100644 --- a/synapse/config/server_notices_config.py +++ b/synapse/config/server_notices_config.py @@ -26,12 +26,13 @@ DEFAULT_CONFIG = """\ # setting, which defines the id of the user which will be used to send the # notices. # -# It's also possible to override the room name, or the display name of the -# "notices" user. +# It's also possible to override the room name, the display name of the +# "notices" user, and the avatar for the user. # # server_notices: # system_mxid_localpart: notices # system_mxid_display_name: "Server Notices" +# system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" # room_name: "Server Notices" """ @@ -48,6 +49,10 @@ class ServerNoticesConfig(Config): The display name to use for the server notices user. None if server notices are not enabled. + server_notices_mxid_avatar_url (str|None): + The display name to use for the server notices user. + None if server notices are not enabled. + server_notices_room_name (str|None): The name to use for the server notices room. None if server notices are not enabled. @@ -56,6 +61,7 @@ class ServerNoticesConfig(Config): super(ServerNoticesConfig, self).__init__() self.server_notices_mxid = None self.server_notices_mxid_display_name = None + self.server_notices_mxid_avatar_url = None self.server_notices_room_name = None def read_config(self, config): @@ -68,7 +74,10 @@ class ServerNoticesConfig(Config): mxid_localpart, self.server_name, ).to_string() self.server_notices_mxid_display_name = c.get( - 'system_mxid_display_name', 'Server Notices', + 'system_mxid_display_name', None, + ) + self.server_notices_mxid_avatar_url = c.get( + 'system_mxid_avatar_url', None, ) # todo: i18n self.server_notices_room_name = c.get('room_name', "Server Notices") diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 7e4055c84..a26deace5 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -113,6 +113,19 @@ class ServerNoticesManager(object): # apparently no existing notice room: create a new one logger.info("Creating server notices room for %s", user_id) + # see if we want to override the profile info for the server user. + # note that if we want to override either the display name or the + # avatar, we have to use both. + join_profile = None + if ( + self._config.server_notices_mxid_display_name is not None or + self._config.server_notices_mxid_avatar_url is not None + ): + join_profile = { + "displayname": self._config.server_notices_mxid_display_name, + "avatar_url": self._config.server_notices_mxid_avatar_url, + } + requester = create_requester(system_mxid) info = yield self._room_creation_handler.create_room( requester, @@ -125,9 +138,7 @@ class ServerNoticesManager(object): "invite": (user_id,) }, ratelimit=False, - creator_join_profile={ - "displayname": self._config.server_notices_mxid_display_name, - }, + creator_join_profile=join_profile, ) room_id = info['room_id'] From 91d95a1d8e4005ebd4805418d331a6ea8f73fdb6 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 14:05:07 +0100 Subject: [PATCH 35/46] bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 353387f15..5bada5e29 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.30.0-rc1" +__version__ = "0.30.0" From bdd2ed5acf7ea34092c9aa7049232ea5e8f29874 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:02:03 +0100 Subject: [PATCH 36/46] update for v0.30.0 --- CHANGES.rst | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 096fa4794..ffadd83e4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,30 @@ +Changes in synapse v0.30.0 (2018-05-24) +========================================== + +'Server Notices' are a new feature introduced in Synapse 0.30. They provide a +channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server policies (see `docs/consent_tracking.md`), +however the intention is that they may also find a use for features such +as "Message of the day". + +This feature is specific to Synapse, but uses standard Matrix communication mechanisms, +so should work with any Matrix client. For more details see ``docs/server_notices.md`` + +Further Server Notices/Consent Tracking Support: + +* Allow overriding the server_notices user's avatar (#3273) +* Use the localpart in the consent uri (#3272) +* Support for putting %(consent_uri)s in messages (#3271) +* Block attempts to send server notices to remote users (#3270) +* Docs on consent bits (#3268) + + + Changes in synapse v0.30.0-rc1 (2018-05-23) ========================================== -This version includes support for Privacy Notice agreement collection. More -details live at docs/privacy_policy_templates/README.md - - -GDPR Support: +Server Notices/Consent Tracking Support: * ConsentResource to gather policy consent from users (PR #3213) * Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) From 6f67163c6358554bfd9568e98e0ac2225436b1c1 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:04:41 +0100 Subject: [PATCH 37/46] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ffadd83e4..6e7f108f8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes in synapse v0.30.0 (2018-05-24) 'Server Notices' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. -They are used as part of communication of the server policies (see `docs/consent_tracking.md`), +They are used as part of communication of the server policies (see ``docs/consent_tracking.md``), however the intention is that they may also find a use for features such as "Message of the day". From 14ca67867422d333a933b3bb0a4fd86da1656a98 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 24 May 2018 15:54:02 +0100 Subject: [PATCH 38/46] Update CHANGES.rst --- CHANGES.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6e7f108f8..0569b581d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,11 +13,11 @@ so should work with any Matrix client. For more details see ``docs/server_notice Further Server Notices/Consent Tracking Support: -* Allow overriding the server_notices user's avatar (#3273) -* Use the localpart in the consent uri (#3272) -* Support for putting %(consent_uri)s in messages (#3271) -* Block attempts to send server notices to remote users (#3270) -* Docs on consent bits (#3268) +* Allow overriding the server_notices user's avatar (PR #3273) +* Use the localpart in the consent uri (PR #3272) +* Support for putting %(consent_uri)s in messages (PR #3271) +* Block attempts to send server notices to remote users (PR #3270) +* Docs on consent bits (PR #3268) From 7a1af504d70e168e87f1e15ecda90179dd75ebc8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 24 May 2018 15:59:58 +0100 Subject: [PATCH 39/46] Remove users from user directory on deactivate --- synapse/handlers/deactivate_account.py | 4 ++++ synapse/handlers/user_directory.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index d58ea6c65..c5e92f621 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -30,6 +30,7 @@ class DeactivateAccountHandler(BaseHandler): self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + self.user_directory_handler = hs.get_user_directory_handler() # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False @@ -65,6 +66,9 @@ class DeactivateAccountHandler(BaseHandler): # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) + # delete from user directory + yield self.user_directory_handler.handle_user_deactivated(user_id) + # Now start the process that goes through that list and # parts users from rooms (if it isn't already running) self._start_user_parting() diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 714f0195c..2f73d664f 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -122,6 +122,13 @@ class UserDirectoryHandler(object): user_id, profile.display_name, profile.avatar_url, None, ) + @defer.inlineCallbacks + def handle_user_deactivated(self, user_id): + """Called when a user ID is deactivated + """ + yield self.store.remove_from_user_dir(user_id) + yield self.store.remove_from_user_in_public_room(user_id) + @defer.inlineCallbacks def _unsafe_process(self): # If self.pos is None then means we haven't fetched it from DB From 095292304f04c0984813a261a7f6042b35cbd2dd Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:52:41 +0200 Subject: [PATCH 40/46] Py3 storage/_base.py Signed-off-by: Adrian Tschira --- synapse/storage/_base.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2262776ab..51cf7b4c6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,6 +27,8 @@ import sys import time import threading +from six import itervalues, iterkeys, iteritems +from six.moves import intern, range logger = logging.getLogger(__name__) @@ -137,7 +139,7 @@ class PerformanceCounters(object): def interval(self, interval_duration, limit=3): counters = [] - for name, (count, cum_time) in self.current_counters.iteritems(): + for name, (count, cum_time) in iteritems(self.current_counters): prev_count, prev_time = self.previous_counters.get(name, (0, 0)) counters.append(( (cum_time - prev_time) / interval_duration, @@ -543,7 +545,7 @@ class SQLBaseStore(object): ", ".join("%s = ?" % (k,) for k in values), " AND ".join("%s = ?" % (k,) for k in keyvalues) ) - sqlargs = values.values() + keyvalues.values() + sqlargs = list(values.values()) + list(keyvalues.values()) txn.execute(sql, sqlargs) if txn.rowcount > 0: @@ -561,7 +563,7 @@ class SQLBaseStore(object): ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues) ) - txn.execute(sql, allvalues.values()) + txn.execute(sql, list(allvalues.values())) # successfully inserted return True @@ -629,8 +631,8 @@ class SQLBaseStore(object): } if keyvalues: - sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys()) - txn.execute(sql, keyvalues.values()) + sql += " WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues)) + txn.execute(sql, list(keyvalues.values())) else: txn.execute(sql) @@ -694,7 +696,7 @@ class SQLBaseStore(object): table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - txn.execute(sql, keyvalues.values()) + txn.execute(sql, list(keyvalues.values())) else: sql = "SELECT %s FROM %s" % ( ", ".join(retcols), @@ -725,9 +727,12 @@ class SQLBaseStore(object): if not iterable: defer.returnValue(results) + # iterables can not be sliced, so convert it to a list first + it_list = list(iterable) + chunks = [ - iterable[i:i + batch_size] - for i in xrange(0, len(iterable), batch_size) + it_list[i:i + batch_size] + for i in range(0, len(it_list), batch_size) ] for chunk in chunks: rows = yield self.runInteraction( @@ -767,7 +772,7 @@ class SQLBaseStore(object): ) values.extend(iterable) - for key, value in keyvalues.iteritems(): + for key, value in iteritems(keyvalues): clauses.append("%s = ?" % (key,)) values.append(value) @@ -790,7 +795,7 @@ class SQLBaseStore(object): @staticmethod def _simple_update_txn(txn, table, keyvalues, updatevalues): if keyvalues: - where = "WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.iterkeys()) + where = "WHERE %s" % " AND ".join("%s = ?" % k for k in iterkeys(keyvalues)) else: where = "" @@ -802,7 +807,7 @@ class SQLBaseStore(object): txn.execute( update_sql, - updatevalues.values() + keyvalues.values() + list(updatevalues.values()) + list(keyvalues.values()) ) return txn.rowcount @@ -850,7 +855,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k,) for k in keyvalues) ) - txn.execute(select_sql, keyvalues.values()) + txn.execute(select_sql, list(keyvalues.values())) row = txn.fetchone() if not row: @@ -888,7 +893,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - txn.execute(sql, keyvalues.values()) + txn.execute(sql, list(keyvalues.values())) if txn.rowcount == 0: raise StoreError(404, "No row found") if txn.rowcount > 1: @@ -906,7 +911,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) - return txn.execute(sql, keyvalues.values()) + return txn.execute(sql, list(keyvalues.values())) def _simple_delete_many(self, table, column, iterable, keyvalues, desc): return self.runInteraction( @@ -938,7 +943,7 @@ class SQLBaseStore(object): ) values.extend(iterable) - for key, value in keyvalues.iteritems(): + for key, value in iteritems(keyvalues): clauses.append("%s = ?" % (key,)) values.append(value) @@ -978,7 +983,7 @@ class SQLBaseStore(object): txn.close() if cache: - min_val = min(cache.itervalues()) + min_val = min(itervalues(cache)) else: min_val = max_value @@ -1093,7 +1098,7 @@ class SQLBaseStore(object): " AND ".join("%s = ?" % (k,) for k in keyvalues), " ? ASC LIMIT ? OFFSET ?" ) - txn.execute(sql, keyvalues.values() + pagevalues) + txn.execute(sql, list(keyvalues.values()) + list(pagevalues)) else: sql = "SELECT %s FROM %s ORDER BY %s" % ( ", ".join(retcols), From 6c16a4ec1b6bc1aa3ef3bbcc080419e1b4bce5bd Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Tue, 15 May 2018 17:45:01 +0200 Subject: [PATCH 41/46] more iteritems --- synapse/storage/roommember.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 6a861943a..7bfc3d91b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,6 +30,8 @@ from synapse.types import get_domain_from_id import logging import simplejson as json +from six import itervalues, iteritems + logger = logging.getLogger(__name__) @@ -272,7 +274,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): users_in_room = {} member_event_ids = [ e_id - for key, e_id in current_state_ids.iteritems() + for key, e_id in iteritems(current_state_ids) if key[0] == EventTypes.Member ] @@ -289,7 +291,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): users_in_room = dict(prev_res) member_event_ids = [ e_id - for key, e_id in context.delta_ids.iteritems() + for key, e_id in iteritems(context.delta_ids) if key[0] == EventTypes.Member ] for etype, state_key in context.delta_ids: @@ -741,7 +743,7 @@ class _JoinedHostsCache(object): if state_entry.state_group == self.state_group: pass elif state_entry.prev_group == self.state_group: - for (typ, state_key), event_id in state_entry.delta_ids.iteritems(): + for (typ, state_key), event_id in iteritems(state_entry.delta_ids): if typ != EventTypes.Member: continue @@ -771,7 +773,7 @@ class _JoinedHostsCache(object): self.state_group = state_entry.state_group else: self.state_group = object() - self._len = sum(len(v) for v in self.hosts_to_joined_users.itervalues()) + self._len = sum(len(v) for v in itervalues(self.hosts_to_joined_users)) defer.returnValue(frozenset(self.hosts_to_joined_users)) def __len__(self): From 17a70cf6e9ae016011972b4d8a97aca0c3d945d4 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Sat, 5 May 2018 22:47:18 +0200 Subject: [PATCH 42/46] Misc. py3 fixes Signed-off-by: Adrian Tschira --- synapse/handlers/federation.py | 9 +++++---- synapse/handlers/message.py | 9 +++++---- synapse/storage/events_worker.py | 2 +- synapse/storage/filtering.py | 2 +- synapse/storage/keys.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2c72beff2..87c061582 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -24,6 +24,7 @@ from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json import six from six.moves import http_client +from six import iteritems from twisted.internet import defer from unpaddedbase64 import decode_base64 @@ -1388,7 +1389,7 @@ class FederationHandler(BaseHandler): ) if state_groups: - _, state = state_groups.items().pop() + _, state = list(iteritems(state_groups)).pop() results = { (e.type, e.state_key): e for e in state } @@ -2034,7 +2035,7 @@ class FederationHandler(BaseHandler): this will not be included in the current_state in the context. """ state_updates = { - k: a.event_id for k, a in auth_events.iteritems() + k: a.event_id for k, a in iteritems(auth_events) if k != event_key } context.current_state_ids = dict(context.current_state_ids) @@ -2044,7 +2045,7 @@ class FederationHandler(BaseHandler): context.delta_ids.update(state_updates) context.prev_state_ids = dict(context.prev_state_ids) context.prev_state_ids.update({ - k: a.event_id for k, a in auth_events.iteritems() + k: a.event_id for k, a in iteritems(auth_events) }) context.state_group = yield self.store.store_state_group( event.event_id, @@ -2096,7 +2097,7 @@ class FederationHandler(BaseHandler): def get_next(it, opt=None): try: - return it.next() + return next(it) except Exception: return opt diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c32b9bcae..81cff0870 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -19,6 +19,7 @@ import sys from canonicaljson import encode_canonical_json import six +from six import string_types, itervalues, iteritems from twisted.internet import defer, reactor from twisted.internet.defer import succeed from twisted.python.failure import Failure @@ -402,7 +403,7 @@ class MessageHandler(BaseHandler): "avatar_url": profile.avatar_url, "display_name": profile.display_name, } - for user_id, profile in users_with_profile.iteritems() + for user_id, profile in iteritems(users_with_profile) }) @@ -667,7 +668,7 @@ class EventCreationHandler(object): spam_error = self.spam_checker.check_event_for_spam(event) if spam_error: - if not isinstance(spam_error, basestring): + if not isinstance(spam_error, string_types): spam_error = "Spam is not permitted here" raise SynapseError( 403, spam_error, Codes.FORBIDDEN @@ -881,7 +882,7 @@ class EventCreationHandler(object): state_to_include_ids = [ e_id - for k, e_id in context.current_state_ids.iteritems() + for k, e_id in iteritems(context.current_state_ids) if k[0] in self.hs.config.room_invite_state_types or k == (EventTypes.Member, event.sender) ] @@ -895,7 +896,7 @@ class EventCreationHandler(object): "content": e.content, "sender": e.sender, } - for e in state_to_include.itervalues() + for e in itervalues(state_to_include) ] invitee = UserID.from_string(event.state_key) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index ba834854e..32d9d00ff 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -337,7 +337,7 @@ class EventsWorkerStore(SQLBaseStore): def _fetch_event_rows(self, txn, events): rows = [] N = 200 - for i in range(1 + len(events) / N): + for i in range(1 + len(events) // N): evs = events[i * N:(i + 1) * N] if not evs: break diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 78b1e3094..2e2763126 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -44,7 +44,7 @@ class FilteringStore(SQLBaseStore): desc="get_user_filter", ) - defer.returnValue(json.loads(str(def_json).decode("utf-8"))) + defer.returnValue(json.loads(bytes(def_json).decode("utf-8"))) def add_user_filter(self, user_localpart, user_filter): def_json = encode_canonical_json(user_filter) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 87aeaf71d..0540c2b0b 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -92,7 +92,7 @@ class KeyStore(SQLBaseStore): if verify_key_bytes: defer.returnValue(decode_verify_key_bytes( - key_id, str(verify_key_bytes) + key_id, bytes(verify_key_bytes) )) @defer.inlineCallbacks From 0e617056615d5db9f95ba458b2fafbbe0103c9d8 Mon Sep 17 00:00:00 2001 From: Adrian Tschira Date: Thu, 24 May 2018 20:58:21 +0200 Subject: [PATCH 43/46] py3-ize state.py --- synapse/state.py | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 26093c843..b8c27c681 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -32,6 +32,8 @@ from frozendict import frozendict import logging import hashlib +from six import iteritems, itervalues + logger = logging.getLogger(__name__) @@ -132,7 +134,7 @@ class StateHandler(object): state_map = yield self.store.get_events(state.values(), get_prev_content=False) state = { - key: state_map[e_id] for key, e_id in state.iteritems() if e_id in state_map + key: state_map[e_id] for key, e_id in iteritems(state) if e_id in state_map } defer.returnValue(state) @@ -338,7 +340,7 @@ class StateHandler(object): ) if len(state_groups_ids) == 1: - name, state_list = state_groups_ids.items().pop() + name, state_list = list(state_groups_ids.items()).pop() prev_group, delta_ids = yield self.store.get_state_group_delta(name) @@ -378,7 +380,7 @@ class StateHandler(object): new_state = resolve_events_with_state_map(state_set_ids, state_map) new_state = { - key: state_map[ev_id] for key, ev_id in new_state.iteritems() + key: state_map[ev_id] for key, ev_id in iteritems(new_state) } return new_state @@ -458,15 +460,15 @@ class StateResolutionHandler(object): # build a map from state key to the event_ids which set that state. # dict[(str, str), set[str]) state = {} - for st in state_groups_ids.itervalues(): - for key, e_id in st.iteritems(): + for st in itervalues(state_groups_ids): + for key, e_id in iteritems(st): state.setdefault(key, set()).add(e_id) # build a map from state key to the event_ids which set that state, # including only those where there are state keys in conflict. conflicted_state = { k: list(v) - for k, v in state.iteritems() + for k, v in iteritems(state) if len(v) > 1 } @@ -474,13 +476,13 @@ class StateResolutionHandler(object): logger.info("Resolving conflicted state for %r", room_id) with Measure(self.clock, "state._resolve_events"): new_state = yield resolve_events_with_factory( - state_groups_ids.values(), + list(state_groups_ids.values()), event_map=event_map, state_map_factory=state_map_factory, ) else: new_state = { - key: e_ids.pop() for key, e_ids in state.iteritems() + key: e_ids.pop() for key, e_ids in iteritems(state) } with Measure(self.clock, "state.create_group_ids"): @@ -489,8 +491,8 @@ class StateResolutionHandler(object): # which will be used as a cache key for future resolutions, but # not get persisted. state_group = None - new_state_event_ids = frozenset(new_state.itervalues()) - for sg, events in state_groups_ids.iteritems(): + new_state_event_ids = frozenset(itervalues(new_state)) + for sg, events in iteritems(state_groups_ids): if new_state_event_ids == frozenset(e_id for e_id in events): state_group = sg break @@ -501,11 +503,11 @@ class StateResolutionHandler(object): prev_group = None delta_ids = None - for old_group, old_ids in state_groups_ids.iteritems(): + for old_group, old_ids in iteritems(state_groups_ids): if not set(new_state) - set(old_ids): n_delta_ids = { k: v - for k, v in new_state.iteritems() + for k, v in iteritems(new_state) if old_ids.get(k) != v } if not delta_ids or len(n_delta_ids) < len(delta_ids): @@ -527,7 +529,7 @@ class StateResolutionHandler(object): def _ordered_events(events): def key_func(e): - return -int(e.depth), hashlib.sha1(e.event_id).hexdigest() + return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest() return sorted(events, key=key_func) @@ -584,7 +586,7 @@ def _seperate(state_sets): conflicted_state = {} for state_set in state_sets[1:]: - for key, value in state_set.iteritems(): + for key, value in iteritems(state_set): # Check if there is an unconflicted entry for the state key. unconflicted_value = unconflicted_state.get(key) if unconflicted_value is None: @@ -640,7 +642,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): needed_events = set( event_id - for event_ids in conflicted_state.itervalues() + for event_ids in itervalues(conflicted_state) for event_id in event_ids ) if event_map is not None: @@ -662,7 +664,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): unconflicted_state, conflicted_state, state_map ) - new_needed_events = set(auth_events.itervalues()) + new_needed_events = set(itervalues(auth_events)) new_needed_events -= needed_events if event_map is not None: new_needed_events -= set(event_map.iterkeys()) @@ -679,7 +681,7 @@ def resolve_events_with_factory(state_sets, event_map, state_map_factory): def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_map): auth_events = {} - for event_ids in conflicted_state.itervalues(): + for event_ids in itervalues(conflicted_state): for event_id in event_ids: if event_id in state_map: keys = event_auth.auth_types_for_event(state_map[event_id]) @@ -694,7 +696,7 @@ def _create_auth_events_from_maps(unconflicted_state, conflicted_state, state_ma def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ids, state_map): conflicted_state = {} - for key, event_ids in conflicted_state_ds.iteritems(): + for key, event_ids in iteritems(conflicted_state_ds): events = [state_map[ev_id] for ev_id in event_ids if ev_id in state_map] if len(events) > 1: conflicted_state[key] = events @@ -703,7 +705,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ auth_events = { key: state_map[ev_id] - for key, ev_id in auth_event_ids.iteritems() + for key, ev_id in iteritems(auth_event_ids) if ev_id in state_map } @@ -716,7 +718,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ raise new_state = unconflicted_state_ids - for key, event in resolved_state.iteritems(): + for key, event in iteritems(resolved_state): new_state[key] = event.event_id return new_state @@ -741,7 +743,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key[0] == EventTypes.JoinRules: logger.debug("Resolving conflicted join rules %r", events) resolved_state[key] = _resolve_auth_events( @@ -751,7 +753,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key[0] == EventTypes.Member: logger.debug("Resolving conflicted member lists %r", events) resolved_state[key] = _resolve_auth_events( @@ -761,7 +763,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.iteritems(): + for key, events in iteritems(conflicted_state): if key not in resolved_state: logger.debug("Resolving conflicted state %r:%r", key, events) resolved_state[key] = _resolve_normal_events( From 757ed2725803ff6d190eb1bb8dea872345862aa3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:07:21 +0100 Subject: [PATCH 44/46] Let users leave the server notice room after joining They still can't reject invites, but we let them leave it. --- docs/server_notices.md | 7 +++++-- synapse/handlers/room_member.py | 24 ++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/server_notices.md b/docs/server_notices.md index 221553b24..58f877631 100644 --- a/docs/server_notices.md +++ b/docs/server_notices.md @@ -5,7 +5,7 @@ Server Notices channel whereby server administrators can send messages to users on the server. They are used as part of communication of the server polices(see -[consent_tracking.md](consent_tracking.md)), however the intention is that +[consent_tracking.md](consent_tracking.md)), however the intention is that they may also find a use for features such as "Message of the day". This is a feature specific to Synapse, but it uses standard Matrix @@ -24,7 +24,10 @@ history; it will appear to have come from the 'server notices user' (see below). The user is prevented from sending any messages in this room by the power -levels. They also cannot leave it. +levels. + +Having joined the room, the user can leave the room if they want. Subsequent +server notices will then cause a new room to be created. Synapse configuration --------------------- diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 82adfc8fd..f930e939e 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -298,16 +298,6 @@ class RoomMemberHandler(object): is_blocked = yield self.store.is_room_blocked(room_id) if is_blocked: raise SynapseError(403, "This room has been blocked on this server") - else: - # we don't allow people to reject invites to, or leave, the - # server notice room. - is_blocked = yield self._is_server_notice_room(room_id) - if is_blocked: - raise SynapseError( - http_client.FORBIDDEN, - "You cannot leave this room", - errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, - ) if effective_membership_state == Membership.INVITE: # block any attempts to invite the server notices mxid @@ -383,6 +373,20 @@ class RoomMemberHandler(object): if same_sender and same_membership and same_content: defer.returnValue(old_state) + # we don't allow people to reject invites to the server notice + # room, but they can leave it once they are joined. + if ( + old_membership == Membership.INVITE and + effective_membership_state == Membership.LEAVE + ): + is_blocked = yield self._is_server_notice_room(room_id) + if is_blocked: + raise SynapseError( + http_client.FORBIDDEN, + "You cannot reject this invite", + errcode=Codes.CANNOT_LEAVE_SERVER_NOTICE_ROOM, + ) + is_host_in_room = yield self._is_host_in_room(current_state_ids) if effective_membership_state == Membership.JOIN: From ba1b1635901f148d793777ea77386dbe3412dfaf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:36:43 +0100 Subject: [PATCH 45/46] Avoid sending consent notice to guest users we think it makes sense not to send the notices to guest users. --- synapse/config/consent_config.py | 8 +++++++- synapse/server_notices/consent_server_notices.py | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index ddcd305f4..3f2934659 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -32,7 +32,8 @@ DEFAULT_CONFIG = """\ # # 'server_notice_content', if enabled, will send a user a "Server Notice" # asking them to consent to the privacy policy. The 'server_notices' section -# must also be configured for this to work. +# must also be configured for this to work. Notices will *not* be sent to +# guest users unless 'send_server_notice_to_guests' is set to true. # # 'block_events_error', if set, will block any attempts to send events # until the user consents to the privacy policy. The value of the setting is @@ -46,6 +47,7 @@ DEFAULT_CONFIG = """\ # body: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s +# send_server_notice_to_guests: True # block_events_error: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s @@ -60,6 +62,7 @@ class ConsentConfig(Config): self.user_consent_version = None self.user_consent_template_dir = None self.user_consent_server_notice_content = None + self.user_consent_server_notice_to_guests = False self.block_events_without_consent_error = None def read_config(self, config): @@ -74,6 +77,9 @@ class ConsentConfig(Config): self.block_events_without_consent_error = consent_config.get( "block_events_error", ) + self.user_consent_server_notice_to_guests = bool(consent_config.get( + "send_server_notice_to_guests", "False" + )) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/server_notices/consent_server_notices.py b/synapse/server_notices/consent_server_notices.py index a70980285..bb74af1af 100644 --- a/synapse/server_notices/consent_server_notices.py +++ b/synapse/server_notices/consent_server_notices.py @@ -42,6 +42,7 @@ class ConsentServerNotices(object): self._current_consent_version = hs.config.user_consent_version self._server_notice_content = hs.config.user_consent_server_notice_content + self._send_to_guests = hs.config.user_consent_server_notice_to_guests if self._server_notice_content is not None: if not self._server_notices_manager.is_enabled(): @@ -78,6 +79,10 @@ class ConsentServerNotices(object): try: u = yield self._store.get_user_by_id(user_id) + if u["is_guest"] and not self._send_to_guests: + # don't send to guests + return + if u["consent_version"] == self._current_consent_version: # user has already consented return From 66bdae986f4bdee300aebb4ad6ebeeba48286e6b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 25 May 2018 11:42:05 +0100 Subject: [PATCH 46/46] Fix default for send_server_notice_to_guests bool("False") == True... --- synapse/config/consent_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index 3f2934659..8f6ed7332 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -78,7 +78,7 @@ class ConsentConfig(Config): "block_events_error", ) self.user_consent_server_notice_to_guests = bool(consent_config.get( - "send_server_notice_to_guests", "False" + "send_server_notice_to_guests", False, )) def default_config(self, **kwargs):