From 2ef3f84945c3a6ef653d6f589c06e2d8882cca05 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 10 Jul 2018 21:14:22 +0200 Subject: [PATCH 01/42] change http links to https Especially useful for the debian repo, as this makes it easier to get the key in a secure way --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 4fe54b0c9..84b8cf572 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ We'd like to invite you to join #matrix:matrix.org (via https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look at the `Matrix spec `_, and experiment with the `APIs `_ and `Client SDKs -`_. +`_. Thanks for using Matrix! @@ -283,7 +283,7 @@ Connecting to Synapse from a client The easiest way to try out your new Synapse installation is by connecting to it from a web client. The easiest option is probably the one at -http://riot.im/app. You will need to specify a "Custom server" when you log on +https://riot.im/app. You will need to specify a "Custom server" when you log on or register: set this to ``https://domain.tld`` if you setup a reverse proxy following the recommended setup, or ``https://localhost:8448`` - remember to specify the port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity @@ -329,7 +329,7 @@ Security Note ============= Matrix serves raw user generated data in some APIs - specifically the `content -repository endpoints `_. +repository endpoints `_. Whilst we have tried to mitigate against possible XSS attacks (e.g. https://github.com/matrix-org/synapse/pull/1021) we recommend running @@ -348,7 +348,7 @@ Platform-Specific Instructions Debian ------ -Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/. +Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/. Note that these packages do not include a client - choose one from https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :) @@ -524,7 +524,7 @@ Troubleshooting Running ----------------------- If synapse fails with ``missing "sodium.h"`` crypto errors, you may need -to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for +to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for encryption and digital signatures. Unfortunately PyNACL currently has a few issues (https://github.com/pyca/pynacl/issues/53) and @@ -672,8 +672,8 @@ useful just for development purposes. See ``_. Using PostgreSQL ================ -As of Synapse 0.9, `PostgreSQL `_ is supported as an -alternative to the `SQLite `_ database that Synapse has +As of Synapse 0.9, `PostgreSQL `_ is supported as an +alternative to the `SQLite `_ database that Synapse has traditionally used for convenience and simplicity. The advantages of Postgres include: @@ -697,7 +697,7 @@ Using a reverse proxy with Synapse It is recommended to put a reverse proxy such as `nginx `_, `Apache `_ or -`HAProxy `_ in front of Synapse. One advantage of +`HAProxy `_ in front of Synapse. One advantage of doing so is that it means that you can expose the default https port (443) to Matrix clients without needing to run Synapse with root privileges. From 5797f5542b687b73ebdd8613f64ce0f38e637b55 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 12 Jul 2018 01:32:39 +0100 Subject: [PATCH 02/42] WIP to announce deleted devices over federation Previously we queued up the poke correctly when the device was deleted, but then the actual EDU wouldn't get sent, as the device was no longer known. Instead, we now send EDUs for deleted devices too if there's a poke for them. --- synapse/notifier.py | 2 +- synapse/storage/devices.py | 40 +++++++++++++++++++++--------- synapse/storage/end_to_end_keys.py | 16 +++++++++++- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 51cbd66f0..e650c3e49 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -274,7 +274,7 @@ class Notifier(object): logger.exception("Error notifying application services of event") def on_new_event(self, stream_key, new_token, users=[], rooms=[]): - """ Used to inform listeners that something has happend event wise. + """ Used to inform listeners that something has happened event wise. Will wake up all listeners for the given users and rooms. """ diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index ec68e39f1..0c797f9f3 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -239,6 +239,7 @@ class DeviceStore(SQLBaseStore): def update_remote_device_list_cache_entry(self, user_id, device_id, content, stream_id): """Updates a single user's device in the cache. + If the content is null, delete the device from the cache. """ return self.runInteraction( "update_remote_device_list_cache_entry", @@ -248,17 +249,32 @@ class DeviceStore(SQLBaseStore): def _update_remote_device_list_cache_entry_txn(self, txn, user_id, device_id, content, stream_id): - self._simple_upsert_txn( - txn, - table="device_lists_remote_cache", - keyvalues={ - "user_id": user_id, - "device_id": device_id, - }, - values={ - "content": json.dumps(content), - } - ) + if content is None: + self._simple_delete_txn( + txn, + table="device_lists_remote_cache", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + ) + + # Do we need this? + txn.call_after( + self.device_id_exists_cache.invalidate, (user_id, device_id,) + ) + else: + self._simple_upsert_txn( + txn, + table="device_lists_remote_cache", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + values={ + "content": json.dumps(content), + } + ) txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id,)) txn.call_after(self._get_cached_devices_for_user.invalidate, (user_id,)) @@ -366,7 +382,7 @@ class DeviceStore(SQLBaseStore): now_stream_id = max(stream_id for stream_id in itervalues(query_map)) devices = self._get_e2e_device_keys_txn( - txn, query_map.keys(), include_all_devices=True + txn, query_map.keys(), include_all_devices=True, include_deleted_devices=True ) prev_sent_id_sql = """ diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 7ae5c6548..f61553cec 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -64,12 +64,17 @@ class EndToEndKeyStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_e2e_device_keys(self, query_list, include_all_devices=False): + def get_e2e_device_keys( + self, query_list, include_all_devices=False, + include_deleted_devices=False + ): """Fetch a list of device keys. Args: query_list(list): List of pairs of user_ids and device_ids. include_all_devices (bool): whether to include entries for devices that don't have device keys + include_deleted_devices (bool): whether to include null entries for + devices which no longer exist (but where in the query_list) Returns: Dict mapping from user-id to dict mapping from device_id to dict containing "key_json", "device_display_name". @@ -82,10 +87,19 @@ class EndToEndKeyStore(SQLBaseStore): query_list, include_all_devices, ) + if include_deleted_devices: + deleted_devices = set(query_list) + for user_id, device_keys in iteritems(results): for device_id, device_info in iteritems(device_keys): + if include_deleted_devices: + deleted_devices -= (user_id, device_id) device_info["keys"] = json.loads(device_info.pop("key_json")) + if include_deleted_devices: + for user_id, device_id in deleted_devices: + results.setdefault(user_id, {})[device_id] = None + defer.returnValue(results) def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices): From 12ec58301f946ced9702afbf6dfbfbc8c3dfd3dd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 12 Jul 2018 11:39:43 +0100 Subject: [PATCH 03/42] shift to using an explicit deleted flag on m.device_list_update EDUs and generally make it work. --- synapse/storage/devices.py | 18 ++++++++++-------- synapse/storage/end_to_end_keys.py | 27 +++++++++++++++------------ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 0c797f9f3..203f50f07 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -239,7 +239,6 @@ class DeviceStore(SQLBaseStore): def update_remote_device_list_cache_entry(self, user_id, device_id, content, stream_id): """Updates a single user's device in the cache. - If the content is null, delete the device from the cache. """ return self.runInteraction( "update_remote_device_list_cache_entry", @@ -249,7 +248,7 @@ class DeviceStore(SQLBaseStore): def _update_remote_device_list_cache_entry_txn(self, txn, user_id, device_id, content, stream_id): - if content is None: + if content.get("deleted"): self._simple_delete_txn( txn, table="device_lists_remote_cache", @@ -409,12 +408,15 @@ class DeviceStore(SQLBaseStore): prev_id = stream_id - key_json = device.get("key_json", None) - if key_json: - result["keys"] = json.loads(key_json) - device_display_name = device.get("device_display_name", None) - if device_display_name: - result["device_display_name"] = device_display_name + if device is not None: + key_json = device.get("key_json", None) + if key_json: + result["keys"] = json.loads(key_json) + device_display_name = device.get("device_display_name", None) + if device_display_name: + result["device_display_name"] = device_display_name + else: + result["deleted"] = True results.append(result) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index f61553cec..6c2871942 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -74,7 +74,7 @@ class EndToEndKeyStore(SQLBaseStore): include_all_devices (bool): whether to include entries for devices that don't have device keys include_deleted_devices (bool): whether to include null entries for - devices which no longer exist (but where in the query_list) + devices which no longer exist (but were in the query_list) Returns: Dict mapping from user-id to dict mapping from device_id to dict containing "key_json", "device_display_name". @@ -84,28 +84,25 @@ class EndToEndKeyStore(SQLBaseStore): results = yield self.runInteraction( "get_e2e_device_keys", self._get_e2e_device_keys_txn, - query_list, include_all_devices, + query_list, include_all_devices, include_deleted_devices, ) - if include_deleted_devices: - deleted_devices = set(query_list) - for user_id, device_keys in iteritems(results): for device_id, device_info in iteritems(device_keys): - if include_deleted_devices: - deleted_devices -= (user_id, device_id) device_info["keys"] = json.loads(device_info.pop("key_json")) - if include_deleted_devices: - for user_id, device_id in deleted_devices: - results.setdefault(user_id, {})[device_id] = None - defer.returnValue(results) - def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices): + def _get_e2e_device_keys_txn( + self, txn, query_list, include_all_devices=False, + include_deleted_devices=False, + ): query_clauses = [] query_params = [] + if include_deleted_devices: + deleted_devices = set(query_list) + for (user_id, device_id) in query_list: query_clause = "user_id = ?" query_params.append(user_id) @@ -133,8 +130,14 @@ class EndToEndKeyStore(SQLBaseStore): result = {} for row in rows: + if include_deleted_devices: + deleted_devices.remove((row["user_id"], row["device_id"])) result.setdefault(row["user_id"], {})[row["device_id"]] = row + if include_deleted_devices: + for user_id, device_id in deleted_devices: + result.setdefault(user_id, {})[device_id] = None + return result @defer.inlineCallbacks From 37c4fba0ac95e0425193b1eaeaf96bc348e094d9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 12 Jul 2018 11:45:33 +0100 Subject: [PATCH 04/42] changelog --- changelog.d/3520.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3520.bugfix diff --git a/changelog.d/3520.bugfix b/changelog.d/3520.bugfix new file mode 100644 index 000000000..9278cb370 --- /dev/null +++ b/changelog.d/3520.bugfix @@ -0,0 +1 @@ +Correctly announce deleted devices over federation From 5f3d02f6ebbce8fbbb2a71ef1ea7f5537257ff15 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Jul 2018 12:52:56 +1000 Subject: [PATCH 05/42] bump to 0.33.0rc1 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 3cde33c0d..6dc16f080 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -17,4 +17,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.32.2" +__version__ = "0.33.0rc1" From 21d3b879433e040babd43c89b62827f92e3ac861 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Jul 2018 12:53:32 +1000 Subject: [PATCH 06/42] 0.33.0rc1 changelog --- CHANGES.rst | 27 +++++++++++++++++++++++++++ changelog.d/3316.feature | 1 - changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 - changelog.d/3497.feature | 1 - changelog.d/3498.misc | 0 changelog.d/3499.misc | 0 changelog.d/3501.misc | 0 changelog.d/3505.feature | 1 - changelog.d/3521.feature | 1 - changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 - changelog.d/3534.misc | 1 - changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 - changelog.d/3541.feature | 1 - changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 - 20 files changed, 27 insertions(+), 10 deletions(-) delete mode 100644 changelog.d/3316.feature delete mode 100644 changelog.d/3351.misc delete mode 100644 changelog.d/3463.misc delete mode 100644 changelog.d/3464.misc delete mode 100644 changelog.d/3496.feature delete mode 100644 changelog.d/3497.feature delete mode 100644 changelog.d/3498.misc delete mode 100644 changelog.d/3499.misc delete mode 100644 changelog.d/3501.misc delete mode 100644 changelog.d/3505.feature delete mode 100644 changelog.d/3521.feature delete mode 100644 changelog.d/3530.misc delete mode 100644 changelog.d/3533.bugfix delete mode 100644 changelog.d/3534.misc delete mode 100644 changelog.d/3535.misc delete mode 100644 changelog.d/3540.misc delete mode 100644 changelog.d/3541.feature delete mode 100644 changelog.d/3544.misc delete mode 100644 changelog.d/3546.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index ba6929c43..f81e7e6cf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,30 @@ +Synapse 0.33.0rc1 (2018-07-18) +============================== + +Features +-------- + +- Enforce the specified API for report_event (`#3316 `_) +- Include CPU time from database threads in request/block metrics. (`#3496 `_) +- Add CPU metrics for _fetch_event_list (`#3497 `_) +- Reduce database consumption when processing large numbers of receipts (`#3505 `_) +- Cache optimisation for /sync requests (`#3521 `_) +- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) + + +Bugfixes +-------- + +- Fix queued federation requests being processed in the wrong order (`#3533 `_) +- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) + + +Misc +---- + +- `#3351 `_, `#3463 `_, `#3464 `_, `#3498 `_, `#3499 `_, `#3501 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ + + Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature deleted file mode 100644 index 50068b722..000000000 --- a/changelog.d/3316.feature +++ /dev/null @@ -1 +0,0 @@ -Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature deleted file mode 100644 index 6a06a7e75..000000000 --- a/changelog.d/3496.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature deleted file mode 100644 index 4e333155b..000000000 --- a/changelog.d/3497.feature +++ /dev/null @@ -1 +0,0 @@ -Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature deleted file mode 100644 index ca1867f52..000000000 --- a/changelog.d/3505.feature +++ /dev/null @@ -1 +0,0 @@ -Reduce database consumption when processing large numbers of receipts diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature deleted file mode 100644 index 6dced5f2a..000000000 --- a/changelog.d/3521.feature +++ /dev/null @@ -1 +0,0 @@ -Cache optimisation for /sync requests \ No newline at end of file diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix deleted file mode 100644 index 04cbbefd5..000000000 --- a/changelog.d/3533.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc deleted file mode 100644 index 949c12dc6..000000000 --- a/changelog.d/3534.misc +++ /dev/null @@ -1 +0,0 @@ -refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc deleted file mode 100644 index 99dcad8e4..000000000 --- a/changelog.d/3540.misc +++ /dev/null @@ -1 +0,0 @@ -check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature deleted file mode 100644 index 24524136e..000000000 --- a/changelog.d/3541.feature +++ /dev/null @@ -1 +0,0 @@ -Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix deleted file mode 100644 index 921dc6e7b..000000000 --- a/changelog.d/3546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file From 6e3fc657b4c8db10e872ba1c7264e82b7fed5ed0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 10:50:33 +0100 Subject: [PATCH 07/42] Resource tracking for background processes This introduces a mechanism for tracking resource usage by background processes, along with an example of how it will be used. This will help address #3518, but more importantly will give us better insights into things which are happening but not being shown up by the request metrics. We *could* do this with Measure blocks, but: - I think having them pulled out as a completely separate metric class will make it easier to distinguish top-level processes from those which are nested. - I want to be able to report on in-flight background processes, and I don't think we want to do this for *all* Measure blocks. --- synapse/federation/transaction_queue.py | 12 +- synapse/metrics/background_process_metrics.py | 179 ++++++++++++++++++ 2 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 synapse/metrics/background_process_metrics.py diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 5a956ecfb..5c5a73b73 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -30,7 +30,8 @@ from synapse.metrics import ( sent_edus_counter, sent_transactions_counter, ) -from synapse.util import PreserveLoggingContext, logcontext +from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.util import logcontext from synapse.util.metrics import measure_func from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter @@ -165,10 +166,11 @@ class TransactionQueue(object): if self._is_processing: return - # fire off a processing loop in the background. It's likely it will - # outlast the current request, so run it in the sentinel logcontext. - with PreserveLoggingContext(): - self._process_event_queue_loop() + # fire off a processing loop in the background + run_as_background_process( + "process_transaction_queue", + self._process_event_queue_loop, + ) @defer.inlineCallbacks def _process_event_queue_loop(self): diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py new file mode 100644 index 000000000..9d820e44a --- /dev/null +++ b/synapse/metrics/background_process_metrics.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily + +from twisted.internet import defer + +from synapse.util.logcontext import LoggingContext, PreserveLoggingContext + +_background_process_start_count = Counter( + "synapse_background_process_start_count", + "Number of background processes started", + ["name"], +) + +# we set registry=None in all of these to stop them getting registered with +# the default registry. Instead we collect them all via the CustomCollector, +# which ensures that we can update them before they are collected. +# +_background_process_ru_utime = Counter( + "synapse_background_process_ru_utime_seconds", + "User CPU time used by background processes, in seconds", + ["name"], + registry=None, +) + +_background_process_ru_stime = Counter( + "synapse_background_process_ru_stime_seconds", + "System CPU time used by background processes, in seconds", + ["name"], + registry=None, +) + +_background_process_db_txn_count = Counter( + "synapse_background_process_db_txn_count", + "Number of database transactions done by background processes", + ["name"], + registry=None, +) + +_background_process_db_txn_duration = Counter( + "synapse_background_process_db_txn_duration_seconds", + ("Seconds spent by background processes waiting for database " + "transactions, excluding scheduling time"), + ["name"], + registry=None, +) + +_background_process_db_sched_duration = Counter( + "synapse_background_process_db_sched_duration_seconds", + "Seconds spent by background processes waiting for database connections", + ["name"], + registry=None, +) + +# map from description to a counter, so that we can name our logcontexts +# incrementally. (It actually duplicates _background_process_start_count, but +# it's much simpler to do so than to try to combine them.) +_background_process_counts = dict() # type: dict[str, int] + +# map from description to the currently running background processes. +# +# it's kept as a dict of sets rather than a big set so that we can keep track +# of process descriptions that no longer have any active processes. +_background_processes = dict() # type: dict[str, set[_BackgroundProcess]] + + +class _Collector(object): + """A custom metrics collector for the background process metrics. + + Ensures that all of the metrics are up-to-date with any in-flight processes + before they are returned. + """ + def collect(self): + background_process_in_flight_count = GaugeMetricFamily( + "synapse_background_process_in_flight_count", + "Number of background processes in flight", + labels=["name"], + ) + + for desc, processes in six.iteritems(_background_processes): + background_process_in_flight_count.add_metric( + (desc,), len(processes), + ) + for process in processes: + process.update_metrics() + + yield background_process_in_flight_count + + # now we need to run collect() over each of the static Counters, and + # yield each metric they return. + for m in ( + _background_process_ru_utime, + _background_process_ru_stime, + _background_process_db_txn_count, + _background_process_db_txn_duration, + _background_process_db_sched_duration, + ): + for r in m.collect(): + yield r + + +REGISTRY.register(_Collector()) + + +class _BackgroundProcess(object): + def __init__(self, desc, ctx): + self.desc = desc + self._context = ctx + self._reported_stats = None + + def update_metrics(self): + """Updates the metrics with values from this process.""" + new_stats = self._context.get_resource_usage() + if self._reported_stats is None: + diff = new_stats + else: + diff = new_stats - self._reported_stats + self._reported_stats = new_stats + + _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime) + _background_process_ru_stime.labels(self.desc).inc(diff.ru_stime) + _background_process_db_txn_count.labels(self.desc).inc( + diff.db_txn_count, + ) + _background_process_db_txn_duration.labels(self.desc).inc( + diff.db_txn_duration_sec, + ) + _background_process_db_sched_duration.labels(self.desc).inc( + diff.db_sched_duration_sec, + ) + + +def run_as_background_process(desc, func, *args, **kwargs): + """Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + Args: + desc (str): a description for this background process type + func: a function, which may return a Deferred + args: positional args for func + kwargs: keyword args for func + + Returns: None + """ + @defer.inlineCallbacks + def run(): + count = _background_process_counts.get(desc, 0) + _background_process_counts[desc] = count + 1 + _background_process_start_count.labels(desc).inc() + + with LoggingContext(desc) as context: + context.request = "%s-%i" % (desc, count) + proc = _BackgroundProcess(desc, context) + _background_processes.setdefault(desc, set()).add(proc) + try: + yield func(*args, **kwargs) + finally: + proc.update_metrics() + _background_processes[desc].remove(proc) + + with PreserveLoggingContext(): + run() From 92aecd557bdd4c6eb60bdb7afa449ac6ba0c4a0f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 11:29:48 +0100 Subject: [PATCH 08/42] changelog --- changelog.d/3553.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3553.feature diff --git a/changelog.d/3553.feature b/changelog.d/3553.feature new file mode 100644 index 000000000..77a294cb9 --- /dev/null +++ b/changelog.d/3553.feature @@ -0,0 +1 @@ +Add metrics to track resource usage by background processes From e45a46b6e4990befe6567aaf26354320dce4337a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 13:59:36 +0100 Subject: [PATCH 09/42] Add response code to response timer metrics --- synapse/http/request_metrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index f24b4b949..c029a69ee 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -38,7 +38,7 @@ outgoing_responses_counter = Counter( ) response_timer = Histogram( - "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag"] + "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"] ) response_ru_utime = Counter( @@ -171,11 +171,13 @@ class RequestMetrics(object): ) return - outgoing_responses_counter.labels(request.method, str(request.code)).inc() + response_code = str(request.code) + + outgoing_responses_counter.labels(request.method, response_code).inc() response_count.labels(request.method, self.name, tag).inc() - response_timer.labels(request.method, self.name, tag).observe( + response_timer.labels(request.method, self.name, tag, response_code).observe( time_sec - self.start ) From 00845c49d287a60100c73790225cc168801eb52f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 14:03:16 +0100 Subject: [PATCH 10/42] Newsfile --- changelog.d/3554.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3554.feature diff --git a/changelog.d/3554.feature b/changelog.d/3554.feature new file mode 100644 index 000000000..b00397872 --- /dev/null +++ b/changelog.d/3554.feature @@ -0,0 +1 @@ +Add `code` label to `synapse_http_server_response_time_seconds` prometheus metric From 5bd0a47fcdea7dada06dc5d42e4340c8748386e8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Jul 2018 14:19:00 +0100 Subject: [PATCH 11/42] pep8 --- synapse/http/request_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index c029a69ee..588e28057 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -38,7 +38,8 @@ outgoing_responses_counter = Counter( ) response_timer = Histogram( - "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"] + "synapse_http_server_response_time_seconds", "sec", + ["method", "servlet", "tag", "code"], ) response_ru_utime = Counter( From 667fba68f3ca808f48143a2a739a54665b0162c6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 14:35:24 +0100 Subject: [PATCH 12/42] Run things as background processes This fixes #3518, and ensures that we get useful logs and metrics for lots of things that happen in the background. (There are certainly more things that happen in the background; these are just the common ones I've found running a single-process synapse locally). --- synapse/federation/transaction_queue.py | 15 ++++++--------- synapse/storage/background_updates.py | 10 ++++++++-- synapse/storage/client_ips.py | 15 +++++++++++---- synapse/storage/events.py | 10 ++++------ synapse/storage/events_worker.py | 10 ++++++---- synapse/util/caches/expiringcache.py | 6 +++++- synapse/util/distributor.py | 4 ++++ 7 files changed, 44 insertions(+), 26 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 5c5a73b73..6996d6b69 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -168,7 +168,7 @@ class TransactionQueue(object): # fire off a processing loop in the background run_as_background_process( - "process_transaction_queue", + "process_event_queue_for_federation", self._process_event_queue_loop, ) @@ -434,14 +434,11 @@ class TransactionQueue(object): logger.debug("TX [%s] Starting transaction loop", destination) - # Drop the logcontext before starting the transaction. It doesn't - # really make sense to log all the outbound transactions against - # whatever path led us to this point: that's pretty arbitrary really. - # - # (this also means we can fire off _perform_transaction without - # yielding) - with logcontext.PreserveLoggingContext(): - self._transaction_transmission_loop(destination) + run_as_background_process( + "federation_transaction_transmission_loop", + self._transaction_transmission_loop, + destination, + ) @defer.inlineCallbacks def _transaction_transmission_loop(self, destination): diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index dc9eca7d1..5fe1ca2de 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -19,6 +19,8 @@ from canonicaljson import json from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process + from . import engines from ._base import SQLBaseStore @@ -87,10 +89,14 @@ class BackgroundUpdateStore(SQLBaseStore): self._background_update_handlers = {} self._all_done = False - @defer.inlineCallbacks def start_doing_background_updates(self): - logger.info("Starting background schema updates") + run_as_background_process( + "background_updates", self._run_background_updates, + ) + @defer.inlineCallbacks + def _run_background_updates(self): + logger.info("Starting background schema updates") while True: yield self.hs.get_clock().sleep( self.BACKGROUND_UPDATE_INTERVAL_MS / 1000.) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index b78eda341..77ae10da3 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -19,6 +19,7 @@ from six import iteritems from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import CACHE_SIZE_FACTOR from . import background_updates @@ -93,10 +94,16 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): self._batch_row_update[key] = (user_agent, device_id, now) def _update_client_ips_batch(self): - to_update = self._batch_row_update - self._batch_row_update = {} - return self.runInteraction( - "_update_client_ips_batch", self._update_client_ips_batch_txn, to_update + def update(): + to_update = self._batch_row_update + self._batch_row_update = {} + return self.runInteraction( + "_update_client_ips_batch", self._update_client_ips_batch_txn, + to_update, + ) + + run_as_background_process( + "update_client_ips", update, ) def _update_client_ips_batch_txn(self, txn, to_update): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2aaab0d02..4ff0fdc4a 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -33,12 +33,13 @@ from synapse.api.errors import SynapseError # these are only included to make the type annotations work from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.events_worker import EventsWorkerStore from synapse.types import RoomStreamToken, get_domain_from_id from synapse.util.async import ObservableDeferred from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.util.frozenutils import frozendict_json_encoder -from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable +from synapse.util.logcontext import make_deferred_yieldable from synapse.util.logutils import log_function from synapse.util.metrics import Measure @@ -155,11 +156,8 @@ class _EventPeristenceQueue(object): self._event_persist_queues[room_id] = queue self._currently_persisting_rooms.discard(room_id) - # set handle_queue_loop off on the background. We don't want to - # attribute work done in it to the current request, so we drop the - # logcontext altogether. - with PreserveLoggingContext(): - handle_queue_loop() + # set handle_queue_loop off in the background + run_as_background_process("persist_events", handle_queue_loop) def _get_drainining_queue(self, room_id): queue = self._event_persist_queues.setdefault(room_id, deque()) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 67433606c..f28239a80 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -25,6 +25,7 @@ from synapse.events import EventBase # noqa: F401 from synapse.events import FrozenEvent from synapse.events.snapshot import EventContext # noqa: F401 from synapse.events.utils import prune_event +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.logcontext import ( LoggingContext, PreserveLoggingContext, @@ -322,10 +323,11 @@ class EventsWorkerStore(SQLBaseStore): should_start = False if should_start: - with PreserveLoggingContext(): - self.runWithConnection( - self._do_fetch - ) + run_as_background_process( + "fetch_events", + self.runWithConnection, + self._do_fetch, + ) logger.debug("Loading %d events", len(events)) with PreserveLoggingContext(): diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 4abca91f6..465adc54a 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -16,6 +16,7 @@ import logging from collections import OrderedDict +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import register_cache logger = logging.getLogger(__name__) @@ -63,7 +64,10 @@ class ExpiringCache(object): return def f(): - self._prune_cache() + run_as_background_process( + "prune_cache_%s" % self._cache_name, + self._prune_cache, + ) self._clock.looping_call(f, self._expiry_ms / 2) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 734331caa..d91ae400e 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -75,6 +75,10 @@ class Distributor(object): self.pre_registration[name].append(observer) def fire(self, name, *args, **kwargs): + """Dispatches the given signal to the registered observers. + + Runs the observers as a background process. Does not return a deferred. + """ if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) From 08436c556a6dd6053ce778d581159f82f8668a63 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 16:14:01 +0100 Subject: [PATCH 13/42] changelog --- changelog.d/3556.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3556.feature diff --git a/changelog.d/3556.feature b/changelog.d/3556.feature new file mode 100644 index 000000000..77a294cb9 --- /dev/null +++ b/changelog.d/3556.feature @@ -0,0 +1 @@ +Add metrics to track resource usage by background processes From 8c69b735e3186875a8bf676241f990470fb7c592 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 15:33:13 +0100 Subject: [PATCH 14/42] Make Distributor run its processes as a background process This is more involved than it might otherwise be, because the current implementation just drops its logcontexts and runs everything in the sentinel context. It turns out that we aren't actually using a bunch of the functionality here (notably suppress_failures and the fact that Distributor.fire returns a deferred), so the easiest way to fix this is actually by simplifying a bunch of code. --- synapse/util/distributor.py | 44 ++++++++++++----------------- tests/test_distributor.py | 56 +++---------------------------------- 2 files changed, 22 insertions(+), 78 deletions(-) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index d91ae400e..194da8763 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -17,20 +17,18 @@ import logging from twisted.internet import defer -from synapse.util import unwrapFirstError -from synapse.util.logcontext import PreserveLoggingContext +from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.util.logcontext import make_deferred_yieldable, run_in_background logger = logging.getLogger(__name__) def user_left_room(distributor, user, room_id): - with PreserveLoggingContext(): - distributor.fire("user_left_room", user=user, room_id=room_id) + distributor.fire("user_left_room", user=user, room_id=room_id) def user_joined_room(distributor, user, room_id): - with PreserveLoggingContext(): - distributor.fire("user_joined_room", user=user, room_id=room_id) + distributor.fire("user_joined_room", user=user, room_id=room_id) class Distributor(object): @@ -44,9 +42,7 @@ class Distributor(object): model will do for today. """ - def __init__(self, suppress_failures=True): - self.suppress_failures = suppress_failures - + def __init__(self): self.signals = {} self.pre_registration = {} @@ -56,7 +52,6 @@ class Distributor(object): self.signals[name] = Signal( name, - suppress_failures=self.suppress_failures, ) if name in self.pre_registration: @@ -82,7 +77,11 @@ class Distributor(object): if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) - return self.signals[name].fire(*args, **kwargs) + run_as_background_process( + name, + self.signals[name].fire, + *args, **kwargs + ) class Signal(object): @@ -95,9 +94,8 @@ class Signal(object): method into all of the observers. """ - def __init__(self, name, suppress_failures): + def __init__(self, name): self.name = name - self.suppress_failures = suppress_failures self.observers = [] def observe(self, observer): @@ -107,7 +105,6 @@ class Signal(object): Each observer callable may return a Deferred.""" self.observers.append(observer) - @defer.inlineCallbacks def fire(self, *args, **kwargs): """Invokes every callable in the observer list, passing in the args and kwargs. Exceptions thrown by observers are logged but ignored. It is @@ -125,22 +122,17 @@ class Signal(object): failure.type, failure.value, failure.getTracebackObject())) - if not self.suppress_failures: - return failure return defer.maybeDeferred(observer, *args, **kwargs).addErrback(eb) - with PreserveLoggingContext(): - deferreds = [ - do(observer) - for observer in self.observers - ] + deferreds = [ + run_in_background(do, o) + for o in self.observers + ] - res = yield defer.gatherResults( - deferreds, consumeErrors=True - ).addErrback(unwrapFirstError) - - defer.returnValue(res) + return make_deferred_yieldable(defer.gatherResults( + deferreds, consumeErrors=True, + )) def __repr__(self): return "" % (self.name,) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 04a88056f..71d11cda7 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,6 @@ from mock import Mock, patch -from twisted.internet import defer - from synapse.util.distributor import Distributor from . import unittest @@ -27,38 +26,15 @@ class DistributorTestCase(unittest.TestCase): def setUp(self): self.dist = Distributor() - @defer.inlineCallbacks def test_signal_dispatch(self): self.dist.declare("alert") observer = Mock() self.dist.observe("alert", observer) - d = self.dist.fire("alert", 1, 2, 3) - yield d - self.assertTrue(d.called) + self.dist.fire("alert", 1, 2, 3) observer.assert_called_with(1, 2, 3) - @defer.inlineCallbacks - def test_signal_dispatch_deferred(self): - self.dist.declare("whine") - - d_inner = defer.Deferred() - - def observer(): - return d_inner - - self.dist.observe("whine", observer) - - d_outer = self.dist.fire("whine") - - self.assertFalse(d_outer.called) - - d_inner.callback(None) - yield d_outer - self.assertTrue(d_outer.called) - - @defer.inlineCallbacks def test_signal_catch(self): self.dist.declare("alarm") @@ -71,9 +47,7 @@ class DistributorTestCase(unittest.TestCase): with patch( "synapse.util.distributor.logger", spec=["warning"] ) as mock_logger: - d = self.dist.fire("alarm", "Go") - yield d - self.assertTrue(d.called) + self.dist.fire("alarm", "Go") observers[0].assert_called_once_with("Go") observers[1].assert_called_once_with("Go") @@ -83,34 +57,12 @@ class DistributorTestCase(unittest.TestCase): mock_logger.warning.call_args[0][0], str ) - @defer.inlineCallbacks - def test_signal_catch_no_suppress(self): - # Gut-wrenching - self.dist.suppress_failures = False - - self.dist.declare("whail") - - class MyException(Exception): - pass - - @defer.inlineCallbacks - def observer(): - raise MyException("Oopsie") - - self.dist.observe("whail", observer) - - d = self.dist.fire("whail") - - yield self.assertFailure(d, MyException) - self.dist.suppress_failures = True - - @defer.inlineCallbacks def test_signal_prereg(self): observer = Mock() self.dist.observe("flare", observer) self.dist.declare("flare") - yield self.dist.fire("flare", 4, 5) + self.dist.fire("flare", 4, 5) observer.assert_called_with(4, 5) From 3f9e649f1720b3419fd8008771955714baa9600b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 20:55:37 +0100 Subject: [PATCH 15/42] add config for pep8 Since, for better or worse, we seem to have configured isort to generate 89-character lines, pycharm is now complaining at me that our lines are too long. So, let's configure pep8 to behave consistently with flake8. --- setup.cfg | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9b5b75cd6..3f84283a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,17 @@ ignore = pylint.cfg tox.ini -[flake8] +[pep8] max-line-length = 90 -# W503 requires that binary operators be at the end, not start, of lines. Erik doesn't like it. -# E203 is contrary to PEP8. +# W503 requires that binary operators be at the end, not start, of lines. Erik +# doesn't like it. E203 is contrary to PEP8. ignore = W503,E203 +[flake8] +# note that flake8 inherits the "ignore" settings from "pep8" (because it uses +# pep8 to do those checks), but not the "max-line-length" setting +max-line-length = 90 + [isort] line_length = 89 not_skip = __init__.py From eed24893fabb95b9ebbda58db54831f17dbdf2e6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 22:12:19 +0100 Subject: [PATCH 16/42] changelog --- changelog.d/3559.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3559.misc diff --git a/changelog.d/3559.misc b/changelog.d/3559.misc new file mode 100644 index 000000000..26df859e4 --- /dev/null +++ b/changelog.d/3559.misc @@ -0,0 +1 @@ +add config for pep8 From 00bc9791379109690fb5e465adf0e0e8e76167aa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 10:51:15 +0100 Subject: [PATCH 17/42] Disable logcontext warning Temporary workaround to #3518 while we release 0.33.0. --- synapse/storage/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 98dde7743..a6a0e6ec9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -343,9 +343,10 @@ class SQLBaseStore(object): """ parent_context = LoggingContext.current_context() if parent_context == LoggingContext.sentinel: - logger.warn( - "Running db txn from sentinel context: metrics will be lost", - ) + # warning disabled for 0.33.0 release; proper fixes will land imminently. + # logger.warn( + # "Running db txn from sentinel context: metrics will be lost", + # ) parent_context = None start_time = time.time() From 18a2b2c0b402af89f818a50ca2f4f0c733f864e4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 10:54:39 +0100 Subject: [PATCH 18/42] changelog --- changelog.d/3561.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3561.bugfix diff --git a/changelog.d/3561.bugfix b/changelog.d/3561.bugfix new file mode 100644 index 000000000..9a197cd67 --- /dev/null +++ b/changelog.d/3561.bugfix @@ -0,0 +1 @@ +Disable a noisy warning about logcontexts From c0685f67c001ab156fb6922877c35f70536100dc Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 19 Jul 2018 10:59:02 +0100 Subject: [PATCH 19/42] spell out that include_deleted_devices requires include_all_devices --- synapse/storage/end_to_end_keys.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 6c2871942..ffe4d7235 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -74,7 +74,8 @@ class EndToEndKeyStore(SQLBaseStore): include_all_devices (bool): whether to include entries for devices that don't have device keys include_deleted_devices (bool): whether to include null entries for - devices which no longer exist (but were in the query_list) + devices which no longer exist (but were in the query_list). + This option only takes effect if include_all_devices is true. Returns: Dict mapping from user-id to dict mapping from device_id to dict containing "key_json", "device_display_name". @@ -100,6 +101,9 @@ class EndToEndKeyStore(SQLBaseStore): query_clauses = [] query_params = [] + if include_all_devices is False: + include_deleted_devices = False + if include_deleted_devices: deleted_devices = set(query_list) From a97c845271f9a89ebdb7186d4c9d04c099bd1beb Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 20:03:33 +1000 Subject: [PATCH 20/42] Move v1-only APIs into their own module & isolate deprecated ones (#3460) --- changelog.d/3460.misc | 0 setup.cfg | 1 + synapse/http/client.py | 6 +- synapse/rest/__init__.py | 43 +++++++-- synapse/rest/client/v1/room.py | 5 +- synapse/rest/client/v1_only/__init__.py | 3 + synapse/rest/client/v1_only/base.py | 39 ++++++++ .../rest/client/{v1 => v1_only}/register.py | 7 +- tests/rest/client/v1/test_events.py | 90 +++---------------- tests/rest/client/v1/test_register.py | 5 +- tests/rest/client/v1/test_rooms.py | 2 +- tests/rest/client/v2_alpha/test_filter.py | 8 +- tests/rest/client/v2_alpha/test_sync.py | 8 +- 13 files changed, 116 insertions(+), 101 deletions(-) create mode 100644 changelog.d/3460.misc create mode 100644 synapse/rest/client/v1_only/__init__.py create mode 100644 synapse/rest/client/v1_only/base.py rename synapse/rest/client/{v1 => v1_only}/register.py (98%) diff --git a/changelog.d/3460.misc b/changelog.d/3460.misc new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index 3f84283a3..c2620be6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,3 +36,4 @@ known_compat = mock,six known_twisted=twisted,OpenSSL multi_line_output=3 include_trailing_comma=true +combine_as_imports=true diff --git a/synapse/http/client.py b/synapse/http/client.py index d6a0d75b2..25b630788 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,9 +26,11 @@ from OpenSSL.SSL import VERIFY_NONE from twisted.internet import defer, protocol, reactor, ssl, task from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.web._newclient import ResponseDone -from twisted.web.client import Agent, BrowserLikeRedirectAgent, ContentDecoderAgent -from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer from twisted.web.client import ( + Agent, + BrowserLikeRedirectAgent, + ContentDecoderAgent, + FileBodyProducer as TwistedFileBodyProducer, GzipDecoder, HTTPConnectionPool, PartialDownloadError, diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 75c2a4ec8..3418f06fd 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,13 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six import PY3 + from synapse.http.server import JsonResource from synapse.rest.client import versions -from synapse.rest.client.v1 import admin, directory, events, initial_sync -from synapse.rest.client.v1 import login as v1_login -from synapse.rest.client.v1 import logout, presence, profile, push_rule, pusher -from synapse.rest.client.v1 import register as v1_register -from synapse.rest.client.v1 import room, voip +from synapse.rest.client.v1 import ( + admin, + directory, + events, + initial_sync, + login as v1_login, + logout, + presence, + profile, + push_rule, + pusher, + room, + voip, +) from synapse.rest.client.v2_alpha import ( account, account_data, @@ -42,6 +54,11 @@ from synapse.rest.client.v2_alpha import ( user_directory, ) +if not PY3: + from synapse.rest.client.v1_only import ( + register as v1_register, + ) + class ClientRestResource(JsonResource): """A resource for version 1 of the matrix client API.""" @@ -54,14 +71,22 @@ class ClientRestResource(JsonResource): def register_servlets(client_resource, hs): versions.register_servlets(client_resource) - # "v1" - room.register_servlets(hs, client_resource) + if not PY3: + # "v1" (Python 2 only) + v1_register.register_servlets(hs, client_resource) + + # Deprecated in r0 + initial_sync.register_servlets(hs, client_resource) + room.register_deprecated_servlets(hs, client_resource) + + # Partially deprecated in r0 events.register_servlets(hs, client_resource) - v1_register.register_servlets(hs, client_resource) + + # "v1" + "r0" + room.register_servlets(hs, client_resource) v1_login.register_servlets(hs, client_resource) profile.register_servlets(hs, client_resource) presence.register_servlets(hs, client_resource) - initial_sync.register_servlets(hs, client_resource) directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 3d6244785..b9512a2b6 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -832,10 +832,13 @@ def register_servlets(hs, http_server): RoomSendEventRestServlet(hs).register(http_server) PublicRoomListRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) - RoomInitialSyncRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) RoomTypingRestServlet(hs).register(http_server) SearchRestServlet(hs).register(http_server) JoinedRoomsRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) + + +def register_deprecated_servlets(hs, http_server): + RoomInitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1_only/__init__.py b/synapse/rest/client/v1_only/__init__.py new file mode 100644 index 000000000..936f902ac --- /dev/null +++ b/synapse/rest/client/v1_only/__init__.py @@ -0,0 +1,3 @@ +""" +REST APIs that are only used in v1 (the legacy API). +""" diff --git a/synapse/rest/client/v1_only/base.py b/synapse/rest/client/v1_only/base.py new file mode 100644 index 000000000..9d4db7437 --- /dev/null +++ b/synapse/rest/client/v1_only/base.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +import re + +from synapse.api.urls import CLIENT_PREFIX + + +def v1_only_client_path_patterns(path_regex, include_in_unstable=True): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + list of SRE_Pattern + """ + patterns = [re.compile("^" + CLIENT_PREFIX + path_regex)] + if include_in_unstable: + unstable_prefix = CLIENT_PREFIX.replace("/api/v1", "/unstable") + patterns.append(re.compile("^" + unstable_prefix + path_regex)) + return patterns diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1_only/register.py similarity index 98% rename from synapse/rest/client/v1/register.py rename to synapse/rest/client/v1_only/register.py index 25a143af8..3439c3c6d 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1_only/register.py @@ -24,9 +24,10 @@ import synapse.util.stringutils as stringutils from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError from synapse.http.servlet import assert_params_in_dict, parse_json_object_from_request +from synapse.rest.client.v1.base import ClientV1RestServlet from synapse.types import create_requester -from .base import ClientV1RestServlet, client_path_patterns +from .base import v1_only_client_path_patterns logger = logging.getLogger(__name__) @@ -49,7 +50,7 @@ class RegisterRestServlet(ClientV1RestServlet): handler doesn't have a concept of multi-stages or sessions. """ - PATTERNS = client_path_patterns("/register$", releases=(), include_in_unstable=False) + PATTERNS = v1_only_client_path_patterns("/register$", include_in_unstable=False) def __init__(self, hs): """ @@ -379,7 +380,7 @@ class CreateUserRestServlet(ClientV1RestServlet): """Handles user creation via a server-to-server interface """ - PATTERNS = client_path_patterns("/createUser$", releases=()) + PATTERNS = v1_only_client_path_patterns("/createUser$") def __init__(self, hs): super(CreateUserRestServlet, self).__init__(hs) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index a5af36a99..50418153f 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -14,100 +14,30 @@ # limitations under the License. """ Tests REST events for /events paths.""" + from mock import Mock, NonCallableMock +from six import PY3 -# twisted imports from twisted.internet import defer -import synapse.rest.client.v1.events -import synapse.rest.client.v1.register -import synapse.rest.client.v1.room - -from tests import unittest - from ....utils import MockHttpResource, setup_test_homeserver from .utils import RestTestCase PATH_PREFIX = "/_matrix/client/api/v1" -class EventStreamPaginationApiTestCase(unittest.TestCase): - """ Tests event streaming query parameters and start/end keys used in the - Pagination stream API. """ - user_id = "sid1" - - def setUp(self): - # configure stream and inject items - pass - - def tearDown(self): - pass - - def TODO_test_long_poll(self): - # stream from 'end' key, send (self+other) message, expect message. - - # stream from 'END', send (self+other) message, expect message. - - # stream from 'end' key, send (self+other) topic, expect topic. - - # stream from 'END', send (self+other) topic, expect topic. - - # stream from 'end' key, send (self+other) invite, expect invite. - - # stream from 'END', send (self+other) invite, expect invite. - - pass - - def TODO_test_stream_forward(self): - # stream from START, expect injected items - - # stream from 'start' key, expect same content - - # stream from 'end' key, expect nothing - - # stream from 'END', expect nothing - - # The following is needed for cases where content is removed e.g. you - # left a room, so the token you're streaming from is > the one that - # would be returned naturally from START>END. - # stream from very new token (higher than end key), expect same token - # returned as end key - pass - - def TODO_test_limits(self): - # stream from a key, expect limit_num items - - # stream from START, expect limit_num items - - pass - - def TODO_test_range(self): - # stream from key to key, expect X items - - # stream from key to END, expect X items - - # stream from START to key, expect X items - - # stream from START to END, expect all items - pass - - def TODO_test_direction(self): - # stream from END to START and fwds, expect newest first - - # stream from END to START and bwds, expect oldest first - - # stream from START to END and fwds, expect oldest first - - # stream from START to END and bwds, expect newest first - - pass - - class EventStreamPermissionsTestCase(RestTestCase): """ Tests event streaming (GET /events). """ + if PY3: + skip = "Skip on Py3 until ported to use not V1 only register." + @defer.inlineCallbacks def setUp(self): + import synapse.rest.client.v1.events + import synapse.rest.client.v1_only.register + import synapse.rest.client.v1.room + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) hs = yield setup_test_homeserver( @@ -125,7 +55,7 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1_only.register.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py index f15fb3621..83a23cd8f 100644 --- a/tests/rest/client/v1/test_register.py +++ b/tests/rest/client/v1/test_register.py @@ -16,11 +16,12 @@ import json from mock import Mock +from six import PY3 from twisted.test.proto_helpers import MemoryReactorClock from synapse.http.server import JsonResource -from synapse.rest.client.v1.register import register_servlets +from synapse.rest.client.v1_only.register import register_servlets from synapse.util import Clock from tests import unittest @@ -31,6 +32,8 @@ class CreateUserServletTestCase(unittest.TestCase): """ Tests for CreateUserRestServlet. """ + if PY3: + skip = "Not ported to Python 3." def setUp(self): self.registration_handler = Mock() diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 6b5764095..00fc79678 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -20,7 +20,6 @@ import json from mock import Mock, NonCallableMock from six.moves.urllib import parse as urlparse -# twisted imports from twisted.internet import defer import synapse.rest.client.v1.room @@ -86,6 +85,7 @@ class RoomBase(unittest.TestCase): self.resource = JsonResource(self.hs) synapse.rest.client.v1.room.register_servlets(self.hs, self.resource) + synapse.rest.client.v1.room.register_deprecated_servlets(self.hs, self.resource) self.helper = RestHelper(self.hs, self.resource, self.user_id) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 5ea9cc825..e890f0fea 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -21,8 +21,12 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest -from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock -from tests.server import make_request, setup_test_homeserver, wait_until_result +from tests.server import ( + ThreadedMemoryReactorClock as MemoryReactorClock, + make_request, + setup_test_homeserver, + wait_until_result, +) PATH_PREFIX = "/_matrix/client/v2_alpha" diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 704cf97a4..03ec3993b 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -20,8 +20,12 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest -from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock -from tests.server import make_request, setup_test_homeserver, wait_until_result +from tests.server import ( + ThreadedMemoryReactorClock as MemoryReactorClock, + make_request, + setup_test_homeserver, + wait_until_result, +) PATH_PREFIX = "/_matrix/client/v2_alpha" From f1a15ea20692e9f4b347cdddf6d714912356ba64 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 11:14:20 +0100 Subject: [PATCH 21/42] revert 00bc979 ... we've fixed the things that caused the warnings, so we should reinstate the warning. --- synapse/storage/_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6a0e6ec9..1d41d8d44 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -343,10 +343,9 @@ class SQLBaseStore(object): """ parent_context = LoggingContext.current_context() if parent_context == LoggingContext.sentinel: - # warning disabled for 0.33.0 release; proper fixes will land imminently. - # logger.warn( - # "Running db txn from sentinel context: metrics will be lost", - # ) + logger.warn( + "Starting db connection from sentinel context: metrics will be lost", + ) parent_context = None start_time = time.time() From 9e40834f742d95c324183f3e71ae73aafe3c6a99 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 19 Jul 2018 11:15:10 +0100 Subject: [PATCH 22/42] yes, we do need to invalidate the device_id_exists_cache when deleting a remote device --- synapse/storage/devices.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 203f50f07..cc3cdf2eb 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -258,7 +258,6 @@ class DeviceStore(SQLBaseStore): }, ) - # Do we need this? txn.call_after( self.device_id_exists_cache.invalidate, (user_id, device_id,) ) From 95ccb6e2ec57f2150a697ea9cde030e8f78d6db9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 20:58:18 +1000 Subject: [PATCH 23/42] Don't spew errors because we can't save metrics (#3563) --- changelog.d/3548.bugfix | 1 + synapse/util/logcontext.py | 11 +++++++++++ synapse/util/metrics.py | 19 +++++++++++++------ 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 changelog.d/3548.bugfix diff --git a/changelog.d/3548.bugfix b/changelog.d/3548.bugfix new file mode 100644 index 000000000..38dc3b123 --- /dev/null +++ b/changelog.d/3548.bugfix @@ -0,0 +1 @@ +Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis. diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index f6c7175f7..8dcae50b3 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -99,6 +99,17 @@ class ContextResourceUsage(object): self.db_sched_duration_sec = 0 self.evt_db_fetch_count = 0 + def __repr__(self): + return ("") % ( + self.ru_stime, + self.ru_utime, + self.db_txn_count, + self.db_txn_duration_sec, + self.db_sched_duration_sec, + self.evt_db_fetch_count,) + def __iadd__(self, other): """Add another ContextResourceUsage's stats to this one's. diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6ba710789..97f126738 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -104,12 +104,19 @@ class Measure(object): logger.warn("Expected context. (%r)", self.name) return - usage = context.get_resource_usage() - self.start_usage - block_ru_utime.labels(self.name).inc(usage.ru_utime) - block_ru_stime.labels(self.name).inc(usage.ru_stime) - block_db_txn_count.labels(self.name).inc(usage.db_txn_count) - block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) - block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) + current = context.get_resource_usage() + usage = current - self.start_usage + try: + block_ru_utime.labels(self.name).inc(usage.ru_utime) + block_ru_stime.labels(self.name).inc(usage.ru_stime) + block_db_txn_count.labels(self.name).inc(usage.db_txn_count) + block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) + block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) + except ValueError: + logger.warn( + "Failed to save metrics! OLD: %r, NEW: %r", + self.start_usage, current + ) if self.created_context: self.start_context.__exit__(exc_type, exc_val, exc_tb) From ce0545eca1daa9f0f50335b644d53ba2ec965a08 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:03:15 +1000 Subject: [PATCH 24/42] Revert "0.33.0rc1 changelog" This reverts commit 21d3b879433e040babd43c89b62827f92e3ac861. --- CHANGES.rst | 27 --------------------------- changelog.d/3316.feature | 1 + changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 + changelog.d/3497.feature | 1 + changelog.d/3498.misc | 0 changelog.d/3499.misc | 0 changelog.d/3501.misc | 0 changelog.d/3505.feature | 1 + changelog.d/3521.feature | 1 + changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 + changelog.d/3534.misc | 1 + changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 + changelog.d/3541.feature | 1 + changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 + 20 files changed, 10 insertions(+), 27 deletions(-) create mode 100644 changelog.d/3316.feature create mode 100644 changelog.d/3351.misc create mode 100644 changelog.d/3463.misc create mode 100644 changelog.d/3464.misc create mode 100644 changelog.d/3496.feature create mode 100644 changelog.d/3497.feature create mode 100644 changelog.d/3498.misc create mode 100644 changelog.d/3499.misc create mode 100644 changelog.d/3501.misc create mode 100644 changelog.d/3505.feature create mode 100644 changelog.d/3521.feature create mode 100644 changelog.d/3530.misc create mode 100644 changelog.d/3533.bugfix create mode 100644 changelog.d/3534.misc create mode 100644 changelog.d/3535.misc create mode 100644 changelog.d/3540.misc create mode 100644 changelog.d/3541.feature create mode 100644 changelog.d/3544.misc create mode 100644 changelog.d/3546.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index f81e7e6cf..ba6929c43 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,30 +1,3 @@ -Synapse 0.33.0rc1 (2018-07-18) -============================== - -Features --------- - -- Enforce the specified API for report_event (`#3316 `_) -- Include CPU time from database threads in request/block metrics. (`#3496 `_) -- Add CPU metrics for _fetch_event_list (`#3497 `_) -- Reduce database consumption when processing large numbers of receipts (`#3505 `_) -- Cache optimisation for /sync requests (`#3521 `_) -- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) - - -Bugfixes --------- - -- Fix queued federation requests being processed in the wrong order (`#3533 `_) -- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) - - -Misc ----- - -- `#3351 `_, `#3463 `_, `#3464 `_, `#3498 `_, `#3499 `_, `#3501 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ - - Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature new file mode 100644 index 000000000..50068b722 --- /dev/null +++ b/changelog.d/3316.feature @@ -0,0 +1 @@ +Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature new file mode 100644 index 000000000..6a06a7e75 --- /dev/null +++ b/changelog.d/3496.feature @@ -0,0 +1 @@ +Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature new file mode 100644 index 000000000..4e333155b --- /dev/null +++ b/changelog.d/3497.feature @@ -0,0 +1 @@ +Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature new file mode 100644 index 000000000..ca1867f52 --- /dev/null +++ b/changelog.d/3505.feature @@ -0,0 +1 @@ +Reduce database consumption when processing large numbers of receipts diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature new file mode 100644 index 000000000..6dced5f2a --- /dev/null +++ b/changelog.d/3521.feature @@ -0,0 +1 @@ +Cache optimisation for /sync requests \ No newline at end of file diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix new file mode 100644 index 000000000..04cbbefd5 --- /dev/null +++ b/changelog.d/3533.bugfix @@ -0,0 +1 @@ +Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc new file mode 100644 index 000000000..949c12dc6 --- /dev/null +++ b/changelog.d/3534.misc @@ -0,0 +1 @@ +refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc new file mode 100644 index 000000000..99dcad8e4 --- /dev/null +++ b/changelog.d/3540.misc @@ -0,0 +1 @@ +check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature new file mode 100644 index 000000000..24524136e --- /dev/null +++ b/changelog.d/3541.feature @@ -0,0 +1 @@ +Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix new file mode 100644 index 000000000..921dc6e7b --- /dev/null +++ b/changelog.d/3546.bugfix @@ -0,0 +1 @@ +Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file From 13d501c773e4c7a5fcf409db7360293a4f1347e0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:11:24 +1000 Subject: [PATCH 25/42] update changelogs --- changelog.d/3498.bugfix | 1 + changelog.d/3498.misc | 0 changelog.d/3501.feature | 1 + changelog.d/3501.misc | 0 changelog.d/3505.feature | 2 +- changelog.d/3521.feature | 2 +- 6 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/3498.bugfix delete mode 100644 changelog.d/3498.misc create mode 100644 changelog.d/3501.feature delete mode 100644 changelog.d/3501.misc diff --git a/changelog.d/3498.bugfix b/changelog.d/3498.bugfix new file mode 100644 index 000000000..d0d01101f --- /dev/null +++ b/changelog.d/3498.bugfix @@ -0,0 +1 @@ +Use more portable syntax in our use of the attrs package, widening the supported versions diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3501.feature b/changelog.d/3501.feature new file mode 100644 index 000000000..6a06a7e75 --- /dev/null +++ b/changelog.d/3501.feature @@ -0,0 +1 @@ +Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature index ca1867f52..a6b686f68 100644 --- a/changelog.d/3505.feature +++ b/changelog.d/3505.feature @@ -1 +1 @@ -Reduce database consumption when processing large numbers of receipts +optimisation for /sync diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature index 6dced5f2a..a6b686f68 100644 --- a/changelog.d/3521.feature +++ b/changelog.d/3521.feature @@ -1 +1 @@ -Cache optimisation for /sync requests \ No newline at end of file +optimisation for /sync From 38f53399a28a75016255509c8e8939d2815cd807 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:11:40 +1000 Subject: [PATCH 26/42] 0.33 final --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 6dc16f080..5c0f2f83a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -17,4 +17,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.33.0rc1" +__version__ = "0.33.0" From d69decd5c78c72abef50b597a689e2bc55a39702 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:12:15 +1000 Subject: [PATCH 27/42] 0.33.0 final changelog --- CHANGES.rst | 28 ++++++++++++++++++++++++++++ changelog.d/3316.feature | 1 - changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 - changelog.d/3497.feature | 1 - changelog.d/3498.bugfix | 1 - changelog.d/3499.misc | 0 changelog.d/3501.feature | 1 - changelog.d/3505.feature | 1 - changelog.d/3521.feature | 1 - changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 - changelog.d/3534.misc | 1 - changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 - changelog.d/3541.feature | 1 - changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 - changelog.d/3561.bugfix | 1 - 21 files changed, 28 insertions(+), 13 deletions(-) delete mode 100644 changelog.d/3316.feature delete mode 100644 changelog.d/3351.misc delete mode 100644 changelog.d/3463.misc delete mode 100644 changelog.d/3464.misc delete mode 100644 changelog.d/3496.feature delete mode 100644 changelog.d/3497.feature delete mode 100644 changelog.d/3498.bugfix delete mode 100644 changelog.d/3499.misc delete mode 100644 changelog.d/3501.feature delete mode 100644 changelog.d/3505.feature delete mode 100644 changelog.d/3521.feature delete mode 100644 changelog.d/3530.misc delete mode 100644 changelog.d/3533.bugfix delete mode 100644 changelog.d/3534.misc delete mode 100644 changelog.d/3535.misc delete mode 100644 changelog.d/3540.misc delete mode 100644 changelog.d/3541.feature delete mode 100644 changelog.d/3544.misc delete mode 100644 changelog.d/3546.bugfix delete mode 100644 changelog.d/3561.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index ba6929c43..afc4fee88 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,31 @@ +Synapse 0.33.0 (2018-07-19) +=========================== + +Features +-------- + +- Enforce the specified API for report_event (`#3316 `_) +- Include CPU time from database threads in request/block metrics. (`#3496 `_, `#3501 `_) +- Add CPU metrics for _fetch_event_list (`#3497 `_) +- optimisation for /sync (`#3505 `_, `#3521 `_) +- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) + + +Bugfixes +-------- + +- Use more portable syntax in our use of the attrs package, widening the supported versions (`#3498 `_) +- Fix queued federation requests being processed in the wrong order (`#3533 `_) +- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) +- Disable a noisy warning about logcontexts (`#3561 `_) + + +Misc +---- + +- `#3351 `_, `#3463 `_, `#3464 `_, `#3499 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ + + Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature deleted file mode 100644 index 50068b722..000000000 --- a/changelog.d/3316.feature +++ /dev/null @@ -1 +0,0 @@ -Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature deleted file mode 100644 index 6a06a7e75..000000000 --- a/changelog.d/3496.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature deleted file mode 100644 index 4e333155b..000000000 --- a/changelog.d/3497.feature +++ /dev/null @@ -1 +0,0 @@ -Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.bugfix b/changelog.d/3498.bugfix deleted file mode 100644 index d0d01101f..000000000 --- a/changelog.d/3498.bugfix +++ /dev/null @@ -1 +0,0 @@ -Use more portable syntax in our use of the attrs package, widening the supported versions diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3501.feature b/changelog.d/3501.feature deleted file mode 100644 index 6a06a7e75..000000000 --- a/changelog.d/3501.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature deleted file mode 100644 index a6b686f68..000000000 --- a/changelog.d/3505.feature +++ /dev/null @@ -1 +0,0 @@ -optimisation for /sync diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature deleted file mode 100644 index a6b686f68..000000000 --- a/changelog.d/3521.feature +++ /dev/null @@ -1 +0,0 @@ -optimisation for /sync diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix deleted file mode 100644 index 04cbbefd5..000000000 --- a/changelog.d/3533.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc deleted file mode 100644 index 949c12dc6..000000000 --- a/changelog.d/3534.misc +++ /dev/null @@ -1 +0,0 @@ -refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc deleted file mode 100644 index 99dcad8e4..000000000 --- a/changelog.d/3540.misc +++ /dev/null @@ -1 +0,0 @@ -check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature deleted file mode 100644 index 24524136e..000000000 --- a/changelog.d/3541.feature +++ /dev/null @@ -1 +0,0 @@ -Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix deleted file mode 100644 index 921dc6e7b..000000000 --- a/changelog.d/3546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file diff --git a/changelog.d/3561.bugfix b/changelog.d/3561.bugfix deleted file mode 100644 index 9a197cd67..000000000 --- a/changelog.d/3561.bugfix +++ /dev/null @@ -1 +0,0 @@ -Disable a noisy warning about logcontexts From 252f80094c5c18c7a822fda1fba36d01e1e174a3 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:25:33 +1000 Subject: [PATCH 28/42] rst -> md --- CHANGES.md | 2454 +++++++++++++++++++++++++++++++++++++++++++ CHANGES.rst | 2867 --------------------------------------------------- 2 files changed, 2454 insertions(+), 2867 deletions(-) create mode 100644 CHANGES.md delete mode 100644 CHANGES.rst diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 000000000..d7e9e0890 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,2454 @@ +Synapse 0.33.0 (2018-07-19) +=========================== + +Features +-------- + +- Enforce the specified API for report\_event ([\#3316](https://github.com/matrix-org/synapse/issues/3316)) +- Include CPU time from database threads in request/block metrics. ([\#3496](https://github.com/matrix-org/synapse/issues/3496), [\#3501](https://github.com/matrix-org/synapse/issues/3501)) +- Add CPU metrics for \_fetch\_event\_list ([\#3497](https://github.com/matrix-org/synapse/issues/3497)) +- optimisation for /sync ([\#3505](https://github.com/matrix-org/synapse/issues/3505), [\#3521](https://github.com/matrix-org/synapse/issues/3521)) +- Optimisation to make handling incoming federation requests more efficient. ([\#3541](https://github.com/matrix-org/synapse/issues/3541)) + +Bugfixes +-------- + +- Use more portable syntax in our use of the attrs package, widening the supported versions ([\#3498](https://github.com/matrix-org/synapse/issues/3498)) +- Fix queued federation requests being processed in the wrong order ([\#3533](https://github.com/matrix-org/synapse/issues/3533)) +- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. ([\#3546](https://github.com/matrix-org/synapse/issues/3546)) +- Disable a noisy warning about logcontexts ([\#3561](https://github.com/matrix-org/synapse/issues/3561)) + +Misc +---- + +- [\#3351](https://github.com/matrix-org/synapse/issues/3351), [\#3463](https://github.com/matrix-org/synapse/issues/3463), [\#3464](https://github.com/matrix-org/synapse/issues/3464), [\#3499](https://github.com/matrix-org/synapse/issues/3499), [\#3530](https://github.com/matrix-org/synapse/issues/3530), [\#3534](https://github.com/matrix-org/synapse/issues/3534), [\#3535](https://github.com/matrix-org/synapse/issues/3535), [\#3540](https://github.com/matrix-org/synapse/issues/3540), [\#3544](https://github.com/matrix-org/synapse/issues/3544) + +Synapse 0.32.2 (2018-07-07) +=========================== + +Bugfixes +-------- + +- Amend the Python dependencies to depend on attrs from PyPI, not attr ([\#3492](https://github.com/matrix-org/synapse/issues/3492)) + +Synapse 0.32.1 (2018-07-06) +=========================== + +Bugfixes +-------- + +- Add explicit dependency on netaddr ([\#3488](https://github.com/matrix-org/synapse/issues/3488)) + +Changes in synapse v0.32.0 (2018-07-06) +======================================= + +No changes since 0.32.0rc1 + +Synapse 0.32.0rc1 (2018-07-05) +============================== + +Features +-------- + +- Add blacklist & whitelist of servers allowed to send events to a room via `m.room.server_acl` event. +- Cache factor override system for specific caches ([\#3334](https://github.com/matrix-org/synapse/issues/3334)) +- Add metrics to track appservice transactions ([\#3344](https://github.com/matrix-org/synapse/issues/3344)) +- Try to log more helpful info when a sig verification fails ([\#3372](https://github.com/matrix-org/synapse/issues/3372)) +- Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy). ([\#3462](https://github.com/matrix-org/synapse/issues/3462)) +- Add optional ip\_range\_whitelist param to AS registration files to lock AS IP access ([\#3465](https://github.com/matrix-org/synapse/issues/3465)) +- Reject invalid server names in federation requests ([\#3480](https://github.com/matrix-org/synapse/issues/3480)) +- Reject invalid server names in homeserver.yaml ([\#3483](https://github.com/matrix-org/synapse/issues/3483)) + +Bugfixes +-------- + +- Strip access\_token from outgoing requests ([\#3327](https://github.com/matrix-org/synapse/issues/3327)) +- Redact AS tokens in logs ([\#3349](https://github.com/matrix-org/synapse/issues/3349)) +- Fix federation backfill from SQLite servers ([\#3355](https://github.com/matrix-org/synapse/issues/3355)) +- Fix event-purge-by-ts admin API ([\#3363](https://github.com/matrix-org/synapse/issues/3363)) +- Fix event filtering in get\_missing\_events handler ([\#3371](https://github.com/matrix-org/synapse/issues/3371)) +- Synapse is now stricter regarding accepting events which it cannot retrieve the prev\_events for. ([\#3456](https://github.com/matrix-org/synapse/issues/3456)) +- Fix bug where synapse would explode when receiving unicode in HTTP User-Agent header ([\#3470](https://github.com/matrix-org/synapse/issues/3470)) +- Invalidate cache on correct thread to avoid race ([\#3473](https://github.com/matrix-org/synapse/issues/3473)) + +Improved Documentation +---------------------- + +- `doc/postgres.rst`: fix display of the last command block. Thanks to \@ArchangeGabriel! ([\#3340](https://github.com/matrix-org/synapse/issues/3340)) + +Deprecations and Removals +------------------------- + +- Remove was\_forgotten\_at ([\#3324](https://github.com/matrix-org/synapse/issues/3324)) + +Misc +---- + +- [\#3332](https://github.com/matrix-org/synapse/issues/3332), [\#3341](https://github.com/matrix-org/synapse/issues/3341), [\#3347](https://github.com/matrix-org/synapse/issues/3347), [\#3348](https://github.com/matrix-org/synapse/issues/3348), [\#3356](https://github.com/matrix-org/synapse/issues/3356), [\#3385](https://github.com/matrix-org/synapse/issues/3385), [\#3446](https://github.com/matrix-org/synapse/issues/3446), [\#3447](https://github.com/matrix-org/synapse/issues/3447), [\#3467](https://github.com/matrix-org/synapse/issues/3467), [\#3474](https://github.com/matrix-org/synapse/issues/3474) + +Changes in synapse v0.31.2 (2018-06-14) +======================================= + +SECURITY UPDATE: Prevent unauthorised users from setting state events in a room when there is no `m.room.power_levels` event in force in the room. (PR \#3397) + +Discussion around the Matrix Spec change proposal for this change can be followed at . + +Changes in synapse v0.31.1 (2018-06-08) +======================================= + +v0.31.1 fixes a security bug in the `get_missing_events` federation API where event visibility rules were not applied correctly. + +We are not aware of it being actively exploited but please upgrade asap. + +Bug Fixes: + +- Fix event filtering in get\_missing\_events handler (PR \#3371) + +Changes in synapse v0.31.0 (2018-06-06) +======================================= + +Most notable change from v0.30.0 is to switch to the python prometheus library to improve system stats reporting. WARNING: this changes a number of prometheus metrics in a backwards-incompatible manner. For more details, see [docs/metrics-howto.rst](docs/metrics-howto.rst#removal-of-deprecated-metrics--time-based-counters-becoming-histograms-in-0310). + +Bug Fixes: + +- Fix metric documentation tables (PR \#3341) +- Fix LaterGauge error handling (694968f) +- Fix replication metrics (b7e7fd2) + +Changes in synapse v0.31.0-rc1 (2018-06-04) ========================================== + +Features: + +- Switch to the Python Prometheus library (PR \#3256, \#3274) +- Let users leave the server notice room after joining (PR \#3287) + +Changes: + +- daily user type phone home stats (PR \#3264) +- Use iter\* methods for \_filter\_events\_for\_server (PR \#3267) +- Docs on consent bits (PR \#3268) +- Remove users from user directory on deactivate (PR \#3277) +- Avoid sending consent notice to guest users (PR \#3288) +- disable CPUMetrics if no /proc/self/stat (PR \#3299) +- Consistently use six\'s iteritems and wrap lazy keys/values in list() if they\'re not meant to be lazy (PR \#3307) +- Add private IPv6 addresses to example config for url preview blacklist (PR \#3317) Thanks to \@thegcat! +- Reduce stuck read-receipts: ignore depth when updating (PR \#3318) +- Put python\'s logs into Trial when running unit tests (PR \#3319) + +Changes, python 3 migration: + +- Replace some more comparisons with six (PR \#3243) Thanks to \@NotAFile! +- replace some iteritems with six (PR \#3244) Thanks to \@NotAFile! +- Add batch\_iter to utils (PR \#3245) Thanks to \@NotAFile! +- use repr, not str (PR \#3246) Thanks to \@NotAFile! +- Misc Python3 fixes (PR \#3247) Thanks to \@NotAFile! +- Py3 storage/\_base.py (PR \#3278) Thanks to \@NotAFile! +- more six iteritems (PR \#3279) Thanks to \@NotAFile! +- More Misc. py3 fixes (PR \#3280) Thanks to \@NotAFile! +- remaining isintance fixes (PR \#3281) Thanks to \@NotAFile! +- py3-ize state.py (PR \#3283) Thanks to \@NotAFile! +- extend tox testing for py3 to avoid regressions (PR \#3302) Thanks to \@krombel! +- use memoryview in py3 (PR \#3303) Thanks to \@NotAFile! + +Bugs: + +- Fix federation backfill bugs (PR \#3261) +- federation: fix LaterGauge usage (PR \#3328) Thanks to \@intelfx! + +Changes in synapse v0.30.0 (2018-05-24) +======================================= + +\'Server Notices\' are a new feature introduced in Synapse 0.30. They provide a channel whereby server administrators can send messages to users on the server. + +They are used as part of communication of the server policies (see `docs/consent_tracking.md`), however the intention is that they may also find a use for features such as \"Message of the day\". + +This feature is specific to Synapse, but uses standard Matrix communication mechanisms, so should work with any Matrix client. For more details see `docs/server_notices.md` + +Further Server Notices/Consent Tracking Support: + +- Allow overriding the server\_notices user\'s avatar (PR \#3273) +- Use the localpart in the consent uri (PR \#3272) +- Support for putting %(consent\_uri)s in messages (PR \#3271) +- Block attempts to send server notices to remote users (PR \#3270) +- Docs on consent bits (PR \#3268) + +Changes in synapse v0.30.0-rc1 (2018-05-23) ========================================== + +Server Notices/Consent Tracking Support: + +- ConsentResource to gather policy consent from users (PR \#3213) +- Move RoomCreationHandler out of synapse.handlers.Handlers (PR \#3225) +- Infrastructure for a server notices room (PR \#3232) +- Send users a server notice about consent (PR \#3236) +- Reject attempts to send event before privacy consent is given (PR \#3257) +- Add a \'has\_consented\' template var to consent forms (PR \#3262) +- Fix dependency on jinja2 (PR \#3263) + +Features: + +- Cohort analytics (PR \#3163, \#3241, \#3251) +- Add lxml to docker image for web previews (PR \#3239) Thanks to \@ptman! +- Add in flight request metrics (PR \#3252) + +Changes: + +- Remove unused update\_external\_syncs (PR \#3233) +- Use stream rather depth ordering for push actions (PR \#3212) +- Make purge\_history operate on tokens (PR \#3221) +- Don\'t support limitless pagination (PR \#3265) + +Bug Fixes: + +- Fix logcontext resource usage tracking (PR \#3258) +- Fix error in handling receipts (PR \#3235) +- Stop the transaction cache caching failures (PR \#3255) + +Changes in synapse v0.29.1 (2018-05-17) +======================================= + +Changes: + +- Update docker documentation (PR \#3222) + +Changes in synapse v0.29.0 (2018-05-16) +======================================= + +Not changes since v0.29.0-rc1 + +Changes in synapse v0.29.0-rc1 (2018-05-14) +=========================================== + +Notable changes, a docker file for running Synapse (Thanks to \@kaiyou!) and a closed spec bug in the Client Server API. Additionally further prep for Python 3 migration. + +Potentially breaking change: + +- Make Client-Server API return 401 for invalid token (PR \#3161). + + This changes the Client-server spec to return a 401 error code instead of 403 when the access token is unrecognised. This is the behaviour required by the specification, but some clients may be relying on the old, incorrect behaviour. + + Thanks to \@NotAFile for fixing this. + +Features: + +- Add a Dockerfile for synapse (PR \#2846) Thanks to \@kaiyou! + +Changes - General: + +- nuke-room-from-db.sh: added postgresql option and help (PR \#2337) Thanks to \@rubo77! +- Part user from rooms on account deactivate (PR \#3201) +- Make \'unexpected logging context\' into warnings (PR \#3007) +- Set Server header in SynapseRequest (PR \#3208) +- remove duplicates from groups tables (PR \#3129) +- Improve exception handling for background processes (PR \#3138) +- Add missing consumeErrors to improve exception handling (PR \#3139) +- reraise exceptions more carefully (PR \#3142) +- Remove redundant call to preserve\_fn (PR \#3143) +- Trap exceptions thrown within run\_in\_background (PR \#3144) + +Changes - Refactors: + +- Refactor /context to reuse pagination storage functions (PR \#3193) +- Refactor recent events func to use pagination func (PR \#3195) +- Refactor pagination DB API to return concrete type (PR \#3196) +- Refactor get\_recent\_events\_for\_room return type (PR \#3198) +- Refactor sync APIs to reuse pagination API (PR \#3199) +- Remove unused code path from member change DB func (PR \#3200) +- Refactor request handling wrappers (PR \#3203) +- transaction\_id, destination defined twice (PR \#3209) Thanks to \@damir-manapov! +- Refactor event storage to prepare for changes in state calculations (PR \#3141) +- Set Server header in SynapseRequest (PR \#3208) +- Use deferred.addTimeout instead of time\_bound\_deferred (PR \#3127, \#3178) +- Use run\_in\_background in preference to preserve\_fn (PR \#3140) + +Changes - Python 3 migration: + +- Construct HMAC as bytes on py3 (PR \#3156) Thanks to \@NotAFile! +- run config tests on py3 (PR \#3159) Thanks to \@NotAFile! +- Open certificate files as bytes (PR \#3084) Thanks to \@NotAFile! +- Open config file in non-bytes mode (PR \#3085) Thanks to \@NotAFile! +- Make event properties raise AttributeError instead (PR \#3102) Thanks to \@NotAFile! +- Use six.moves.urlparse (PR \#3108) Thanks to \@NotAFile! +- Add py3 tests to tox with folders that work (PR \#3145) Thanks to \@NotAFile! +- Don\'t yield in list comprehensions (PR \#3150) Thanks to \@NotAFile! +- Move more xrange to six (PR \#3151) Thanks to \@NotAFile! +- make imports local (PR \#3152) Thanks to \@NotAFile! +- move httplib import to six (PR \#3153) Thanks to \@NotAFile! +- Replace stringIO imports with six (PR \#3154, \#3168) Thanks to \@NotAFile! +- more bytes strings (PR \#3155) Thanks to \@NotAFile! + +Bug Fixes: + +- synapse fails to start under Twisted \>= 18.4 (PR \#3157) +- Fix a class of logcontext leaks (PR \#3170) +- Fix a couple of logcontext leaks in unit tests (PR \#3172) +- Fix logcontext leak in media repo (PR \#3174) +- Escape label values in prometheus metrics (PR \#3175, \#3186) +- Fix \'Unhandled Error\' logs with Twisted 18.4 (PR \#3182) Thanks to \@Half-Shot! +- Fix logcontext leaks in rate limiter (PR \#3183) +- notifications: Convert next\_token to string according to the spec (PR \#3190) Thanks to \@mujx! +- nuke-room-from-db.sh: fix deletion from search table (PR \#3194) Thanks to \@rubo77! +- add guard for None on purge\_history api (PR \#3160) Thanks to \@krombel! + +Changes in synapse v0.28.1 (2018-05-01) +======================================= + +SECURITY UPDATE + +- Clamp the allowed values of event depth received over federation to be \[0, 2\^63 - 1\]. This mitigates an attack where malicious events injected with depth = 2\^63 - 1 render rooms unusable. Depth is used to determine the cosmetic ordering of events within a room, and so the ordering of events in such a room will default to using stream\_ordering rather than depth (topological\_ordering). + + This is a temporary solution to mitigate abuse in the wild, whilst a long term solution is being implemented to improve how the depth parameter is used. + + Full details at + +- Pin Twisted to \<18.4 until we stop using the private \_OpenSSLECCurve API. + +Changes in synapse v0.28.0 (2018-04-26) +======================================= + +Bug Fixes: + +- Fix quarantine media admin API and search reindex (PR \#3130) +- Fix media admin APIs (PR \#3134) + +Changes in synapse v0.28.0-rc1 (2018-04-24) +=========================================== + +Minor performance improvement to federation sending and bug fixes. + +(Note: This release does not include the delta state resolution implementation discussed in matrix live) + +Features: + +- Add metrics for event processing lag (PR \#3090) +- Add metrics for ResponseCache (PR \#3092) + +Changes: + +- Synapse on PyPy (PR \#2760) Thanks to \@Valodim! +- move handling of auto\_join\_rooms to RegisterHandler (PR \#2996) Thanks to \@krombel! +- Improve handling of SRV records for federation connections (PR \#3016) Thanks to \@silkeh! +- Document the behaviour of ResponseCache (PR \#3059) +- Preparation for py3 (PR \#3061, \#3073, \#3074, \#3075, \#3103, \#3104, \#3106, \#3107, \#3109, \#3110) Thanks to \@NotAFile! +- update prometheus dashboard to use new metric names (PR \#3069) Thanks to \@krombel! +- use python3-compatible prints (PR \#3074) Thanks to \@NotAFile! +- Send federation events concurrently (PR \#3078) +- Limit concurrent event sends for a room (PR \#3079) +- Improve R30 stat definition (PR \#3086) +- Send events to ASes concurrently (PR \#3088) +- Refactor ResponseCache usage (PR \#3093) +- Clarify that SRV may not point to a CNAME (PR \#3100) Thanks to \@silkeh! +- Use str(e) instead of e.message (PR \#3103) Thanks to \@NotAFile! +- Use six.itervalues in some places (PR \#3106) Thanks to \@NotAFile! +- Refactor store.have\_events (PR \#3117) + +Bug Fixes: + +- Return 401 for invalid access\_token on logout (PR \#2938) Thanks to \@dklug! +- Return a 404 rather than a 500 on rejoining empty rooms (PR \#3080) +- fix federation\_domain\_whitelist (PR \#3099) +- Avoid creating events with huge numbers of prev\_events (PR \#3113) +- Reject events which have lots of prev\_events (PR \#3118) + +Changes in synapse v0.27.4 (2018-04-13) ====================================== + +Changes: + +- Update canonicaljson dependency (\#3095) + +Changes in synapse v0.27.3 (2018-04-11) ====================================== + +Bug fixes: + +- URL quote path segments over federation (\#3082) + +Changes in synapse v0.27.3-rc2 (2018-04-09) ========================================== + +v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored. + +Changes in synapse v0.27.3-rc1 (2018-04-09) ======================================= + +Notable changes include API support for joinability of groups. Also new metrics and phone home stats. Phone home stats include better visibility of system usage so we can tweak synpase to work better for all users rather than our own experience with matrix.org. Also, recording \'r30\' stat which is the measure we use to track overal growth of the Matrix ecosystem. It is defined as:- + +Counts the number of native 30 day retained users, defined as:- \* Users who have created their accounts more than 30 days + +: - Where last seen at most 30 days ago + - Where account creation and last\_seen are \> 30 days\" + +Features: + +- Add joinability for groups (PR \#3045) +- Implement group join API (PR \#3046) +- Add counter metrics for calculating state delta (PR \#3033) +- R30 stats (PR \#3041) +- Measure time it takes to calculate state group ID (PR \#3043) +- Add basic performance statistics to phone home (PR \#3044) +- Add response size metrics (PR \#3071) +- phone home cache size configurations (PR \#3063) + +Changes: + +- Add a blurb explaining the main synapse worker (PR \#2886) Thanks to \@turt2live! +- Replace old style error catching with \'as\' keyword (PR \#3000) Thanks to \@NotAFile! +- Use .iter\* to avoid copies in StateHandler (PR \#3006) +- Linearize calls to \_generate\_user\_id (PR \#3029) +- Remove last usage of ujson (PR \#3030) +- Use simplejson throughout (PR \#3048) +- Use static JSONEncoders (PR \#3049) +- Remove uses of events.content (PR \#3060) +- Improve database cache performance (PR \#3068) + +Bug fixes: + +- Add room\_id to the response of rooms/{roomId}/join (PR \#2986) Thanks to \@jplatte! +- Fix replication after switch to simplejson (PR \#3015) +- 404 correctly on missing paths via NoResource (PR \#3022) +- Fix error when claiming e2e keys from offline servers (PR \#3034) +- fix tests/storage/test\_user\_directory.py (PR \#3042) +- use PUT instead of POST for federating groups/m.join\_policy (PR \#3070) Thanks to \@krombel! +- postgres port script: fix state\_groups\_pkey error (PR \#3072) + +Changes in synapse v0.27.2 (2018-03-26) +======================================= + +Bug fixes: + +- Fix bug which broke TCP replication between workers (PR \#3015) + +Changes in synapse v0.27.1 (2018-03-26) +======================================= + +Meta release as v0.27.0 temporarily pointed to the wrong commit + +Changes in synapse v0.27.0 (2018-03-26) +======================================= + +No changes since v0.27.0-rc2 + +Changes in synapse v0.27.0-rc2 (2018-03-19) +=========================================== + +Pulls in v0.26.1 + +Bug fixes: + +- Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR \#3005) + +Changes in synapse v0.26.1 (2018-03-15) +======================================= + +Bug fixes: + +- Fix bug where an invalid event caused server to stop functioning correctly, due to parsing and serializing bugs in ujson library (PR \#3008) + +Changes in synapse v0.27.0-rc1 (2018-03-14) +=========================================== + +The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using `-a` option with workers. A new worker file should be added with `worker_app: synapse.app.homeserver`. + +This release also begins the process of renaming a number of the metrics reported to prometheus. See [docs/metrics-howto.rst](docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0-27-0). Note that the v0.28.0 release will remove the deprecated metric names. + +Features: + +- Add ability for ASes to override message send time (PR \#2754) +- Add support for custom storage providers for media repository (PR \#2867, \#2777, \#2783, \#2789, \#2791, \#2804, \#2812, \#2814, \#2857, \#2868, \#2767) +- Add purge API features, see [docs/admin\_api/purge\_history\_api.rst](docs/admin_api/purge_history_api.rst) for full details (PR \#2858, \#2867, \#2882, \#2946, \#2962, \#2943) +- Add support for whitelisting 3PIDs that users can register. (PR \#2813) +- Add `/room/{id}/event/{id}` API (PR \#2766) +- Add an admin API to get all the media in a room (PR \#2818) Thanks to \@turt2live! +- Add `federation_domain_whitelist` option (PR \#2820, \#2821) + +Changes: + +- Continue to factor out processing from main process and into worker processes. See updated [docs/workers.rst](docs/workers.rst) (PR \#2892 - \#2904, \#2913, \#2920 - \#2926, \#2947, \#2847, \#2854, \#2872, \#2873, \#2874, \#2928, \#2929, \#2934, \#2856, \#2976 - \#2984, \#2987 - \#2989, \#2991 - \#2993, \#2995, \#2784) +- Ensure state cache is used when persisting events (PR \#2864, \#2871, \#2802, \#2835, \#2836, \#2841, \#2842, \#2849) +- Change the default config to bind on both IPv4 and IPv6 on all platforms (PR \#2435) Thanks to \@silkeh! +- No longer require a specific version of saml2 (PR \#2695) Thanks to \@okurz! +- Remove `verbosity`/`log_file` from generated config (PR \#2755) +- Add and improve metrics and logging (PR \#2770, \#2778, \#2785, \#2786, \#2787, \#2793, \#2794, \#2795, \#2809, \#2810, \#2833, \#2834, \#2844, \#2965, \#2927, \#2975, \#2790, \#2796, \#2838) +- When using synctl with workers, don\'t start the main synapse automatically (PR \#2774) +- Minor performance improvements (PR \#2773, \#2792) +- Use a connection pool for non-federation outbound connections (PR \#2817) +- Make it possible to run unit tests against postgres (PR \#2829) +- Update pynacl dependency to 1.2.1 or higher (PR \#2888) Thanks to \@bachp! +- Remove ability for AS users to call /events and /sync (PR \#2948) +- Use bcrypt.checkpw (PR \#2949) Thanks to \@krombel! + +Bug fixes: + +- Fix broken `ldap_config` config option (PR \#2683) Thanks to \@seckrv! +- Fix error message when user is not allowed to unban (PR \#2761) Thanks to \@turt2live! +- Fix publicised groups GET API (singular) over federation (PR \#2772) +- Fix user directory when using `user_directory_search_all_users` config option (PR \#2803, \#2831) +- Fix error on `/publicRooms` when no rooms exist (PR \#2827) +- Fix bug in quarantine\_media (PR \#2837) +- Fix url\_previews when no Content-Type is returned from URL (PR \#2845) +- Fix rare race in sync API when joining room (PR \#2944) +- Fix slow event search, switch back from GIST to GIN indexes (PR \#2769, \#2848) + +Changes in synapse v0.26.0 (2018-01-05) +======================================= + +No changes since v0.26.0-rc1 + +Changes in synapse v0.26.0-rc1 (2017-12-13) +=========================================== + +Features: + +- Add ability for ASes to publicise groups for their users (PR \#2686) +- Add all local users to the user\_directory and optionally search them (PR \#2723) +- Add support for custom login types for validating users (PR \#2729) + +Changes: + +- Update example Prometheus config to new format (PR \#2648) Thanks to \@krombel! +- Rename redact\_content option to include\_content in Push API (PR \#2650) +- Declare support for r0.3.0 (PR \#2677) +- Improve upserts (PR \#2684, \#2688, \#2689, \#2713) +- Improve documentation of workers (PR \#2700) +- Improve tracebacks on exceptions (PR \#2705) +- Allow guest access to group APIs for reading (PR \#2715) +- Support for posting content in federation\_client script (PR \#2716) +- Delete devices and pushers on logouts etc (PR \#2722) + +Bug fixes: + +- Fix database port script (PR \#2673) +- Fix internal server error on login with ldap\_auth\_provider (PR \#2678) Thanks to \@jkolo! +- Fix error on sqlite 3.7 (PR \#2697) +- Fix OPTIONS on preview\_url (PR \#2707) +- Fix error handling on dns lookup (PR \#2711) +- Fix wrong avatars when inviting multiple users when creating room (PR \#2717) +- Fix 500 when joining matrix-dev (PR \#2719) + +Changes in synapse v0.25.1 (2017-11-17) +======================================= + +Bug fixes: + +- Fix login with LDAP and other password provider modules (PR \#2678). Thanks to \@jkolo! + +Changes in synapse v0.25.0 (2017-11-15) +======================================= + +Bug fixes: + +- Fix port script (PR \#2673) + +Changes in synapse v0.25.0-rc1 (2017-11-14) +=========================================== + +Features: + +- Add is\_public to groups table to allow for private groups (PR \#2582) +- Add a route for determining who you are (PR \#2668) Thanks to \@turt2live! +- Add more features to the password providers (PR \#2608, \#2610, \#2620, \#2622, \#2623, \#2624, \#2626, \#2628, \#2629) +- Add a hook for custom rest endpoints (PR \#2627) +- Add API to update group room visibility (PR \#2651) + +Changes: + +- Ignore \ tags when generating URL preview descriptions (PR \#2576) Thanks to \@maximevaillancourt! +- Register some /unstable endpoints in /r0 as well (PR \#2579) Thanks to \@krombel! +- Support /keys/upload on /r0 as well as /unstable (PR \#2585) +- Front-end proxy: pass through auth header (PR \#2586) +- Allow ASes to deactivate their own users (PR \#2589) +- Remove refresh tokens (PR \#2613) +- Automatically set default displayname on register (PR \#2617) +- Log login requests (PR \#2618) +- Always return is\_public in the /groups/:group\_id/rooms API (PR \#2630) +- Avoid no-op media deletes (PR \#2637) Thanks to \@spantaleev! +- Fix various embarrassing typos around user\_directory and add some doc. (PR \#2643) +- Return whether a user is an admin within a group (PR \#2647) +- Namespace visibility options for groups (PR \#2657) +- Downcase UserIDs on registration (PR \#2662) +- Cache failures when fetching URL previews (PR \#2669) + +Bug fixes: + +- Fix port script (PR \#2577) +- Fix error when running synapse with no logfile (PR \#2581) +- Fix UI auth when deleting devices (PR \#2591) +- Fix typo when checking if user is invited to group (PR \#2599) +- Fix the port script to drop NUL values in all tables (PR \#2611) +- Fix appservices being backlogged and not receiving new events due to a bug in notify\_interested\_services (PR \#2631) Thanks to \@xyzz! +- Fix updating rooms avatar/display name when modified by admin (PR \#2636) Thanks to \@farialima! +- Fix bug in state group storage (PR \#2649) +- Fix 500 on invalid utf-8 in request (PR \#2663) + +Changes in synapse v0.24.1 (2017-10-24) +======================================= + +Bug fixes: + +- Fix updating group profiles over federation (PR \#2567) + +Changes in synapse v0.24.0 (2017-10-23) +======================================= + +No changes since v0.24.0-rc1 + +Changes in synapse v0.24.0-rc1 (2017-10-19) +=========================================== + +Features: + +- Add Group Server (PR \#2352, \#2363, \#2374, \#2377, \#2378, \#2382, \#2410, \#2426, \#2430, \#2454, \#2471, \#2472, \#2544) +- Add support for channel notifications (PR \#2501) +- Add basic implementation of backup media store (PR \#2538) +- Add config option to auto-join new users to rooms (PR \#2545) + +Changes: + +- Make the spam checker a module (PR \#2474) +- Delete expired url cache data (PR \#2478) +- Ignore incoming events for rooms that we have left (PR \#2490) +- Allow spam checker to reject invites too (PR \#2492) +- Add room creation checks to spam checker (PR \#2495) +- Spam checking: add the invitee to user\_may\_invite (PR \#2502) +- Process events from federation for different rooms in parallel (PR \#2520) +- Allow error strings from spam checker (PR \#2531) +- Improve error handling for missing files in config (PR \#2551) + +Bug fixes: + +- Fix handling SERVFAILs when doing AAAA lookups for federation (PR \#2477) +- Fix incompatibility with newer versions of ujson (PR \#2483) Thanks to \@jeremycline! +- Fix notification keywords that start/end with non-word chars (PR \#2500) +- Fix stack overflow and logcontexts from linearizer (PR \#2532) +- Fix 500 error when fields missing from power\_levels event (PR \#2552) +- Fix 500 error when we get an error handling a PDU (PR \#2553) + +Changes in synapse v0.23.1 (2017-10-02) +======================================= + +Changes: + +- Make \'affinity\' package optional, as it is not supported on some platforms + +Changes in synapse v0.23.0 (2017-10-02) +======================================= + +No changes since v0.23.0-rc2 + +Changes in synapse v0.23.0-rc2 (2017-09-26) +=========================================== + +Bug fixes: + +- Fix regression in performance of syncs (PR \#2470) + +Changes in synapse v0.23.0-rc1 (2017-09-25) +=========================================== + +Features: + +- Add a frontend proxy worker (PR \#2344) +- Add support for event\_id\_only push format (PR \#2450) +- Add a PoC for filtering spammy events (PR \#2456) +- Add a config option to block all room invites (PR \#2457) + +Changes: + +- Use bcrypt module instead of py-bcrypt (PR \#2288) Thanks to \@kyrias! +- Improve performance of generating push notifications (PR \#2343, \#2357, \#2365, \#2366, \#2371) +- Improve DB performance for device list handling in sync (PR \#2362) +- Include a sample prometheus config (PR \#2416) +- Document known to work postgres version (PR \#2433) Thanks to \@ptman! + +Bug fixes: + +- Fix caching error in the push evaluator (PR \#2332) +- Fix bug where pusherpool didn\'t start and broke some rooms (PR \#2342) +- Fix port script for user directory tables (PR \#2375) +- Fix device lists notifications when user rejoins a room (PR \#2443, \#2449) +- Fix sync to always send down current state events in timeline (PR \#2451) +- Fix bug where guest users were incorrectly kicked (PR \#2453) +- Fix bug talking to IPv6 only servers using SRV records (PR \#2462) + +Changes in synapse v0.22.1 (2017-07-06) +======================================= + +Bug fixes: + +- Fix bug where pusher pool didn\'t start and caused issues when interacting with some rooms (PR \#2342) + +Changes in synapse v0.22.0 (2017-07-06) +======================================= + +No changes since v0.22.0-rc2 + +Changes in synapse v0.22.0-rc2 (2017-07-04) +=========================================== + +Changes: + +- Improve performance of storing user IPs (PR \#2307, \#2308) +- Slightly improve performance of verifying access tokens (PR \#2320) +- Slightly improve performance of event persistence (PR \#2321) +- Increase default cache factor size from 0.1 to 0.5 (PR \#2330) + +Bug fixes: + +- Fix bug with storing registration sessions that caused frequent CPU churn (PR \#2319) + +Changes in synapse v0.22.0-rc1 (2017-06-26) +=========================================== + +Features: + +- Add a user directory API (PR \#2252, and many more) +- Add shutdown room API to remove room from local server (PR \#2291) +- Add API to quarantine media (PR \#2292) +- Add new config option to not send event contents to push servers (PR \#2301) Thanks to \@cjdelisle! + +Changes: + +- Various performance fixes (PR \#2177, \#2233, \#2230, \#2238, \#2248, \#2256, \#2274) +- Deduplicate sync filters (PR \#2219) Thanks to \@krombel! +- Correct a typo in UPGRADE.rst (PR \#2231) Thanks to \@aaronraimist! +- Add count of one time keys to sync stream (PR \#2237) +- Only store event\_auth for state events (PR \#2247) +- Store URL cache preview downloads separately (PR \#2299) + +Bug fixes: + +- Fix users not getting notifications when AS listened to that user\_id (PR \#2216) Thanks to \@slipeer! +- Fix users without push set up not getting notifications after joining rooms (PR \#2236) +- Fix preview url API to trim long descriptions (PR \#2243) +- Fix bug where we used cached but unpersisted state group as prev group, resulting in broken state of restart (PR \#2263) +- Fix removing of pushers when using workers (PR \#2267) +- Fix CORS headers to allow Authorization header (PR \#2285) Thanks to \@krombel! + +Changes in synapse v0.21.1 (2017-06-15) +======================================= + +Bug fixes: + +- Fix bug in anonymous usage statistic reporting (PR \#2281) + +Changes in synapse v0.21.0 (2017-05-18) +======================================= + +No changes since v0.21.0-rc3 + +Changes in synapse v0.21.0-rc3 (2017-05-17) +=========================================== + +Features: + +- Add per user rate-limiting overrides (PR \#2208) +- Add config option to limit maximum number of events requested by `/sync` and `/messages` (PR \#2221) Thanks to \@psaavedra! + +Changes: + +- Various small performance fixes (PR \#2201, \#2202, \#2224, \#2226, \#2227, \#2228, \#2229) +- Update username availability checker API (PR \#2209, \#2213) +- When purging, don\'t de-delta state groups we\'re about to delete (PR \#2214) +- Documentation to check synapse version (PR \#2215) Thanks to \@hamber-dick! +- Add an index to event\_search to speed up purge history API (PR \#2218) + +Bug fixes: + +- Fix API to allow clients to upload one-time-keys with new sigs (PR \#2206) + +Changes in synapse v0.21.0-rc2 (2017-05-08) +=========================================== + +Changes: + +- Always mark remotes as up if we receive a signed request from them (PR \#2190) + +Bug fixes: + +- Fix bug where users got pushed for rooms they had muted (PR \#2200) + +Changes in synapse v0.21.0-rc1 (2017-05-08) +=========================================== + +Features: + +- Add username availability checker API (PR \#2183) +- Add read marker API (PR \#2120) + +Changes: + +- Enable guest access for the 3pl/3pid APIs (PR \#1986) +- Add setting to support TURN for guests (PR \#2011) +- Various performance improvements (PR \#2075, \#2076, \#2080, \#2083, \#2108, \#2158, \#2176, \#2185) +- Make synctl a bit more user friendly (PR \#2078, \#2127) Thanks \@APwhitehat! +- Replace HTTP replication with TCP replication (PR \#2082, \#2097, \#2098, \#2099, \#2103, \#2014, \#2016, \#2115, \#2116, \#2117) +- Support authenticated SMTP (PR \#2102) Thanks \@DanielDent! +- Add a counter metric for successfully-sent transactions (PR \#2121) +- Propagate errors sensibly from proxied IS requests (PR \#2147) +- Add more granular event send metrics (PR \#2178) + +Bug fixes: + +- Fix nuke-room script to work with current schema (PR \#1927) Thanks \@zuckschwerdt! +- Fix db port script to not assume postgres tables are in the public schema (PR \#2024) Thanks \@jerrykan! +- Fix getting latest device IP for user with no devices (PR \#2118) +- Fix rejection of invites to unreachable servers (PR \#2145) +- Fix code for reporting old verify keys in synapse (PR \#2156) +- Fix invite state to always include all events (PR \#2163) +- Fix bug where synapse would always fetch state for any missing event (PR \#2170) +- Fix a leak with timed out HTTP connections (PR \#2180) +- Fix bug where we didn\'t time out HTTP requests to ASes (PR \#2192) + +Docs: + +- Clarify doc for SQLite to PostgreSQL port (PR \#1961) Thanks \@benhylau! +- Fix typo in synctl help (PR \#2107) Thanks \@HarHarLinks! +- `web_client_location` documentation fix (PR \#2131) Thanks \@matthewjwolff! +- Update README.rst with FreeBSD changes (PR \#2132) Thanks \@feld! +- Clarify setting up metrics (PR \#2149) Thanks \@encks! + +Changes in synapse v0.20.0 (2017-04-11) +======================================= + +Bug fixes: + +- Fix joining rooms over federation where not all servers in the room saw the new server had joined (PR \#2094) + +Changes in synapse v0.20.0-rc1 (2017-03-30) +=========================================== + +Features: + +- Add delete\_devices API (PR \#1993) +- Add phone number registration/login support (PR \#1994, \#2055) + +Changes: + +- Use JSONSchema for validation of filters. Thanks \@pik! (PR \#1783) +- Reread log config on SIGHUP (PR \#1982) +- Speed up public room list (PR \#1989) +- Add helpful texts to logger config options (PR \#1990) +- Minor `/sync` performance improvements. (PR \#2002, \#2013, \#2022) +- Add some debug to help diagnose weird federation issue (PR \#2035) +- Correctly limit retries for all federation requests (PR \#2050, \#2061) +- Don\'t lock table when persisting new one time keys (PR \#2053) +- Reduce some CPU work on DB threads (PR \#2054) +- Cache hosts in room (PR \#2060) +- Batch sending of device list pokes (PR \#2063) +- Speed up persist event path in certain edge cases (PR \#2070) + +Bug fixes: + +- Fix bug where current\_state\_events renamed to current\_state\_ids (PR \#1849) +- Fix routing loop when fetching remote media (PR \#1992) +- Fix current\_state\_events table to not lie (PR \#1996) +- Fix CAS login to handle PartialDownloadError (PR \#1997) +- Fix assertion to stop transaction queue getting wedged (PR \#2010) +- Fix presence to fallback to last\_active\_ts if it beats the last sync time. Thanks \@Half-Shot! (PR \#2014) +- Fix bug when federation received a PDU while a room join is in progress (PR \#2016) +- Fix resetting state on rejected events (PR \#2025) +- Fix installation issues in readme. Thanks \@ricco386 (PR \#2037) +- Fix caching of remote servers\' signature keys (PR \#2042) +- Fix some leaking log context (PR \#2048, \#2049, \#2057, \#2058) +- Fix rejection of invites not reaching sync (PR \#2056) + +Changes in synapse v0.19.3 (2017-03-20) +======================================= + +No changes since v0.19.3-rc2 + +Changes in synapse v0.19.3-rc2 (2017-03-13) +=========================================== + +Bug fixes: + +- Fix bug in handling of incoming device list updates over federation. + +Changes in synapse v0.19.3-rc1 (2017-03-08) +=========================================== + +Features: + +- Add some administration functionalities. Thanks to morteza-araby! (PR \#1784) + +Changes: + +- Reduce database table sizes (PR \#1873, \#1916, \#1923, \#1963) +- Update contrib/ to not use syutil. Thanks to andrewshadura! (PR \#1907) +- Don\'t fetch current state when sending an event in common case (PR \#1955) + +Bug fixes: + +- Fix synapse\_port\_db failure. Thanks to Pneumaticat! (PR \#1904) +- Fix caching to not cache error responses (PR \#1913) +- Fix APIs to make kick & ban reasons work (PR \#1917) +- Fix bugs in the /keys/changes api (PR \#1921) +- Fix bug where users couldn\'t forget rooms they were banned from (PR \#1922) +- Fix issue with long language values in pushers API (PR \#1925) +- Fix a race in transaction queue (PR \#1930) +- Fix dynamic thumbnailing to preserve aspect ratio. Thanks to jkolo! (PR \#1945) +- Fix device list update to not constantly resync (PR \#1964) +- Fix potential for huge memory usage when getting device that have changed (PR \#1969) + +Changes in synapse v0.19.2 (2017-02-20) +======================================= + +- Fix bug with event visibility check in /context/ API. Thanks to Tokodomo for pointing it out! (PR \#1929) + +Changes in synapse v0.19.1 (2017-02-09) +======================================= + +- Fix bug where state was incorrectly reset in a room when synapse received an event over federation that did not pass auth checks (PR \#1892) + +Changes in synapse v0.19.0 (2017-02-04) +======================================= + +No changes since RC 4. + +Changes in synapse v0.19.0-rc4 (2017-02-02) +=========================================== + +- Bump cache sizes for common membership queries (PR \#1879) + +Changes in synapse v0.19.0-rc3 (2017-02-02) +=========================================== + +- Fix email push in pusher worker (PR \#1875) +- Make presence.get\_new\_events a bit faster (PR \#1876) +- Make /keys/changes a bit more performant (PR \#1877) + +Changes in synapse v0.19.0-rc2 (2017-02-02) +=========================================== + +- Include newly joined users in /keys/changes API (PR \#1872) + +Changes in synapse v0.19.0-rc1 (2017-02-02) +=========================================== + +Features: + +- Add support for specifying multiple bind addresses (PR \#1709, \#1712, \#1795, \#1835). Thanks to \@kyrias! +- Add /account/3pid/delete endpoint (PR \#1714) +- Add config option to configure the Riot URL used in notification emails (PR \#1811). Thanks to \@aperezdc! +- Add username and password config options for turn server (PR \#1832). Thanks to \@xsteadfastx! +- Implement device lists updates over federation (PR \#1857, \#1861, \#1864) +- Implement /keys/changes (PR \#1869, \#1872) + +Changes: + +- Improve IPv6 support (PR \#1696). Thanks to \@kyrias and \@glyph! +- Log which files we saved attachments to in the media\_repository (PR \#1791) +- Linearize updates to membership via PUT /state/ to better handle multiple joins (PR \#1787) +- Limit number of entries to prefill from cache on startup (PR \#1792) +- Remove full\_twisted\_stacktraces option (PR \#1802) +- Measure size of some caches by sum of the size of cached values (PR \#1815) +- Measure metrics of string\_cache (PR \#1821) +- Reduce logging verbosity (PR \#1822, \#1823, \#1824) +- Don\'t clobber a displayname or avatar\_url if provided by an m.room.member event (PR \#1852) +- Better handle 401/404 response for federation /send/ (PR \#1866, \#1871) + +Fixes: + +- Fix ability to change password to a non-ascii one (PR \#1711) +- Fix push getting stuck due to looking at the wrong view of state (PR \#1820) +- Fix email address comparison to be case insensitive (PR \#1827) +- Fix occasional inconsistencies of room membership (PR \#1836, \#1840) + +Performance: + +- Don\'t block messages sending on bumping presence (PR \#1789) +- Change device\_inbox stream index to include user (PR \#1793) +- Optimise state resolution (PR \#1818) +- Use DB cache of joined users for presence (PR \#1862) +- Add an index to make membership queries faster (PR \#1867) + +Changes in synapse v0.18.7 (2017-01-09) +======================================= + +No changes from v0.18.7-rc2 + +Changes in synapse v0.18.7-rc2 (2017-01-07) +=========================================== + +Bug fixes: + +- Fix error in rc1\'s discarding invalid inbound traffic logic that was incorrectly discarding missing events + +Changes in synapse v0.18.7-rc1 (2017-01-06) +=========================================== + +Bug fixes: + +- Fix error in \#PR 1764 to actually fix the nightmare \#1753 bug. +- Improve deadlock logging further +- Discard inbound federation traffic from invalid domains, to immunise against \#1753 + +Changes in synapse v0.18.6 (2017-01-06) +======================================= + +Bug fixes: + +- Fix bug when checking if a guest user is allowed to join a room (PR \#1772) Thanks to Patrik Oldsberg for diagnosing and the fix! + +Changes in synapse v0.18.6-rc3 (2017-01-05) +=========================================== + +Bug fixes: + +- Fix bug where we failed to send ban events to the banned server (PR \#1758) +- Fix bug where we sent event that didn\'t originate on this server to other servers (PR \#1764) +- Fix bug where processing an event from a remote server took a long time because we were making long HTTP requests (PR \#1765, PR \#1744) + +Changes: + +- Improve logging for debugging deadlocks (PR \#1766, PR \#1767) + +Changes in synapse v0.18.6-rc2 (2016-12-30) +=========================================== + +Bug fixes: + +- Fix memory leak in twisted by initialising logging correctly (PR \#1731) +- Fix bug where fetching missing events took an unacceptable amount of time in large rooms (PR \#1734) + +Changes in synapse v0.18.6-rc1 (2016-12-29) +=========================================== + +Bug fixes: + +- Make sure that outbound connections are closed (PR \#1725) + +Changes in synapse v0.18.5 (2016-12-16) +======================================= + +Bug fixes: + +- Fix federation /backfill returning events it shouldn\'t (PR \#1700) +- Fix crash in url preview (PR \#1701) + +Changes in synapse v0.18.5-rc3 (2016-12-13) +=========================================== + +Features: + +- Add support for E2E for guests (PR \#1653) +- Add new API appservice specific public room list (PR \#1676) +- Add new room membership APIs (PR \#1680) + +Changes: + +- Enable guest access for private rooms by default (PR \#653) +- Limit the number of events that can be created on a given room concurrently (PR \#1620) +- Log the args that we have on UI auth completion (PR \#1649) +- Stop generating refresh\_tokens (PR \#1654) +- Stop putting a time caveat on access tokens (PR \#1656) +- Remove unspecced GET endpoints for e2e keys (PR \#1694) + +Bug fixes: + +- Fix handling of 500 and 429\'s over federation (PR \#1650) +- Fix Content-Type header parsing (PR \#1660) +- Fix error when previewing sites that include unicode, thanks to kyrias (PR \#1664) +- Fix some cases where we drop read receipts (PR \#1678) +- Fix bug where calls to `/sync` didn\'t correctly timeout (PR \#1683) +- Fix bug where E2E key query would fail if a single remote host failed (PR \#1686) + +Changes in synapse v0.18.5-rc2 (2016-11-24) +=========================================== + +Bug fixes: + +- Don\'t send old events over federation, fixes bug in -rc1. + +Changes in synapse v0.18.5-rc1 (2016-11-24) +=========================================== + +Features: + +- Implement \"event\_fields\" in filters (PR \#1638) + +Changes: + +- Use external ldap auth pacakge (PR \#1628) +- Split out federation transaction sending to a worker (PR \#1635) +- Fail with a coherent error message if /sync?filter= is invalid (PR \#1636) +- More efficient notif count queries (PR \#1644) + +Changes in synapse v0.18.4 (2016-11-22) +======================================= + +Bug fixes: + +- Add workaround for buggy clients that the fail to register (PR \#1632) + +Changes in synapse v0.18.4-rc1 (2016-11-14) +=========================================== + +Changes: + +- Various database efficiency improvements (PR \#1188, \#1192) +- Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR \#1198) +- Allow specifying duration in minutes in config, thanks to Daniel Dent (PR \#1625) + +Bug fixes: + +- Fix media repo to set CORs headers on responses (PR \#1190) +- Fix registration to not error on non-ascii passwords (PR \#1191) +- Fix create event code to limit the number of prev\_events (PR \#1615) +- Fix bug in transaction ID deduplication (PR \#1624) + +Changes in synapse v0.18.3 (2016-11-08) +======================================= + +SECURITY UPDATE + +Explicitly require authentication when using LDAP3. This is the default on versions of `ldap3` above 1.0, but some distributions will package an older version. + +If you are using LDAP3 login and have a version of `ldap3` older than 1.0 it is **CRITICAL to updgrade**. + +Changes in synapse v0.18.2 (2016-11-01) +======================================= + +No changes since v0.18.2-rc5 + +Changes in synapse v0.18.2-rc5 (2016-10-28) +=========================================== + +Bug fixes: + +- Fix prometheus process metrics in worker processes (PR \#1184) + +Changes in synapse v0.18.2-rc4 (2016-10-27) +=========================================== + +Bug fixes: + +- Fix `user_threepids` schema delta, which in some instances prevented startup after upgrade (PR \#1183) + +Changes in synapse v0.18.2-rc3 (2016-10-27) +=========================================== + +Changes: + +- Allow clients to supply access tokens as headers (PR \#1098) +- Clarify error codes for GET /filter/, thanks to Alexander Maznev (PR \#1164) +- Make password reset email field case insensitive (PR \#1170) +- Reduce redundant database work in email pusher (PR \#1174) +- Allow configurable rate limiting per AS (PR \#1175) +- Check whether to ratelimit sooner to avoid work (PR \#1176) +- Standardise prometheus metrics (PR \#1177) + +Bug fixes: + +- Fix incredibly slow back pagination query (PR \#1178) +- Fix infinite typing bug (PR \#1179) + +Changes in synapse v0.18.2-rc2 (2016-10-25) +=========================================== + +(This release did not include the changes advertised and was identical to RC1) + +Changes in synapse v0.18.2-rc1 (2016-10-17) +=========================================== + +Changes: + +- Remove redundant event\_auth index (PR \#1113) +- Reduce DB hits for replication (PR \#1141) +- Implement pluggable password auth (PR \#1155) +- Remove rate limiting from app service senders and fix get\_or\_create\_user requester, thanks to Patrik Oldsberg (PR \#1157) +- window.postmessage for Interactive Auth fallback (PR \#1159) +- Use sys.executable instead of hardcoded python, thanks to Pedro Larroy (PR \#1162) +- Add config option for adding additional TLS fingerprints (PR \#1167) +- User-interactive auth on delete device (PR \#1168) + +Bug fixes: + +- Fix not being allowed to set your own state\_key, thanks to Patrik Oldsberg (PR \#1150) +- Fix interactive auth to return 401 from for incorrect password (PR \#1160, \#1166) +- Fix email push notifs being dropped (PR \#1169) + +Changes in synapse v0.18.1 (2016-10-05) ====================================== + +No changes since v0.18.1-rc1 + +Changes in synapse v0.18.1-rc1 (2016-09-30) +=========================================== + +Features: + +- Add total\_room\_count\_estimate to `/publicRooms` (PR \#1133) + +Changes: + +- Time out typing over federation (PR \#1140) +- Restructure LDAP authentication (PR \#1153) + +Bug fixes: + +- Fix 3pid invites when server is already in the room (PR \#1136) +- Fix upgrading with SQLite taking lots of CPU for a few days after upgrade (PR \#1144) +- Fix upgrading from very old database versions (PR \#1145) +- Fix port script to work with recently added tables (PR \#1146) + +Changes in synapse v0.18.0 (2016-09-19) +======================================= + +The release includes major changes to the state storage database schemas, which significantly reduce database size. Synapse will attempt to upgrade the current data in the background. Servers with large SQLite database may experience degradation of performance while this upgrade is in progress, therefore you may want to consider migrating to using Postgres before upgrading very large SQLite databases + +Changes: + +- Make public room search case insensitive (PR \#1127) + +Bug fixes: + +- Fix and clean up publicRooms pagination (PR \#1129) + +Changes in synapse v0.18.0-rc1 (2016-09-16) +=========================================== + +Features: + +- Add `only=highlight` on `/notifications` (PR \#1081) +- Add server param to /publicRooms (PR \#1082) +- Allow clients to ask for the whole of a single state event (PR \#1094) +- Add is\_direct param to /createRoom (PR \#1108) +- Add pagination support to publicRooms (PR \#1121) +- Add very basic filter API to /publicRooms (PR \#1126) +- Add basic direct to device messaging support for E2E (PR \#1074, \#1084, \#1104, \#1111) + +Changes: + +- Move to storing state\_groups\_state as deltas, greatly reducing DB size (PR \#1065) +- Reduce amount of state pulled out of the DB during common requests (PR \#1069) +- Allow PDF to be rendered from media repo (PR \#1071) +- Reindex state\_groups\_state after pruning (PR \#1085) +- Clobber EDUs in send queue (PR \#1095) +- Conform better to the CAS protocol specification (PR \#1100) +- Limit how often we ask for keys from dead servers (PR \#1114) + +Bug fixes: + +- Fix /notifications API when used with `from` param (PR \#1080) +- Fix backfill when cannot find an event. (PR \#1107) + +Changes in synapse v0.17.3 (2016-09-09) +======================================= + +This release fixes a major bug that stopped servers from handling rooms with over 1000 members. + +Changes in synapse v0.17.2 (2016-09-08) +======================================= + +This release contains security bug fixes. Please upgrade. + +No changes since v0.17.2-rc1 + +Changes in synapse v0.17.2-rc1 (2016-09-05) +=========================================== + +Features: + +- Start adding store-and-forward direct-to-device messaging (PR \#1046, \#1050, \#1062, \#1066) + +Changes: + +- Avoid pulling the full state of a room out so often (PR \#1047, \#1049, \#1063, \#1068) +- Don\'t notify for online to online presence transitions. (PR \#1054) +- Occasionally persist unpersisted presence updates (PR \#1055) +- Allow application services to have an optional \'url\' (PR \#1056) +- Clean up old sent transactions from DB (PR \#1059) + +Bug fixes: + +- Fix None check in backfill (PR \#1043) +- Fix membership changes to be idempotent (PR \#1067) +- Fix bug in get\_pdu where it would sometimes return events with incorrect signature + +Changes in synapse v0.17.1 (2016-08-24) +======================================= + +Changes: + +- Delete old received\_transactions rows (PR \#1038) +- Pass through user-supplied content in /join/\$room\_id (PR \#1039) + +Bug fixes: + +- Fix bug with backfill (PR \#1040) + +Changes in synapse v0.17.1-rc1 (2016-08-22) +=========================================== + +Features: + +- Add notification API (PR \#1028) + +Changes: + +- Don\'t print stack traces when failing to get remote keys (PR \#996) +- Various federation /event/ perf improvements (PR \#998) +- Only process one local membership event per room at a time (PR \#1005) +- Move default display name push rule (PR \#1011, \#1023) +- Fix up preview URL API. Add tests. (PR \#1015) +- Set `Content-Security-Policy` on media repo (PR \#1021) +- Make notify\_interested\_services faster (PR \#1022) +- Add usage stats to prometheus monitoring (PR \#1037) + +Bug fixes: + +- Fix token login (PR \#993) +- Fix CAS login (PR \#994, \#995) +- Fix /sync to not clobber status\_msg (PR \#997) +- Fix redacted state events to include prev\_content (PR \#1003) +- Fix some bugs in the auth/ldap handler (PR \#1007) +- Fix backfill request to limit URI length, so that remotes don\'t reject the requests due to path length limits (PR \#1012) +- Fix AS push code to not send duplicate events (PR \#1025) + +Changes in synapse v0.17.0 (2016-08-08) +======================================= + +This release contains significant security bug fixes regarding authenticating events received over federation. PLEASE UPGRADE. + +This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details. + +Changes: + +- Add federation /version API (PR \#990) +- Make psutil dependency optional (PR \#992) + +Bug fixes: + +- Fix URL preview API to exclude HTML comments in description (PR \#988) +- Fix error handling of remote joins (PR \#991) + +Changes in synapse v0.17.0-rc4 (2016-08-05) +=========================================== + +Changes: + +- Change the way we summarize URLs when previewing (PR \#973) +- Add new `/state_ids/` federation API (PR \#979) +- Speed up processing of `/state/` response (PR \#986) + +Bug fixes: + +- Fix event persistence when event has already been partially persisted (PR \#975, \#983, \#985) +- Fix port script to also copy across backfilled events (PR \#982) + +Changes in synapse v0.17.0-rc3 (2016-08-02) +=========================================== + +Changes: + +- Forbid non-ASes from registering users whose names begin with \'\_\' (PR \#958) +- Add some basic admin API docs (PR \#963) + +Bug fixes: + +- Send the correct host header when fetching keys (PR \#941) +- Fix joining a room that has missing auth events (PR \#964) +- Fix various push bugs (PR \#966, \#970) +- Fix adding emails on registration (PR \#968) + +Changes in synapse v0.17.0-rc2 (2016-08-02) +=========================================== + +(This release did not include the changes advertised and was identical to RC1) + +Changes in synapse v0.17.0-rc1 (2016-07-28) +=========================================== + +This release changes the LDAP configuration format in a backwards incompatible way, see PR \#843 for details. + +Features: + +- Add purge\_media\_cache admin API (PR \#902) +- Add deactivate account admin API (PR \#903) +- Add optional pepper to password hashing (PR \#907, \#910 by KentShikama) +- Add an admin option to shared secret registration (breaks backwards compat) (PR \#909) +- Add purge local room history API (PR \#911, \#923, \#924) +- Add requestToken endpoints (PR \#915) +- Add an /account/deactivate endpoint (PR \#921) +- Add filter param to /messages. Add \'contains\_url\' to filter. (PR \#922) +- Add device\_id support to /login (PR \#929) +- Add device\_id support to /v2/register flow. (PR \#937, \#942) +- Add GET /devices endpoint (PR \#939, \#944) +- Add GET /device/{deviceId} (PR \#943) +- Add update and delete APIs for devices (PR \#949) + +Changes: + +- Rewrite LDAP Authentication against ldap3 (PR \#843 by mweinelt) +- Linearize some federation endpoints based on (origin, room\_id) (PR \#879) +- Remove the legacy v0 content upload API. (PR \#888) +- Use similar naming we use in email notifs for push (PR \#894) +- Optionally include password hash in createUser endpoint (PR \#905 by KentShikama) +- Use a query that postgresql optimises better for get\_events\_around (PR \#906) +- Fall back to \'username\' if \'user\' is not given for appservice registration. (PR \#927 by Half-Shot) +- Add metrics for psutil derived memory usage (PR \#936) +- Record device\_id in client\_ips (PR \#938) +- Send the correct host header when fetching keys (PR \#941) +- Log the hostname the reCAPTCHA was completed on (PR \#946) +- Make the device id on e2e key upload optional (PR \#956) +- Add r0.2.0 to the \"supported versions\" list (PR \#960) +- Don\'t include name of room for invites in push (PR \#961) + +Bug fixes: + +- Fix substitution failure in mail template (PR \#887) +- Put most recent 20 messages in email notif (PR \#892) +- Ensure that the guest user is in the database when upgrading accounts (PR \#914) +- Fix various edge cases in auth handling (PR \#919) +- Fix 500 ISE when sending alias event without a state\_key (PR \#925) +- Fix bug where we stored rejections in the state\_group, persist all rejections (PR \#948) +- Fix lack of check of if the user is banned when handling 3pid invites (PR \#952) +- Fix a couple of bugs in the transaction and keyring code (PR \#954, \#955) + +Changes in synapse v0.16.1-r1 (2016-07-08) +========================================== + +THIS IS A CRITICAL SECURITY UPDATE. + +This fixes a bug which allowed users\' accounts to be accessed by unauthorised users. + +Changes in synapse v0.16.1 (2016-06-20) +======================================= + +Bug fixes: + +- Fix assorted bugs in `/preview_url` (PR \#872) +- Fix TypeError when setting unicode passwords (PR \#873) + +Performance improvements: + +- Turn `use_frozen_events` off by default (PR \#877) +- Disable responding with canonical json for federation (PR \#878) + +Changes in synapse v0.16.1-rc1 (2016-06-15) +=========================================== + +Features: None + +Changes: + +- Log requester for `/publicRoom` endpoints when possible (PR \#856) +- 502 on `/thumbnail` when can\'t connect to remote server (PR \#862) +- Linearize fetching of gaps on incoming events (PR \#871) + +Bugs fixes: + +- Fix bug where rooms where marked as published by default (PR \#857) +- Fix bug where joining room with an event with invalid sender (PR \#868) +- Fix bug where backfilled events were sent down sync streams (PR \#869) +- Fix bug where outgoing connections could wedge indefinitely, causing push notifications to be unreliable (PR \#870) + +Performance improvements: + +- Improve `/publicRooms` performance(PR \#859) + +Changes in synapse v0.16.0 (2016-06-09) +======================================= + +NB: As of v0.14 all AS config files must have an ID field. + +Bug fixes: + +- Don\'t make rooms published by default (PR \#857) + +Changes in synapse v0.16.0-rc2 (2016-06-08) +=========================================== + +Features: + +- Add configuration option for tuning GC via `gc.set_threshold` (PR \#849) + +Changes: + +- Record metrics about GC (PR \#771, \#847, \#852) +- Add metric counter for number of persisted events (PR \#841) + +Bug fixes: + +- Fix \'From\' header in email notifications (PR \#843) +- Fix presence where timeouts were not being fired for the first 8h after restarts (PR \#842) +- Fix bug where synapse sent malformed transactions to AS\'s when retrying transactions (Commits 310197b, 8437906) + +Performance improvements: + +- Remove event fetching from DB threads (PR \#835) +- Change the way we cache events (PR \#836) +- Add events to cache when we persist them (PR \#840) + +Changes in synapse v0.16.0-rc1 (2016-06-03) +=========================================== + +Version 0.15 was not released. See v0.15.0-rc1 below for additional changes. + +Features: + +- Add email notifications for missed messages (PR \#759, \#786, \#799, \#810, \#815, \#821) +- Add a `url_preview_ip_range_whitelist` config param (PR \#760) +- Add /report endpoint (PR \#762) +- Add basic ignore user API (PR \#763) +- Add an openidish mechanism for proving that you own a given user\_id (PR \#765) +- Allow clients to specify a server\_name to avoid \'No known servers\' (PR \#794) +- Add secondary\_directory\_servers option to fetch room list from other servers (PR \#808, \#813) + +Changes: + +- Report per request metrics for all of the things using request\_handler (PR \#756) +- Correctly handle `NULL` password hashes from the database (PR \#775) +- Allow receipts for events we haven\'t seen in the db (PR \#784) +- Make synctl read a cache factor from config file (PR \#785) +- Increment badge count per missed convo, not per msg (PR \#793) +- Special case m.room.third\_party\_invite event auth to match invites (PR \#814) + +Bug fixes: + +- Fix typo in event\_auth servlet path (PR \#757) +- Fix password reset (PR \#758) + +Performance improvements: + +- Reduce database inserts when sending transactions (PR \#767) +- Queue events by room for persistence (PR \#768) +- Add cache to `get_user_by_id` (PR \#772) +- Add and use `get_domain_from_id` (PR \#773) +- Use tree cache for `get_linearized_receipts_for_room` (PR \#779) +- Remove unused indices (PR \#782) +- Add caches to `bulk_get_push_rules*` (PR \#804) +- Cache `get_event_reference_hashes` (PR \#806) +- Add `get_users_with_read_receipts_in_room` cache (PR \#809) +- Use state to calculate `get_users_in_room` (PR \#811) +- Load push rules in storage layer so that they get cached (PR \#825) +- Make `get_joined_hosts_for_room` use get\_users\_in\_room (PR \#828) +- Poke notifier on next reactor tick (PR \#829) +- Change CacheMetrics to be quicker (PR \#830) + +Changes in synapse v0.15.0-rc1 (2016-04-26) +=========================================== + +Features: + +- Add login support for Javascript Web Tokens, thanks to Niklas Riekenbrauck (PR \#671,\#687) +- Add URL previewing support (PR \#688) +- Add login support for LDAP, thanks to Christoph Witzany (PR \#701) +- Add GET endpoint for pushers (PR \#716) + +Changes: + +- Never notify for member events (PR \#667) +- Deduplicate identical `/sync` requests (PR \#668) +- Require user to have left room to forget room (PR \#673) +- Use DNS cache if within TTL (PR \#677) +- Let users see their own leave events (PR \#699) +- Deduplicate membership changes (PR \#700) +- Increase performance of pusher code (PR \#705) +- Respond with error status 504 if failed to talk to remote server (PR \#731) +- Increase search performance on postgres (PR \#745) + +Bug fixes: + +- Fix bug where disabling all notifications still resulted in push (PR \#678) +- Fix bug where users couldn\'t reject remote invites if remote refused (PR \#691) +- Fix bug where synapse attempted to backfill from itself (PR \#693) +- Fix bug where profile information was not correctly added when joining remote rooms (PR \#703) +- Fix bug where register API required incorrect key name for AS registration (PR \#727) + +Changes in synapse v0.14.0 (2016-03-30) +======================================= + +No changes from v0.14.0-rc2 + +Changes in synapse v0.14.0-rc2 (2016-03-23) +=========================================== + +Features: + +- Add published room list API (PR \#657) + +Changes: + +- Change various caches to consume less memory (PR \#656, \#658, \#660, \#662, \#663, \#665) +- Allow rooms to be published without requiring an alias (PR \#664) +- Intern common strings in caches to reduce memory footprint (\#666) + +Bug fixes: + +- Fix reject invites over federation (PR \#646) +- Fix bug where registration was not idempotent (PR \#649) +- Update aliases event after deleting aliases (PR \#652) +- Fix unread notification count, which was sometimes wrong (PR \#661) + +Changes in synapse v0.14.0-rc1 (2016-03-14) +=========================================== + +Features: + +- Add event\_id to response to state event PUT (PR \#581) +- Allow guest users access to messages in rooms they have joined (PR \#587) +- Add config for what state is included in a room invite (PR \#598) +- Send the inviter\'s member event in room invite state (PR \#607) +- Add error codes for malformed/bad JSON in /login (PR \#608) +- Add support for changing the actions for default rules (PR \#609) +- Add environment variable SYNAPSE\_CACHE\_FACTOR, default it to 0.1 (PR \#612) +- Add ability for alias creators to delete aliases (PR \#614) +- Add profile information to invites (PR \#624) + +Changes: + +- Enforce user\_id exclusivity for AS registrations (PR \#572) +- Make adding push rules idempotent (PR \#587) +- Improve presence performance (PR \#582, \#586) +- Change presence semantics for `last_active_ago` (PR \#582, \#586) +- Don\'t allow `m.room.create` to be changed (PR \#596) +- Add 800x600 to default list of valid thumbnail sizes (PR \#616) +- Always include kicks and bans in full /sync (PR \#625) +- Send history visibility on boundary changes (PR \#626) +- Register endpoint now returns a refresh\_token (PR \#637) + +Bug fixes: + +- Fix bug where we returned incorrect state in /sync (PR \#573) +- Always return a JSON object from push rule API (PR \#606) +- Fix bug where registering without a user id sometimes failed (PR \#610) +- Report size of ExpiringCache in cache size metrics (PR \#611) +- Fix rejection of invites to empty rooms (PR \#615) +- Fix usage of `bcrypt` to not use `checkpw` (PR \#619) +- Pin `pysaml2` dependency (PR \#634) +- Fix bug in `/sync` where timeline order was incorrect for backfilled events (PR \#635) + +Changes in synapse v0.13.3 (2016-02-11) +======================================= + +- Fix bug where `/sync` would occasionally return events in the wrong room. + +Changes in synapse v0.13.2 (2016-02-11) +======================================= + +- Fix bug where `/events` would fail to skip some events if there had been more events than the limit specified since the last request (PR \#570) + +Changes in synapse v0.13.1 (2016-02-10) +======================================= + +- Bump matrix-angular-sdk (matrix web console) dependency to 0.6.8 to pull in the fix for SYWEB-361 so that the default client can display HTML messages again(!) + +Changes in synapse v0.13.0 (2016-02-10) +======================================= + +This version includes an upgrade of the schema, specifically adding an index to the `events` table. This may cause synapse to pause for several minutes the first time it is started after the upgrade. + +Changes: + +- Improve general performance (PR \#540, \#543. \#544, \#54, \#549, \#567) +- Change guest user ids to be incrementing integers (PR \#550) +- Improve performance of public room list API (PR \#552) +- Change profile API to omit keys rather than return null (PR \#557) +- Add `/media/r0` endpoint prefix, which is equivalent to `/media/v1/` (PR \#595) + +Bug fixes: + +- Fix bug with upgrading guest accounts where it would fail if you opened the registration email on a different device (PR \#547) +- Fix bug where unread count could be wrong (PR \#568) + +Changes in synapse v0.12.1-rc1 (2016-01-29) +=========================================== + +Features: + +- Add unread notification counts in `/sync` (PR \#456) +- Add support for inviting 3pids in `/createRoom` (PR \#460) +- Add ability for guest accounts to upgrade (PR \#462) +- Add `/versions` API (PR \#468) +- Add `event` to `/context` API (PR \#492) +- Add specific error code for invalid user names in `/register` (PR \#499) +- Add support for push badge counts (PR \#507) +- Add support for non-guest users to peek in rooms using `/events` (PR \#510) + +Changes: + +- Change `/sync` so that guest users only get rooms they\'ve joined (PR \#469) +- Change to require unbanning before other membership changes (PR \#501) +- Change default push rules to notify for all messages (PR \#486) +- Change default push rules to not notify on membership changes (PR \#514) +- Change default push rules in one to one rooms to only notify for events that are messages (PR \#529) +- Change `/sync` to reject requests with a `from` query param (PR \#512) +- Change server manhole to use SSH rather than telnet (PR \#473) +- Change server to require AS users to be registered before use (PR \#487) +- Change server not to start when ASes are invalidly configured (PR \#494) +- Change server to require ID and `as_token` to be unique for AS\'s (PR \#496) +- Change maximum pagination limit to 1000 (PR \#497) + +Bug fixes: + +- Fix bug where `/sync` didn\'t return when something under the leave key changed (PR \#461) +- Fix bug where we returned smaller rather than larger than requested thumbnails when `method=crop` (PR \#464) +- Fix thumbnails API to only return cropped thumbnails when asking for a cropped thumbnail (PR \#475) +- Fix bug where we occasionally still logged access tokens (PR \#477) +- Fix bug where `/events` would always return immediately for guest users (PR \#480) +- Fix bug where `/sync` unexpectedly returned old left rooms (PR \#481) +- Fix enabling and disabling push rules (PR \#498) +- Fix bug where `/register` returned 500 when given unicode username (PR \#513) + +Changes in synapse v0.12.0 (2016-01-04) +======================================= + +- Expose `/login` under `r0` (PR \#459) + +Changes in synapse v0.12.0-rc3 (2015-12-23) +=========================================== + +- Allow guest accounts access to `/sync` (PR \#455) +- Allow filters to include/exclude rooms at the room level rather than just from the components of the sync for each room. (PR \#454) +- Include urls for room avatars in the response to `/publicRooms` (PR \#453) +- Don\'t set a identicon as the avatar for a user when they register (PR \#450) +- Add a `display_name` to third-party invites (PR \#449) +- Send more information to the identity server for third-party invites so that it can send richer messages to the invitee (PR \#446) +- Cache the responses to `/initialSync` for 5 minutes. If a client retries a request to `/initialSync` before the a response was computed to the first request then the same response is used for both requests (PR \#457) +- Fix a bug where synapse would always request the signing keys of remote servers even when the key was cached locally (PR \#452) +- Fix 500 when pagination search results (PR \#447) +- Fix a bug where synapse was leaking raw email address in third-party invites (PR \#448) + +Changes in synapse v0.12.0-rc2 (2015-12-14) +=========================================== + +- Add caches for whether rooms have been forgotten by a user (PR \#434) +- Remove instructions to use `--process-dependency-link` since all of the dependencies of synapse are on PyPI (PR \#436) +- Parallelise the processing of `/sync` requests (PR \#437) +- Fix race updating presence in `/events` (PR \#444) +- Fix bug back-populating search results (PR \#441) +- Fix bug calculating state in `/sync` requests (PR \#442) + +Changes in synapse v0.12.0-rc1 (2015-12-10) +=========================================== + +- Host the client APIs released as r0 by on paths prefixed by `/_matrix/client/r0`. (PR \#430, PR \#415, PR \#400) +- Updates the client APIs to match r0 of the matrix specification. + - All APIs return events in the new event format, old APIs also include the fields needed to parse the event using the old format for compatibility. (PR \#402) + - Search results are now given as a JSON array rather than a JSON object (PR \#405) + - Miscellaneous changes to search (PR \#403, PR \#406, PR \#412) + - Filter JSON objects may now be passed as query parameters to `/sync` (PR \#431) + - Fix implementation of `/admin/whois` (PR \#418) + - Only include the rooms that user has left in `/sync` if the client requests them in the filter (PR \#423) + - Don\'t push for `m.room.message` by default (PR \#411) + - Add API for setting per account user data (PR \#392) + - Allow users to forget rooms (PR \#385) +- Performance improvements and monitoring: + - Add per-request counters for CPU time spent on the main python thread. (PR \#421, PR \#420) + - Add per-request counters for time spent in the database (PR \#429) + - Make state updates in the C+S API idempotent (PR \#416) + - Only fire `user_joined_room` if the user has actually joined. (PR \#410) + - Reuse a single http client, rather than creating new ones (PR \#413) +- Fixed a bug upgrading from older versions of synapse on postgresql (PR \#417) + +Changes in synapse v0.11.1 (2015-11-20) +======================================= + +- Add extra options to search API (PR \#394) +- Fix bug where we did not correctly cap federation retry timers. This meant it could take several hours for servers to start talking to ressurected servers, even when they were receiving traffic from them (PR \#393) +- Don\'t advertise login token flow unless CAS is enabled. This caused issues where some clients would always use the fallback API if they did not recognize all login flows (PR \#391) +- Change /v2 sync API to rename `private_user_data` to `account_data` (PR \#386) +- Change /v2 sync API to remove the `event_map` and rename keys in `rooms` object (PR \#389) + +Changes in synapse v0.11.0-r2 (2015-11-19) +========================================== + +- Fix bug in database port script (PR \#387) + +Changes in synapse v0.11.0-r1 (2015-11-18) +========================================== + +- Retry and fail federation requests more aggressively for requests that block client side requests (PR \#384) + +Changes in synapse v0.11.0 (2015-11-17) +======================================= + +- Change CAS login API (PR \#349) + +Changes in synapse v0.11.0-rc2 (2015-11-13) +=========================================== + +- Various changes to /sync API response format (PR \#373) +- Fix regression when setting display name in newly joined room over federation (PR \#368) +- Fix problem where /search was slow when using SQLite (PR \#366) + +Changes in synapse v0.11.0-rc1 (2015-11-11) +=========================================== + +- Add Search API (PR \#307, \#324, \#327, \#336, \#350, \#359) +- Add \'archived\' state to v2 /sync API (PR \#316) +- Add ability to reject invites (PR \#317) +- Add config option to disable password login (PR \#322) +- Add the login fallback API (PR \#330) +- Add room context API (PR \#334) +- Add room tagging support (PR \#335) +- Update v2 /sync API to match spec (PR \#305, \#316, \#321, \#332, \#337, \#341) +- Change retry schedule for application services (PR \#320) +- Change retry schedule for remote servers (PR \#340) +- Fix bug where we hosted static content in the incorrect place (PR \#329) +- Fix bug where we didn\'t increment retry interval for remote servers (PR \#343) + +Changes in synapse v0.10.1-rc1 (2015-10-15) +=========================================== + +- Add support for CAS, thanks to Steven Hammerton (PR \#295, \#296) +- Add support for using macaroons for `access_token` (PR \#256, \#229) +- Add support for `m.room.canonical_alias` (PR \#287) +- Add support for viewing the history of rooms that they have left. (PR \#276, \#294) +- Add support for refresh tokens (PR \#240) +- Add flag on creation which disables federation of the room (PR \#279) +- Add some room state to invites. (PR \#275) +- Atomically persist events when joining a room over federation (PR \#283) +- Change default history visibility for private rooms (PR \#271) +- Allow users to redact their own sent events (PR \#262) +- Use tox for tests (PR \#247) +- Split up syutil into separate libraries (PR \#243) + +Changes in synapse v0.10.0-r2 (2015-09-16) +========================================== + +- Fix bug where we always fetched remote server signing keys instead of using ones in our cache. +- Fix adding threepids to an existing account. +- Fix bug with invinting over federation where remote server was already in the room. (PR \#281, SYN-392) + +Changes in synapse v0.10.0-r1 (2015-09-08) +========================================== + +- Fix bug with python packaging + +Changes in synapse v0.10.0 (2015-09-03) +======================================= + +No change from release candidate. + +Changes in synapse v0.10.0-rc6 (2015-09-02) +=========================================== + +- Remove some of the old database upgrade scripts. +- Fix database port script to work with newly created sqlite databases. + +Changes in synapse v0.10.0-rc5 (2015-08-27) +=========================================== + +- Fix bug that broke downloading files with ascii filenames across federation. + +Changes in synapse v0.10.0-rc4 (2015-08-27) +=========================================== + +- Allow UTF-8 filenames for upload. (PR \#259) + +Changes in synapse v0.10.0-rc3 (2015-08-25) +=========================================== + +- Add `--keys-directory` config option to specify where files such as certs and signing keys should be stored in, when using `--generate-config` or `--generate-keys`. (PR \#250) +- Allow `--config-path` to specify a directory, causing synapse to use all \*.yaml files in the directory as config files. (PR \#249) +- Add `web_client_location` config option to specify static files to be hosted by synapse under `/_matrix/client`. (PR \#245) +- Add helper utility to synapse to read and parse the config files and extract the value of a given key. For example: + + $ python -m synapse.config read server_name -c homeserver.yaml + localhost + + (PR \#246) + +Changes in synapse v0.10.0-rc2 (2015-08-24) +=========================================== + +- Fix bug where we incorrectly populated the `event_forward_extremities` table, resulting in problems joining large remote rooms (e.g. `#matrix:matrix.org`) +- Reduce the number of times we wake up pushers by not listening for presence or typing events, reducing the CPU cost of each pusher. + +Changes in synapse v0.10.0-rc1 (2015-08-21) +=========================================== + +Also see v0.9.4-rc1 changelog, which has been amalgamated into this release. + +General: + +- Upgrade to Twisted 15 (PR \#173) +- Add support for serving and fetching encryption keys over federation. (PR \#208) +- Add support for logging in with email address (PR \#234) +- Add support for new `m.room.canonical_alias` event. (PR \#233) +- Change synapse to treat user IDs case insensitively during registration and login. (If two users already exist with case insensitive matching user ids, synapse will continue to require them to specify their user ids exactly.) +- Error if a user tries to register with an email already in use. (PR \#211) +- Add extra and improve existing caches (PR \#212, \#219, \#226, \#228) +- Batch various storage request (PR \#226, \#228) +- Fix bug where we didn\'t correctly log the entity that triggered the request if the request came in via an application service (PR \#230) +- Fix bug where we needlessly regenerated the full list of rooms an AS is interested in. (PR \#232) +- Add support for AS\'s to use v2\_alpha registration API (PR \#210) + +Configuration: + +- Add `--generate-keys` that will generate any missing cert and key files in the configuration files. This is equivalent to running `--generate-config` on an existing configuration file. (PR \#220) +- `--generate-config` now no longer requires a `--server-name` parameter when used on existing configuration files. (PR \#220) +- Add `--print-pidfile` flag that controls the printing of the pid to stdout of the demonised process. (PR \#213) + +Media Repository: + +- Fix bug where we picked a lower resolution image than requested. (PR \#205) +- Add support for specifying if a the media repository should dynamically thumbnail images or not. (PR \#206) + +Metrics: + +- Add statistics from the reactor to the metrics API. (PR \#224, \#225) + +Demo Homeservers: + +- Fix starting the demo homeservers without rate-limiting enabled. (PR \#182) +- Fix enabling registration on demo homeservers (PR \#223) + +Changes in synapse v0.9.4-rc1 (2015-07-21) +========================================== + +General: + +- Add basic implementation of receipts. (SPEC-99) +- Add support for configuration presets in room creation API. (PR \#203) +- Add auth event that limits the visibility of history for new users. (SPEC-134) +- Add SAML2 login/registration support. (PR \#201. Thanks Muthu Subramanian!) +- Add client side key management APIs for end to end encryption. (PR \#198) +- Change power level semantics so that you cannot kick, ban or change power levels of users that have equal or greater power level than you. (SYN-192) +- Improve performance by bulk inserting events where possible. (PR \#193) +- Improve performance by bulk verifying signatures where possible. (PR \#194) + +Configuration: + +- Add support for including TLS certificate chains. + +Media Repository: + +- Add Content-Disposition headers to content repository responses. (SYN-150) + +Changes in synapse v0.9.3 (2015-07-01) +====================================== + +No changes from v0.9.3 Release Candidate 1. + +Changes in synapse v0.9.3-rc1 (2015-06-23) +========================================== + +General: + +- Fix a memory leak in the notifier. (SYN-412) +- Improve performance of room initial sync. (SYN-418) +- General improvements to logging. +- Remove `access_token` query params from `INFO` level logging. + +Configuration: + +- Add support for specifying and configuring multiple listeners. (SYN-389) + +Application services: + +- Fix bug where synapse failed to send user queries to application services. + +Changes in synapse v0.9.2-r2 (2015-06-15) +========================================= + +Fix packaging so that schema delta python files get included in the package. + +Changes in synapse v0.9.2 (2015-06-12) +====================================== + +General: + +- Use ultrajson for json (de)serialisation when a canonical encoding is not required. Ultrajson is significantly faster than simplejson in certain circumstances. +- Use connection pools for outgoing HTTP connections. +- Process thumbnails on separate threads. + +Configuration: + +- Add option, `gzip_responses`, to disable HTTP response compression. + +Federation: + +- Improve resilience of backfill by ensuring we fetch any missing auth events. +- Improve performance of backfill and joining remote rooms by removing unnecessary computations. This included handling events we\'d previously handled as well as attempting to compute the current state for outliers. + +Changes in synapse v0.9.1 (2015-05-26) +====================================== + +General: + +- Add support for backfilling when a client paginates. This allows servers to request history for a room from remote servers when a client tries to paginate history the server does not have - SYN-36 +- Fix bug where you couldn\'t disable non-default pushrules - SYN-378 +- Fix `register_new_user` script - SYN-359 +- Improve performance of fetching events from the database, this improves both initialSync and sending of events. +- Improve performance of event streams, allowing synapse to handle more simultaneous connected clients. + +Federation: + +- Fix bug with existing backfill implementation where it returned the wrong selection of events in some circumstances. +- Improve performance of joining remote rooms. + +Configuration: + +- Add support for changing the bind host of the metrics listener via the `metrics_bind_host` option. + +Changes in synapse v0.9.0-r5 (2015-05-21) +========================================= + +- Add more database caches to reduce amount of work done for each pusher. This radically reduces CPU usage when multiple pushers are set up in the same room. + +Changes in synapse v0.9.0 (2015-05-07) +====================================== + +General: + +- Add support for using a PostgreSQL database instead of SQLite. See [docs/postgres.rst](docs/postgres.rst) for details. +- Add password change and reset APIs. See [Registration](https://github.com/matrix-org/matrix-doc/blob/master/specification/10_client_server_api.rst#registration) in the spec. +- Fix memory leak due to not releasing stale notifiers - SYN-339. +- Fix race in caches that occasionally caused some presence updates to be dropped - SYN-369. +- Check server name has not changed on restart. +- Add a sample systemd unit file and a logger configuration in contrib/systemd. Contributed Ivan Shapovalov. + +Federation: + +- Add key distribution mechanisms for fetching public keys of unavailable remote home servers. See [Retrieving Server Keys](https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys) in the spec. + +Configuration: + +- Add support for multiple config files. +- Add support for dictionaries in config files. +- Remove support for specifying config options on the command line, except for: + - `--daemonize` - Daemonize the home server. + - `--manhole` - Turn on the twisted telnet manhole service on the given port. + - `--database-path` - The path to a sqlite database to use. + - `--verbose` - The verbosity level. + - `--log-file` - File to log to. + - `--log-config` - Python logging config file. + - `--enable-registration` - Enable registration for new users. + +Application services: + +- Reliably retry sending of events from Synapse to application services, as per [Application Services](https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api) spec. +- Application services can no longer register via the `/register` API, instead their configuration should be saved to a file and listed in the synapse `app_service_config_files` config option. The AS configuration file has the same format as the old `/register` request. See [docs/application\_services.rst](docs/application_services.rst) for more information. + +Changes in synapse v0.8.1 (2015-03-18) +====================================== + +- Disable registration by default. New users can be added using the command `register_new_matrix_user` or by enabling registration in the config. +- Add metrics to synapse. To enable metrics use config options `enable_metrics` and `metrics_port`. +- Fix bug where banning only kicked the user. + +Changes in synapse v0.8.0 (2015-03-06) +====================================== + +General: + +- Add support for registration fallback. This is a page hosted on the server which allows a user to register for an account, regardless of what client they are using (e.g. mobile devices). +- Added new default push rules and made them configurable by clients: + - Suppress all notice messages. + - Notify when invited to a new room. + - Notify for messages that don\'t match any rule. + - Notify on incoming call. + +Federation: + +- Added per host server side rate-limiting of incoming federation requests. +- Added a `/get_missing_events/` API to federation to reduce number of `/events/` requests. + +Configuration: + +- Added configuration option to disable registration: `disable_registration`. +- Added configuration option to change soft limit of number of open file descriptors: `soft_file_limit`. +- Make `tls_private_key_path` optional when running with `no_tls`. + +Application services: + +- Application services can now poll on the CS API `/events` for their events, by providing their application service `access_token`. +- Added exclusive namespace support to application services API. + +Changes in synapse v0.7.1 (2015-02-19) +====================================== + +- Initial alpha implementation of parts of the Application Services API. Including: + - AS Registration / Unregistration + - User Query API + - Room Alias Query API + - Push transport for receiving events. + - User/Alias namespace admin control +- Add cache when fetching events from remote servers to stop repeatedly fetching events with bad signatures. +- Respect the per remote server retry scheme when fetching both events and server keys to reduce the number of times we send requests to dead servers. +- Inform remote servers when the local server fails to handle a received event. +- Turn off python bytecode generation due to problems experienced when upgrading from previous versions. + +Changes in synapse v0.7.0 (2015-02-12) +====================================== + +- Add initial implementation of the query auth federation API, allowing servers to agree on whether an event should be allowed or rejected. +- Persist events we have rejected from federation, fixing the bug where servers would keep requesting the same events. +- Various federation performance improvements, including: + - Add in memory caches on queries such as: + + > - Computing the state of a room at a point in time, used for authorization on federation requests. + > - Fetching events from the database. + > - User\'s room membership, used for authorizing presence updates. + + - Upgraded JSON library to improve parsing and serialisation speeds. + +- Add default avatars to new user accounts using pydenticon library. +- Correctly time out federation requests. +- Retry federation requests against different servers. +- Add support for push and push rules. +- Add alpha versions of proposed new CSv2 APIs, including `/sync` API. + +Changes in synapse 0.6.1 (2015-01-07) +===================================== + +- Major optimizations to improve performance of initial sync and event sending in large rooms (by up to 10x) +- Media repository now includes a Content-Length header on media downloads. +- Improve quality of thumbnails by changing resizing algorithm. + +Changes in synapse 0.6.0 (2014-12-16) +===================================== + +- Add new API for media upload and download that supports thumbnailing. +- Replicate media uploads over multiple homeservers so media is always served to clients from their local homeserver. This obsoletes the \--content-addr parameter and confusion over accessing content directly from remote homeservers. +- Implement exponential backoff when retrying federation requests when sending to remote homeservers which are offline. +- Implement typing notifications. +- Fix bugs where we sent events with invalid signatures due to bugs where we incorrectly persisted events. +- Improve performance of database queries involving retrieving events. + +Changes in synapse 0.5.4a (2014-12-13) +====================================== + +- Fix bug while generating the error message when a file path specified in the config doesn\'t exist. + +Changes in synapse 0.5.4 (2014-12-03) +===================================== + +- Fix presence bug where some rooms did not display presence updates for remote users. +- Do not log SQL timing log lines when started with \"-v\" +- Fix potential memory leak. + +Changes in synapse 0.5.3c (2014-12-02) +====================================== + +- Change the default value for the content\_addr option to use the HTTP listener, as by default the HTTPS listener will be using a self-signed certificate. + +Changes in synapse 0.5.3 (2014-11-27) +===================================== + +- Fix bug that caused joining a remote room to fail if a single event was not signed correctly. +- Fix bug which caused servers to continuously try and fetch events from other servers. + +Changes in synapse 0.5.2 (2014-11-26) +===================================== + +Fix major bug that caused rooms to disappear from peoples initial sync. + +Changes in synapse 0.5.1 (2014-11-26) +===================================== + +See UPGRADES.rst for specific instructions on how to upgrade. + +> - Fix bug where we served up an Event that did not match its signatures. +> - Fix regression where we no longer correctly handled the case where a homeserver receives an event for a room it doesn\'t recognise (but is in.) + +Changes in synapse 0.5.0 (2014-11-19) +===================================== + +This release includes changes to the federation protocol and client-server API that is not backwards compatible. + +This release also changes the internal database schemas and so requires servers to drop their current history. See UPGRADES.rst for details. + +Homeserver: + +: - Add authentication and authorization to the federation protocol. Events are now signed by their originating homeservers. + - Implement the new authorization model for rooms. + - Split out web client into a seperate repository: matrix-angular-sdk. + - Change the structure of PDUs. + - Fix bug where user could not join rooms via an alias containing 4-byte UTF-8 characters. + - Merge concept of PDUs and Events internally. + - Improve logging by adding request ids to log lines. + - Implement a very basic room initial sync API. + - Implement the new invite/join federation APIs. + +Webclient: + +: - The webclient has been moved to a seperate repository. + +Changes in synapse 0.4.2 (2014-10-31) +===================================== + +Homeserver: + +: - Fix bugs where we did not notify users of correct presence updates. + - Fix bug where we did not handle sub second event stream timeouts. + +Webclient: + +: - Add ability to click on messages to see JSON. + - Add ability to redact messages. + - Add ability to view and edit all room state JSON. + - Handle incoming redactions. + - Improve feedback on errors. + - Fix bugs in mobile CSS. + - Fix bugs with desktop notifications. + +Changes in synapse 0.4.1 (2014-10-17) +===================================== + +Webclient: + +: - Fix bug with display of timestamps. + +Changes in synpase 0.4.0 (2014-10-17) +===================================== + +This release includes changes to the federation protocol and client-server API that is not backwards compatible. + +The Matrix specification has been moved to a separate git repository: + +You will also need an updated syutil and config. See UPGRADES.rst. + +Homeserver: + +: - Sign federation transactions to assert strong identity over federation. + - Rename timestamp keys in PDUs and events from \'ts\' and \'hsob\_ts\' to \'origin\_server\_ts\'. + +Changes in synapse 0.3.4 (2014-09-25) +===================================== + +This version adds support for using a TURN server. See docs/turn-howto.rst on how to set one up. + +Homeserver: + +: - Add support for redaction of messages. + - Fix bug where inviting a user on a remote home server could take up to 20-30s. + - Implement a get current room state API. + - Add support specifying and retrieving turn server configuration. + +Webclient: + +: - Add button to send messages to users from the home page. + - Add support for using TURN for VoIP calls. + - Show display name change messages. + - Fix bug where the client didn\'t get the state of a newly joined room until after it has been refreshed. + - Fix bugs with tab complete. + - Fix bug where holding down the down arrow caused chrome to chew 100% CPU. + - Fix bug where desktop notifications occasionally used \"Undefined\" as the display name. + - Fix more places where we sometimes saw room IDs incorrectly. + - Fix bug which caused lag when entering text in the text box. + +Changes in synapse 0.3.3 (2014-09-22) +===================================== + +Homeserver: + +: - Fix bug where you continued to get events for rooms you had left. + +Webclient: + +: - Add support for video calls with basic UI. + - Fix bug where one to one chats were named after your display name rather than the other person\'s. + - Fix bug which caused lag when typing in the textarea. + - Refuse to run on browsers we know won\'t work. + - Trigger pagination when joining new rooms. + - Fix bug where we sometimes didn\'t display invitations in recents. + - Automatically join room when accepting a VoIP call. + - Disable outgoing and reject incoming calls on browsers we don\'t support VoIP in. + - Don\'t display desktop notifications for messages in the room you are non-idle and speaking in. + +Changes in synapse 0.3.2 (2014-09-18) +===================================== + +Webclient: + +: - Fix bug where an empty \"bing words\" list in old accounts didn\'t send notifications when it should have done. + +Changes in synapse 0.3.1 (2014-09-18) +===================================== + +This is a release to hotfix v0.3.0 to fix two regressions. + +Webclient: + +: - Fix a regression where we sometimes displayed duplicate events. + - Fix a regression where we didn\'t immediately remove rooms you were banned in from the recents list. + +Changes in synapse 0.3.0 (2014-09-18) +===================================== + +See UPGRADE for information about changes to the client server API, including breaking backwards compatibility with VoIP calls and registration API. + +Homeserver: + +: - When a user changes their displayname or avatar the server will now update all their join states to reflect this. + - The server now adds \"age\" key to events to indicate how old they are. This is clock independent, so at no point does any server or webclient have to assume their clock is in sync with everyone else. + - Fix bug where we didn\'t correctly pull in missing PDUs. + - Fix bug where prev\_content key wasn\'t always returned. + - Add support for password resets. + +Webclient: + +: - Improve page content loading. + - Join/parts now trigger desktop notifications. + - Always show room aliases in the UI if one is present. + - No longer show user-count in the recents side panel. + - Add up & down arrow support to the text box for message sending to step through your sent history. + - Don\'t display notifications for our own messages. + - Emotes are now formatted correctly in desktop notifications. + - The recents list now differentiates between public & private rooms. + - Fix bug where when switching between rooms the pagination flickered before the view jumped to the bottom of the screen. + - Add bing word support. + +Registration API: + +: - The registration API has been overhauled to function like the login API. In practice, this means registration requests must now include the following: \'type\':\'m.login.password\'. See UPGRADE for more information on this. + - The \'user\_id\' key has been renamed to \'user\' to better match the login API. + - There is an additional login type: \'m.login.email.identity\'. + - The command client and web client have been updated to reflect these changes. + +Changes in synapse 0.2.3 (2014-09-12) +===================================== + +Homeserver: + +: - Fix bug where we stopped sending events to remote home servers if a user from that home server left, even if there were some still in the room. + - Fix bugs in the state conflict resolution where it was incorrectly rejecting events. + +Webclient: + +: - Display room names and topics. + - Allow setting/editing of room names and topics. + - Display information about rooms on the main page. + - Handle ban and kick events in real time. + - VoIP UI and reliability improvements. + - Add glare support for VoIP. + - Improvements to initial startup speed. + - Don\'t display duplicate join events. + - Local echo of messages. + - Differentiate sending and sent of local echo. + - Various minor bug fixes. + +Changes in synapse 0.2.2 (2014-09-06) +===================================== + +Homeserver: + +: - When the server returns state events it now also includes the previous content. + - Add support for inviting people when creating a new room. + - Make the homeserver inform the room via m.room.aliases when a new alias is added for a room. + - Validate m.room.power\_level events. + +Webclient: + +: - Add support for captchas on registration. + - Handle m.room.aliases events. + - Asynchronously send messages and show a local echo. + - Inform the UI when a message failed to send. + - Only autoscroll on receiving a new message if the user was already at the bottom of the screen. + - Add support for ban/kick reasons. + +Changes in synapse 0.2.1 (2014-09-03) +===================================== + +Homeserver: + +: - Added support for signing up with a third party id. + - Add synctl scripts. + - Added rate limiting. + - Add option to change the external address the content repo uses. + - Presence bug fixes. + +Webclient: + +: - Added support for signing up with a third party id. + - Added support for banning and kicking users. + - Added support for displaying and setting ops. + - Added support for room names. + - Fix bugs with room membership event display. + +Changes in synapse 0.2.0 (2014-09-02) +===================================== + +This update changes many configuration options, updates the database schema and mandates SSL for server-server connections. + +Homeserver: + +: - Require SSL for server-server connections. + - Add SSL listener for client-server connections. + - Add ability to use config files. + - Add support for kicking/banning and power levels. + - Allow setting of room names and topics on creation. + - Change presence to include last seen time of the user. + - Change url path prefix to /\_matrix/\... + - Bug fixes to presence. + +Webclient: + +: - Reskin the CSS for registration and login. + - Various improvements to rooms CSS. + - Support changes in client-server API. + - Bug fixes to VOIP UI. + - Various bug fixes to handling of changes to room member list. + +Changes in synapse 0.1.2 (2014-08-29) +===================================== + +Webclient: + +: - Add basic call state UI for VoIP calls. + +Changes in synapse 0.1.1 (2014-08-29) +===================================== + +Homeserver: + +: - Fix bug that caused the event stream to not notify some clients about changes. + +Changes in synapse 0.1.0 (2014-08-29) +===================================== + +Presence has been reenabled in this release. + +Homeserver: + +: - + + Update client to server API, including: + + : - Use a more consistent url scheme. + - Provide more useful information in the initial sync api. + + - Change the presence handling to be much more efficient. + - Change the presence server to server API to not require explicit polling of all users who share a room with a user. + - Fix races in the event streaming logic. + +Webclient: + +: - Update to use new client to server API. + - Add basic VOIP support. + - Add idle timers that change your status to away. + - Add recent rooms column when viewing a room. + - Various network efficiency improvements. + - Add basic mobile browser support. + - Add a settings page. + +Changes in synapse 0.0.1 (2014-08-22) +===================================== + +Presence has been disabled in this release due to a bug that caused the homeserver to spam other remote homeservers. + +Homeserver: + +: - Completely change the database schema to support generic event types. + - Improve presence reliability. + - Improve reliability of joining remote rooms. + - Fix bug where room join events were duplicated. + - Improve initial sync API to return more information to the client. + - Stop generating fake messages for room membership events. + +Webclient: + +: - Add tab completion of names. + - Add ability to upload and send images. + - Add profile pages. + - Improve CSS layout of room. + - Disambiguate identical display names. + - Don\'t get remote users display names and avatars individually. + - Use the new initial sync API to reduce number of round trips to the homeserver. + - Change url scheme to use room aliases instead of room ids where known. + - Increase longpoll timeout. + +Changes in synapse 0.0.0 (2014-08-13) +===================================== + +> - Initial alpha release diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index afc4fee88..000000000 --- a/CHANGES.rst +++ /dev/null @@ -1,2867 +0,0 @@ -Synapse 0.33.0 (2018-07-19) -=========================== - -Features --------- - -- Enforce the specified API for report_event (`#3316 `_) -- Include CPU time from database threads in request/block metrics. (`#3496 `_, `#3501 `_) -- Add CPU metrics for _fetch_event_list (`#3497 `_) -- optimisation for /sync (`#3505 `_, `#3521 `_) -- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) - - -Bugfixes --------- - -- Use more portable syntax in our use of the attrs package, widening the supported versions (`#3498 `_) -- Fix queued federation requests being processed in the wrong order (`#3533 `_) -- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) -- Disable a noisy warning about logcontexts (`#3561 `_) - - -Misc ----- - -- `#3351 `_, `#3463 `_, `#3464 `_, `#3499 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ - - -Synapse 0.32.2 (2018-07-07) -=========================== - -Bugfixes --------- - -- Amend the Python dependencies to depend on attrs from PyPI, not attr (`#3492 `_) - - -Synapse 0.32.1 (2018-07-06) -=========================== - -Bugfixes --------- - -- Add explicit dependency on netaddr (`#3488 `_) - - -Changes in synapse v0.32.0 (2018-07-06) -=========================================== -No changes since 0.32.0rc1 - -Synapse 0.32.0rc1 (2018-07-05) -============================== - -Features --------- - -- Add blacklist & whitelist of servers allowed to send events to a room via ``m.room.server_acl`` event. -- Cache factor override system for specific caches (`#3334 `_) -- Add metrics to track appservice transactions (`#3344 `_) -- Try to log more helpful info when a sig verification fails (`#3372 `_) -- Synapse now uses the best performing JSON encoder/decoder according to your runtime (simplejson on CPython, stdlib json on PyPy). (`#3462 `_) -- Add optional ip_range_whitelist param to AS registration files to lock AS IP access (`#3465 `_) -- Reject invalid server names in federation requests (`#3480 `_) -- Reject invalid server names in homeserver.yaml (`#3483 `_) - - -Bugfixes --------- - -- Strip access_token from outgoing requests (`#3327 `_) -- Redact AS tokens in logs (`#3349 `_) -- Fix federation backfill from SQLite servers (`#3355 `_) -- Fix event-purge-by-ts admin API (`#3363 `_) -- Fix event filtering in get_missing_events handler (`#3371 `_) -- Synapse is now stricter regarding accepting events which it cannot retrieve the prev_events for. (`#3456 `_) -- Fix bug where synapse would explode when receiving unicode in HTTP User-Agent header (`#3470 `_) -- Invalidate cache on correct thread to avoid race (`#3473 `_) - - -Improved Documentation ----------------------- - -- ``doc/postgres.rst``: fix display of the last command block. Thanks to @ArchangeGabriel! (`#3340 `_) - - -Deprecations and Removals -------------------------- - -- Remove was_forgotten_at (`#3324 `_) - - -Misc ----- - -- `#3332 `_, `#3341 `_, `#3347 `_, `#3348 `_, `#3356 `_, `#3385 `_, `#3446 `_, `#3447 `_, `#3467 `_, `#3474 `_ - - -Changes in synapse v0.31.2 (2018-06-14) -======================================= - -SECURITY UPDATE: Prevent unauthorised users from setting state events in a room -when there is no ``m.room.power_levels`` event in force in the room. (PR #3397) - -Discussion around the Matrix Spec change proposal for this change can be -followed at https://github.com/matrix-org/matrix-doc/issues/1304. - -Changes in synapse v0.31.1 (2018-06-08) -======================================= - -v0.31.1 fixes a security bug in the ``get_missing_events`` federation API -where event visibility rules were not applied correctly. - -We are not aware of it being actively exploited but please upgrade asap. - -Bug Fixes: - -* Fix event filtering in get_missing_events handler (PR #3371) - -Changes in synapse v0.31.0 (2018-06-06) -======================================= - -Most notable change from v0.30.0 is to switch to the python prometheus library to improve system -stats reporting. WARNING: this changes a number of prometheus metrics in a -backwards-incompatible manner. For more details, see -`docs/metrics-howto.rst `_. - -Bug Fixes: - -* Fix metric documentation tables (PR #3341) -* Fix LaterGauge error handling (694968f) -* Fix replication metrics (b7e7fd2) - -Changes in synapse v0.31.0-rc1 (2018-06-04) -========================================== - -Features: - -* Switch to the Python Prometheus library (PR #3256, #3274) -* Let users leave the server notice room after joining (PR #3287) - - -Changes: - -* daily user type phone home stats (PR #3264) -* Use iter* methods for _filter_events_for_server (PR #3267) -* Docs on consent bits (PR #3268) -* Remove users from user directory on deactivate (PR #3277) -* Avoid sending consent notice to guest users (PR #3288) -* disable CPUMetrics if no /proc/self/stat (PR #3299) -* Consistently use six's iteritems and wrap lazy keys/values in list() if they're not meant to be lazy (PR #3307) -* Add private IPv6 addresses to example config for url preview blacklist (PR #3317) Thanks to @thegcat! -* Reduce stuck read-receipts: ignore depth when updating (PR #3318) -* Put python's logs into Trial when running unit tests (PR #3319) - -Changes, python 3 migration: - -* Replace some more comparisons with six (PR #3243) Thanks to @NotAFile! -* replace some iteritems with six (PR #3244) Thanks to @NotAFile! -* Add batch_iter to utils (PR #3245) Thanks to @NotAFile! -* use repr, not str (PR #3246) Thanks to @NotAFile! -* Misc Python3 fixes (PR #3247) Thanks to @NotAFile! -* Py3 storage/_base.py (PR #3278) Thanks to @NotAFile! -* more six iteritems (PR #3279) Thanks to @NotAFile! -* More Misc. py3 fixes (PR #3280) Thanks to @NotAFile! -* remaining isintance fixes (PR #3281) Thanks to @NotAFile! -* py3-ize state.py (PR #3283) Thanks to @NotAFile! -* extend tox testing for py3 to avoid regressions (PR #3302) Thanks to @krombel! -* use memoryview in py3 (PR #3303) Thanks to @NotAFile! - -Bugs: - -* Fix federation backfill bugs (PR #3261) -* federation: fix LaterGauge usage (PR #3328) Thanks to @intelfx! - - -Changes in synapse v0.30.0 (2018-05-24) -========================================== - -'Server Notices' are a new feature introduced in Synapse 0.30. They provide a -channel whereby server administrators can send messages to users on the server. - -They are used as part of communication of the server policies (see ``docs/consent_tracking.md``), -however the intention is that they may also find a use for features such -as "Message of the day". - -This feature is specific to Synapse, but uses standard Matrix communication mechanisms, -so should work with any Matrix client. For more details see ``docs/server_notices.md`` - -Further Server Notices/Consent Tracking Support: - -* Allow overriding the server_notices user's avatar (PR #3273) -* Use the localpart in the consent uri (PR #3272) -* Support for putting %(consent_uri)s in messages (PR #3271) -* Block attempts to send server notices to remote users (PR #3270) -* Docs on consent bits (PR #3268) - - - -Changes in synapse v0.30.0-rc1 (2018-05-23) -========================================== - -Server Notices/Consent Tracking Support: - -* ConsentResource to gather policy consent from users (PR #3213) -* Move RoomCreationHandler out of synapse.handlers.Handlers (PR #3225) -* Infrastructure for a server notices room (PR #3232) -* Send users a server notice about consent (PR #3236) -* Reject attempts to send event before privacy consent is given (PR #3257) -* Add a 'has_consented' template var to consent forms (PR #3262) -* Fix dependency on jinja2 (PR #3263) - -Features: - -* Cohort analytics (PR #3163, #3241, #3251) -* Add lxml to docker image for web previews (PR #3239) Thanks to @ptman! -* Add in flight request metrics (PR #3252) - -Changes: - -* Remove unused `update_external_syncs` (PR #3233) -* Use stream rather depth ordering for push actions (PR #3212) -* Make purge_history operate on tokens (PR #3221) -* Don't support limitless pagination (PR #3265) - -Bug Fixes: - -* Fix logcontext resource usage tracking (PR #3258) -* Fix error in handling receipts (PR #3235) -* Stop the transaction cache caching failures (PR #3255) - - -Changes in synapse v0.29.1 (2018-05-17) -========================================== -Changes: - -* Update docker documentation (PR #3222) - -Changes in synapse v0.29.0 (2018-05-16) -=========================================== -Not changes since v0.29.0-rc1 - -Changes in synapse v0.29.0-rc1 (2018-05-14) -=========================================== - -Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a -closed spec bug in the Client Server API. Additionally further prep for Python 3 -migration. - -Potentially breaking change: - -* Make Client-Server API return 401 for invalid token (PR #3161). - - This changes the Client-server spec to return a 401 error code instead of 403 - when the access token is unrecognised. This is the behaviour required by the - specification, but some clients may be relying on the old, incorrect - behaviour. - - Thanks to @NotAFile for fixing this. - -Features: - -* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! - -Changes - General: - -* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! -* Part user from rooms on account deactivate (PR #3201) -* Make 'unexpected logging context' into warnings (PR #3007) -* Set Server header in SynapseRequest (PR #3208) -* remove duplicates from groups tables (PR #3129) -* Improve exception handling for background processes (PR #3138) -* Add missing consumeErrors to improve exception handling (PR #3139) -* reraise exceptions more carefully (PR #3142) -* Remove redundant call to preserve_fn (PR #3143) -* Trap exceptions thrown within run_in_background (PR #3144) - -Changes - Refactors: - -* Refactor /context to reuse pagination storage functions (PR #3193) -* Refactor recent events func to use pagination func (PR #3195) -* Refactor pagination DB API to return concrete type (PR #3196) -* Refactor get_recent_events_for_room return type (PR #3198) -* Refactor sync APIs to reuse pagination API (PR #3199) -* Remove unused code path from member change DB func (PR #3200) -* Refactor request handling wrappers (PR #3203) -* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! -* Refactor event storage to prepare for changes in state calculations (PR #3141) -* Set Server header in SynapseRequest (PR #3208) -* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) -* Use run_in_background in preference to preserve_fn (PR #3140) - -Changes - Python 3 migration: - -* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! -* run config tests on py3 (PR #3159) Thanks to @NotAFile! -* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! -* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! -* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! -* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! -* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! -* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! -* Move more xrange to six (PR #3151) Thanks to @NotAFile! -* make imports local (PR #3152) Thanks to @NotAFile! -* move httplib import to six (PR #3153) Thanks to @NotAFile! -* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile! -* more bytes strings (PR #3155) Thanks to @NotAFile! - -Bug Fixes: - -* synapse fails to start under Twisted >= 18.4 (PR #3157) -* Fix a class of logcontext leaks (PR #3170) -* Fix a couple of logcontext leaks in unit tests (PR #3172) -* Fix logcontext leak in media repo (PR #3174) -* Escape label values in prometheus metrics (PR #3175, #3186) -* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! -* Fix logcontext leaks in rate limiter (PR #3183) -* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! -* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! -* add guard for None on purge_history api (PR #3160) Thanks to @krombel! - -Changes in synapse v0.28.1 (2018-05-01) -======================================= - -SECURITY UPDATE - -* Clamp the allowed values of event depth received over federation to be - [0, 2^63 - 1]. This mitigates an attack where malicious events - injected with depth = 2^63 - 1 render rooms unusable. Depth is used to - determine the cosmetic ordering of events within a room, and so the ordering - of events in such a room will default to using stream_ordering rather than depth - (topological_ordering). - - This is a temporary solution to mitigate abuse in the wild, whilst a long term solution - is being implemented to improve how the depth parameter is used. - - Full details at - https://docs.google.com/document/d/1I3fi2S-XnpO45qrpCsowZv8P8dHcNZ4fsBsbOW7KABI - -* Pin Twisted to <18.4 until we stop using the private _OpenSSLECCurve API. - - -Changes in synapse v0.28.0 (2018-04-26) -======================================= - -Bug Fixes: - -* Fix quarantine media admin API and search reindex (PR #3130) -* Fix media admin APIs (PR #3134) - - -Changes in synapse v0.28.0-rc1 (2018-04-24) -=========================================== - -Minor performance improvement to federation sending and bug fixes. - -(Note: This release does not include the delta state resolution implementation discussed in matrix live) - - -Features: - -* Add metrics for event processing lag (PR #3090) -* Add metrics for ResponseCache (PR #3092) - -Changes: - -* Synapse on PyPy (PR #2760) Thanks to @Valodim! -* move handling of auto_join_rooms to RegisterHandler (PR #2996) Thanks to @krombel! -* Improve handling of SRV records for federation connections (PR #3016) Thanks to @silkeh! -* Document the behaviour of ResponseCache (PR #3059) -* Preparation for py3 (PR #3061, #3073, #3074, #3075, #3103, #3104, #3106, #3107, #3109, #3110) Thanks to @NotAFile! -* update prometheus dashboard to use new metric names (PR #3069) Thanks to @krombel! -* use python3-compatible prints (PR #3074) Thanks to @NotAFile! -* Send federation events concurrently (PR #3078) -* Limit concurrent event sends for a room (PR #3079) -* Improve R30 stat definition (PR #3086) -* Send events to ASes concurrently (PR #3088) -* Refactor ResponseCache usage (PR #3093) -* Clarify that SRV may not point to a CNAME (PR #3100) Thanks to @silkeh! -* Use str(e) instead of e.message (PR #3103) Thanks to @NotAFile! -* Use six.itervalues in some places (PR #3106) Thanks to @NotAFile! -* Refactor store.have_events (PR #3117) - -Bug Fixes: - -* Return 401 for invalid access_token on logout (PR #2938) Thanks to @dklug! -* Return a 404 rather than a 500 on rejoining empty rooms (PR #3080) -* fix federation_domain_whitelist (PR #3099) -* Avoid creating events with huge numbers of prev_events (PR #3113) -* Reject events which have lots of prev_events (PR #3118) - - -Changes in synapse v0.27.4 (2018-04-13) -====================================== - -Changes: - -* Update canonicaljson dependency (#3095) - - -Changes in synapse v0.27.3 (2018-04-11) -====================================== - -Bug fixes: - -* URL quote path segments over federation (#3082) - -Changes in synapse v0.27.3-rc2 (2018-04-09) -========================================== - -v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates -the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored. - -Changes in synapse v0.27.3-rc1 (2018-04-09) -======================================= - -Notable changes include API support for joinability of groups. Also new metrics -and phone home stats. Phone home stats include better visibility of system usage -so we can tweak synpase to work better for all users rather than our own experience -with matrix.org. Also, recording 'r30' stat which is the measure we use to track -overal growth of the Matrix ecosystem. It is defined as:- - -Counts the number of native 30 day retained users, defined as:- - * Users who have created their accounts more than 30 days - * Where last seen at most 30 days ago - * Where account creation and last_seen are > 30 days" - - -Features: - -* Add joinability for groups (PR #3045) -* Implement group join API (PR #3046) -* Add counter metrics for calculating state delta (PR #3033) -* R30 stats (PR #3041) -* Measure time it takes to calculate state group ID (PR #3043) -* Add basic performance statistics to phone home (PR #3044) -* Add response size metrics (PR #3071) -* phone home cache size configurations (PR #3063) - -Changes: - -* Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live! -* Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile! -* Use .iter* to avoid copies in StateHandler (PR #3006) -* Linearize calls to _generate_user_id (PR #3029) -* Remove last usage of ujson (PR #3030) -* Use simplejson throughout (PR #3048) -* Use static JSONEncoders (PR #3049) -* Remove uses of events.content (PR #3060) -* Improve database cache performance (PR #3068) - -Bug fixes: - -* Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte! -* Fix replication after switch to simplejson (PR #3015) -* 404 correctly on missing paths via NoResource (PR #3022) -* Fix error when claiming e2e keys from offline servers (PR #3034) -* fix tests/storage/test_user_directory.py (PR #3042) -* use PUT instead of POST for federating groups/m.join_policy (PR #3070) Thanks to @krombel! -* postgres port script: fix state_groups_pkey error (PR #3072) - - -Changes in synapse v0.27.2 (2018-03-26) -======================================= - -Bug fixes: - -* Fix bug which broke TCP replication between workers (PR #3015) - - -Changes in synapse v0.27.1 (2018-03-26) -======================================= - -Meta release as v0.27.0 temporarily pointed to the wrong commit - - -Changes in synapse v0.27.0 (2018-03-26) -======================================= - -No changes since v0.27.0-rc2 - - -Changes in synapse v0.27.0-rc2 (2018-03-19) -=========================================== - -Pulls in v0.26.1 - -Bug fixes: - -* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005) - - -Changes in synapse v0.26.1 (2018-03-15) -======================================= - -Bug fixes: - -* Fix bug where an invalid event caused server to stop functioning correctly, - due to parsing and serializing bugs in ujson library (PR #3008) - - -Changes in synapse v0.27.0-rc1 (2018-03-14) -=========================================== - -The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. - -This release also begins the process of renaming a number of the metrics -reported to prometheus. See `docs/metrics-howto.rst `_. -Note that the v0.28.0 release will remove the deprecated metric names. - -Features: - -* Add ability for ASes to override message send time (PR #2754) -* Add support for custom storage providers for media repository (PR #2867, #2777, #2783, #2789, #2791, #2804, #2812, #2814, #2857, #2868, #2767) -* Add purge API features, see `docs/admin_api/purge_history_api.rst `_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943) -* Add support for whitelisting 3PIDs that users can register. (PR #2813) -* Add ``/room/{id}/event/{id}`` API (PR #2766) -* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live! -* Add ``federation_domain_whitelist`` option (PR #2820, #2821) - - -Changes: - -* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) -* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) -* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! -* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz! -* Remove ``verbosity``/``log_file`` from generated config (PR #2755) -* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838) -* When using synctl with workers, don't start the main synapse automatically (PR #2774) -* Minor performance improvements (PR #2773, #2792) -* Use a connection pool for non-federation outbound connections (PR #2817) -* Make it possible to run unit tests against postgres (PR #2829) -* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! -* Remove ability for AS users to call /events and /sync (PR #2948) -* Use bcrypt.checkpw (PR #2949) Thanks to @krombel! - -Bug fixes: - -* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv! -* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live! -* Fix publicised groups GET API (singular) over federation (PR #2772) -* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831) -* Fix error on ``/publicRooms`` when no rooms exist (PR #2827) -* Fix bug in quarantine_media (PR #2837) -* Fix url_previews when no Content-Type is returned from URL (PR #2845) -* Fix rare race in sync API when joining room (PR #2944) -* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) - - -Changes in synapse v0.26.0 (2018-01-05) -======================================= - -No changes since v0.26.0-rc1 - - -Changes in synapse v0.26.0-rc1 (2017-12-13) -=========================================== - -Features: - -* Add ability for ASes to publicise groups for their users (PR #2686) -* Add all local users to the user_directory and optionally search them (PR - #2723) -* Add support for custom login types for validating users (PR #2729) - - -Changes: - -* Update example Prometheus config to new format (PR #2648) Thanks to - @krombel! -* Rename redact_content option to include_content in Push API (PR #2650) -* Declare support for r0.3.0 (PR #2677) -* Improve upserts (PR #2684, #2688, #2689, #2713) -* Improve documentation of workers (PR #2700) -* Improve tracebacks on exceptions (PR #2705) -* Allow guest access to group APIs for reading (PR #2715) -* Support for posting content in federation_client script (PR #2716) -* Delete devices and pushers on logouts etc (PR #2722) - - -Bug fixes: - -* Fix database port script (PR #2673) -* Fix internal server error on login with ldap_auth_provider (PR #2678) Thanks - to @jkolo! -* Fix error on sqlite 3.7 (PR #2697) -* Fix OPTIONS on preview_url (PR #2707) -* Fix error handling on dns lookup (PR #2711) -* Fix wrong avatars when inviting multiple users when creating room (PR #2717) -* Fix 500 when joining matrix-dev (PR #2719) - - -Changes in synapse v0.25.1 (2017-11-17) -======================================= - -Bug fixes: - -* Fix login with LDAP and other password provider modules (PR #2678). Thanks to - @jkolo! - -Changes in synapse v0.25.0 (2017-11-15) -======================================= - -Bug fixes: - -* Fix port script (PR #2673) - - -Changes in synapse v0.25.0-rc1 (2017-11-14) -=========================================== - -Features: - -* Add is_public to groups table to allow for private groups (PR #2582) -* Add a route for determining who you are (PR #2668) Thanks to @turt2live! -* Add more features to the password providers (PR #2608, #2610, #2620, #2622, - #2623, #2624, #2626, #2628, #2629) -* Add a hook for custom rest endpoints (PR #2627) -* Add API to update group room visibility (PR #2651) - - -Changes: - -* Ignore