From 4a54b821bba9d8faff401771c8ba2a20a78c5f65 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 13:59:58 +0000 Subject: [PATCH 1/7] 1.23.0rc1 --- CHANGES.md | 86 ++++++++++++++++++++++++++++++++++++++++ changelog.d/8286.feature | 1 - changelog.d/8455.bugfix | 1 - changelog.d/8519.feature | 1 - changelog.d/8539.feature | 1 - changelog.d/8559.misc | 1 - changelog.d/8580.bugfix | 1 - changelog.d/8582.doc | 1 - changelog.d/8595.misc | 1 - changelog.d/8607.feature | 1 - changelog.d/8610.feature | 1 - changelog.d/8614.misc | 1 - changelog.d/8615.misc | 1 - changelog.d/8616.misc | 1 - changelog.d/8620.bugfix | 1 - changelog.d/8621.misc | 1 - changelog.d/8627.bugfix | 1 - changelog.d/8628.bugfix | 1 - changelog.d/8632.bugfix | 1 - changelog.d/8633.misc | 1 - changelog.d/8634.misc | 1 - changelog.d/8635.doc | 1 - changelog.d/8636.misc | 1 - changelog.d/8639.misc | 1 - changelog.d/8640.misc | 1 - changelog.d/8643.bugfix | 1 - changelog.d/8644.misc | 1 - changelog.d/8647.feature | 1 - changelog.d/8655.misc | 1 - changelog.d/8657.doc | 1 - changelog.d/8664.misc | 1 - changelog.d/8665.doc | 1 - changelog.d/8666.doc | 1 - changelog.d/8667.doc | 1 - changelog.d/8668.misc | 1 - changelog.d/8669.misc | 1 - changelog.d/8670.misc | 1 - changelog.d/8671.misc | 1 - changelog.d/8679.misc | 1 - changelog.d/8680.misc | 1 - changelog.d/8682.bugfix | 1 - changelog.d/8684.misc | 1 - changelog.d/8685.feature | 1 - changelog.d/8688.misc | 1 - changelog.d/8689.feature | 1 - changelog.d/8690.misc | 1 - changelog.d/8693.misc | 1 - changelog.d/8694.misc | 1 - changelog.d/8697.misc | 1 - changelog.d/8698.misc | 1 - changelog.d/8700.feature | 1 - changelog.d/8701.doc | 1 - changelog.d/8702.misc | 1 - changelog.d/8705.misc | 1 - changelog.d/8706.doc | 1 - changelog.d/8708.misc | 1 - changelog.d/8712.misc | 1 - changelog.d/8713.misc | 1 - changelog.d/8714.doc | 1 - changelog.d/8718.misc | 1 - changelog.d/8719.misc | 1 - changelog.d/8722.feature | 1 - changelog.d/8726.bugfix | 1 - changelog.d/8728.bugfix | 1 - changelog.d/8729.bugfix | 1 - changelog.d/8730.bugfix | 1 - changelog.d/8742.doc | 1 - changelog.d/8746.doc | 1 - changelog.d/8752.misc | 1 - changelog.d/8755.bugfix | 1 - synapse/__init__.py | 2 +- 71 files changed, 87 insertions(+), 70 deletions(-) delete mode 100644 changelog.d/8286.feature delete mode 100644 changelog.d/8455.bugfix delete mode 100644 changelog.d/8519.feature delete mode 100644 changelog.d/8539.feature delete mode 100644 changelog.d/8559.misc delete mode 100644 changelog.d/8580.bugfix delete mode 100644 changelog.d/8582.doc delete mode 100644 changelog.d/8595.misc delete mode 100644 changelog.d/8607.feature delete mode 100644 changelog.d/8610.feature delete mode 100644 changelog.d/8614.misc delete mode 100644 changelog.d/8615.misc delete mode 100644 changelog.d/8616.misc delete mode 100644 changelog.d/8620.bugfix delete mode 100644 changelog.d/8621.misc delete mode 100644 changelog.d/8627.bugfix delete mode 100644 changelog.d/8628.bugfix delete mode 100644 changelog.d/8632.bugfix delete mode 100644 changelog.d/8633.misc delete mode 100644 changelog.d/8634.misc delete mode 100644 changelog.d/8635.doc delete mode 100644 changelog.d/8636.misc delete mode 100644 changelog.d/8639.misc delete mode 100644 changelog.d/8640.misc delete mode 100644 changelog.d/8643.bugfix delete mode 100644 changelog.d/8644.misc delete mode 100644 changelog.d/8647.feature delete mode 100644 changelog.d/8655.misc delete mode 100644 changelog.d/8657.doc delete mode 100644 changelog.d/8664.misc delete mode 100644 changelog.d/8665.doc delete mode 100644 changelog.d/8666.doc delete mode 100644 changelog.d/8667.doc delete mode 100644 changelog.d/8668.misc delete mode 100644 changelog.d/8669.misc delete mode 100644 changelog.d/8670.misc delete mode 100644 changelog.d/8671.misc delete mode 100644 changelog.d/8679.misc delete mode 100644 changelog.d/8680.misc delete mode 100644 changelog.d/8682.bugfix delete mode 100644 changelog.d/8684.misc delete mode 100644 changelog.d/8685.feature delete mode 100644 changelog.d/8688.misc delete mode 100644 changelog.d/8689.feature delete mode 100644 changelog.d/8690.misc delete mode 100644 changelog.d/8693.misc delete mode 100644 changelog.d/8694.misc delete mode 100644 changelog.d/8697.misc delete mode 100644 changelog.d/8698.misc delete mode 100644 changelog.d/8700.feature delete mode 100644 changelog.d/8701.doc delete mode 100644 changelog.d/8702.misc delete mode 100644 changelog.d/8705.misc delete mode 100644 changelog.d/8706.doc delete mode 100644 changelog.d/8708.misc delete mode 100644 changelog.d/8712.misc delete mode 100644 changelog.d/8713.misc delete mode 100644 changelog.d/8714.doc delete mode 100644 changelog.d/8718.misc delete mode 100644 changelog.d/8719.misc delete mode 100644 changelog.d/8722.feature delete mode 100644 changelog.d/8726.bugfix delete mode 100644 changelog.d/8728.bugfix delete mode 100644 changelog.d/8729.bugfix delete mode 100644 changelog.d/8730.bugfix delete mode 100644 changelog.d/8742.doc delete mode 100644 changelog.d/8746.doc delete mode 100644 changelog.d/8752.misc delete mode 100644 changelog.d/8755.bugfix diff --git a/CHANGES.md b/CHANGES.md index 8bd782508..dc7d24581 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,89 @@ +Synapse 1.23.0rc1 (2020-11-13) +============================== + +Features +-------- + +- Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) +- Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) +- Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) +- Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) +- Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) +- Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) +- Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) +- Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) + + +Bugfixes +-------- + +- Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. ([\#8455](https://github.com/matrix-org/synapse/issues/8455)) +- Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. ([\#8580](https://github.com/matrix-org/synapse/issues/8580)) +- Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. ([\#8620](https://github.com/matrix-org/synapse/issues/8620)) +- Fix email notifications for invites without local state. ([\#8627](https://github.com/matrix-org/synapse/issues/8627)) +- Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. ([\#8628](https://github.com/matrix-org/synapse/issues/8628)) +- Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) +- Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) +- Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) +- Fix bug where Synapse would not recover after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) +- Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) +- Fix port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) +- Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) +- Fix port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) + + +Improved Documentation +---------------------- + +- Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. ([\#8582](https://github.com/matrix-org/synapse/issues/8582)) +- Improve the sample configuration for single sign-on providers. ([\#8635](https://github.com/matrix-org/synapse/issues/8635)) +- Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. ([\#8657](https://github.com/matrix-org/synapse/issues/8657)) +- Note support for Python 3.9. ([\#8665](https://github.com/matrix-org/synapse/issues/8665)) +- Minor updates to docs on running tests. ([\#8666](https://github.com/matrix-org/synapse/issues/8666)) +- Interlink prometheus/grafana documentation. ([\#8667](https://github.com/matrix-org/synapse/issues/8667)) +- Notes on SSO logins and media_repository worker. ([\#8701](https://github.com/matrix-org/synapse/issues/8701)) +- Document experimental support for running multiple event persisters. ([\#8706](https://github.com/matrix-org/synapse/issues/8706)) +- Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. ([\#8714](https://github.com/matrix-org/synapse/issues/8714)) +- Migrate documentation `docs/admin_api/event_reports` to markdown. ([\#8742](https://github.com/matrix-org/synapse/issues/8742)) +- Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. ([\#8746](https://github.com/matrix-org/synapse/issues/8746)) + + +Internal Changes +---------------- + +- Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559)) +- Implement and use an @lru_cache decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) +- Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614)) +- Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615)) +- Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616)) +- Remove unused OPTIONS handlers. ([\#8621](https://github.com/matrix-org/synapse/issues/8621)) +- Run `mypy` as part of the lint.sh script. ([\#8633](https://github.com/matrix-org/synapse/issues/8633)) +- Correct Synapse's PyPI package name in the OpenID Connect installation instructions. ([\#8634](https://github.com/matrix-org/synapse/issues/8634)) +- Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. ([\#8636](https://github.com/matrix-org/synapse/issues/8636)) +- Fix typos and spelling errors in the code. ([\#8639](https://github.com/matrix-org/synapse/issues/8639)) +- Reduce number of OpenTracing spans started. ([\#8640](https://github.com/matrix-org/synapse/issues/8640), [\#8668](https://github.com/matrix-org/synapse/issues/8668), [\#8670](https://github.com/matrix-org/synapse/issues/8670)) +- Add field `total` to device list in admin API. ([\#8644](https://github.com/matrix-org/synapse/issues/8644)) +- Add more type hints to the application services code. ([\#8655](https://github.com/matrix-org/synapse/issues/8655), [\#8693](https://github.com/matrix-org/synapse/issues/8693)) +- Tell Black to format code for Python 3.5. ([\#8664](https://github.com/matrix-org/synapse/issues/8664)) +- Don't pull event from DB when handling replication traffic. ([\#8669](https://github.com/matrix-org/synapse/issues/8669)) +- Abstract some invite-related code in preparation for landing knocking. ([\#8671](https://github.com/matrix-org/synapse/issues/8671), [\#8688](https://github.com/matrix-org/synapse/issues/8688)) +- Clarify representation of events in logfiles. ([\#8679](https://github.com/matrix-org/synapse/issues/8679)) +- Don't require `hiredis` package to be installed to run unit tests. ([\#8680](https://github.com/matrix-org/synapse/issues/8680)) +- Fix typing info on cache call signature to accept `on_invalidate`. ([\#8684](https://github.com/matrix-org/synapse/issues/8684)) +- Fail tests if they do not await coroutines. ([\#8690](https://github.com/matrix-org/synapse/issues/8690)) +- Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. ([\#8694](https://github.com/matrix-org/synapse/issues/8694)) +- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8697](https://github.com/matrix-org/synapse/issues/8697)) +- Use Python 3.8 in Docker images by default. ([\#8698](https://github.com/matrix-org/synapse/issues/8698)) +- Remove the "draft" status of the Room Details Admin API. ([\#8702](https://github.com/matrix-org/synapse/issues/8702)) +- Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. ([\#8705](https://github.com/matrix-org/synapse/issues/8705)) +- Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. ([\#8708](https://github.com/matrix-org/synapse/issues/8708)) +- Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. ([\#8712](https://github.com/matrix-org/synapse/issues/8712)) +- Consolidate duplicated lists of purged tables that are checked in tests. ([\#8713](https://github.com/matrix-org/synapse/issues/8713)) +- Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. ([\#8718](https://github.com/matrix-org/synapse/issues/8718)) +- Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. ([\#8719](https://github.com/matrix-org/synapse/issues/8719)) +- Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. ([\#8752](https://github.com/matrix-org/synapse/issues/8752)) + + Synapse 1.22.1 (2020-10-30) =========================== diff --git a/changelog.d/8286.feature b/changelog.d/8286.feature deleted file mode 100644 index 2c371419a..000000000 --- a/changelog.d/8286.feature +++ /dev/null @@ -1 +0,0 @@ -Add a push rule that highlights when a jitsi conference is created in a room. diff --git a/changelog.d/8455.bugfix b/changelog.d/8455.bugfix deleted file mode 100644 index 561e73f5e..000000000 --- a/changelog.d/8455.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. diff --git a/changelog.d/8519.feature b/changelog.d/8519.feature deleted file mode 100644 index e2ab54868..000000000 --- a/changelog.d/8519.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8539.feature b/changelog.d/8539.feature deleted file mode 100644 index 15ce02fb8..000000000 --- a/changelog.d/8539.feature +++ /dev/null @@ -1 +0,0 @@ -Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8559.misc b/changelog.d/8559.misc deleted file mode 100644 index d7bd00964..000000000 --- a/changelog.d/8559.misc +++ /dev/null @@ -1 +0,0 @@ -Optimise `/createRoom` with multiple invited users. diff --git a/changelog.d/8580.bugfix b/changelog.d/8580.bugfix deleted file mode 100644 index 31734fd97..000000000 --- a/changelog.d/8580.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. diff --git a/changelog.d/8582.doc b/changelog.d/8582.doc deleted file mode 100644 index 041f16871..000000000 --- a/changelog.d/8582.doc +++ /dev/null @@ -1 +0,0 @@ -Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. diff --git a/changelog.d/8595.misc b/changelog.d/8595.misc deleted file mode 100644 index 24fab65cd..000000000 --- a/changelog.d/8595.misc +++ /dev/null @@ -1 +0,0 @@ -Implement and use an @lru_cache decorator. diff --git a/changelog.d/8607.feature b/changelog.d/8607.feature deleted file mode 100644 index fef1eccb9..000000000 --- a/changelog.d/8607.feature +++ /dev/null @@ -1 +0,0 @@ -Support generating structured logs via the standard logging configuration. diff --git a/changelog.d/8610.feature b/changelog.d/8610.feature deleted file mode 100644 index ed8d92696..000000000 --- a/changelog.d/8610.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8614.misc b/changelog.d/8614.misc deleted file mode 100644 index 1bf9ea08f..000000000 --- a/changelog.d/8614.misc +++ /dev/null @@ -1 +0,0 @@ -Don't instansiate Requester directly. diff --git a/changelog.d/8615.misc b/changelog.d/8615.misc deleted file mode 100644 index 79fa7b7ff..000000000 --- a/changelog.d/8615.misc +++ /dev/null @@ -1 +0,0 @@ -Type hints for `RegistrationStore`. diff --git a/changelog.d/8616.misc b/changelog.d/8616.misc deleted file mode 100644 index 385b14063..000000000 --- a/changelog.d/8616.misc +++ /dev/null @@ -1 +0,0 @@ -Change schema to support access tokens belonging to one user but granting access to another. diff --git a/changelog.d/8620.bugfix b/changelog.d/8620.bugfix deleted file mode 100644 index c1078a3fb..000000000 --- a/changelog.d/8620.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. diff --git a/changelog.d/8621.misc b/changelog.d/8621.misc deleted file mode 100644 index 5720b665f..000000000 --- a/changelog.d/8621.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused OPTIONS handlers. diff --git a/changelog.d/8627.bugfix b/changelog.d/8627.bugfix deleted file mode 100644 index 143cf95f9..000000000 --- a/changelog.d/8627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix email notifications for invites without local state. diff --git a/changelog.d/8628.bugfix b/changelog.d/8628.bugfix deleted file mode 100644 index 1316136ca..000000000 --- a/changelog.d/8628.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. diff --git a/changelog.d/8632.bugfix b/changelog.d/8632.bugfix deleted file mode 100644 index 7d834aa2e..000000000 --- a/changelog.d/8632.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. diff --git a/changelog.d/8633.misc b/changelog.d/8633.misc deleted file mode 100644 index 8e1d006b3..000000000 --- a/changelog.d/8633.misc +++ /dev/null @@ -1 +0,0 @@ -Run `mypy` as part of the lint.sh script. diff --git a/changelog.d/8634.misc b/changelog.d/8634.misc deleted file mode 100644 index c4f74ba7c..000000000 --- a/changelog.d/8634.misc +++ /dev/null @@ -1 +0,0 @@ -Correct Synapse's PyPI package name in the OpenID Connect installation instructions. \ No newline at end of file diff --git a/changelog.d/8635.doc b/changelog.d/8635.doc deleted file mode 100644 index 00fb1e61a..000000000 --- a/changelog.d/8635.doc +++ /dev/null @@ -1 +0,0 @@ -Improve the sample configuration for single sign-on providers. diff --git a/changelog.d/8636.misc b/changelog.d/8636.misc deleted file mode 100644 index df4dca42f..000000000 --- a/changelog.d/8636.misc +++ /dev/null @@ -1 +0,0 @@ -Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. diff --git a/changelog.d/8639.misc b/changelog.d/8639.misc deleted file mode 100644 index 20a213df3..000000000 --- a/changelog.d/8639.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typos and spelling errors in the code. diff --git a/changelog.d/8640.misc b/changelog.d/8640.misc deleted file mode 100644 index cf6023f78..000000000 --- a/changelog.d/8640.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8643.bugfix b/changelog.d/8643.bugfix deleted file mode 100644 index fcda1ca87..000000000 --- a/changelog.d/8643.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. diff --git a/changelog.d/8644.misc b/changelog.d/8644.misc deleted file mode 100644 index 87f2b7292..000000000 --- a/changelog.d/8644.misc +++ /dev/null @@ -1 +0,0 @@ -Add field `total` to device list in admin API. \ No newline at end of file diff --git a/changelog.d/8647.feature b/changelog.d/8647.feature deleted file mode 100644 index 79e98f6e9..000000000 --- a/changelog.d/8647.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8655.misc b/changelog.d/8655.misc deleted file mode 100644 index b588bdd3e..000000000 --- a/changelog.d/8655.misc +++ /dev/null @@ -1 +0,0 @@ -Add more type hints to the application services code. diff --git a/changelog.d/8657.doc b/changelog.d/8657.doc deleted file mode 100644 index 3dcbb221a..000000000 --- a/changelog.d/8657.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. diff --git a/changelog.d/8664.misc b/changelog.d/8664.misc deleted file mode 100644 index 278cf53ad..000000000 --- a/changelog.d/8664.misc +++ /dev/null @@ -1 +0,0 @@ -Tell Black to format code for Python 3.5. diff --git a/changelog.d/8665.doc b/changelog.d/8665.doc deleted file mode 100644 index 3b75307dc..000000000 --- a/changelog.d/8665.doc +++ /dev/null @@ -1 +0,0 @@ -Note support for Python 3.9. diff --git a/changelog.d/8666.doc b/changelog.d/8666.doc deleted file mode 100644 index dee86b4a2..000000000 --- a/changelog.d/8666.doc +++ /dev/null @@ -1 +0,0 @@ -Minor updates to docs on running tests. diff --git a/changelog.d/8667.doc b/changelog.d/8667.doc deleted file mode 100644 index 422d697da..000000000 --- a/changelog.d/8667.doc +++ /dev/null @@ -1 +0,0 @@ -Interlink prometheus/grafana documentation. diff --git a/changelog.d/8668.misc b/changelog.d/8668.misc deleted file mode 100644 index cf6023f78..000000000 --- a/changelog.d/8668.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8669.misc b/changelog.d/8669.misc deleted file mode 100644 index 5228105cd..000000000 --- a/changelog.d/8669.misc +++ /dev/null @@ -1 +0,0 @@ -Don't pull event from DB when handling replication traffic. diff --git a/changelog.d/8670.misc b/changelog.d/8670.misc deleted file mode 100644 index cf6023f78..000000000 --- a/changelog.d/8670.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8671.misc b/changelog.d/8671.misc deleted file mode 100644 index bef8dc425..000000000 --- a/changelog.d/8671.misc +++ /dev/null @@ -1 +0,0 @@ -Abstract some invite-related code in preparation for landing knocking. \ No newline at end of file diff --git a/changelog.d/8679.misc b/changelog.d/8679.misc deleted file mode 100644 index 662eced4c..000000000 --- a/changelog.d/8679.misc +++ /dev/null @@ -1 +0,0 @@ -Clarify representation of events in logfiles. diff --git a/changelog.d/8680.misc b/changelog.d/8680.misc deleted file mode 100644 index 2ca297546..000000000 --- a/changelog.d/8680.misc +++ /dev/null @@ -1 +0,0 @@ -Don't require `hiredis` package to be installed to run unit tests. diff --git a/changelog.d/8682.bugfix b/changelog.d/8682.bugfix deleted file mode 100644 index e61276aa0..000000000 --- a/changelog.d/8682.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. diff --git a/changelog.d/8684.misc b/changelog.d/8684.misc deleted file mode 100644 index 1d23d4292..000000000 --- a/changelog.d/8684.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typing info on cache call signature to accept `on_invalidate`. diff --git a/changelog.d/8685.feature b/changelog.d/8685.feature deleted file mode 100644 index fef1eccb9..000000000 --- a/changelog.d/8685.feature +++ /dev/null @@ -1 +0,0 @@ -Support generating structured logs via the standard logging configuration. diff --git a/changelog.d/8688.misc b/changelog.d/8688.misc deleted file mode 100644 index bef8dc425..000000000 --- a/changelog.d/8688.misc +++ /dev/null @@ -1 +0,0 @@ -Abstract some invite-related code in preparation for landing knocking. \ No newline at end of file diff --git a/changelog.d/8689.feature b/changelog.d/8689.feature deleted file mode 100644 index ed8d92696..000000000 --- a/changelog.d/8689.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8690.misc b/changelog.d/8690.misc deleted file mode 100644 index 0f38ba1f5..000000000 --- a/changelog.d/8690.misc +++ /dev/null @@ -1 +0,0 @@ -Fail tests if they do not await coroutines. diff --git a/changelog.d/8693.misc b/changelog.d/8693.misc deleted file mode 100644 index b588bdd3e..000000000 --- a/changelog.d/8693.misc +++ /dev/null @@ -1 +0,0 @@ -Add more type hints to the application services code. diff --git a/changelog.d/8694.misc b/changelog.d/8694.misc deleted file mode 100644 index c90a6375a..000000000 --- a/changelog.d/8694.misc +++ /dev/null @@ -1 +0,0 @@ -Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. diff --git a/changelog.d/8697.misc b/changelog.d/8697.misc deleted file mode 100644 index 7982a4e46..000000000 --- a/changelog.d/8697.misc +++ /dev/null @@ -1 +0,0 @@ - Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. diff --git a/changelog.d/8698.misc b/changelog.d/8698.misc deleted file mode 100644 index 6b777fb29..000000000 --- a/changelog.d/8698.misc +++ /dev/null @@ -1 +0,0 @@ -Use Python 3.8 in Docker images by default. diff --git a/changelog.d/8700.feature b/changelog.d/8700.feature deleted file mode 100644 index 47d63dce0..000000000 --- a/changelog.d/8700.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API for local user media statistics. Contributed by @dklimpel. diff --git a/changelog.d/8701.doc b/changelog.d/8701.doc deleted file mode 100644 index e2e8b2f79..000000000 --- a/changelog.d/8701.doc +++ /dev/null @@ -1 +0,0 @@ -Notes on SSO logins and media_repository worker. \ No newline at end of file diff --git a/changelog.d/8702.misc b/changelog.d/8702.misc deleted file mode 100644 index f20085cbe..000000000 --- a/changelog.d/8702.misc +++ /dev/null @@ -1 +0,0 @@ -Remove the "draft" status of the Room Details Admin API. \ No newline at end of file diff --git a/changelog.d/8705.misc b/changelog.d/8705.misc deleted file mode 100644 index 1189464a0..000000000 --- a/changelog.d/8705.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. \ No newline at end of file diff --git a/changelog.d/8706.doc b/changelog.d/8706.doc deleted file mode 100644 index 96a0427e7..000000000 --- a/changelog.d/8706.doc +++ /dev/null @@ -1 +0,0 @@ -Document experimental support for running multiple event persisters. diff --git a/changelog.d/8708.misc b/changelog.d/8708.misc deleted file mode 100644 index be679fb0f..000000000 --- a/changelog.d/8708.misc +++ /dev/null @@ -1 +0,0 @@ -Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. diff --git a/changelog.d/8712.misc b/changelog.d/8712.misc deleted file mode 100644 index 90d63a9a2..000000000 --- a/changelog.d/8712.misc +++ /dev/null @@ -1 +0,0 @@ -Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. diff --git a/changelog.d/8713.misc b/changelog.d/8713.misc deleted file mode 100644 index c5d3f3216..000000000 --- a/changelog.d/8713.misc +++ /dev/null @@ -1 +0,0 @@ -Consolidate duplicated lists of purged tables that are checked in tests. \ No newline at end of file diff --git a/changelog.d/8714.doc b/changelog.d/8714.doc deleted file mode 100644 index bda22714e..000000000 --- a/changelog.d/8714.doc +++ /dev/null @@ -1 +0,0 @@ -Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. \ No newline at end of file diff --git a/changelog.d/8718.misc b/changelog.d/8718.misc deleted file mode 100644 index f1868ce70..000000000 --- a/changelog.d/8718.misc +++ /dev/null @@ -1 +0,0 @@ -Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. diff --git a/changelog.d/8719.misc b/changelog.d/8719.misc deleted file mode 100644 index 9aabef8fc..000000000 --- a/changelog.d/8719.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. diff --git a/changelog.d/8722.feature b/changelog.d/8722.feature deleted file mode 100644 index 0413d8838..000000000 --- a/changelog.d/8722.feature +++ /dev/null @@ -1 +0,0 @@ -Add `displayname` to Shared-Secret Registration for admins. \ No newline at end of file diff --git a/changelog.d/8726.bugfix b/changelog.d/8726.bugfix deleted file mode 100644 index 831f773a2..000000000 --- a/changelog.d/8726.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where Synapse would not recover after losing connection to the database. diff --git a/changelog.d/8728.bugfix b/changelog.d/8728.bugfix deleted file mode 100644 index 8064aad0f..000000000 --- a/changelog.d/8728.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. diff --git a/changelog.d/8729.bugfix b/changelog.d/8729.bugfix deleted file mode 100644 index 7f59a3b9e..000000000 --- a/changelog.d/8729.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script fails when DB has no backfilled events. Broke in v1.21.0. diff --git a/changelog.d/8730.bugfix b/changelog.d/8730.bugfix deleted file mode 100644 index dcc42bc98..000000000 --- a/changelog.d/8730.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. diff --git a/changelog.d/8742.doc b/changelog.d/8742.doc deleted file mode 100644 index cbae6cfdc..000000000 --- a/changelog.d/8742.doc +++ /dev/null @@ -1 +0,0 @@ -Migrate documentation `docs/admin_api/event_reports` to markdown. \ No newline at end of file diff --git a/changelog.d/8746.doc b/changelog.d/8746.doc deleted file mode 100644 index 6baf58ba7..000000000 --- a/changelog.d/8746.doc +++ /dev/null @@ -1 +0,0 @@ -Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. \ No newline at end of file diff --git a/changelog.d/8752.misc b/changelog.d/8752.misc deleted file mode 100644 index eac92e9d1..000000000 --- a/changelog.d/8752.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. diff --git a/changelog.d/8755.bugfix b/changelog.d/8755.bugfix deleted file mode 100644 index 42bbed3ac..000000000 --- a/changelog.d/8755.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script so that it can be run again after a failure. Broke in v1.21.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 3e1df2b03..537f2239e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.22.1" +__version__ = "1.23.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 0a5185495bcb7c946214b4398d3a705f80c5354e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 14:06:52 +0000 Subject: [PATCH 2/7] Fix changelog --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dc7d24581..f05bbe786 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,10 +5,10 @@ Features -------- - Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) -- Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) +- Add an admin api to delete a single file or files that were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) - Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) - Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) -- Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) +- Add an admin API to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) - Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) - Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) - Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) From 34226ec7614027046e98fc3447b2ed7f2d47226e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 14:14:09 +0000 Subject: [PATCH 3/7] Fix changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f05bbe786..75871979c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,11 +25,11 @@ Bugfixes - Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) - Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) - Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) -- Fix bug where Synapse would not recover after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) +- Fix bug that prevented Synapse from recovering after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) - Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) -- Fix port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) -- Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) -- Fix port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) +- Fix PostgreSQL port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) +- Fix PostgreSQL port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) +- Fix PostgreSQL port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) Improved Documentation From e8d08537394a49f3e66e9cbea3627e3c25818a7d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 13 Nov 2020 16:24:04 +0000 Subject: [PATCH 4/7] Generalise _maybe_store_room_on_invite (#8754) There's a handy function called maybe_store_room_on_invite which allows us to create an entry in the rooms table for a room and its version for which we aren't joined to yet, but we can reference when ingesting events about. This is currently used for invites where we receive some stripped state about the room and pass it down via /sync to the client, without us being in the room yet. There is a similar requirement for knocking, where we will eventually do the same thing, and need an entry in the rooms table as well. Thus, reusing this function works, however its name needs to be generalised a bit. Separated out from #6739. --- changelog.d/8754.misc | 1 + synapse/handlers/federation.py | 10 ++++++---- synapse/replication/http/federation.py | 10 +++++----- synapse/storage/databases/main/room.py | 10 ++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 changelog.d/8754.misc diff --git a/changelog.d/8754.misc b/changelog.d/8754.misc new file mode 100644 index 000000000..0436bb1be --- /dev/null +++ b/changelog.d/8754.misc @@ -0,0 +1 @@ +Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. \ No newline at end of file diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c38695770..69bc5ba44 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -67,7 +67,7 @@ from synapse.replication.http.devices import ReplicationUserDevicesResyncRestSer from synapse.replication.http.federation import ( ReplicationCleanRoomRestServlet, ReplicationFederationSendEventsRestServlet, - ReplicationStoreRoomOnInviteRestServlet, + ReplicationStoreRoomOnOutlierMembershipRestServlet, ) from synapse.state import StateResolutionStore from synapse.storage.databases.main.events_worker import EventRedactBehaviour @@ -152,12 +152,14 @@ class FederationHandler(BaseHandler): self._user_device_resync = ReplicationUserDevicesResyncRestServlet.make_client( hs ) - self._maybe_store_room_on_invite = ReplicationStoreRoomOnInviteRestServlet.make_client( + self._maybe_store_room_on_outlier_membership = ReplicationStoreRoomOnOutlierMembershipRestServlet.make_client( hs ) else: self._device_list_updater = hs.get_device_handler().device_list_updater - self._maybe_store_room_on_invite = self.store.maybe_store_room_on_invite + self._maybe_store_room_on_outlier_membership = ( + self.store.maybe_store_room_on_outlier_membership + ) # When joining a room we need to queue any events for that room up. # For each room, a list of (pdu, origin) tuples. @@ -1617,7 +1619,7 @@ class FederationHandler(BaseHandler): # keep a record of the room version, if we don't yet know it. # (this may get overwritten if we later get a different room version in a # join dance). - await self._maybe_store_room_on_invite( + await self._maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=room_version ) diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index b4f4a68b5..7a0dbb5b1 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -254,20 +254,20 @@ class ReplicationCleanRoomRestServlet(ReplicationEndpoint): return 200, {} -class ReplicationStoreRoomOnInviteRestServlet(ReplicationEndpoint): +class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. Request format: - POST /_synapse/replication/store_room_on_invite/:room_id/:txn_id + POST /_synapse/replication/store_room_on_outlier_membership/:room_id/:txn_id { "room_version": "1", } """ - NAME = "store_room_on_invite" + NAME = "store_room_on_outlier_membership" PATH_ARGS = ("room_id",) def __init__(self, hs): @@ -282,7 +282,7 @@ class ReplicationStoreRoomOnInviteRestServlet(ReplicationEndpoint): async def _handle_request(self, request, room_id): content = parse_json_object_from_request(request) room_version = KNOWN_ROOM_VERSIONS[content["room_version"]] - await self.store.maybe_store_room_on_invite(room_id, room_version) + await self.store.maybe_store_room_on_outlier_membership(room_id, room_version) return 200, {} @@ -291,4 +291,4 @@ def register_servlets(hs, http_server): ReplicationFederationSendEduRestServlet(hs).register(http_server) ReplicationGetQueryRestServlet(hs).register(http_server) ReplicationCleanRoomRestServlet(hs).register(http_server) - ReplicationStoreRoomOnInviteRestServlet(hs).register(http_server) + ReplicationStoreRoomOnOutlierMembershipRestServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index dc0c4b549..6b89db15c 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1240,13 +1240,15 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore): logger.error("store_room with room_id=%s failed: %s", room_id, e) raise StoreError(500, "Problem creating room.") - async def maybe_store_room_on_invite(self, room_id: str, room_version: RoomVersion): + async def maybe_store_room_on_outlier_membership( + self, room_id: str, room_version: RoomVersion + ): """ - When we receive an invite over federation, store the version of the room if we - don't already know the room version. + When we receive an invite or any other event over federation that may relate to a room + we are not in, store the version of the room if we don't already know the room version. """ await self.db_pool.simple_upsert( - desc="maybe_store_room_on_invite", + desc="maybe_store_room_on_outlier_membership", table="rooms", keyvalues={"room_id": room_id}, values={}, From f1de4bb58b5b68b0fbb0033bd42fa30df0944281 Mon Sep 17 00:00:00 2001 From: Adrian Wannenmacher Date: Sun, 15 Nov 2020 00:09:36 +0100 Subject: [PATCH 5/7] Clarify the usecase for an msisdn delegate (#8734) Signed-off-by: Adrian Wannenmacher --- changelog.d/8734.doc | 1 + docs/sample_config.yaml | 5 +++-- synapse/config/registration.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/8734.doc diff --git a/changelog.d/8734.doc b/changelog.d/8734.doc new file mode 100644 index 000000000..3bff9021c --- /dev/null +++ b/changelog.d/8734.doc @@ -0,0 +1 @@ +Clarify the usecase for an msisdn delegate. Contributed by Adrian Wannenmacher. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index c0cd00923..e9e77ca94 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1230,8 +1230,9 @@ account_validity: # email will be globally disabled. # # Additionally, if `msisdn` is not set, registration and password resets via msisdn -# will be disabled regardless. This is due to Synapse currently not supporting any -# method of sending SMS messages on its own. +# will be disabled regardless, and users will not be able to associate an msisdn +# identifier to their account. This is due to Synapse currently not supporting +# any method of sending SMS messages on its own. # # To enable using an identity server for operations regarding a particular third-party # identifier type, set the value to the URL of that identity server as shown in the diff --git a/synapse/config/registration.py b/synapse/config/registration.py index b0a77a2e4..cc5f75123 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -347,8 +347,9 @@ class RegistrationConfig(Config): # email will be globally disabled. # # Additionally, if `msisdn` is not set, registration and password resets via msisdn - # will be disabled regardless. This is due to Synapse currently not supporting any - # method of sending SMS messages on its own. + # will be disabled regardless, and users will not be able to associate an msisdn + # identifier to their account. This is due to Synapse currently not supporting + # any method of sending SMS messages on its own. # # To enable using an identity server for operations regarding a particular third-party # identifier type, set the value to the URL of that identity server as shown in the From ebc405446e6615d6187a2e29cb33f27dd5bd0841 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Nov 2020 14:45:22 +0000 Subject: [PATCH 6/7] Add a `custom_headers` param to `make_request` (#8760) Some tests want to set some custom HTTP request headers, so provide a way to do that before calling requestReceived(). --- changelog.d/8760.misc | 1 + tests/rest/client/v1/utils.py | 10 ++++++---- tests/server.py | 11 ++++++++++- tests/storage/test_client_ips.py | 13 +++++++------ 4 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 changelog.d/8760.misc diff --git a/changelog.d/8760.misc b/changelog.d/8760.misc new file mode 100644 index 000000000..54502e9b9 --- /dev/null +++ b/changelog.d/8760.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index afaf9f7b8..1b2d0497a 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -296,10 +296,12 @@ class RestHelper: image_length = len(image_data) path = "/_matrix/media/r0/upload?filename=%s" % (filename,) request, channel = make_request( - self.hs.get_reactor(), "POST", path, content=image_data, access_token=tok - ) - request.requestHeaders.addRawHeader( - b"Content-Length", str(image_length).encode("UTF-8") + self.hs.get_reactor(), + "POST", + path, + content=image_data, + access_token=tok, + custom_headers=[(b"Content-Length", str(image_length))], ) request.render(resource) self.hs.get_reactor().pump([100]) diff --git a/tests/server.py b/tests/server.py index 3dd2cfc07..ef03109a6 100644 --- a/tests/server.py +++ b/tests/server.py @@ -2,7 +2,7 @@ import json import logging from collections import deque from io import SEEK_END, BytesIO -from typing import Callable +from typing import Callable, Iterable, Optional, Tuple, Union import attr from typing_extensions import Deque @@ -139,6 +139,9 @@ def make_request( shorthand=True, federation_auth_origin=None, content_is_form=False, + custom_headers: Optional[ + Iterable[Tuple[Union[bytes, str], Union[bytes, str]]] + ] = None, ): """ Make a web request using the given method and path, feed it the @@ -157,6 +160,8 @@ def make_request( content_is_form: Whether the content is URL encoded form data. Adds the 'Content-Type': 'application/x-www-form-urlencoded' header. + custom_headers: (name, value) pairs to add as request headers + Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ @@ -211,6 +216,10 @@ def make_request( # Assume the body is JSON req.requestHeaders.addRawHeader(b"Content-Type", b"application/json") + if custom_headers: + for k, v in custom_headers: + req.requestHeaders.addRawHeader(k, v) + req.requestReceived(method, path, b"1.1") return req, channel diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index e96ca1c8c..efca43ec7 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -21,6 +21,7 @@ from synapse.http.site import XForwardedForRequest from synapse.rest.client.v1 import login from tests import unittest +from tests.server import make_request from tests.test_utils import make_awaitable from tests.unittest import override_config @@ -408,17 +409,17 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): # Advance to a known time self.reactor.advance(123456 - self.reactor.seconds()) - request, channel = self.make_request( + headers1 = {b"User-Agent": b"Mozzila pizza"} + headers1.update(headers) + + request, channel = make_request( + self.reactor, "GET", "/_matrix/client/r0/admin/users/" + self.user_id, access_token=access_token, + custom_headers=headers1.items(), **make_request_args, ) - request.requestHeaders.addRawHeader(b"User-Agent", b"Mozzila pizza") - - # Add the optional headers - for h, v in headers.items(): - request.requestHeaders.addRawHeader(h, v) self.render(request) # Advance so the save loop occurs From 791d7cd6f065e166576538b9cba3d90febf83ea4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Nov 2020 14:45:52 +0000 Subject: [PATCH 7/7] Rename `create_test_json_resource` to `create_test_resource` (#8759) The root resource isn't necessarily a JsonResource, so rename this method accordingly, and update a couple of test classes to use the method rather than directly manipulating self.resource. --- changelog.d/8759.misc | 1 + tests/replication/_base.py | 4 ++-- tests/rest/admin/test_admin.py | 2 +- tests/rest/key/v2/test_remote_key_resource.py | 2 +- tests/rest/test_health.py | 6 ++---- tests/rest/test_well_known.py | 6 ++---- tests/unittest.py | 18 +++++++----------- 7 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 changelog.d/8759.misc diff --git a/changelog.d/8759.misc b/changelog.d/8759.misc new file mode 100644 index 000000000..54502e9b9 --- /dev/null +++ b/changelog.d/8759.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 5c633ac6d..bc56b13dc 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -240,8 +240,8 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): lambda: self._handle_http_replication_attempt(self.hs, 8765), ) - def create_test_json_resource(self): - """Overrides `HomeserverTestCase.create_test_json_resource`. + def create_test_resource(self): + """Overrides `HomeserverTestCase.create_test_resource`. """ # We override this so that it automatically registers all the HTTP # replication servlets, without having to explicitly do that in all diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 0f1144fe1..6804f9337 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -35,7 +35,7 @@ from tests import unittest class VersionTestCase(unittest.HomeserverTestCase): url = "/_synapse/admin/v1/server_version" - def create_test_json_resource(self): + def create_test_resource(self): resource = JsonResource(self.hs) VersionServlet(self.hs).register(resource) return resource diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index 6850c666b..6671cbd32 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -41,7 +41,7 @@ class BaseRemoteKeyResourceTestCase(unittest.HomeserverTestCase): self.http_client = Mock() return self.setup_test_homeserver(http_client=self.http_client) - def create_test_json_resource(self): + def create_test_resource(self): return create_resource_tree( {"/_matrix/key/v2": KeyApiV2Resource(self.hs)}, root_resource=NoResource() ) diff --git a/tests/rest/test_health.py b/tests/rest/test_health.py index 2d021f656..f4d06e220 100644 --- a/tests/rest/test_health.py +++ b/tests/rest/test_health.py @@ -20,11 +20,9 @@ from tests import unittest class HealthCheckTests(unittest.HomeserverTestCase): - def setUp(self): - super().setUp() - + def create_test_resource(self): # replace the JsonResource with a HealthResource. - self.resource = HealthResource() + return HealthResource() def test_health(self): request, channel = self.make_request("GET", "/health", shorthand=False) diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index dcd65c2a5..a3746e713 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -20,11 +20,9 @@ from tests import unittest class WellKnownTests(unittest.HomeserverTestCase): - def setUp(self): - super().setUp() - + def create_test_resource(self): # replace the JsonResource with a WellKnownResource - self.resource = WellKnownResource(self.hs) + return WellKnownResource(self.hs) def test_well_known(self): self.hs.config.public_baseurl = "https://tesths" diff --git a/tests/unittest.py b/tests/unittest.py index e36ac8919..c630760e5 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -30,6 +30,7 @@ from twisted.internet.defer import Deferred, ensureDeferred, succeed from twisted.python.failure import Failure from twisted.python.threadpool import ThreadPool from twisted.trial import unittest +from twisted.web.resource import Resource from synapse.api.constants import EventTypes, Membership from synapse.config.homeserver import HomeServerConfig @@ -239,10 +240,8 @@ class HomeserverTestCase(TestCase): if not isinstance(self.hs, HomeServer): raise Exception("A homeserver wasn't returned, but %r" % (self.hs,)) - # Register the resources - self.resource = self.create_test_json_resource() - - # create a site to wrap the resource. + # create the root resource, and a site to wrap it. + self.resource = self.create_test_resource() self.site = SynapseSite( logger_name="synapse.access.http.fake", site_tag=self.hs.config.server.server_name, @@ -323,15 +322,12 @@ class HomeserverTestCase(TestCase): hs = self.setup_test_homeserver() return hs - def create_test_json_resource(self): + def create_test_resource(self) -> Resource: """ - Create a test JsonResource, with the relevant servlets registerd to it + Create a the root resource for the test server. - The default implementation calls each function in `servlets` to do the - registration. - - Returns: - JsonResource: + The default implementation creates a JsonResource and calls each function in + `servlets` to register servletes against it """ resource = JsonResource(self.hs)