From 053ecae4db7c51c7fa3d56f5ea2b1289d73fa10d Mon Sep 17 00:00:00 2001 From: Andrew Conrad Date: Tue, 14 Nov 2017 15:09:50 -0600 Subject: [PATCH 01/49] Mention SyTest in the README, after Development Signed-off-by: Andrew Conrad --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 9da8c7f7a..cda0794e5 100644 --- a/README.rst +++ b/README.rst @@ -883,6 +883,17 @@ This should end with a 'PASSED' result:: PASSED (successes=143) +Running the Integration Tests +============================= + +Synapse is accompanied by `SyTest `_, +a Matrix homeserver integration testing suite, which uses HTTP requests to +access the API as a Matrix client would. It is able to run Synapse directly from +the source tree, so installation of the server is not required. + +Testing with SyTest is recommended for verifying that changes related to the +Client-Server API are functioning correctly. See the `installation instructions +`_ for details. Building Internal API Documentation =================================== From 4102468da98543ddb899857b67f5666ae4cfae92 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Tue, 16 Jan 2018 16:13:28 -0500 Subject: [PATCH 02/49] Note that Synapse is available in Fedora Signed-off-by: Jeremy Cline --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index ef1eed458..ef27b92dd 100644 --- a/README.rst +++ b/README.rst @@ -354,6 +354,10 @@ https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one Fedora ------ +Synapse is in the Fedora repositories as ``matrix-synapse``:: + + sudo dnf install matrix-synapse + Oleg Girko provides Fedora RPMs at https://obs.infoserver.lv/project/monitor/matrix-synapse From cb2c7c0669c7f4d4cc41d65a7f2328589fa04fc5 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 14 Mar 2018 16:09:20 +0000 Subject: [PATCH 03/49] Update CHANGES.rst WIP, need to add most recent PRs --- CHANGES.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index a7ed49e10..0bcd5a717 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,97 @@ This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. +Changes in synapse v0.27.0-rc1 (2018-03-14) +=========================================== + +Features: + * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) + * Add shiney new purge API. New implementation is:- + * Faster + * Supports clearing by timestamp + * Supports deleting of local events + * Is transactional +(PR #2858,2867,2882, 2946,2962,2943) + + * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813) + * Add /room/{id}/event/{id} to synapse (PR #2766) + * Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live! + * Add federation_domain_whitelist option (PR #2820,2821) + + +Changes: + + * Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are:- + * Event Creation + * Push Actions + +With more to follow + + (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856) + + * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) + * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! + * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! + * Better logging when login can't find a 3pid (PR #2744) + * add ?ts massaging for ASes (PR #2754) + * Remove 'verbosity'/'log_file' from generated cfg (PR #2755) + * Make indentation of generated log config consistent (PR #2762) + * Remove dead code related to default thumbnails (PR #2764) + * Update http request metrics before calling servlet (PR #2770) + * Do bcrypt hashing in a background thread (PR #2773) + * When using synctl with workers, don't start the main synapse automatically (PR #2774) + * Make Counter render floats (PR #2778) + * Store state groups separately from events (PR #2784) + * Reorganise request and block metrics (PR #2785) + * Metrics for number of RDATA commands received (PR #2786) + * Metrics for events processed in appservice and fed sender (PR #2787) + * Optimise LoggingContext creation and copying (PR #2792) + * Track db txn time in millisecs (PR #2793) + * Track DB scheduling delay per-request (PR #2794, #2795) + * Sanity checking for user ids (PR #2797) + * better exception logging in callbackmetrics (PR #2809) + * Fix bugs in block metrics (PR #2810) + * Add some comments about the reactor tick time metric (PR #2816) + * Use a connection pool for the SimpleHttpClient (PR #2817) + * Remove unused/bitrotted MemoryDataStore (PR #2828) + * Make it possible to run tests against postgres (PR #2829) + * Factor out get_db_conn to HomeServer base class (PR #2830) + * Logging and metrics for the http pusher (PR #2833) + * Improve exception handling in persist_event (PR #2834) + * montoring metrics for number of cache evictions (PR #2844) + * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! + * Remove ability for AS users to call /events and /sync (PR #2948) + * use bcrypt.checkpw (PR #2949) Thanks to @krombel! + * Factor run_in_background out from preserve_fn (PR #2961) + * Add a metric which increments when a request is received (PR #2965) + * Improve caching for read_marker API (PR #2927) + + +Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270) + + + +Bug fixes: + + * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv! + * Check missing fields in event_from_pdu_json (PR #2745) + * Fix templating error with unban permission message (PR #2761) Thanks to @turt2live! + * Fix flaky test_rooms UTs (PR #2765) + * Fix publicised groups GET API (singular) over federation (PR #2772) + * Fix a logcontext leak in persist_events (PR #2790) + * Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796) + * fix SQL when searching all users (PR #2803) + * Fix server 500 on public rooms call when no rooms exist (PR #2827) + * Fix SQL for user search (PR #2831) + * Fix sql error in quarantine_media (PR #2837) + * Handle url_previews with no content-type (PR #2845) + * Add missing yield during 3pid signature checks (PR #2933) + * Fix race in sync when joining room (PR #2944) + * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) + * Fix scary-looking dns resolution errors (PR #2838) + + + Changes in synapse v0.26.0 (2018-01-05) ======================================= From a492b17fe2cbbb10687d9a7cbe9eda67b0562656 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 14 Mar 2018 16:18:09 +0000 Subject: [PATCH 04/49] Update CHANGES.rst clean formatting --- CHANGES.rst | 128 ++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 68 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0bcd5a717..4d1b4ba13 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,66 +12,58 @@ Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== Features: - * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) - * Add shiney new purge API. New implementation is:- +* Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) +* Add shiney new purge API. New implementation is:- * Faster * Supports clearing by timestamp * Supports deleting of local events - * Is transactional -(PR #2858,2867,2882, 2946,2962,2943) + * Is transactional (PR #2858,2867,2882, 2946,2962,2943) - * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813) - * Add /room/{id}/event/{id} to synapse (PR #2766) - * Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live! - * Add federation_domain_whitelist option (PR #2820,2821) +* Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813) +* Add /room/{id}/event/{id} to synapse (PR #2766) +* Add an admin route to get all the media in a room (PR #2818) Thanks to @turt2live! +* Add federation_domain_whitelist option (PR #2820,2821) Changes: - * Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are:- - * Event Creation - * Push Actions - -With more to follow - - (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856) - - * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) - * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! - * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! - * Better logging when login can't find a 3pid (PR #2744) - * add ?ts massaging for ASes (PR #2754) - * Remove 'verbosity'/'log_file' from generated cfg (PR #2755) - * Make indentation of generated log config consistent (PR #2762) - * Remove dead code related to default thumbnails (PR #2764) - * Update http request metrics before calling servlet (PR #2770) - * Do bcrypt hashing in a background thread (PR #2773) - * When using synctl with workers, don't start the main synapse automatically (PR #2774) - * Make Counter render floats (PR #2778) - * Store state groups separately from events (PR #2784) - * Reorganise request and block metrics (PR #2785) - * Metrics for number of RDATA commands received (PR #2786) - * Metrics for events processed in appservice and fed sender (PR #2787) - * Optimise LoggingContext creation and copying (PR #2792) - * Track db txn time in millisecs (PR #2793) - * Track DB scheduling delay per-request (PR #2794, #2795) - * Sanity checking for user ids (PR #2797) - * better exception logging in callbackmetrics (PR #2809) - * Fix bugs in block metrics (PR #2810) - * Add some comments about the reactor tick time metric (PR #2816) - * Use a connection pool for the SimpleHttpClient (PR #2817) - * Remove unused/bitrotted MemoryDataStore (PR #2828) - * Make it possible to run tests against postgres (PR #2829) - * Factor out get_db_conn to HomeServer base class (PR #2830) - * Logging and metrics for the http pusher (PR #2833) - * Improve exception handling in persist_event (PR #2834) - * montoring metrics for number of cache evictions (PR #2844) - * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! - * Remove ability for AS users to call /events and /sync (PR #2948) - * use bcrypt.checkpw (PR #2949) Thanks to @krombel! - * Factor run_in_background out from preserve_fn (PR #2961) - * Add a metric which increments when a request is received (PR #2965) - * Improve caching for read_marker API (PR #2927) +* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are XXXXXX, (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856) +* Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) +* Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! +* Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! +* Better logging when login can't find a 3pid (PR #2744) +* add ?ts massaging for ASes (PR #2754) +* Remove 'verbosity'/'log_file' from generated cfg (PR #2755) +* Make indentation of generated log config consistent (PR #2762) +* Remove dead code related to default thumbnails (PR #2764) +* Update http request metrics before calling servlet (PR #2770) +* Do bcrypt hashing in a background thread (PR #2773) +* When using synctl with workers, don't start the main synapse automatically (PR #2774) +* Make Counter render floats (PR #2778) +* Store state groups separately from events (PR #2784) +* Reorganise request and block metrics (PR #2785) +* Metrics for number of RDATA commands received (PR #2786) +* Metrics for events processed in appservice and fed sender (PR #2787) +* Optimise LoggingContext creation and copying (PR #2792) +* Track db txn time in millisecs (PR #2793) +* Track DB scheduling delay per-request (PR #2794, #2795) +* Sanity checking for user ids (PR #2797) +* better exception logging in callbackmetrics (PR #2809) +* Fix bugs in block metrics (PR #2810) +* Add some comments about the reactor tick time metric (PR #2816) +* Use a connection pool for the SimpleHttpClient (PR #2817) +* Remove unused/bitrotted MemoryDataStore (PR #2828) +* Make it possible to run tests against postgres (PR #2829) +* Factor out get_db_conn to HomeServer base class (PR #2830) +* Logging and metrics for the http pusher (PR #2833) +* Improve exception handling in persist_event (PR #2834) +* montoring metrics for number of cache evictions (PR #2844) +* Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! +* Remove ability for AS users to call /events and /sync (PR #2948) +* use bcrypt.checkpw (PR #2949) Thanks to @krombel! +* Factor run_in_background out from preserve_fn (PR #2961) +* Add a metric which increments when a request is received (PR #2965) +* Improve caching for read_marker API (PR #2927) Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270) @@ -80,22 +72,22 @@ Synapse 0.27.0 begins the process of rationalising metric names. To enable a gra Bug fixes: - * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv! - * Check missing fields in event_from_pdu_json (PR #2745) - * Fix templating error with unban permission message (PR #2761) Thanks to @turt2live! - * Fix flaky test_rooms UTs (PR #2765) - * Fix publicised groups GET API (singular) over federation (PR #2772) - * Fix a logcontext leak in persist_events (PR #2790) - * Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796) - * fix SQL when searching all users (PR #2803) - * Fix server 500 on public rooms call when no rooms exist (PR #2827) - * Fix SQL for user search (PR #2831) - * Fix sql error in quarantine_media (PR #2837) - * Handle url_previews with no content-type (PR #2845) - * Add missing yield during 3pid signature checks (PR #2933) - * Fix race in sync when joining room (PR #2944) - * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) - * Fix scary-looking dns resolution errors (PR #2838) +* synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv! +* Check missing fields in event_from_pdu_json (PR #2745) +* Fix templating error with unban permission message (PR #2761) Thanks to @turt2live! +* Fix flaky test_rooms UTs (PR #2765) +* Fix publicised groups GET API (singular) over federation (PR #2772) +* Fix a logcontext leak in persist_events (PR #2790) +* Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796) +* fix SQL when searching all users (PR #2803) +* Fix server 500 on public rooms call when no rooms exist (PR #2827) +* Fix SQL for user search (PR #2831) +* Fix sql error in quarantine_media (PR #2837) +* Handle url_previews with no content-type (PR #2845) +* Add missing yield during 3pid signature checks (PR #2933) +* Fix race in sync when joining room (PR #2944) +* Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) +* Fix scary-looking dns resolution errors (PR #2838) From fb647164f2a008cdba56577ad0f41c4b4c1147d1 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 14 Mar 2018 16:20:36 +0000 Subject: [PATCH 05/49] Update CHANGES.rst --- CHANGES.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4d1b4ba13..53a97a769 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,10 +14,10 @@ Changes in synapse v0.27.0-rc1 (2018-03-14) Features: * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) * Add shiney new purge API. New implementation is:- - * Faster - * Supports clearing by timestamp - * Supports deleting of local events - * Is transactional (PR #2858,2867,2882, 2946,2962,2943) + * Faster + * Supports clearing by timestamp + * Supports deleting of local events + * Is transactional (PR #2858,2867,2882, 2946,2962,2943) * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813) * Add /room/{id}/event/{id} to synapse (PR #2766) From c33c1ceddd5da8195b38059dce31255209075ba2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 14 Mar 2018 11:09:08 -0600 Subject: [PATCH 06/49] OCD: Make the event_creator routes regex a code block All the others are code blocks, so this one should be to (currently it is a blockquote). Signed-off-by: Travis Ralston --- docs/workers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workers.rst b/docs/workers.rst index dee04bbf3..80f8d2181 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -230,7 +230,7 @@ file. For example:: ``synapse.app.event_creator`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Handles non-state event creation. It can handle REST endpoints matching: +Handles non-state event creation. It can handle REST endpoints matching:: ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send From 5ccb57d3ff6e3affd32cc796f625b2378483d871 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 14 Mar 2018 17:12:58 +0000 Subject: [PATCH 07/49] Update CHANGES.rst --- CHANGES.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 53a97a769..beac2117d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,16 +1,17 @@ Unreleased ========== + + +Changes in synapse v0.27.0-rc1 (2018-03-14) +=========================================== + synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. - -Changes in synapse v0.27.0-rc1 (2018-03-14) -=========================================== - Features: * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) * Add shiney new purge API. New implementation is:- @@ -27,7 +28,7 @@ Features: Changes: -* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. New workers are XXXXXX, (PR #2892, #2893, #2894, #2896, #2897, #2898, #2899, #2900, #2901, #2902, #2903, #2904, #2913, #2920, #2921, #2922, #2923, #2924, #2925, #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856) +* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more [here](https://github.com/matrix-org/synapse/blob/master/docs/workers.rst), (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995) * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! @@ -64,12 +65,9 @@ Changes: * Factor run_in_background out from preserve_fn (PR #2961) * Add a metric which increments when a request is received (PR #2965) * Improve caching for read_marker API (PR #2927) +* Add Measure block for persist_events (PR #2975) -Synapse 0.27.0 begins the process of rationalising metric names. To enable a graceful migration path, this release just adds new names for the metrics being renamed. A future release will remove the old ones. For more info see [here](https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.rst#block-and-response-metrics-renamed-for-0270) - - - Bug fixes: * synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv! From 10fdcf561d614235ba8f6ef3f980ac16ca0d7df7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Mar 2018 17:30:17 +0000 Subject: [PATCH 08/49] Fix up rst formatting --- CHANGES.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index beac2117d..42fe461b2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,12 +13,14 @@ This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. Features: + * Add support for a remote media repository backed by S3 (PR 2867, 2777, 2783, 2789, 2791, 2804, 2812,2814, 2857, 2868, 2767) -* Add shiney new purge API. New implementation is:- - * Faster - * Supports clearing by timestamp - * Supports deleting of local events - * Is transactional (PR #2858,2867,2882, 2946,2962,2943) +* Add shiney new purge API. New implementation is: + + * Faster + * Supports clearing by timestamp + * Supports deleting of local events + * Is transactional (PR #2858,2867,2882, 2946,2962,2943) * Let homeservers specify a whitelist for the format of 3PIDs that users are allowed to register with or add to their HS accounts. (PR #2813) * Add /room/{id}/event/{id} to synapse (PR #2766) @@ -28,7 +30,7 @@ Features: Changes: -* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more [here](https://github.com/matrix-org/synapse/blob/master/docs/workers.rst), (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995) +* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more at `docs/workers.rst `_ (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995) * Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) * Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! * Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! From 2059b8573f997e3210e0fa995d03f9168f8aeba6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Mar 2018 18:11:21 +0000 Subject: [PATCH 09/49] Update CHANGES.rst --- CHANGES.rst | 86 +++++++++++++++-------------------------------------- 1 file changed, 24 insertions(+), 62 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 42fe461b2..38cd3bec6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,80 +14,42 @@ reported to prometheus. See `docs/metrics-howto.rst `_ for full details (PR #2858, #2867, #2882, #2946, #2962, #2943) +* Add support for whitelisting 3PIDs that users can register. (PR #2813) +* Add ``/room/{id}/event/{id}`` API (PR #2766) +* Add an admin API to get all the media in a room (PR #2818) Thanks to @turt2live! +* Add ``federation_domain_whitelist`` option (PR #2820, #2821) Changes: -* Major refactor to move computation out of the main process and into a series of worker processes with a view to taking advantage of multicore machines. See more at `docs/workers.rst `_ (PR #2892-#2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995) -* Use StateResolutionHandler to resolve state in persist_events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) -* Adapt the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! -* Allow use of higher versions of saml2 (PR #2695) Thanks to @okurz! -* Better logging when login can't find a 3pid (PR #2744) -* add ?ts massaging for ASes (PR #2754) -* Remove 'verbosity'/'log_file' from generated cfg (PR #2755) -* Make indentation of generated log config consistent (PR #2762) -* Remove dead code related to default thumbnails (PR #2764) -* Update http request metrics before calling servlet (PR #2770) -* Do bcrypt hashing in a background thread (PR #2773) +* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) +* Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) +* Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! +* No longer require a specific version of saml2 (PR #2695) Thanks to @okurz! +* Remove ``verbosity``/``log_file`` from generated config (PR #2755) +* Add and improve metrics and logging (PR #2770, #2778, #2785, #2786, #2787, #2793, #2794, #2795, #2809, #2810, #2833, #2834, #2844, #2965, #2927, #2975, #2790, #2796, #2838) * When using synctl with workers, don't start the main synapse automatically (PR #2774) -* Make Counter render floats (PR #2778) -* Store state groups separately from events (PR #2784) -* Reorganise request and block metrics (PR #2785) -* Metrics for number of RDATA commands received (PR #2786) -* Metrics for events processed in appservice and fed sender (PR #2787) -* Optimise LoggingContext creation and copying (PR #2792) -* Track db txn time in millisecs (PR #2793) -* Track DB scheduling delay per-request (PR #2794, #2795) -* Sanity checking for user ids (PR #2797) -* better exception logging in callbackmetrics (PR #2809) -* Fix bugs in block metrics (PR #2810) -* Add some comments about the reactor tick time metric (PR #2816) -* Use a connection pool for the SimpleHttpClient (PR #2817) -* Remove unused/bitrotted MemoryDataStore (PR #2828) -* Make it possible to run tests against postgres (PR #2829) -* Factor out get_db_conn to HomeServer base class (PR #2830) -* Logging and metrics for the http pusher (PR #2833) -* Improve exception handling in persist_event (PR #2834) -* montoring metrics for number of cache evictions (PR #2844) +* Minor performance improvements (PR #2773, #2792) +* Use a connection pool for non-federation outbound connections (PR #2817) +* Make it possible to run unit tests against postgres (PR #2829) * Update pynacl dependency to 1.2.1 or higher (PR #2888) Thanks to @bachp! * Remove ability for AS users to call /events and /sync (PR #2948) -* use bcrypt.checkpw (PR #2949) Thanks to @krombel! -* Factor run_in_background out from preserve_fn (PR #2961) -* Add a metric which increments when a request is received (PR #2965) -* Improve caching for read_marker API (PR #2927) -* Add Measure block for persist_events (PR #2975) - +* Use bcrypt.checkpw (PR #2949) Thanks to @krombel! Bug fixes: -* synapse/config/password_auth_providers: Fixed bracket typo (PR #2683) Thanks to @seckrv! -* Check missing fields in event_from_pdu_json (PR #2745) -* Fix templating error with unban permission message (PR #2761) Thanks to @turt2live! -* Fix flaky test_rooms UTs (PR #2765) +* Fix broken ``ldap_config`` config option (PR #2683) Thanks to @seckrv! +* Fix error message when user is not allowed to unban (PR #2761) Thanks to @turt2live! * Fix publicised groups GET API (singular) over federation (PR #2772) -* Fix a logcontext leak in persist_events (PR #2790) -* Fix 'NoneType' object has no attribute 'writeHeaders' (PR #2796) -* fix SQL when searching all users (PR #2803) -* Fix server 500 on public rooms call when no rooms exist (PR #2827) -* Fix SQL for user search (PR #2831) -* Fix sql error in quarantine_media (PR #2837) -* Handle url_previews with no content-type (PR #2845) -* Add missing yield during 3pid signature checks (PR #2933) -* Fix race in sync when joining room (PR #2944) +* Fix user directory when using ``user_directory_search_all_users`` config option (PR #2803, #2831) +* Fix error on ``/publicRooms`` when no rooms exist (PR #2827) +* Fix bug in quarantine_media (PR #2837) +* Fix url_previews when no Content-Type is returned from URL (PR #2845) +* Fix rare race in sync API when joining room (PR #2944) * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) -* Fix scary-looking dns resolution errors (PR #2838) From 7d26591048ac5ab6263ddb24a840c58edfd85f50 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 15 Mar 2018 10:33:24 +0000 Subject: [PATCH 10/49] Update CHANGES.rst --- CHANGES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 38cd3bec6..a50d3ea5b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,12 +6,13 @@ Unreleased Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== -synctl no longer starts the main synapse when using ``-a`` option with workers. -A new worker file should be added with ``worker_app: synapse.app.homeserver``. +The common case for running Synapse is not to run separate workers, but for those that do, synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. +Note, old metrics will be removed from the 0.28.0 release. + Features: * Add ability for ASes to override message send time (PR #2754) From 7367a4a8239a04047fa648694c08f3d1c63dd3ee Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 15 Mar 2018 10:33:52 +0000 Subject: [PATCH 11/49] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a50d3ea5b..48791cec3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,7 @@ Unreleased Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== -The common case for running Synapse is not to run separate workers, but for those that do, synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. +The common case for running Synapse is not to run separate workers, but for those that do, be aware that synctl no longer starts the main synapse when using ``-a`` option with workers. A new worker file should be added with ``worker_app: synapse.app.homeserver``. This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. From b29d1abab6ca42f693e3b83ab31440fee95d00f2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 15 Mar 2018 10:34:15 +0000 Subject: [PATCH 12/49] Update CHANGES.rst --- CHANGES.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 48791cec3..931ecb4d3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,3 @@ -Unreleased -========== - - - Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== From 068c21ab102b22aeac7fc9aa1f4d2d229a807872 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 15 Mar 2018 10:36:31 +0000 Subject: [PATCH 13/49] CHANGES.rst: reword metric deprecation --- CHANGES.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 931ecb4d3..e25327f28 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,8 +5,7 @@ The common case for running Synapse is not to run separate workers, but for thos This release also begins the process of renaming a number of the metrics reported to prometheus. See `docs/metrics-howto.rst `_. - -Note, old metrics will be removed from the 0.28.0 release. +Note that the v0.28.0 release will remove the deprecated metric names. Features: From 0ad5125814dc18a79423740ac54f96e16a427758 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 15 Mar 2018 11:05:42 +0000 Subject: [PATCH 14/49] Update purge_history_api.rst clarify that `purge_history` will not purge state --- docs/admin_api/purge_history_api.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/admin_api/purge_history_api.rst b/docs/admin_api/purge_history_api.rst index ea2922da5..2da833c82 100644 --- a/docs/admin_api/purge_history_api.rst +++ b/docs/admin_api/purge_history_api.rst @@ -16,9 +16,11 @@ including an ``access_token`` of a server admin. By default, events sent by local users are not deleted, as they may represent the only copies of this content in existence. (Events sent by remote users are -deleted, and room state data before the cutoff is always removed). +deleted.) -To delete local events as well, set ``delete_local_events`` in the body: +Room state data (such as joins, leaves, topic) is always preserved. + +To delete local message events as well, set ``delete_local_events`` in the body: .. code:: json From 5ea624b0f5f6ad11b1640729b96272f8e7c94d46 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 15 Mar 2018 11:48:35 +0000 Subject: [PATCH 15/49] CONTRIBUTING.rst: fix CI info --- CONTRIBUTING.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a88647ca..c6ee16efc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,8 +30,12 @@ use github's pull request workflow to review the contribution, and either ask you to make any refinements needed or merge it and make them ourselves. The changes will then land on master when we next do a release. -We use Jenkins for continuous integration (http://matrix.org/jenkins), and -typically all pull requests get automatically tested Jenkins: if your change breaks the build, Jenkins will yell about it in #matrix-dev:matrix.org so please lurk there and keep an eye open. +We use `Jenkins `_ and +`Travis `_ for continuous +integration. All pull requests to synapse get automatically tested by Travis; +the Jenkins builds require an adminstrator to start them. If your change +breaks the build, this will be shown in github, so please keep an eye on the +pull request for feedback. Code style ~~~~~~~~~~ @@ -115,4 +119,4 @@ can't be accepted. Git makes this trivial - just use the -s flag when you do Conclusion ~~~~~~~~~~ -That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! \ No newline at end of file +That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! From ddb00efc1ddec646d02e8def6053003f04d077d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 14:41:30 +0000 Subject: [PATCH 16/49] Bump version number --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index ef8853bd2..d72d88b47 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.26.0" +__version__ = "0.27.0-rc1" From 7c7706f42b56dd61f5eb17679aa12247f7058ed5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 15:40:13 +0000 Subject: [PATCH 17/49] Fix bug where state cache used lots of memory The state cache bases its size on the sum of the size of entries. The size of the entry is calculated once on insertion, so it is important that the size of entries does not change. The DictionaryCache modified the entries size, which caused the state cache to incorrectly think it was smaller than it actually was. --- synapse/util/caches/dictionary_cache.py | 6 +++++- synapse/util/caches/lrucache.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index d4105822b..1709e8b42 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -132,9 +132,13 @@ class DictionaryCache(object): self._update_or_insert(key, value, known_absent) def _update_or_insert(self, key, value, known_absent): - entry = self.cache.setdefault(key, DictionaryEntry(False, set(), {})) + # We pop and reinsert as we need to tell the cache the size may have + # changed + + entry = self.cache.pop(key, DictionaryEntry(False, set(), {})) entry.value.update(value) entry.known_absent.update(known_absent) + self.cache[key] = entry def _insert(self, key, value, known_absent): self.cache[key] = DictionaryEntry(True, known_absent, value) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index f088dd430..a4bf8fa6a 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -154,14 +154,14 @@ class LruCache(object): def cache_set(key, value, callbacks=[]): node = cache.get(key, None) if node is not None: - if value != node.value: + if node.callbacks and value != node.value: for cb in node.callbacks: cb() node.callbacks.clear() - if size_callback: - cached_cache_len[0] -= size_callback(node.value) - cached_cache_len[0] += size_callback(value) + if size_callback: + cached_cache_len[0] -= size_callback(node.value) + cached_cache_len[0] += size_callback(value) node.callbacks.update(callbacks) From 9cf519769b70311d06b43a97e47cfb5a06e6ade4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 17:48:42 +0000 Subject: [PATCH 18/49] Use .iter* to avoid copies in StateHandler --- synapse/state.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 932f60250..18386556a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -132,7 +132,7 @@ class StateHandler(object): state_map = yield self.store.get_events(state.values(), get_prev_content=False) state = { - key: state_map[e_id] for key, e_id in state.items() if e_id in state_map + key: state_map[e_id] for key, e_id in state.iteritems() if e_id in state_map } defer.returnValue(state) @@ -378,7 +378,7 @@ class StateHandler(object): new_state = resolve_events_with_state_map(state_set_ids, state_map) new_state = { - key: state_map[ev_id] for key, ev_id in new_state.items() + key: state_map[ev_id] for key, ev_id in new_state.iteritems() } return new_state @@ -458,15 +458,15 @@ class StateResolutionHandler(object): # build a map from state key to the event_ids which set that state. # dict[(str, str), set[str]) state = {} - for st in state_groups_ids.values(): - for key, e_id in st.items(): + for st in state_groups_ids.itervalues(): + for key, e_id in st.iteritems(): state.setdefault(key, set()).add(e_id) # build a map from state key to the event_ids which set that state, # including only those where there are state keys in conflict. conflicted_state = { k: list(v) - for k, v in state.items() + for k, v in state.iteritems() if len(v) > 1 } @@ -480,7 +480,7 @@ class StateResolutionHandler(object): ) else: new_state = { - key: e_ids.pop() for key, e_ids in state.items() + key: e_ids.pop() for key, e_ids in state.iteritems() } # if the new state matches any of the input state groups, we can @@ -488,8 +488,8 @@ class StateResolutionHandler(object): # which will be used as a cache key for future resolutions, but # not get persisted. state_group = None - new_state_event_ids = frozenset(new_state.values()) - for sg, events in state_groups_ids.items(): + new_state_event_ids = frozenset(new_state.iteritems()) + for sg, events in state_groups_ids.iteritems(): if new_state_event_ids == frozenset(e_id for e_id in events): state_group = sg break @@ -702,7 +702,7 @@ def _resolve_with_state(unconflicted_state_ids, conflicted_state_ds, auth_event_ auth_events = { key: state_map[ev_id] - for key, ev_id in auth_event_ids.items() + for key, ev_id in auth_event_ids.iteritems() if ev_id in state_map } @@ -740,7 +740,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.items(): + for key, events in conflicted_state.iteritems(): if key[0] == EventTypes.JoinRules: logger.debug("Resolving conflicted join rules %r", events) resolved_state[key] = _resolve_auth_events( @@ -750,7 +750,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.items(): + for key, events in conflicted_state.iteritems(): if key[0] == EventTypes.Member: logger.debug("Resolving conflicted member lists %r", events) resolved_state[key] = _resolve_auth_events( @@ -760,7 +760,7 @@ def _resolve_state_events(conflicted_state, auth_events): auth_events.update(resolved_state) - for key, events in conflicted_state.items(): + for key, events in conflicted_state.iteritems(): if key not in resolved_state: logger.debug("Resolving conflicted state %r:%r", key, events) resolved_state[key] = _resolve_normal_events( From 926ba76e23ea9d55638baff541cdfaeb9e01ac47 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 23:43:31 +0000 Subject: [PATCH 19/49] Replace ujson with simplejson --- synapse/api/filtering.py | 2 +- synapse/handlers/e2e_keys.py | 2 +- synapse/handlers/message.py | 6 +++--- synapse/http/server.py | 5 ++--- synapse/replication/tcp/commands.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v1/preview_url_resource.py | 2 +- synapse/storage/account_data.py | 2 +- synapse/storage/background_updates.py | 2 +- synapse/storage/deviceinbox.py | 12 ++++++------ synapse/storage/devices.py | 2 +- synapse/storage/end_to_end_keys.py | 2 +- synapse/storage/event_push_actions.py | 2 +- synapse/storage/events.py | 4 ++-- synapse/storage/receipts.py | 2 +- synapse/storage/room.py | 2 +- synapse/storage/roommember.py | 2 +- synapse/storage/schema/delta/25/fts.py | 4 ++-- synapse/storage/schema/delta/27/ts.py | 2 +- synapse/storage/schema/delta/31/search_update.py | 4 ++-- synapse/storage/schema/delta/33/event_fields.py | 4 ++-- synapse/storage/search.py | 2 +- synapse/storage/tags.py | 2 +- synapse/storage/transactions.py | 2 +- 25 files changed, 37 insertions(+), 38 deletions(-) diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 83206348e..db43219d2 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -17,7 +17,7 @@ from synapse.storage.presence import UserPresenceState from synapse.types import UserID, RoomID from twisted.internet import defer -import ujson as json +import simplejson as json import jsonschema from jsonschema import FormatChecker diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 668a90e49..ce2c87e40 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ujson as json +import simplejson as json import logging from canonicaljson import encode_canonical_json diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 21f1717dd..d7413833e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,7 +35,7 @@ from canonicaljson import encode_canonical_json import logging import random -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -561,8 +561,8 @@ class MessageHandler(BaseHandler): # Ensure that we can round trip before trying to persist in db try: - dump = ujson.dumps(unfreeze(event.content)) - ujson.loads(dump) + dump = simplejson.dumps(unfreeze(event.content)) + simplejson.loads(dump) except Exception: logger.exception("Failed to encode content: %r", event.content) raise diff --git a/synapse/http/server.py b/synapse/http/server.py index 25466cd29..f1e9002e4 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -36,7 +36,7 @@ from twisted.web.util import redirectTo import collections import logging import urllib -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -370,8 +370,7 @@ def respond_with_json(request, code, json_object, send_cors=False, if canonical_json or synapse.events.USE_FROZEN_DICTS: json_bytes = encode_canonical_json(json_object) else: - # ujson doesn't like frozen_dicts. - json_bytes = ujson.dumps(json_object, ensure_ascii=False) + json_bytes = simplejson.dumps(json_object) return respond_with_json_bytes( request, code, json_bytes, diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 171227cce..9633404f7 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -19,7 +19,7 @@ allowed to be sent by which side. """ import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 75b735b47..80989731f 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -29,7 +29,7 @@ from synapse.http.servlet import ( import logging import urllib -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index a0a8e4b8e..eb91c0b29 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit import itertools import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 40d2e664e..a413cb622 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -35,7 +35,7 @@ import os import re import fnmatch import cgi -import ujson as json +import simplejson as json import urlparse import itertools import datetime diff --git a/synapse/storage/account_data.py b/synapse/storage/account_data.py index 56a0bde54..40a2ad8d0 100644 --- a/synapse/storage/account_data.py +++ b/synapse/storage/account_data.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 11a1b942f..8f3bff311 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -19,7 +19,7 @@ from . import engines from twisted.internet import defer -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index 548e795da..a879e5bfc 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -14,7 +14,7 @@ # limitations under the License. import logging -import ujson +import simplejson from twisted.internet import defer @@ -85,7 +85,7 @@ class DeviceInboxStore(BackgroundUpdateStore): ) rows = [] for destination, edu in remote_messages_by_destination.items(): - edu_json = ujson.dumps(edu) + edu_json = simplejson.dumps(edu) rows.append((destination, stream_id, now_ms, edu_json)) txn.executemany(sql, rows) @@ -177,7 +177,7 @@ class DeviceInboxStore(BackgroundUpdateStore): " WHERE user_id = ?" ) txn.execute(sql, (user_id,)) - message_json = ujson.dumps(messages_by_device["*"]) + message_json = simplejson.dumps(messages_by_device["*"]) for row in txn: # Add the message for all devices for this user on this # server. @@ -199,7 +199,7 @@ class DeviceInboxStore(BackgroundUpdateStore): # Only insert into the local inbox if the device exists on # this server device = row[0] - message_json = ujson.dumps(messages_by_device[device]) + message_json = simplejson.dumps(messages_by_device[device]) messages_json_for_user[device] = message_json if messages_json_for_user: @@ -253,7 +253,7 @@ class DeviceInboxStore(BackgroundUpdateStore): messages = [] for row in txn: stream_pos = row[0] - messages.append(ujson.loads(row[1])) + messages.append(simplejson.loads(row[1])) if len(messages) < limit: stream_pos = current_stream_id return (messages, stream_pos) @@ -389,7 +389,7 @@ class DeviceInboxStore(BackgroundUpdateStore): messages = [] for row in txn: stream_pos = row[0] - messages.append(ujson.loads(row[1])) + messages.append(simplejson.loads(row[1])) if len(messages) < limit: stream_pos = current_stream_id return (messages, stream_pos) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index bd2effdf3..712106b83 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import ujson as json +import simplejson as json from twisted.internet import defer diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 2cebb203c..ff8538ddf 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -17,7 +17,7 @@ from twisted.internet import defer from synapse.util.caches.descriptors import cached from canonicaljson import encode_canonical_json -import ujson as json +import simplejson as json from ._base import SQLBaseStore diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 8efe2fd4b..575d710d5 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -21,7 +21,7 @@ from synapse.types import RoomStreamToken from .stream import lower_bound import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index d08f7571d..bcca56329 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -38,7 +38,7 @@ from functools import wraps import synapse.metrics import logging -import ujson as json +import simplejson as json # these are only included to make the type annotations work from synapse.events import EventBase # noqa: F401 @@ -56,7 +56,7 @@ event_counter = metrics.register_counter( def encode_json(json_object): if USE_FROZEN_DICTS: - # ujson doesn't like frozen_dicts + # simplejson doesn't like frozen_dicts return encode_canonical_json(json_object) else: return json.dumps(json_object, ensure_ascii=False) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 12b3cc7f5..2c3aa3369 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -20,7 +20,7 @@ from synapse.util.caches.stream_change_cache import StreamChangeCache from twisted.internet import defer import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 23688430b..2051d8506 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -23,7 +23,7 @@ from .engines import PostgresEngine, Sqlite3Engine import collections import logging -import ujson as json +import simplejson as json import re logger = logging.getLogger(__name__) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 3e77fd390..c1ca29928 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -27,7 +27,7 @@ from synapse.api.constants import Membership, EventTypes from synapse.types import get_domain_from_id import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/schema/delta/25/fts.py b/synapse/storage/schema/delta/25/fts.py index 4269ac69a..e7351c3ae 100644 --- a/synapse/storage/schema/delta/25/fts.py +++ b/synapse/storage/schema/delta/25/fts.py @@ -17,7 +17,7 @@ import logging from synapse.storage.prepare_database import get_statements from synapse.storage.engines import PostgresEngine, Sqlite3Engine -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def run_create(cur, database_engine, *args, **kwargs): "max_stream_id_exclusive": max_stream_id + 1, "rows_inserted": 0, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/schema/delta/27/ts.py b/synapse/storage/schema/delta/27/ts.py index 71b12a273..c0176c41e 100644 --- a/synapse/storage/schema/delta/27/ts.py +++ b/synapse/storage/schema/delta/27/ts.py @@ -16,7 +16,7 @@ import logging from synapse.storage.prepare_database import get_statements -import ujson +import simplejson logger = logging.getLogger(__name__) diff --git a/synapse/storage/schema/delta/31/search_update.py b/synapse/storage/schema/delta/31/search_update.py index 470ae0c00..fe6b7d196 100644 --- a/synapse/storage/schema/delta/31/search_update.py +++ b/synapse/storage/schema/delta/31/search_update.py @@ -16,7 +16,7 @@ from synapse.storage.engines import PostgresEngine from synapse.storage.prepare_database import get_statements import logging -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def run_create(cur, database_engine, *args, **kwargs): "rows_inserted": 0, "have_added_indexes": False, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/schema/delta/33/event_fields.py b/synapse/storage/schema/delta/33/event_fields.py index 83066cccc..1e002f9db 100644 --- a/synapse/storage/schema/delta/33/event_fields.py +++ b/synapse/storage/schema/delta/33/event_fields.py @@ -15,7 +15,7 @@ from synapse.storage.prepare_database import get_statements import logging -import ujson +import simplejson logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def run_create(cur, database_engine, *args, **kwargs): "max_stream_id_exclusive": max_stream_id + 1, "rows_inserted": 0, } - progress_json = ujson.dumps(progress) + progress_json = simplejson.dumps(progress) sql = ( "INSERT into background_updates (update_name, progress_json)" diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 479b04c63..c19e4ea44 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -21,7 +21,7 @@ from synapse.storage.engines import PostgresEngine, Sqlite3Engine import logging import re -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/tags.py b/synapse/storage/tags.py index bff73f3f0..982a50052 100644 --- a/synapse/storage/tags.py +++ b/synapse/storage/tags.py @@ -17,7 +17,7 @@ from ._base import SQLBaseStore from synapse.util.caches.descriptors import cached from twisted.internet import defer -import ujson as json +import simplejson as json import logging logger = logging.getLogger(__name__) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 8f61f7ffa..f825264ea 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -23,7 +23,7 @@ from canonicaljson import encode_canonical_json from collections import namedtuple import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) From ba48755d56ba134590f7b5dcb5da44371639fc4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 23:57:26 +0000 Subject: [PATCH 20/49] Bump version and changelog --- CHANGES.rst | 8 ++++++++ synapse/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index dcf9adc95..38ba4249b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +Changes in synapse v0.26.1 (2018-03-15) +======================================= + +Bug fixes: + +* Fix bug where an invalid event caused server to stop functioning correctly, + due to parsing and serializing bugs in ujson library. + Changes in synapse v0.26.0 (2018-01-05) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index ef8853bd2..80efad3a6 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.26.0" +__version__ = "0.26.1" From 5b631ff41a3230d8b6ed662e5aeb406c86989f9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Mar 2018 00:07:08 +0000 Subject: [PATCH 21/49] Remove wrong comment --- synapse/storage/events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index bcca56329..73658a992 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -56,7 +56,6 @@ event_counter = metrics.register_counter( def encode_json(json_object): if USE_FROZEN_DICTS: - # simplejson doesn't like frozen_dicts return encode_canonical_json(json_object) else: return json.dumps(json_object, ensure_ascii=False) From 0092cf38ae95173519ffcc403b20b4fd2dec5c8d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 16 Mar 2018 00:11:58 +0000 Subject: [PATCH 22/49] Newline --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 38ba4249b..3b3292823 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,7 @@ Bug fixes: * Fix bug where an invalid event caused server to stop functioning correctly, due to parsing and serializing bugs in ujson library. + Changes in synapse v0.26.0 (2018-01-05) ======================================= From a8ce159be43560e9aea8f3be65110eea49d1f50e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 15 Mar 2018 23:38:43 +0000 Subject: [PATCH 23/49] Replace some ujson with simplejson to make it work --- synapse/http/server.py | 3 ++- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/events.py | 2 +- synapse/storage/events_worker.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 4b567215c..3c7a0ef97 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -38,6 +38,7 @@ import collections import logging import urllib import ujson +import simplejson logger = logging.getLogger(__name__) @@ -462,7 +463,7 @@ def respond_with_json(request, code, json_object, send_cors=False, json_bytes = encode_canonical_json(json_object) else: # ujson doesn't like frozen_dicts. - json_bytes = ujson.dumps(json_object, ensure_ascii=False) + json_bytes = simplejson.dumps(json_object) return respond_with_json_bytes( request, code, json_bytes, diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index a0a8e4b8e..eb91c0b29 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -33,7 +33,7 @@ from ._base import set_timeline_upper_limit import itertools import logging -import ujson as json +import simplejson as json logger = logging.getLogger(__name__) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 389087817..9fc65229f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -38,7 +38,7 @@ from functools import wraps import synapse.metrics import logging -import ujson as json +import simplejson as json # these are only included to make the type annotations work from synapse.events import EventBase # noqa: F401 diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 86c3b48ad..2e23dd78b 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -28,7 +28,7 @@ from synapse.api.errors import SynapseError from collections import namedtuple import logging -import ujson as json +import simplejson as json # these are only included to make the type annotations work from synapse.events import EventBase # noqa: F401 From 38f952b9bc96cf72d96bab3510e8f428a8247542 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 19 Mar 2018 09:27:36 +0000 Subject: [PATCH 24/49] spell out not to massively increase bcrypt rounds --- synapse/config/registration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 336959094..c5384b3ad 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -77,7 +77,9 @@ class RegistrationConfig(Config): # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. - # The default number of rounds is 12. + # The default number is 12 (which equates to 2^12 rounds). + # N.B. that increasing this will exponentially increase the time required + # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. bcrypt_rounds: 12 # Allows users to register as guests without a password/email/etc, and From 9a0d783c113ae74c55e409d33219cd77f3662b9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 19 Mar 2018 11:35:53 +0000 Subject: [PATCH 25/49] Add comments --- synapse/util/caches/lrucache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index a4bf8fa6a..1c5a98209 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -154,11 +154,18 @@ class LruCache(object): def cache_set(key, value, callbacks=[]): node = cache.get(key, None) if node is not None: + # We sometimes store large objects, e.g. dicts, which cause + # the inequality check to take a long time. So let's only do + # the check if we have some callbacks to call. if node.callbacks and value != node.value: for cb in node.callbacks: cb() node.callbacks.clear() + # We don't bother to protect this by value != node.value as + # generally size_callback will be cheap compared with equality + # checks. (For example, taking the size of two dicts is quicker + # than comparing them for equality.) if size_callback: cached_cache_len[0] -= size_callback(node.value) cached_cache_len[0] += size_callback(value) From 1a3aa957ca1164268c9bbbd564491c9d46fbb7d2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 19 Mar 2018 15:11:00 +0000 Subject: [PATCH 26/49] Update CHANGES.rst --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e25327f28..55358287a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +Changes in synapse v0.27.0-rc2 (2018-03-19) +=========================================== +Bugs: + +* Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005) + + Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== From c384705ee877ba40e1a736b68ebe07434662cbaa Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 19 Mar 2018 15:11:58 +0000 Subject: [PATCH 27/49] Update __init__.py bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index d72d88b47..71b409ba8 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.0-rc1" +__version__ = "0.27.0-rc2" From 610accbb7f58d7a07007dedaa054c8ad7a9f5851 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 19 Mar 2018 16:06:02 +0000 Subject: [PATCH 28/49] Fix replication after switch to simplejson Turns out that simplejson serialises namedtuple's as dictionaries rather than tuples by default. --- synapse/replication/tcp/commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 9633404f7..1b4b65721 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -107,7 +107,7 @@ class RdataCommand(Command): return " ".join(( self.stream_name, str(self.token) if self.token is not None else "batch", - json.dumps(self.row), + json.dumps(self.row, namedtuple_as_object=False), )) @@ -301,7 +301,9 @@ class InvalidateCacheCommand(Command): return cls(cache_func, json.loads(keys_json)) def to_line(self): - return " ".join((self.cache_func, json.dumps(self.keys))) + return " ".join(( + self.cache_func, json.dumps(self.keys, namedtuple_as_object=False) + )) class UserIpCommand(Command): From 9aa5a0af514b3f2719dd01aa221039c5ba1c0c80 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 20 Mar 2018 09:58:13 +0000 Subject: [PATCH 29/49] Explicitly use simplejson --- synapse/replication/tcp/commands.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/replication/tcp/commands.py b/synapse/replication/tcp/commands.py index 1b4b65721..0005ad587 100644 --- a/synapse/replication/tcp/commands.py +++ b/synapse/replication/tcp/commands.py @@ -19,7 +19,7 @@ allowed to be sent by which side. """ import logging -import simplejson as json +import simplejson logger = logging.getLogger(__name__) @@ -100,14 +100,14 @@ class RdataCommand(Command): return cls( stream_name, None if token == "batch" else int(token), - json.loads(row_json) + simplejson.loads(row_json) ) def to_line(self): return " ".join(( self.stream_name, str(self.token) if self.token is not None else "batch", - json.dumps(self.row, namedtuple_as_object=False), + simplejson.dumps(self.row, namedtuple_as_object=False), )) @@ -298,11 +298,11 @@ class InvalidateCacheCommand(Command): def from_line(cls, line): cache_func, keys_json = line.split(" ", 1) - return cls(cache_func, json.loads(keys_json)) + return cls(cache_func, simplejson.loads(keys_json)) def to_line(self): return " ".join(( - self.cache_func, json.dumps(self.keys, namedtuple_as_object=False) + self.cache_func, simplejson.dumps(self.keys, namedtuple_as_object=False) )) @@ -327,14 +327,14 @@ class UserIpCommand(Command): def from_line(cls, line): user_id, jsn = line.split(" ", 1) - access_token, ip, user_agent, device_id, last_seen = json.loads(jsn) + access_token, ip, user_agent, device_id, last_seen = simplejson.loads(jsn) return cls( user_id, access_token, ip, user_agent, device_id, last_seen ) def to_line(self): - return self.user_id + " " + json.dumps(( + return self.user_id + " " + simplejson.dumps(( self.access_token, self.ip, self.user_agent, self.device_id, self.last_seen, )) From f5aa027c2f9d836df7b98da6c1c5fee6b22d0a3d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 20 Mar 2018 15:06:22 +0000 Subject: [PATCH 30/49] Update CHANGES.rst rearrange ordering of releases to match chronology --- CHANGES.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 788150bcf..bac78d3c1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,21 @@ Changes in synapse v0.27.0-rc2 (2018-03-19) =========================================== -Bugs: + +Pulls in v0.26.1 + +Bug fixes: * Fix bug introduced in v0.27.0-rc1 that causes much increased memory usage in state cache (PR #3005) + + +Changes in synapse v0.26.1 (2018-03-15) +======================================= + +Bug fixes: + * Fix bug where an invalid event caused server to stop functioning correctly, - due to parsing and serializing bugs in ujson library. (pulled in from 0.26.1) + due to parsing and serializing bugs in ujson library (PR #3008) + Changes in synapse v0.27.0-rc1 (2018-03-14) =========================================== @@ -55,16 +66,6 @@ Bug fixes: * Fix slow event search, switch back from GIST to GIN indexes (PR #2769, #2848) -Changes in synapse v0.26.1 (2018-03-15) -======================================= - -Bug fixes: - -* Fix bug where an invalid event caused server to stop functioning correctly, - due to parsing and serializing bugs in ujson library. - - - Changes in synapse v0.26.0 (2018-01-05) ======================================= From 1c41b05c8c98f0b9157c791b5b8ebf5f9fe85acf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Mar 2018 17:46:26 +0000 Subject: [PATCH 31/49] Add Cache-Control headers to all JSON APIs It is especially important that sync requests don't get cached, as if a sync returns the same token given then the client will call sync with the same parameters again. If the previous response was cached it will get reused, resulting in the client tight looping making the same request and never making any progress. In general, clients will expect to get up to date data when requesting APIs, and so its safer to do a blanket no cache policy than only whitelisting APIs that we know will break things if they get cached. --- synapse/http/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/http/server.py b/synapse/http/server.py index 1551db239..f19c068ef 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -488,6 +488,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) + request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") if send_cors: set_cors_headers(request) From fde8e8f09fb0e62d4670e33b75319b5bd57f484f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 22 Mar 2018 11:42:16 +0000 Subject: [PATCH 32/49] Fix s/iteriterms/itervalues --- synapse/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/state.py b/synapse/state.py index 18386556a..a7f20350f 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -488,7 +488,7 @@ class StateResolutionHandler(object): # which will be used as a cache key for future resolutions, but # not get persisted. state_group = None - new_state_event_ids = frozenset(new_state.iteritems()) + new_state_event_ids = frozenset(new_state.itervalues()) for sg, events in state_groups_ids.iteritems(): if new_state_event_ids == frozenset(e_id for e_id in events): state_group = sg From 8cbbfaefc1fc597cbbef52a10dbfb8ecd4d8a8cd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 23 Mar 2018 10:32:50 +0000 Subject: [PATCH 33/49] 404 correctly on missing paths via NoResource fixes https://github.com/matrix-org/synapse/issues/2043 and https://github.com/matrix-org/synapse/issues/2029 --- synapse/app/appservice.py | 4 ++-- synapse/app/client_reader.py | 4 ++-- synapse/app/event_creator.py | 4 ++-- synapse/app/federation_reader.py | 4 ++-- synapse/app/federation_sender.py | 4 ++-- synapse/app/frontend_proxy.py | 4 ++-- synapse/app/homeserver.py | 4 ++-- synapse/app/media_repository.py | 4 ++-- synapse/app/pusher.py | 4 ++-- synapse/app/synchrotron.py | 4 ++-- synapse/app/user_dir.py | 4 ++-- synapse/util/httpresourcetree.py | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index c6fe4516d..f2540023a 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -36,7 +36,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.appservice") @@ -64,7 +64,7 @@ class AppserviceServer(HomeServer): if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(self) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 0a8ce9bc6..267d34c88 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -44,7 +44,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.client_reader") @@ -88,7 +88,7 @@ class ClientReaderServer(HomeServer): "/_matrix/client/api/v1": resource, }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 172e989b5..b915d12d5 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -52,7 +52,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.event_creator") @@ -104,7 +104,7 @@ class EventCreatorServer(HomeServer): "/_matrix/client/api/v1": resource, }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 20d157911..c1dc66dd1 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -41,7 +41,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.federation_reader") @@ -77,7 +77,7 @@ class FederationReaderServer(HomeServer): FEDERATION_PREFIX: TransportLayerServer(self), }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index f760826d2..0cc333151 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -42,7 +42,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.federation_sender") @@ -91,7 +91,7 @@ class FederationSenderServer(HomeServer): if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(self) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index 816c080d1..de889357c 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -44,7 +44,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.frontend_proxy") @@ -142,7 +142,7 @@ class FrontendProxyServer(HomeServer): "/_matrix/client/api/v1": resource, }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e477c7ced..c00afbba2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,7 @@ from synapse.util.rlimit import change_resource_limit from synapse.util.versionstring import get_version_string from twisted.application import service from twisted.internet import defer, reactor -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -126,7 +126,7 @@ class SynapseHomeServer(HomeServer): if WEB_CLIENT_PREFIX in resources: root_resource = RootRedirect(WEB_CLIENT_PREFIX) else: - root_resource = Resource() + root_resource = NoResource() root_resource = create_resource_tree(resources, root_resource) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 84c5791b3..fc8282bbc 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -43,7 +43,7 @@ from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.media_repository") @@ -84,7 +84,7 @@ class MediaRepositoryServer(HomeServer): ), }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 98a4a7c62..d5c3a8519 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -37,7 +37,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.pusher") @@ -94,7 +94,7 @@ class PusherServer(HomeServer): if name == "metrics": resources[METRICS_PREFIX] = MetricsResource(self) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index abe91dcfb..508b66613 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -56,7 +56,7 @@ from synapse.util.manhole import manhole from synapse.util.stringutils import random_string from synapse.util.versionstring import get_version_string from twisted.internet import defer, reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.synchrotron") @@ -269,7 +269,7 @@ class SynchrotronServer(HomeServer): "/_matrix/client/api/v1": resource, }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 494ccb702..5f845e80d 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -43,7 +43,7 @@ from synapse.util.logcontext import LoggingContext, preserve_fn from synapse.util.manhole import manhole from synapse.util.versionstring import get_version_string from twisted.internet import reactor -from twisted.web.resource import Resource +from twisted.web.resource import NoResource logger = logging.getLogger("synapse.app.user_dir") @@ -116,7 +116,7 @@ class UserDirectoryServer(HomeServer): "/_matrix/client/api/v1": resource, }) - root_resource = create_resource_tree(resources, Resource()) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( bind_addresses, diff --git a/synapse/util/httpresourcetree.py b/synapse/util/httpresourcetree.py index 45be47159..d74784955 100644 --- a/synapse/util/httpresourcetree.py +++ b/synapse/util/httpresourcetree.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.web.resource import Resource +from twisted.web.resource import NoResource import logging @@ -45,7 +45,7 @@ def create_resource_tree(desired_tree, root_resource): for path_seg in full_path.split('/')[1:-1]: if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" - child_resource = Resource() + child_resource = NoResource() last_resource.putChild(path_seg, child_resource) res_id = _resource_id(last_resource, path_seg) resource_mappings[res_id] = child_resource From 8efe773ef1dc09c295fe494befbef568257095db Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 23 Mar 2018 21:38:42 +0000 Subject: [PATCH 34/49] fix typo --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bac78d3c1..00c8a2b77 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -39,7 +39,7 @@ Features: Changes: -* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) +* Continue to factor out processing from main process and into worker processes. See updated `docs/workers.rst `_ (PR #2892 - #2904, #2913, #2920 - #2926, #2947, #2847, #2854, #2872, #2873, #2874, #2928, #2929, #2934, #2856, #2976 - #2984, #2987 - #2989, #2991 - #2993, #2995, #2784) * Ensure state cache is used when persisting events (PR #2864, #2871, #2802, #2835, #2836, #2841, #2842, #2849) * Change the default config to bind on both IPv4 and IPv6 on all platforms (PR #2435) Thanks to @silkeh! * No longer require a specific version of saml2 (PR #2695) Thanks to @okurz! From a052aa42e7d500f32b034bbbf1e8ad11fed02aa9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 12:02:20 +0100 Subject: [PATCH 35/49] Linearize calls to _generate_user_id --- synapse/handlers/register.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index ed5939880..d07a8801d 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -24,7 +24,7 @@ from synapse.api.errors import ( from synapse.http.client import CaptchaServerHttpClient from synapse import types from synapse.types import UserID -from synapse.util.async import run_on_reactor +from synapse.util.async import run_on_reactor, Linearizer from synapse.util.threepids import check_3pid_allowed from ._base import BaseHandler @@ -46,6 +46,8 @@ class RegistrationHandler(BaseHandler): self.macaroon_gen = hs.get_macaroon_generator() + self._generate_user_id_linearizer = Linearizer(name="_generate_user_id_linearizer") + @defer.inlineCallbacks def check_username(self, localpart, guest_access_token=None, assigned_user_id=None): @@ -345,9 +347,10 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _generate_user_id(self, reseed=False): if reseed or self._next_generated_user_id is None: - self._next_generated_user_id = ( - yield self.store.find_next_generated_user_id_localpart() - ) + with (yield self._generate_user_id_linearizer.queue(())): + self._next_generated_user_id = ( + yield self.store.find_next_generated_user_id_localpart() + ) id = self._next_generated_user_id self._next_generated_user_id += 1 From 8d6dc106d188308f687db1f696cad7ad1b9ae83d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 12:02:44 +0100 Subject: [PATCH 36/49] Don't use _cursor_to_dict in find_next_generated_user_id_localpart --- synapse/storage/registration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d809b2ba4..6b557ca0c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -460,14 +460,12 @@ class RegistrationStore(RegistrationWorkerStore, """ def _find_next_generated_user_id(txn): txn.execute("SELECT name FROM users") - rows = self.cursor_to_dict(txn) regex = re.compile("^@(\d+):") found = set() - for r in rows: - user_id = r["name"] + for user_id, in txn: match = regex.search(user_id) if match: found.add(int(match.group(1))) From 44cd6e135806387552bab5cf7e8be256d53632b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 12:06:48 +0100 Subject: [PATCH 37/49] PEP8 --- synapse/handlers/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d07a8801d..f1bf81ed7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -46,7 +46,9 @@ class RegistrationHandler(BaseHandler): self.macaroon_gen = hs.get_macaroon_generator() - self._generate_user_id_linearizer = Linearizer(name="_generate_user_id_linearizer") + self._generate_user_id_linearizer = Linearizer( + name="_generate_user_id_linearizer", + ) @defer.inlineCallbacks def check_username(self, localpart, guest_access_token=None, From fecb45e0c38912f72655d53faa953e7713511e7f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 13:31:49 +0100 Subject: [PATCH 38/49] Remove last usage of ujson --- synapse/python_dependencies.py | 1 - synapse/storage/group_server.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 91179ce53..40eedb63c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -34,7 +34,6 @@ REQUIREMENTS = { "bcrypt": ["bcrypt>=3.1.0"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], - "ujson": ["ujson"], "blist": ["blist"], "pysaml2>=3.0.0": ["saml2>=3.0.0"], "pymacaroons-pynacl": ["pymacaroons"], diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py index 8fde1aab8..d03858234 100644 --- a/synapse/storage/group_server.py +++ b/synapse/storage/group_server.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from ._base import SQLBaseStore -import ujson as json +import simplejson as json # The category ID for the "default" category. We don't store as null in the From 51406dab96cb3a4de2fad02a31984cfcb613f809 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 26 Mar 2018 14:48:19 +0100 Subject: [PATCH 39/49] version bump --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bac78d3c1..92c19b13e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.27.0 (2018-03-26) +======================================= + +No changes since v0.27.0-rc2 + + Changes in synapse v0.27.0-rc2 (2018-03-19) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 71b409ba8..70a029705 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.0-rc2" +__version__ = "0.27.0" From 91878620027d17b478ca678482f501d3cfb2edaf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 16:19:38 +0100 Subject: [PATCH 40/49] Bump version and changelog --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 92c19b13e..e5e8e31a1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.27.1 (2018-03-19) +======================================= + +Meta release as v0.27.0 temporarily pointed to the wrong commit + + Changes in synapse v0.27.0 (2018-03-26) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index 70a029705..2cb6ef465 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.0" +__version__ = "0.27.1" From 01f72e2fc7379db7d02d5bc1eae96167aa320389 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 16:21:26 +0100 Subject: [PATCH 41/49] Fix date --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e5e8e31a1..ddef8c205 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.27.1 (2018-03-19) +Changes in synapse v0.27.1 (2018-03-26) ======================================= Meta release as v0.27.0 temporarily pointed to the wrong commit From a9d7d98d3f87cc20d94c9cc4ee9285e6a2dddb2c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 16:36:53 +0100 Subject: [PATCH 42/49] Bum version and changelog --- CHANGES.rst | 8 ++++++++ synapse/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index ddef8c205..1372de424 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +Changes in synapse v0.27.2 (2018-03-26) +======================================= + +Bug fixes: + +* Fix bug which broke TCP replication between workers (PR #3015) + + Changes in synapse v0.27.1 (2018-03-26) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index 2cb6ef465..a9d5198ab 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.1" +__version__ = "0.27.2" From 3f49e131d9a31d5bf198f2aa6b102562b2e747d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 10:30:03 +0100 Subject: [PATCH 43/49] Add counter metrics for calculating state delta This will allow us to measure how often we calculate state deltas in event persistence that we would have been able to calculate at the same time we calculated the state for the event. --- synapse/storage/events.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 85ce6bea1..f3a362db2 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,6 +52,15 @@ persist_event_counter = metrics.register_counter("persisted_events") event_counter = metrics.register_counter( "persisted_events_sep", labels=["type", "origin_type", "origin_entity"] ) +state_delta_counter = metrics.register_counter( + "state_delta", +) +state_delta_single_event_counter = metrics.register_counter( + "state_delta_single_event", +) +state_delta_reuse_delta_counter = metrics.register_counter( + "state_delta_reuse_delta", +) def encode_json(json_object): @@ -368,7 +377,8 @@ class EventsStore(EventsWorkerStore): room_id, ev_ctx_rm, latest_event_ids ) - if new_latest_event_ids == set(latest_event_ids): + latest_event_ids = set(latest_event_ids) + if new_latest_event_ids == latest_event_ids: # No change in extremities, so no change in state continue @@ -389,6 +399,25 @@ class EventsStore(EventsWorkerStore): if all_single_prev_not_state: continue + state_delta_counter.inc() + if len(new_latest_event_ids) == 1: + state_delta_single_event_counter.inc() + + # This is a fairly handwavey check to see if we could + # have guessed what the delta would have been when + # processing one of these events. + # What we're interested in is if the latest extremities + # were the same when we created the event as they are + # now. We guess this by looking at the prev events and + # checking if they match up, as when this server creates + # a new event it will use the extremities as the prev + # events. + for ev, _ in ev_ctx_rm: + prev_event_ids = set(e for e, _ in ev.prev_events) + if latest_event_ids == prev_event_ids: + state_delta_reuse_delta_counter.inc() + break + logger.info( "Calculating state delta for room %s", room_id, ) From 3e0c0660b31827f9d13c1fb5153a77e70813d5d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:01:34 +0100 Subject: [PATCH 44/49] Also do check inside linearizer --- synapse/handlers/register.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index f1bf81ed7..dd0370527 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -350,9 +350,10 @@ class RegistrationHandler(BaseHandler): def _generate_user_id(self, reseed=False): if reseed or self._next_generated_user_id is None: with (yield self._generate_user_id_linearizer.queue(())): - self._next_generated_user_id = ( - yield self.store.find_next_generated_user_id_localpart() - ) + if reseed or self._next_generated_user_id is None: + self._next_generated_user_id = ( + yield self.store.find_next_generated_user_id_localpart() + ) id = self._next_generated_user_id self._next_generated_user_id += 1 From e70287cff3adeeeff2a7a1e95fc5845feb365710 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:13:38 +0100 Subject: [PATCH 45/49] Comment --- synapse/storage/events.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f3a362db2..ee8b3c46d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,12 +52,19 @@ persist_event_counter = metrics.register_counter("persisted_events") event_counter = metrics.register_counter( "persisted_events_sep", labels=["type", "origin_type", "origin_entity"] ) + +# The number of times we are recalculating the current state state_delta_counter = metrics.register_counter( "state_delta", ) +# The number of times we are recalculating state when there is only a +# single forward extremity state_delta_single_event_counter = metrics.register_counter( "state_delta_single_event", ) +# The number of times we are reculating state when we could have resonably +# calculated the delta when we calculated the state for an event we were +# persisting. state_delta_reuse_delta_counter = metrics.register_counter( "state_delta_reuse_delta", ) From 152c2ac19e6af373a9e79a8f4da80d9e77a5396f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:13:46 +0100 Subject: [PATCH 46/49] Fix indent --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index ee8b3c46d..95b36d669 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -423,7 +423,7 @@ class EventsStore(EventsWorkerStore): prev_event_ids = set(e for e, _ in ev.prev_events) if latest_event_ids == prev_event_ids: state_delta_reuse_delta_counter.inc() - break + break logger.info( "Calculating state delta for room %s", room_id, From 800cfd5774d9726a00c53f2d84ce0286ab64c17b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:30:39 +0100 Subject: [PATCH 47/49] Comment --- synapse/storage/events.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 95b36d669..f3d65f433 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -415,10 +415,11 @@ class EventsStore(EventsWorkerStore): # processing one of these events. # What we're interested in is if the latest extremities # were the same when we created the event as they are - # now. We guess this by looking at the prev events and - # checking if they match up, as when this server creates - # a new event it will use the extremities as the prev - # events. + # now. When this server creates a new event (as opposed + # to receiving it over federation) it will use the + # forward extremities as the prev_events, so we can + # guess this by looking at the prev_events and checking + # if they match the current forward extremities. for ev, _ in ev_ctx_rm: prev_event_ids = set(e for e, _ in ev.prev_events) if latest_event_ids == prev_event_ids: From a9cb1a35c85f62bb0114dabd62d118c80d66e415 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 28 Mar 2018 10:57:27 +0100 Subject: [PATCH 48/49] fix tests/storage/test_user_directory.py --- tests/storage/test_user_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index 0891308f2..88add4521 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -62,7 +62,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase): self.assertFalse(r["limited"]) self.assertEqual(1, len(r["results"])) self.assertDictEqual(r["results"][0], { - "user_id": BOB, + "d.user_id": BOB, "display_name": "bob", "avatar_url": None, }) From 545001b9e4b1d6710145d3efe2117fbdf823fb38 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 28 Mar 2018 11:19:45 +0100 Subject: [PATCH 49/49] Fix search_user_dir multiple sqlite versions do different things --- synapse/storage/user_directory.py | 4 ++-- tests/storage/test_user_directory.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index dfdcbb318..d6e289ffb 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -667,7 +667,7 @@ class UserDirectoryStore(SQLBaseStore): # The array of numbers are the weights for the various part of the # search: (domain, _, display name, localpart) sql = """ - SELECT d.user_id, display_name, avatar_url + SELECT d.user_id AS user_id, display_name, avatar_url FROM user_directory_search INNER JOIN user_directory AS d USING (user_id) %s @@ -702,7 +702,7 @@ class UserDirectoryStore(SQLBaseStore): search_query = _parse_query_sqlite(search_term) sql = """ - SELECT d.user_id, display_name, avatar_url + SELECT d.user_id AS user_id, display_name, avatar_url FROM user_directory_search INNER JOIN user_directory AS d USING (user_id) %s diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index 88add4521..0891308f2 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -62,7 +62,7 @@ class UserDirectoryStoreTestCase(unittest.TestCase): self.assertFalse(r["limited"]) self.assertEqual(1, len(r["results"])) self.assertDictEqual(r["results"][0], { - "d.user_id": BOB, + "user_id": BOB, "display_name": "bob", "avatar_url": None, })