From 4e6be6304ded28370d3e8cf244f57d1617ea58f0 Mon Sep 17 00:00:00 2001 From: deathrow Date: Fri, 4 Nov 2022 16:28:02 -0400 Subject: [PATCH] Add Sync --- README.md | 5 +- docker-compose.yml | 71 +++++++++++++++--- files/log/sync4.yaml | 84 ++++++++++++++++++++++ files/log/sync5.yaml | 84 ++++++++++++++++++++++ files/log/worker4.yaml | 84 ++++++++++++++++++++++ files/workers/sync4.yaml | 17 +++++ files/workers/sync5.yaml | 17 +++++ files/workers/worker4.yaml | 17 +++++ swag/nginx/include.d/sync_worker.conf | 30 ++++++++ swag/nginx/include.d/upstream_workers.conf | 36 +++++++--- swag/nginx/site-confs/adminer.conf | 17 ----- 11 files changed, 424 insertions(+), 38 deletions(-) create mode 100644 files/log/sync4.yaml create mode 100644 files/log/sync5.yaml create mode 100644 files/log/worker4.yaml create mode 100644 files/workers/sync4.yaml create mode 100644 files/workers/sync5.yaml create mode 100644 files/workers/worker4.yaml create mode 100644 swag/nginx/include.d/sync_worker.conf delete mode 100644 swag/nginx/site-confs/adminer.conf diff --git a/README.md b/README.md index b808b0b..6ca303b 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,13 @@ A Synapse Docker deployment with: - Manage server via `synadm` - Images built locally - Matrix Maubot -- Postgres web UI - Matrix integration manager ### Getting Started -Dependencies: `cargo docker docker-compose git python ` +Dependencies: `cargo` `docker` `docker-compose` `git` `python ` -Subdomains: `matrix` `dimension` `adminer` `maubot` +Subdomains: `matrix` `dimension` `maubot` Clone the repository: ``` diff --git a/docker-compose.yml b/docker-compose.yml index 92c9f44..19d0cdb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,6 @@ services: networks: - matrix - registration - - adminer pantalaimon: build: ./images/pantalaimon @@ -112,14 +111,6 @@ services: networks: - matrix - adminer: - image: adminer - container_name: adminer - restart: unless-stopped - networks: - - matrix - - adminer - matrix-registration: build: ./images/synapse-captcha container_name: matrix-registration @@ -187,6 +178,45 @@ services: # WORKERS + sync5: + build: ./images/synapse-worker-docker + container_name: sync5 + user: "991:991" + environment: + SYNAPSE_WORKER: synapse.app.generic_worker + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/sync5.yaml"] + volumes: + - ./files:/data:Z + - ./bridges:/bridges:Z + depends_on: + - synapse + ports: + - 127.0.0.1:8094:8081 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + sync4: + build: ./images/synapse-worker-docker + container_name: sync4 + user: "991:991" + environment: + SYNAPSE_WORKER: synapse.app.generic_worker + restart: unless-stopped + entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/sync4.yaml"] + volumes: + - ./files:/data:Z + - ./bridges:/bridges:Z + depends_on: + - synapse + ports: + - 127.0.0.1:8093:8081 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + sync3: build: ./images/synapse-worker-docker container_name: sync3 @@ -333,6 +363,28 @@ services: cap_drop: - ALL + synapse-generic-worker-4: + build: ./images/workers/synapse + container_name: synapse-generic-worker-4 + user: "991:991" + restart: unless-stopped + environment: + SYNAPSE_WORKER: synapse.app.generic_worker + entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/worker4.yaml"] + volumes: + - ./files:/data:Z + - ./bridges:/bridges:Z + depends_on: + - synapse + networks: + - matrix + ports: + - 127.0.0.1:8083:8081 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + synapse-generic-worker-3: build: ./images/workers/synapse container_name: synapse-generic-worker-3 @@ -460,6 +512,5 @@ services: networks: matrix: registration: - adminer: dimension: \ No newline at end of file diff --git a/files/log/sync4.yaml b/files/log/sync4.yaml new file mode 100644 index 0000000..00067b0 --- /dev/null +++ b/files/log/sync4.yaml @@ -0,0 +1,84 @@ +# Log configuration for Synapse. +# +# This is a YAML file containing a standard Python logging configuration +# dictionary. See [1] for details on the valid settings. +# +# Synapse also supports structured logging for machine readable logs which can +# be ingested by ELK stacks. See [2] for details. +# +# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema +# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html + +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + file: + class: logging.handlers.TimedRotatingFileHandler + formatter: precise + filename: /data/workers/log/worker_generic_sync4.log + when: midnight + backupCount: 2 # Does not include the current log file. + encoding: utf8 + + # Default to buffering writes to log file for efficiency. + # WARNING/ERROR logs will still be flushed immediately, but there will be a + # delay (of up to `period` seconds, or until the buffer is full with + # `capacity` messages) before INFO/DEBUG logs get written. + buffer: + class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler + target: file + + # The capacity is the maximum number of log lines that are buffered + # before being written to disk. Increasing this will lead to better + # performance, at the expensive of it taking longer for log lines to + # be written to disk. + # This parameter is required. + capacity: 10 + + # Logs with a level at or above the flush level will cause the buffer to + # be flushed immediately. + # Default value: 40 (ERROR) + # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG) + flushLevel: 30 # Flush immediately for WARNING logs and higher + + # The period of time, in seconds, between forced flushes. + # Messages will not be delayed for longer than this time. + # Default value: 5 seconds + period: 5 + + # A handler that writes logs to stderr. Unused by default, but can be used + # instead of "buffer" and "file" in the logger handlers. + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARN + + twisted: + # We send the twisted logging directly to the file handler, + # to work around https://github.com/matrix-org/synapse/issues/3471 + # when using "buffer" logger. Use "console" to log to stderr instead. + handlers: [file] + propagate: false + +root: + level: WARN + + # Write logs to the `buffer` handler, which will buffer them together in memory, + # then write them to a file. + # + # Replace "buffer" with "console" to log to stderr instead. (Note that you'll + # also need to update the configuration for the `twisted` logger above, in + # this case.) + # + handlers: [buffer] + +disable_existing_loggers: false diff --git a/files/log/sync5.yaml b/files/log/sync5.yaml new file mode 100644 index 0000000..16b0bd4 --- /dev/null +++ b/files/log/sync5.yaml @@ -0,0 +1,84 @@ +# Log configuration for Synapse. +# +# This is a YAML file containing a standard Python logging configuration +# dictionary. See [1] for details on the valid settings. +# +# Synapse also supports structured logging for machine readable logs which can +# be ingested by ELK stacks. See [2] for details. +# +# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema +# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html + +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + file: + class: logging.handlers.TimedRotatingFileHandler + formatter: precise + filename: /data/workers/log/worker_generic_sync5.log + when: midnight + backupCount: 2 # Does not include the current log file. + encoding: utf8 + + # Default to buffering writes to log file for efficiency. + # WARNING/ERROR logs will still be flushed immediately, but there will be a + # delay (of up to `period` seconds, or until the buffer is full with + # `capacity` messages) before INFO/DEBUG logs get written. + buffer: + class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler + target: file + + # The capacity is the maximum number of log lines that are buffered + # before being written to disk. Increasing this will lead to better + # performance, at the expensive of it taking longer for log lines to + # be written to disk. + # This parameter is required. + capacity: 10 + + # Logs with a level at or above the flush level will cause the buffer to + # be flushed immediately. + # Default value: 40 (ERROR) + # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG) + flushLevel: 30 # Flush immediately for WARNING logs and higher + + # The period of time, in seconds, between forced flushes. + # Messages will not be delayed for longer than this time. + # Default value: 5 seconds + period: 5 + + # A handler that writes logs to stderr. Unused by default, but can be used + # instead of "buffer" and "file" in the logger handlers. + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARN + + twisted: + # We send the twisted logging directly to the file handler, + # to work around https://github.com/matrix-org/synapse/issues/3471 + # when using "buffer" logger. Use "console" to log to stderr instead. + handlers: [file] + propagate: false + +root: + level: WARN + + # Write logs to the `buffer` handler, which will buffer them together in memory, + # then write them to a file. + # + # Replace "buffer" with "console" to log to stderr instead. (Note that you'll + # also need to update the configuration for the `twisted` logger above, in + # this case.) + # + handlers: [buffer] + +disable_existing_loggers: false diff --git a/files/log/worker4.yaml b/files/log/worker4.yaml new file mode 100644 index 0000000..9a70c92 --- /dev/null +++ b/files/log/worker4.yaml @@ -0,0 +1,84 @@ +# Log configuration for Synapse. +# +# This is a YAML file containing a standard Python logging configuration +# dictionary. See [1] for details on the valid settings. +# +# Synapse also supports structured logging for machine readable logs which can +# be ingested by ELK stacks. See [2] for details. +# +# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema +# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html + +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + file: + class: logging.handlers.TimedRotatingFileHandler + formatter: precise + filename: /data/workers/log/worker4.log + when: midnight + backupCount: 2 # Does not include the current log file. + encoding: utf8 + + # Default to buffering writes to log file for efficiency. + # WARNING/ERROR logs will still be flushed immediately, but there will be a + # delay (of up to `period` seconds, or until the buffer is full with + # `capacity` messages) before INFO/DEBUG logs get written. + buffer: + class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler + target: file + + # The capacity is the maximum number of log lines that are buffered + # before being written to disk. Increasing this will lead to better + # performance, at the expensive of it taking longer for log lines to + # be written to disk. + # This parameter is required. + capacity: 10 + + # Logs with a level at or above the flush level will cause the buffer to + # be flushed immediately. + # Default value: 40 (ERROR) + # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG) + flushLevel: 30 # Flush immediately for WARNING logs and higher + + # The period of time, in seconds, between forced flushes. + # Messages will not be delayed for longer than this time. + # Default value: 5 seconds + period: 5 + + # A handler that writes logs to stderr. Unused by default, but can be used + # instead of "buffer" and "file" in the logger handlers. + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARN + + twisted: + # We send the twisted logging directly to the file handler, + # to work around https://github.com/matrix-org/synapse/issues/3471 + # when using "buffer" logger. Use "console" to log to stderr instead. + handlers: [file] + propagate: false + +root: + level: WARN + + # Write logs to the `buffer` handler, which will buffer them together in memory, + # then write them to a file. + # + # Replace "buffer" with "console" to log to stderr instead. (Note that you'll + # also need to update the configuration for the `twisted` logger above, in + # this case.) + # + handlers: [buffer] + +disable_existing_loggers: false diff --git a/files/workers/sync4.yaml b/files/workers/sync4.yaml new file mode 100644 index 0000000..6143315 --- /dev/null +++ b/files/workers/sync4.yaml @@ -0,0 +1,17 @@ +worker_app: synapse.app.generic_worker +worker_name: sync4 + +# The replication listener on the synapse to talk to. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_main_http_uri: http://synapse:8008/ + +worker_log_config: /data/log/sync4.yaml + +worker_listeners: + - type: http + port: 8081 + x_forwarded: true + resources: + - names: [client, federation] \ No newline at end of file diff --git a/files/workers/sync5.yaml b/files/workers/sync5.yaml new file mode 100644 index 0000000..cd41822 --- /dev/null +++ b/files/workers/sync5.yaml @@ -0,0 +1,17 @@ +worker_app: synapse.app.generic_worker +worker_name: sync5 + +# The replication listener on the synapse to talk to. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_main_http_uri: http://synapse:8008/ + +worker_log_config: /data/log/sync5.yaml + +worker_listeners: + - type: http + port: 8081 + x_forwarded: true + resources: + - names: [client, federation] \ No newline at end of file diff --git a/files/workers/worker4.yaml b/files/workers/worker4.yaml new file mode 100644 index 0000000..9fd1c8f --- /dev/null +++ b/files/workers/worker4.yaml @@ -0,0 +1,17 @@ +worker_app: synapse.app.generic_worker +worker_name: worker4 + +# The replication listener on the synapse to talk to. +worker_replication_host: synapse +worker_replication_http_port: 9093 + +worker_main_http_uri: http://synapse:8008/ + +worker_log_config: /data/log/worker4.yaml + +worker_listeners: + - type: http + port: 8081 + x_forwarded: true + resources: + - names: [client, federation] \ No newline at end of file diff --git a/swag/nginx/include.d/sync_worker.conf b/swag/nginx/include.d/sync_worker.conf new file mode 100644 index 0000000..7ec80b0 --- /dev/null +++ b/swag/nginx/include.d/sync_worker.conf @@ -0,0 +1,30 @@ +# Choose sync worker based on the existence of "since" query parameter +map $arg_since $sync { + default sync_worker; + '' sync_init; +} + # Sync initial/normal + location ~ ^/_matrix/client/(r0|v3)/sync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://$sync; + proxy_read_timeout 1h; + } + + # Normal sync + location ~ ^/_matrix/client/(api/v1|r0|v3)/events$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://sync_worker; + } + + # Initial_sync + location ~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://sync_init; + proxy_read_timeout 1h; + } + + location ~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://sync_init; + proxy_read_timeout 1h; + } diff --git a/swag/nginx/include.d/upstream_workers.conf b/swag/nginx/include.d/upstream_workers.conf index 5f808c6..ecf54b1 100644 --- a/swag/nginx/include.d/upstream_workers.conf +++ b/swag/nginx/include.d/upstream_workers.conf @@ -2,23 +2,43 @@ upstream generic_worker_ih { ip_hash; server synapse-generic-worker-1:8080; server synapse-generic-worker-2:8081; - server sync1:8090; - server sync2:8091; - server sync3:8092; + server synapse-generic-worker-3:8082; + server synapse-generic-worker-4:8083; } upstream generic_worker_lc { least_conn; server synapse-generic-worker-1:8080; server synapse-generic-worker-2:8081; + server synapse-generic-worker-3:8082; + server synapse-generic-worker-4:8083; +} + +# extract username from token get parameter +map $arg_access_token $token_from_arg { + default $arg_access_token; + "~syt_(?.*?)_.*" $username; +} + +# extract username part from bearer token, fallback to access_token +map $http_authorization $proxy_username_label { + default $http_authorization; + "~Bearer syt_(?.*?)_.*" $username; + "" $token_from_arg; +} + + +upstream sync_worker { + # pin with username extracted from bearer token or access_token + hash $proxy_username_label consistent; server sync1:8090; server sync2:8091; server sync3:8092; } -upstream sync_worker { - ip_hash; - server sync1:8090; - server sync2:8091; - server sync3:8092; +upstream sync_init { + # Use the username mapper result for hash key + hash $mxid_localpart consistent; + server sync4:8093; + server sync5:8094; } \ No newline at end of file diff --git a/swag/nginx/site-confs/adminer.conf b/swag/nginx/site-confs/adminer.conf deleted file mode 100644 index b30b376..0000000 --- a/swag/nginx/site-confs/adminer.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - server_name adminer.*; - - include /config/nginx/ssl.conf; - - location / { - include /config/nginx/proxy.conf; - include /config/nginx/resolver.conf; - set $upstream_app adminer; - set $upstream_port 8080; - set $upstream_proto http; - proxy_pass $upstream_proto://$upstream_app:$upstream_port; - } -}