diff --git a/etc/matrix-synapse/homeserver.yaml b/etc/matrix-synapse/homeserver.yaml index b207ac9..164e359 100644 --- a/etc/matrix-synapse/homeserver.yaml +++ b/etc/matrix-synapse/homeserver.yaml @@ -2764,6 +2764,12 @@ instance_map: generic_worker4: host: localhost port: 9097 + additional1: + host: localhost + port: 9098 + presence: + host: localhost + port: 9100 # Experimental: When using workers you can define which workers should # handle event persistence and typing notifications. Any worker @@ -2778,6 +2784,16 @@ stream_writers: - generic_worker2 - generic_worker3 - generic_worker4 + typing: + - additional1 + to_device: + - additional1 + account_data: + - additional1 + receipts: + - additional1 + presence: + - presence # The worker that is used to run background tasks (e.g. cleaning up expired # data). If not provided this defaults to the main process. diff --git a/etc/matrix-synapse/workers/additional1.yaml b/etc/matrix-synapse/workers/additional1.yaml new file mode 100644 index 0000000..5f6e9ff --- /dev/null +++ b/etc/matrix-synapse/workers/additional1.yaml @@ -0,0 +1,28 @@ +worker_app: synapse.app.generic_worker +worker_name: additional1 + +# The replication listener on the synapse to talk to. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/workers/additional1_log_config.yaml + +worker_listeners: + - type: http + bind_addresses: ['127.0.0.1'] + port: 8514 + tls: false + x_forwarded: true + resources: + - names: [client, federation] + compress: false + + - type: http + bind_addresses: ['127.0.0.1'] + port: 9098 + resources: + - names: [replication] + + - type: metrics + bind_addresses: ['0.0.0.0'] + port: 9135 diff --git a/etc/matrix-synapse/workers/additional1_log_config.yaml b/etc/matrix-synapse/workers/additional1_log_config.yaml new file mode 100644 index 0000000..8ccaa7a --- /dev/null +++ b/etc/matrix-synapse/workers/additional1_log_config.yaml @@ -0,0 +1,84 @@ +# Log configuration for Synapse. +# +# This is a YAML file containing a standard Python logging configuration +# dictionary. See [1] for details on the valid settings. +# +# Synapse also supports structured logging for machine readable logs which can +# be ingested by ELK stacks. See [2] for details. +# +# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema +# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html + +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + file: + class: logging.handlers.TimedRotatingFileHandler + formatter: precise + filename: /var/log/matrix-synapse/additional1.log + when: midnight + backupCount: 2 # Does not include the current log file. + encoding: utf8 + + # Default to buffering writes to log file for efficiency. + # WARNING/ERROR logs will still be flushed immediately, but there will be a + # delay (of up to `period` seconds, or until the buffer is full with + # `capacity` messages) before INFO/DEBUG logs get written. + buffer: + class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler + target: file + + # The capacity is the maximum number of log lines that are buffered + # before being written to disk. Increasing this will lead to better + # performance, at the expensive of it taking longer for log lines to + # be written to disk. + # This parameter is required. + capacity: 10 + + # Logs with a level at or above the flush level will cause the buffer to + # be flushed immediately. + # Default value: 40 (ERROR) + # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG) + flushLevel: 30 # Flush immediately for WARNING logs and higher + + # The period of time, in seconds, between forced flushes. + # Messages will not be delayed for longer than this time. + # Default value: 5 seconds + period: 5 + + # A handler that writes logs to stderr. Unused by default, but can be used + # instead of "buffer" and "file" in the logger handlers. + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARN + + twisted: + # We send the twisted logging directly to the file handler, + # to work around https://github.com/matrix-org/synapse/issues/3471 + # when using "buffer" logger. Use "console" to log to stderr instead. + handlers: [file] + propagate: false + +root: + level: WARN + + # Write logs to the `buffer` handler, which will buffer them together in memory, + # then write them to a file. + # + # Replace "buffer" with "console" to log to stderr instead. (Note that you'll + # also need to update the configuration for the `twisted` logger above, in + # this case.) + # + handlers: [buffer] + +disable_existing_loggers: false diff --git a/etc/matrix-synapse/workers/appservice_log_config.yaml b/etc/matrix-synapse/workers/appservice_log_config.yaml index 0972f75..1b77c0b 100644 --- a/etc/matrix-synapse/workers/appservice_log_config.yaml +++ b/etc/matrix-synapse/workers/appservice_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_appservice.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/federation_sender1_log_config.yaml b/etc/matrix-synapse/workers/federation_sender1_log_config.yaml index 8fae005..9611f03 100644 --- a/etc/matrix-synapse/workers/federation_sender1_log_config.yaml +++ b/etc/matrix-synapse/workers/federation_sender1_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_federation_sender1.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/federation_sender2_log_config.yaml b/etc/matrix-synapse/workers/federation_sender2_log_config.yaml index f10610f..3cbdca3 100644 --- a/etc/matrix-synapse/workers/federation_sender2_log_config.yaml +++ b/etc/matrix-synapse/workers/federation_sender2_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_federation_sender2.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/federation_sender3_log_config.yaml b/etc/matrix-synapse/workers/federation_sender3_log_config.yaml index 8e8db54..2abe88b 100644 --- a/etc/matrix-synapse/workers/federation_sender3_log_config.yaml +++ b/etc/matrix-synapse/workers/federation_sender3_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_federation_sender3.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/generic_worker1_log_config.yaml b/etc/matrix-synapse/workers/generic_worker1_log_config.yaml index ac13ef1..5f04602 100644 --- a/etc/matrix-synapse/workers/generic_worker1_log_config.yaml +++ b/etc/matrix-synapse/workers/generic_worker1_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_generic_worker1.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/generic_worker2_log_config.yaml b/etc/matrix-synapse/workers/generic_worker2_log_config.yaml index 1d46c3f..d6aeb8b 100644 --- a/etc/matrix-synapse/workers/generic_worker2_log_config.yaml +++ b/etc/matrix-synapse/workers/generic_worker2_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_generic_worker2.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/generic_worker3_log_config.yaml b/etc/matrix-synapse/workers/generic_worker3_log_config.yaml index faa756f..aca9f30 100644 --- a/etc/matrix-synapse/workers/generic_worker3_log_config.yaml +++ b/etc/matrix-synapse/workers/generic_worker3_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_generic_worker3.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/generic_worker4_log_config.yaml b/etc/matrix-synapse/workers/generic_worker4_log_config.yaml index a2506d6..1fdca02 100644 --- a/etc/matrix-synapse/workers/generic_worker4_log_config.yaml +++ b/etc/matrix-synapse/workers/generic_worker4_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_generic_worker4.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/matrix-synapse/workers/presence.yaml b/etc/matrix-synapse/workers/presence.yaml new file mode 100644 index 0000000..0e1735c --- /dev/null +++ b/etc/matrix-synapse/workers/presence.yaml @@ -0,0 +1,28 @@ +worker_app: synapse.app.generic_worker +worker_name: presence + +# The replication listener on the synapse to talk to. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/workers/presence_log_config.yaml + +worker_listeners: + - type: http + bind_addresses: ['127.0.0.1'] + port: 8516 + tls: false + x_forwarded: true + resources: + - names: [client, federation] + compress: false + + - type: http + bind_addresses: ['127.0.0.1'] + port: 9100 + resources: + - names: [replication] + + - type: metrics + bind_addresses: ['0.0.0.0'] + port: 9137 diff --git a/etc/matrix-synapse/workers/presence_log_config.yaml b/etc/matrix-synapse/workers/presence_log_config.yaml new file mode 100644 index 0000000..3ce9e76 --- /dev/null +++ b/etc/matrix-synapse/workers/presence_log_config.yaml @@ -0,0 +1,84 @@ +# Log configuration for Synapse. +# +# This is a YAML file containing a standard Python logging configuration +# dictionary. See [1] for details on the valid settings. +# +# Synapse also supports structured logging for machine readable logs which can +# be ingested by ELK stacks. See [2] for details. +# +# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema +# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html + +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + file: + class: logging.handlers.TimedRotatingFileHandler + formatter: precise + filename: /var/log/matrix-synapse/presence.log + when: midnight + backupCount: 2 # Does not include the current log file. + encoding: utf8 + + # Default to buffering writes to log file for efficiency. + # WARNING/ERROR logs will still be flushed immediately, but there will be a + # delay (of up to `period` seconds, or until the buffer is full with + # `capacity` messages) before INFO/DEBUG logs get written. + buffer: + class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler + target: file + + # The capacity is the maximum number of log lines that are buffered + # before being written to disk. Increasing this will lead to better + # performance, at the expensive of it taking longer for log lines to + # be written to disk. + # This parameter is required. + capacity: 10 + + # Logs with a level at or above the flush level will cause the buffer to + # be flushed immediately. + # Default value: 40 (ERROR) + # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG) + flushLevel: 30 # Flush immediately for WARNING logs and higher + + # The period of time, in seconds, between forced flushes. + # Messages will not be delayed for longer than this time. + # Default value: 5 seconds + period: 5 + + # A handler that writes logs to stderr. Unused by default, but can be used + # instead of "buffer" and "file" in the logger handlers. + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: WARN + + twisted: + # We send the twisted logging directly to the file handler, + # to work around https://github.com/matrix-org/synapse/issues/3471 + # when using "buffer" logger. Use "console" to log to stderr instead. + handlers: [file] + propagate: false + +root: + level: WARN + + # Write logs to the `buffer` handler, which will buffer them together in memory, + # then write them to a file. + # + # Replace "buffer" with "console" to log to stderr instead. (Note that you'll + # also need to update the configuration for the `twisted` logger above, in + # this case.) + # + handlers: [buffer] + +disable_existing_loggers: false diff --git a/etc/matrix-synapse/workers/pusher_log_config.yaml b/etc/matrix-synapse/workers/pusher_log_config.yaml index b51e4ec..cbabf65 100644 --- a/etc/matrix-synapse/workers/pusher_log_config.yaml +++ b/etc/matrix-synapse/workers/pusher_log_config.yaml @@ -21,7 +21,7 @@ handlers: formatter: precise filename: /var/log/matrix-synapse/worker_pusher.log when: midnight - backupCount: 3 # Does not include the current log file. + backupCount: 2 # Does not include the current log file. encoding: utf8 # Default to buffering writes to log file for efficiency. diff --git a/etc/nginx/include.d/generic_worker.conf b/etc/nginx/include.d/generic_worker.conf index 6438dd2..8a421bd 100644 --- a/etc/nginx/include.d/generic_worker.conf +++ b/etc/nginx/include.d/generic_worker.conf @@ -314,3 +314,43 @@ location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ { include include.d/synapse-proxy.conf; proxy_pass http://generic_worker_lc; } + +## Typing +location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +## Device requests +location ~ ^/_matrix/client/(r0|v3|unstable)/sendToDevice/ { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +## Account data requests +location ~ ^/_matrix/client/(r0|v3|unstable)/.*/tags { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +location ~ ^/_matrix/client/(r0|v3|unstable)/.*/account_data { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +## Receipts requests +location ~ ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +location ~ ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers { + include include.d/synapse-proxy.conf; + proxy_pass http://additional; +} + +## Presence requests +location ~ ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ { + include include.d/synapse-proxy.conf; + proxy_pass http://presence; +} diff --git a/etc/nginx/include.d/upstream_workers.conf b/etc/nginx/include.d/upstream_workers.conf index dfd3cad..b82d6d7 100644 --- a/etc/nginx/include.d/upstream_workers.conf +++ b/etc/nginx/include.d/upstream_workers.conf @@ -13,3 +13,13 @@ upstream generic_worker_lc { server localhost:8512; server localhost:8513; } + +upstream additional { + least_conn; + server localhost:8514; +} + +upstream presence { + least_conn; + server localhost:8516; +}