add generic_worker5

etc/matrix-synapse/homeserver.yaml

@@ -2850,9 +2850,12 @@ instance_map:
   generic_worker4:
     host: localhost
     port: 9097
-  additional1:
+  generic_worker5:
     host: localhost
     port: 9098
+  additional1:
+    host: localhost
+    port: 9099
   presence:
     host: localhost
     port: 9100
@@ -2870,6 +2873,7 @@ stream_writers:
     - generic_worker2
     - generic_worker3
     - generic_worker4
+    - generic_worker5
   typing:
     - additional1
   to_device:

etc/matrix-synapse/workers/additional1.yaml

@@ -6,7 +6,7 @@ worker_log_config: /etc/matrix-synapse/workers/additional1_log_config.yaml
 worker_listeners:
   - type: http
     bind_addresses: ['127.0.0.1']
-    port: 8514
+    port: 8515
     tls: false
     x_forwarded: true
     resources:
@@ -15,10 +15,10 @@ worker_listeners:
 
   - type: http
     bind_addresses: ['127.0.0.1']
-    port: 9098
+    port: 9099
     resources:
       - names: [replication]
 
   - type: metrics
     bind_addresses: ['0.0.0.0']
-    port: 9135
+    port: 9136

etc/matrix-synapse/workers/additional1_log_config.yaml

@@ -62,13 +62,6 @@ loggers:
         # information such as access tokens.
         level: WARN
 
-    twisted:
-        # We send the twisted logging directly to the file handler,
-        # to work around https://github.com/matrix-org/synapse/issues/3471
-        # when using "buffer" logger. Use "console" to log to stderr instead.
-        handlers: [file]
-        propagate: false
-
 root:
     level: WARN
 

etc/matrix-synapse/workers/generic_worker5.yaml

@@ -0,0 +1,24 @@
+worker_app: synapse.app.generic_worker
+worker_name: generic_worker5
+
+worker_log_config: /etc/matrix-synapse/workers/generic_worker5_log_config.yaml
+
+worker_listeners:
+  - type: http
+    bind_addresses: ['127.0.0.1']
+    port: 8514
+    tls: false
+    x_forwarded: true
+    resources:
+      - names: [client, federation]
+        compress: false
+
+  - type: http
+    bind_addresses: ['127.0.0.1']
+    port: 9098
+    resources:
+      - names: [replication]
+
+  - type: metrics
+    bind_addresses: ['0.0.0.0']
+    port: 9135

etc/matrix-synapse/workers/generic_worker5_log_config.yaml

@@ -0,0 +1,77 @@
+# Log configuration for Synapse.
+#
+# This is a YAML file containing a standard Python logging configuration
+# dictionary. See [1] for details on the valid settings.
+#
+# Synapse also supports structured logging for machine readable logs which can
+# be ingested by ELK stacks. See [2] for details.
+#
+# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: precise
+        filename: /var/log/matrix-synapse/worker_generic_worker5.log
+        when: midnight
+        backupCount: 2  # Does not include the current log file.
+        encoding: utf8
+
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
+    buffer:
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+        target: file
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
+        # performance, at the expensive of it taking longer for log lines to
+        # be written to disk.
+        # This parameter is required.
+        capacity: 10
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
+
+    # A handler that writes logs to stderr. Unused by default, but can be used
+    # instead of "buffer" and "file" in the logger handlers.
+    console:
+        class: logging.StreamHandler
+        formatter: precise
+
+loggers:
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: WARN
+
+root:
+    level: WARN
+
+    # Write logs to the `buffer` handler, which will buffer them together in memory,
+    # then write them to a file.
+    #
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
+    #
+    handlers: [buffer]
+
+disable_existing_loggers: false

etc/matrix-synchrotron-balancer/config.yaml

@@ -9,5 +9,7 @@ synchrotrons:
     pid_file: /etc/matrix-synapse/workers/generic_worker3.pid
   - address: localhost:8513
     pid_file: /etc/matrix-synapse/workers/generic_worker4.pid
+  - address: localhost:8514
+    pid_file: /etc/matrix-synapse/workers/generic_worker5.pid
 balancer:
   interval: 2

etc/nginx/include.d/upstream_workers.conf

@@ -4,6 +4,7 @@ upstream generic_worker_ih {
        server localhost:8511;
        server localhost:8512;
        server localhost:8513;
+       server localhost:8514;
 }
 
 upstream generic_worker_lc {
@@ -12,11 +13,12 @@ upstream generic_worker_lc {
        server localhost:8511;
        server localhost:8512;
        server localhost:8513;
+       server localhost:8514;
 }
 
 upstream additional {
        least_conn;
-       server localhost:8514;
+       server localhost:8515;
 }
 
 upstream presence {

etc/prometheus/prometheus.yml

@@ -74,6 +74,13 @@ scrape_configs:
           group: synapse
           worker: federation_sender3
 
+      - targets: ['matrix.envs.net:9104']
+        labels:
+          job: federation_sender
+          instance: All
+          group: synapse
+          worker: federation_sender4
+
       - targets: ['matrix.envs.net:9111']
         labels:
           job: pusher
@@ -81,13 +88,6 @@ scrape_configs:
           group: synapse
           worker: pusher
 
-      - targets: ['matrix.envs.net:9121']
-        labels:
-          job: appservice
-          instance: All
-          group: synapse
-          worker: appservice
-
       - targets: ['matrix.envs.net:9131']
         labels:
           job: generic_worker
@@ -116,6 +116,27 @@ scrape_configs:
           group: synapse
           worker: generic_worker4
 
+      - targets: ['matrix.envs.net:9135']
+        labels:
+          job: generic_worker
+          instance: All
+          group: synapse
+          worker: generic_worker5
+
+      - targets: ['matrix.envs.net:9136']
+        labels:
+          job: additional
+          instance: All
+          group: synapse
+          worker: additional1
+
+      - targets: ['matrix.envs.net:9137']
+        labels:
+          job: presence
+          instance: All
+          group: synapse
+          worker: presence
+
   - job_name: 'media-repo'
     metrics_path: '/metrics'
     static_configs: