Add Sync

README.md

@@ -12,14 +12,13 @@ A Synapse Docker deployment with:
 - Manage server via `synadm`
 - Images built locally
 - Matrix Maubot
-- Postgres web UI
 - Matrix integration manager
 
 ### Getting Started
 
-Dependencies: `cargo docker docker-compose git python `
+Dependencies: `cargo` `docker` `docker-compose` `git` `python `
 
-Subdomains: `matrix` `dimension` `adminer` `maubot`
+Subdomains: `matrix` `dimension` `maubot`
 
 Clone the repository:
 ```

docker-compose.yml

@@ -70,7 +70,6 @@ services:
     networks:
       - matrix
       - registration
-      - adminer
 
   pantalaimon:
     build: ./images/pantalaimon
@@ -112,14 +111,6 @@ services:
     networks:
       - matrix
 
-  adminer:
-    image: adminer
-    container_name: adminer
-    restart: unless-stopped
-    networks:
-      - matrix
-      - adminer 
-
   matrix-registration:
     build: ./images/synapse-captcha
     container_name: matrix-registration
@@ -187,6 +178,45 @@ services:
 
 # WORKERS 
 
+  sync5:
+    build: ./images/synapse-worker-docker
+    container_name: sync5
+    user: "991:991"
+    environment:
+      SYNAPSE_WORKER: synapse.app.generic_worker
+    restart: unless-stopped
+    entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/sync5.yaml"]
+    volumes:
+      - ./files:/data:Z
+      - ./bridges:/bridges:Z
+    depends_on:
+      - synapse
+    ports:
+     - 127.0.0.1:8094:8081
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+  sync4:
+    build: ./images/synapse-worker-docker
+    container_name: sync4
+    user: "991:991"
+    environment:
+      SYNAPSE_WORKER: synapse.app.generic_worker
+    restart: unless-stopped
+    entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/sync4.yaml"]
+    volumes:
+      - ./files:/data:Z
+      - ./bridges:/bridges:Z
+    depends_on:
+      - synapse
+    ports:
+     - 127.0.0.1:8093:8081
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
   sync3:
     build: ./images/synapse-worker-docker
     container_name: sync3
@@ -333,6 +363,28 @@ services:
     cap_drop:
       - ALL
 
+  synapse-generic-worker-4:
+    build: ./images/workers/synapse
+    container_name: synapse-generic-worker-4
+    user: "991:991"
+    restart: unless-stopped
+    environment:
+      SYNAPSE_WORKER: synapse.app.generic_worker
+    entrypoint: ["/start.py", "run", "--config-path=/data/homeserver.yaml", "--config-path=/data/workers/worker4.yaml"]
+    volumes:  
+      - ./files:/data:Z
+      - ./bridges:/bridges:Z
+    depends_on:
+      - synapse
+    networks:
+      - matrix
+    ports:
+     - 127.0.0.1:8083:8081
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+
   synapse-generic-worker-3:
     build: ./images/workers/synapse
     container_name: synapse-generic-worker-3
@@ -460,6 +512,5 @@ services:
 networks:
   matrix:
   registration:
-  adminer:
   dimension:
   

files/log/sync4.yaml

@@ -0,0 +1,84 @@
+# Log configuration for Synapse.
+#
+# This is a YAML file containing a standard Python logging configuration
+# dictionary. See [1] for details on the valid settings.
+#
+# Synapse also supports structured logging for machine readable logs which can
+# be ingested by ELK stacks. See [2] for details.
+#
+# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: precise
+        filename: /data/workers/log/worker_generic_sync4.log
+        when: midnight
+        backupCount: 2  # Does not include the current log file.
+        encoding: utf8
+
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
+    buffer:
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+        target: file
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
+        # performance, at the expensive of it taking longer for log lines to
+        # be written to disk.
+        # This parameter is required.
+        capacity: 10
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
+
+    # A handler that writes logs to stderr. Unused by default, but can be used
+    # instead of "buffer" and "file" in the logger handlers.
+    console:
+        class: logging.StreamHandler
+        formatter: precise
+
+loggers:
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: WARN
+
+    twisted:
+        # We send the twisted logging directly to the file handler,
+        # to work around https://github.com/matrix-org/synapse/issues/3471
+        # when using "buffer" logger. Use "console" to log to stderr instead.
+        handlers: [file]
+        propagate: false
+
+root:
+    level: WARN
+
+    # Write logs to the `buffer` handler, which will buffer them together in memory,
+    # then write them to a file.
+    #
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
+    #
+    handlers: [buffer]
+
+disable_existing_loggers: false

files/log/sync5.yaml

@@ -0,0 +1,84 @@
+# Log configuration for Synapse.
+#
+# This is a YAML file containing a standard Python logging configuration
+# dictionary. See [1] for details on the valid settings.
+#
+# Synapse also supports structured logging for machine readable logs which can
+# be ingested by ELK stacks. See [2] for details.
+#
+# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: precise
+        filename: /data/workers/log/worker_generic_sync5.log
+        when: midnight
+        backupCount: 2  # Does not include the current log file.
+        encoding: utf8
+
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
+    buffer:
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+        target: file
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
+        # performance, at the expensive of it taking longer for log lines to
+        # be written to disk.
+        # This parameter is required.
+        capacity: 10
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
+
+    # A handler that writes logs to stderr. Unused by default, but can be used
+    # instead of "buffer" and "file" in the logger handlers.
+    console:
+        class: logging.StreamHandler
+        formatter: precise
+
+loggers:
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: WARN
+
+    twisted:
+        # We send the twisted logging directly to the file handler,
+        # to work around https://github.com/matrix-org/synapse/issues/3471
+        # when using "buffer" logger. Use "console" to log to stderr instead.
+        handlers: [file]
+        propagate: false
+
+root:
+    level: WARN
+
+    # Write logs to the `buffer` handler, which will buffer them together in memory,
+    # then write them to a file.
+    #
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
+    #
+    handlers: [buffer]
+
+disable_existing_loggers: false

files/log/worker4.yaml

@@ -0,0 +1,84 @@
+# Log configuration for Synapse.
+#
+# This is a YAML file containing a standard Python logging configuration
+# dictionary. See [1] for details on the valid settings.
+#
+# Synapse also supports structured logging for machine readable logs which can
+# be ingested by ELK stacks. See [2] for details.
+#
+# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+handlers:
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: precise
+        filename: /data/workers/log/worker4.log
+        when: midnight
+        backupCount: 2  # Does not include the current log file.
+        encoding: utf8
+
+    # Default to buffering writes to log file for efficiency.
+    # WARNING/ERROR logs will still be flushed immediately, but there will be a
+    # delay (of up to `period` seconds, or until the buffer is full with
+    # `capacity` messages) before INFO/DEBUG logs get written.
+    buffer:
+        class: synapse.logging.handlers.PeriodicallyFlushingMemoryHandler
+        target: file
+
+        # The capacity is the maximum number of log lines that are buffered
+        # before being written to disk. Increasing this will lead to better
+        # performance, at the expensive of it taking longer for log lines to
+        # be written to disk.
+        # This parameter is required.
+        capacity: 10
+
+        # Logs with a level at or above the flush level will cause the buffer to
+        # be flushed immediately.
+        # Default value: 40 (ERROR)
+        # Other values: 50 (CRITICAL), 30 (WARNING), 20 (INFO), 10 (DEBUG)
+        flushLevel: 30  # Flush immediately for WARNING logs and higher
+
+        # The period of time, in seconds, between forced flushes.
+        # Messages will not be delayed for longer than this time.
+        # Default value: 5 seconds
+        period: 5
+
+    # A handler that writes logs to stderr. Unused by default, but can be used
+    # instead of "buffer" and "file" in the logger handlers.
+    console:
+        class: logging.StreamHandler
+        formatter: precise
+
+loggers:
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: WARN
+
+    twisted:
+        # We send the twisted logging directly to the file handler,
+        # to work around https://github.com/matrix-org/synapse/issues/3471
+        # when using "buffer" logger. Use "console" to log to stderr instead.
+        handlers: [file]
+        propagate: false
+
+root:
+    level: WARN
+
+    # Write logs to the `buffer` handler, which will buffer them together in memory,
+    # then write them to a file.
+    #
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
+    #
+    handlers: [buffer]
+
+disable_existing_loggers: false

files/workers/sync4.yaml

@@ -0,0 +1,17 @@
+worker_app: synapse.app.generic_worker
+worker_name: sync4
+
+# The replication listener on the synapse to talk to.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_main_http_uri: http://synapse:8008/
+
+worker_log_config: /data/log/sync4.yaml
+
+worker_listeners:
+  - type: http
+    port: 8081
+    x_forwarded: true
+    resources:
+      - names: [client, federation]

files/workers/sync5.yaml

@@ -0,0 +1,17 @@
+worker_app: synapse.app.generic_worker
+worker_name: sync5
+
+# The replication listener on the synapse to talk to.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_main_http_uri: http://synapse:8008/
+
+worker_log_config: /data/log/sync5.yaml
+
+worker_listeners:
+  - type: http
+    port: 8081
+    x_forwarded: true
+    resources:
+      - names: [client, federation]

files/workers/worker4.yaml

@@ -0,0 +1,17 @@
+worker_app: synapse.app.generic_worker
+worker_name: worker4
+
+# The replication listener on the synapse to talk to.
+worker_replication_host: synapse
+worker_replication_http_port: 9093
+
+worker_main_http_uri: http://synapse:8008/
+
+worker_log_config: /data/log/worker4.yaml
+
+worker_listeners:
+  - type: http
+    port: 8081
+    x_forwarded: true
+    resources:
+      - names: [client, federation]

swag/nginx/include.d/sync_worker.conf

@@ -0,0 +1,30 @@
+# Choose sync worker based on the existence of "since" query parameter
+map $arg_since $sync {
+        default sync_worker;
+        '' sync_init;
+}
+	# Sync initial/normal
+        location ~ ^/_matrix/client/(r0|v3)/sync$ {
+		include snippets/matrix-proxy-headers.conf;
+		proxy_pass http://$sync;
+		proxy_read_timeout 1h;
+        }
+
+	# Normal sync
+        location ~ ^/_matrix/client/(api/v1|r0|v3)/events$ {
+		include snippets/matrix-proxy-headers.conf;
+		proxy_pass http://sync_worker;
+        }
+
+        # Initial_sync
+	location ~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$ {
+		include snippets/matrix-proxy-headers.conf;
+		proxy_pass http://sync_init;
+		proxy_read_timeout 1h;
+        }
+
+        location ~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ {
+		include snippets/matrix-proxy-headers.conf;
+		proxy_pass http://sync_init;
+		proxy_read_timeout 1h;
+        }

swag/nginx/include.d/upstream_workers.conf

@@ -2,23 +2,43 @@ upstream generic_worker_ih {
        ip_hash;
        server synapse-generic-worker-1:8080;
        server synapse-generic-worker-2:8081;
-       server sync1:8090;
-       server sync2:8091;
-       server sync3:8092;
+       server synapse-generic-worker-3:8082;
+       server synapse-generic-worker-4:8083;
 }
 
 upstream generic_worker_lc {
        least_conn;
        server synapse-generic-worker-1:8080;
        server synapse-generic-worker-2:8081;
+       server synapse-generic-worker-3:8082;
+       server synapse-generic-worker-4:8083;
+}
+
+# extract username from token get parameter
+map $arg_access_token $token_from_arg {
+  default   $arg_access_token;
+  "~syt_(?<username>.*?)_.*"           $username;
+}
+
+# extract username part from bearer token, fallback to access_token
+map $http_authorization $proxy_username_label {
+    default                              $http_authorization;
+    "~Bearer syt_(?<username>.*?)_.*"    $username;
+    ""                                   $token_from_arg;
+}
+
+
+upstream sync_worker {
+       # pin with username extracted from bearer token or access_token
+       hash $proxy_username_label consistent;
        server sync1:8090;
        server sync2:8091;
        server sync3:8092;
 }
 
-upstream sync_worker {
-       ip_hash;
-       server sync1:8090;
-       server sync2:8091;
-       server sync3:8092;
+upstream sync_init {
+        # Use the username mapper result for hash key
+        hash $mxid_localpart consistent;
+        server sync4:8093;
+        server sync5:8094;
 }

swag/nginx/site-confs/adminer.conf

@@ -1,17 +0,0 @@
-server {  
-    listen 443 ssl http2;
-    listen [::]:443 ssl http2;
-
-    server_name adminer.*;
-
-    include /config/nginx/ssl.conf;
-
-    location / {
-        include /config/nginx/proxy.conf;
-        include /config/nginx/resolver.conf;
-        set $upstream_app adminer;
-        set $upstream_port 8080;
-        set $upstream_proto http;
-        proxy_pass $upstream_proto://$upstream_app:$upstream_port;
-    }
-}