From e7f7e3d9095eadb0a08050e3e4fca431af895e04 Mon Sep 17 00:00:00 2001 From: deathrow Date: Tue, 8 Nov 2022 12:53:58 -0500 Subject: [PATCH 1/4] Rename entrypoint --- Worker.dockerfile | 2 +- rootfs/{start.py => configure_workers_and_start.py.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename rootfs/{start.py => configure_workers_and_start.py.py} (100%) diff --git a/Worker.dockerfile b/Worker.dockerfile index c674e32..51e99dd 100644 --- a/Worker.dockerfile +++ b/Worker.dockerfile @@ -109,7 +109,7 @@ VOLUME /data EXPOSE 8008/tcp -ENTRYPOINT ["python3", "start.py"] +ENTRYPOINT ["python3", "configure_workers_and_start.py"] HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD /bin/sh /healthcheck.sh \ No newline at end of file diff --git a/rootfs/start.py b/rootfs/configure_workers_and_start.py.py similarity index 100% rename from rootfs/start.py rename to rootfs/configure_workers_and_start.py.py From 0d319fdd678662ba7096786a407ee667870d1336 Mon Sep 17 00:00:00 2001 From: deathrow Date: Tue, 8 Nov 2022 12:55:00 -0500 Subject: [PATCH 2/4] Updating entrypoint --- rootfs/configure_workers_and_start.py.py | 76 +++++++++++++++++++++--- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/rootfs/configure_workers_and_start.py.py b/rootfs/configure_workers_and_start.py.py index 1ea456b..da25912 100644 --- a/rootfs/configure_workers_and_start.py.py +++ b/rootfs/configure_workers_and_start.py.py @@ -50,7 +50,12 @@ from jinja2 import Environment, FileSystemLoader MAIN_PROCESS_HTTP_LISTENER_PORT = 8080 - +# Workers with exposed endpoints needs either "client", "federation", or "media" listener_resources +# Watching /_matrix/client needs a "client" listener +# Watching /_matrix/federation needs a "federation" listener +# Watching /_matrix/media and related needs a "media" listener +# Stream Writers require "client" and "replication" listeners because they +# have to attach by instance_map to the master process and have client endpoints. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", @@ -209,6 +214,49 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { % (MAIN_PROCESS_HTTP_LISTENER_PORT,) ), }, + "account_data": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/.*/tags", + "^/_matrix/client/(r0|v3|unstable)/.*/account_data", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "presence": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "receipts": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt", + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "to_device": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "typing": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing" + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files @@ -271,7 +319,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def add_sharding_to_shared_config( +def add_worker_roles_to_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -309,6 +357,20 @@ def add_sharding_to_shared_config( "port": worker_port, } + elif worker_type in ["account_data", "presence", "receipts", "to_device", "typing"]: + # Update the list of stream writers + # It's convienent that the name of the worker type is the same as the event stream + shared_config.setdefault("stream_writers", {}).setdefault( + worker_type, [] + ).append(worker_name) + + # Map of stream writer instance names to host/ports combos + # For now, all stream writers need http replication ports + instance_map[worker_name] = { + "host": "localhost", + "port": worker_port, + } + elif worker_type == "media_repository": # The first configured media worker will run the media background jobs shared_config.setdefault("media_instance_running_background_jobs", worker_name) @@ -441,11 +503,11 @@ def generate_worker_files( # Check if more than one instance of this worker type has been specified worker_type_total_count = worker_types.count(worker_type) - if worker_type_total_count > 1: - # Update the shared config with sharding-related options if necessary - add_sharding_to_shared_config( - shared_config, worker_type, worker_name, worker_port - ) + + # Update the shared config with sharding-related options if necessary + add_worker_roles_to_shared_config( + shared_config, worker_type, worker_name, worker_port + ) # Enable the worker in supervisord worker_descriptors.append(worker_config) From 12de822fc131948bc4b0959014fad23a2c128a4f Mon Sep 17 00:00:00 2001 From: deathrow Date: Tue, 8 Nov 2022 12:56:56 -0500 Subject: [PATCH 3/4] Add missing start.py --- rootfs/start.py | 219 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 rootfs/start.py diff --git a/rootfs/start.py b/rootfs/start.py new file mode 100644 index 0000000..14dee17 --- /dev/null +++ b/rootfs/start.py @@ -0,0 +1,219 @@ +#!/usr/local/bin/python + +import codecs +import glob +import os +import subprocess +import sys + +import jinja2 + + +# Utility functions +def log(txt): + print(txt, file=sys.stderr) + + +def error(txt): + log(txt) + sys.exit(2) + + +def convert(src, dst, environ): + """Generate a file from a template + + Args: + src (str): path to input file + dst (str): path to file to write + environ (dict): environment dictionary, for replacement mappings. + """ + with open(src) as infile: + template = infile.read() + rendered = jinja2.Template(template).render(**environ) + with open(dst, "w") as outfile: + outfile.write(rendered) + + +def generate_config_from_template(config_dir, config_path, environ): + """Generate a homeserver.yaml from environment variables + + Args: + config_dir (str): where to put generated config files + config_path (str): where to put the main config file + environ (dict): environment dictionary + """ + for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): + if v not in environ: + error( + "Environment variable '%s' is mandatory when generating a config file." + % (v,) + ) + + # populate some params from data files (if they exist, else create new ones) + environ = environ.copy() + secrets = { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY", + } + + for name, secret in secrets.items(): + if secret not in environ: + filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + + # if the file already exists, load in the existing value; otherwise, + # generate a new secret and write it to a file + + if os.path.exists(filename): + log("Reading %s from %s" % (secret, filename)) + with open(filename) as handle: + value = handle.read() + else: + log("Generating a random secret for {}".format(secret)) + value = codecs.encode(os.urandom(32), "hex").decode() + with open(filename, "w") as handle: + handle.write(value) + environ[secret] = value + + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + if not os.path.exists(config_dir): + os.mkdir(config_dir) + + # Convert SYNAPSE_NO_TLS to boolean if exists + if "SYNAPSE_NO_TLS" in environ: + tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"]) + if tlsanswerstring in ("true", "on", "1", "yes"): + environ["SYNAPSE_NO_TLS"] = True + else: + if tlsanswerstring in ("false", "off", "0", "no"): + environ["SYNAPSE_NO_TLS"] = False + else: + error( + 'Environment variable "SYNAPSE_NO_TLS" found but value "' + + tlsanswerstring + + '" unrecognized; exiting.' + ) + + if "SYNAPSE_LOG_CONFIG" not in environ: + environ["SYNAPSE_LOG_CONFIG"] = config_dir + "/log.config" + + log("Generating synapse config file " + config_path) + convert("/conf/homeserver.yaml", config_path, environ) + + log_config_file = environ["SYNAPSE_LOG_CONFIG"] + log("Generating log config file " + log_config_file) + convert("/conf/log.config", log_config_file, environ) + + # Hopefully we already have a signing key, but generate one if not. + args = [ + "python", + "-m", + "synapse.app.homeserver", + "--config-path", + config_path, + # tell synapse to put generated keys in /data rather than /compiled + "--keys-directory", + config_dir, + "--generate-keys", + ] + + subprocess.check_output(args) + + +def run_generate_config(environ): + """Run synapse with a --generate-config param to generate a template config file + + Args: + environ (dict): env var dict + + Never returns. + """ + for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): + if v not in environ: + error("Environment variable '%s' is mandatory in `generate` mode." % (v,)) + + server_name = environ["SYNAPSE_SERVER_NAME"] + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") + data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") + + # create a suitable log config from our template + log_config_file = "%s/%s.log.config" % (config_dir, server_name) + if not os.path.exists(log_config_file): + log("Creating log config %s" % (log_config_file,)) + convert("/conf/log.config", log_config_file, environ) + + args = [ + "python", + "-m", + "synapse.app.homeserver", + "--server-name", + server_name, + "--report-stats", + environ["SYNAPSE_REPORT_STATS"], + "--config-path", + config_path, + "--config-directory", + config_dir, + "--data-directory", + data_dir, + "--generate-config", + "--open-private-ports", + ] + # log("running %s" % (args, )) + + os.execv("/usr/local/bin/python", args) + + +def main(args, environ): + mode = args[1] if len(args) > 1 else None + synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver") + + # In generate mode, generate a configuration and missing keys, then exit + if mode == "generate": + return run_generate_config(environ) + + if mode == "migrate_config": + # generate a config based on environment vars. + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get( + "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml" + ) + return generate_config_from_template( + config_dir, config_path, environ + ) + + if mode is not None: + error("Unknown execution mode '%s'" % (mode,)) + + config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") + config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") + + if not os.path.exists(config_path): + if "SYNAPSE_SERVER_NAME" in environ: + error( + """\ +Config file '%s' does not exist. + +The synapse docker image no longer supports generating a config file on-the-fly +based on environment variables. You can migrate to a static config file by +running with 'migrate_config'. See the README for more details. +""" + % (config_path,) + ) + + error( + "Config file '%s' does not exist. You should either create a new " + "config file by running with the `generate` argument (and then edit " + "the resulting file before restarting) or specify the path to an " + "existing config file with the SYNAPSE_CONFIG_PATH variable." + % (config_path,) + ) + + log("Starting synapse with config file " + config_path) + + args = ["python", "-m", synapse_worker, "--config-path", config_path] + os.execv("/usr/local/bin/python", args) + + +if __name__ == "__main__": + main(sys.argv, os.environ) \ No newline at end of file From dc1c5dff15bbbcb315d5d19cc984c12ccfa46cdf Mon Sep 17 00:00:00 2001 From: deathrow Date: Tue, 8 Nov 2022 12:57:24 -0500 Subject: [PATCH 4/4] Rename README --- readme.md => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename readme.md => README.md (100%) diff --git a/readme.md b/README.md similarity index 100% rename from readme.md rename to README.md