Update start.py

Dockerfile

@@ -99,9 +99,11 @@ COPY --from=build-malloc /tmp/hardened_malloc/out/libhardened_malloc.so /usr/loc
 COPY --from=builder /install /usr/local
 COPY --chown=synapse:synapse rootfs /
 COPY --from=redis_base /usr/local/bin/redis-server /usr/local/bin
+
+COPY ./rootfs/start.py /start.py
 COPY ./rootfs/conf-workers/* /conf/
+COPY ./rootfs/configure_workers_and_start.py /configure_workers_and_start.py
 COPY ./prefix-log /usr/local/bin/
-COPY ./rootfs /
 
 RUN chown -R synapse:synapse /start.py
 RUN chmod 755 /start.py

rootfs/start.py

@@ -1,31 +1,38 @@
+
 #!/usr/local/bin/python
 
 import codecs
 import glob
 import os
+import platform
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional
 
 import jinja2
 
 
 # Utility functions
-def log(txt):
+def log(txt: str) -> None:
+    print(txt)
+
+
+def error(txt: str) -> NoReturn:
     print(txt, file=sys.stderr)
-
-
-def error(txt):
-    log(txt)
     sys.exit(2)
 
 
-def convert(src, dst, environ):
+def flush_buffers() -> None:
-    """Generate a file from a template
+    sys.stdout.flush()
+    sys.stderr.flush()
 
+
+def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
+    """Generate a file from a template
     Args:
-        src (str): path to input file
+        src: path to input file
-        dst (str): path to file to write
+        dst: path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
+        environ: environment dictionary, for replacement mappings.
     """
     with open(src) as infile:
         template = infile.read()
@@ -34,23 +41,29 @@ def convert(src, dst, environ):
         outfile.write(rendered)
 
 
-def generate_config_from_template(config_dir, config_path, environ):
+def generate_config_from_template(
+    config_dir: str,
+    config_path: str,
+    os_environ: Mapping[str, str],
+    ownership: Optional[str],
+) -> None:
     """Generate a homeserver.yaml from environment variables
-
     Args:
-        config_dir (str): where to put generated config files
+        config_dir: where to put generated config files
-        config_path (str): where to put the main config file
+        config_path: where to put the main config file
-        environ (dict): environment dictionary
+        os_environ: environment mapping
+        ownership: "<user>:<group>" string which will be used to set
+            ownership of the generated configs. If None, ownership will not change.
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
+        if v not in os_environ:
             error(
                 "Environment variable '%s' is mandatory when generating a config file."
                 % (v,)
             )
 
     # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
+    environ: Dict[str, Any] = dict(os_environ)
     secrets = {
         "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
         "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@@ -101,11 +114,15 @@ def generate_config_from_template(config_dir, config_path, environ):
 
     log_config_file = environ["SYNAPSE_LOG_CONFIG"]
     log("Generating log config file " + log_config_file)
-    convert("/conf/log.config", log_config_file, environ)
+    convert(
+        "/conf/log.config",
+        log_config_file,
+        {**environ, "include_worker_name_in_log_line": False},
+    )
 
     # Hopefully we already have a signing key, but generate one if not.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--config-path",
@@ -116,15 +133,19 @@ def generate_config_from_template(config_dir, config_path, environ):
         "--generate-keys",
     ]
 
-    subprocess.check_output(args)
+    if ownership is not None:
+        log(f"Setting ownership on /data to {ownership}")
+        subprocess.run(["chown", "-R", ownership, "/data"], check=True)
+        args = ["gosu", ownership] + args
+
+    subprocess.run(args, check=True)
 
 
-def run_generate_config(environ):
+def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
     """Run synapse with a --generate-config param to generate a template config file
-
     Args:
-        environ (dict): env var dict
+        environ: env vars from `os.enrivon`.
-
+        ownership: "userid:groupid" arg for chmod. If None, ownership will not change.
     Never returns.
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
@@ -136,14 +157,20 @@ def run_generate_config(environ):
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
 
+    if ownership is not None:
+        # make sure that synapse has perms to write to the data dir.
+        log(f"Setting ownership on {data_dir} to {ownership}")
+        subprocess.run(["chown", ownership, data_dir], check=True)
+
     # create a suitable log config from our template
     log_config_file = "%s/%s.log.config" % (config_dir, server_name)
     if not os.path.exists(log_config_file):
         log("Creating log config %s" % (log_config_file,))
         convert("/conf/log.config", log_config_file, environ)
 
+    # generate the main config file, and a signing key.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--server-name",
@@ -160,17 +187,28 @@ def run_generate_config(environ):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-
+    flush_buffers()
-    os.execv("/usr/local/bin/python", args)
+    os.execv(sys.executable, args)
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
-    mode = args[1] if len(args) > 1 else None
+    mode = args[1] if len(args) > 1 else "run"
+
+    # if we were given an explicit user to switch to, do so
+    ownership = None
+    if "UID" in environ:
+        desired_uid = int(environ["UID"])
+        desired_gid = int(environ.get("GID", "991"))
+        ownership = f"{desired_uid}:{desired_gid}"
+    elif os.getuid() == 0:
+        # otherwise, if we are running as root, use user 991
+        ownership = "991:991"
+
     synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
 
     # In generate mode, generate a configuration and missing keys, then exit
     if mode == "generate":
-        return run_generate_config(environ)
+        return run_generate_config(environ, ownership)
 
     if mode == "migrate_config":
         # generate a config based on environment vars.
@@ -179,21 +217,36 @@ def main(args, environ):
             "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
         )
         return generate_config_from_template(
-            config_dir, config_path, environ
+            config_dir, config_path, environ, ownership
         )
 
-    if mode is not None:
+    if mode != "run":
         error("Unknown execution mode '%s'" % (mode,))
 
+    args = args[2:]
+
+    if "-m" not in args:
+        args = ["-m", synapse_worker] + args
+
+    jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
+
+    if os.path.isfile(jemallocpath):
+        environ["LD_PRELOAD"] = jemallocpath
+    else:
+        log("Could not find %s, will not use" % (jemallocpath,))
+
+    # if there are no config files passed to synapse, try adding the default file
+    if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
         config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+        config_path = environ.get(
+            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
+        )
 
         if not os.path.exists(config_path):
             if "SYNAPSE_SERVER_NAME" in environ:
                 error(
                     """\
 Config file '%s' does not exist.
-
 The synapse docker image no longer supports generating a config file on-the-fly
 based on environment variables. You can migrate to a static config file by
 running with 'migrate_config'. See the README for more details.
@@ -209,10 +262,18 @@ running with 'migrate_config'. See the README for more details.
                 % (config_path,)
             )
 
-    log("Starting synapse with config file " + config_path)
+        args += ["--config-path", config_path]
 
-    args = ["python", "-m", synapse_worker, "--config-path", config_path]
+    log("Starting synapse with args " + " ".join(args))
-    os.execv("/usr/local/bin/python", args)
+
+    args = [sys.executable] + args
+    if ownership is not None:
+        args = ["gosu", ownership] + args
+        flush_buffers()
+        os.execve("/usr/sbin/gosu", args, environ)
+    else:
+        flush_buffers()
+        os.execve(sys.executable, args, environ)
 
 
 if __name__ == "__main__":