Pass config_dir_path and data_dir_path into Config.read_config. (#5522)

* Pull config_dir_path and data_dir_path calculation out of read_config_files

* Pass config_dir_path and data_dir_path into read_config

changelog.d/5522.misc

@@ -0,0 +1 @@
+Pass config_dir_path and data_dir_path into Config.read_config.

synapse/config/_base.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2017-2018 New Vector Ltd
+# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -216,7 +218,7 @@ class Config(object):
             "--keys-directory",
             metavar="DIRECTORY",
             help="Where files such as certs and signing keys are stored when"
-            " their location is given explicitly in the config."
+            " their location is not given explicitly in the config."
             " Defaults to the directory containing the last config file",
         )
 
@@ -228,10 +230,22 @@ class Config(object):
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
+        if not config_files:
+            config_parser.error("Must supply a config file.")
+
+        if config_args.keys_directory:
+            config_dir_path = config_args.keys_directory
+        else:
+            config_dir_path = os.path.dirname(config_files[-1])
+        config_dir_path = os.path.abspath(config_dir_path)
+        data_dir_path = os.getcwd()
+
         config_dict = obj.read_config_files(
-            config_files, keys_directory=config_args.keys_directory
+            config_files, config_dir_path=config_dir_path, data_dir_path=data_dir_path
+        )
+        obj.parse_config_dict(
+            config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path
         )
-        obj.parse_config_dict(config_dict)
 
         obj.invoke_all("read_arguments", config_args)
 
@@ -282,7 +296,7 @@ class Config(object):
             metavar="DIRECTORY",
             help=(
                 "Specify where additional config files such as signing keys and log"
-                " config should be stored. Defaults to the same directory as the main"
+                " config should be stored. Defaults to the same directory as the last"
                 " config file."
             ),
         )
@@ -290,6 +304,20 @@ class Config(object):
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
+        if not config_files:
+            config_parser.error(
+                "Must supply a config file.\nA config file can be automatically"
+                ' generated using "--generate-config -H SERVER_NAME'
+                ' -c CONFIG-FILE"'
+            )
+
+        if config_args.config_directory:
+            config_dir_path = config_args.config_directory
+        else:
+            config_dir_path = os.path.dirname(config_files[-1])
+        config_dir_path = os.path.abspath(config_dir_path)
+        data_dir_path = os.getcwd()
+
         generate_missing_configs = config_args.generate_missing_configs
 
         obj = cls()
@@ -300,20 +328,10 @@ class Config(object):
                     "Please specify either --report-stats=yes or --report-stats=no\n\n"
                     + MISSING_REPORT_STATS_SPIEL
                 )
-            if not config_files:
+
-                config_parser.error(
-                    "Must supply a config file.\nA config file can be automatically"
-                    ' generated using "--generate-config -H SERVER_NAME'
-                    ' -c CONFIG-FILE"'
-                )
             (config_path,) = config_files
             if not cls.path_exists(config_path):
                 print("Generating config file %s" % (config_path,))
-                if config_args.config_directory:
-                    config_dir_path = config_args.config_directory
-                else:
-                    config_dir_path = os.path.dirname(config_path)
-                config_dir_path = os.path.abspath(config_dir_path)
 
                 server_name = config_args.server_name
                 if not server_name:
@@ -324,7 +342,7 @@ class Config(object):
 
                 config_str = obj.generate_config(
                     config_dir_path=config_dir_path,
-                    data_dir_path=os.getcwd(),
+                    data_dir_path=data_dir_path,
                     server_name=server_name,
                     report_stats=(config_args.report_stats == "yes"),
                     generate_secrets=True,
@@ -367,35 +385,37 @@ class Config(object):
         obj.invoke_all("add_arguments", parser)
         args = parser.parse_args(remaining_args)
 
-        if not config_files:
-            config_parser.error(
-                "Must supply a config file.\nA config file can be automatically"
-                ' generated using "--generate-config -H SERVER_NAME'
-                ' -c CONFIG-FILE"'
-            )
-
         config_dict = obj.read_config_files(
-            config_files, keys_directory=config_args.config_directory
+            config_files, config_dir_path=config_dir_path, data_dir_path=data_dir_path
         )
 
         if generate_missing_configs:
             obj.generate_missing_files(config_dict)
             return None
 
-        obj.parse_config_dict(config_dict)
+        obj.parse_config_dict(
+            config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path
+        )
         obj.invoke_all("read_arguments", args)
 
         return obj
 
-    def read_config_files(self, config_files, keys_directory=None):
+    def read_config_files(self, config_files, config_dir_path, data_dir_path):
         """Read the config files into a dict
 
+        Args:
+            config_files (iterable[str]): A list of the config files to read
+
+            config_dir_path (str): The path where the config files are kept. Used to
+                create filenames for things like the log config and the signing key.
+
+            data_dir_path (str): The path where the data files are kept. Used to create
+                filenames for things like the database and media store.
+
         Returns: dict
         """
-        if not keys_directory:
+        # FIXME: get rid of this
-            keys_directory = os.path.dirname(config_files[-1])
+        self.config_dir_path = config_dir_path
-
-        self.config_dir_path = os.path.abspath(keys_directory)
 
         # first we read the config files into a dict
         specified_config = {}
@@ -409,8 +429,8 @@ class Config(object):
             raise ConfigError(MISSING_SERVER_NAME)
         server_name = specified_config["server_name"]
         config_string = self.generate_config(
-            config_dir_path=self.config_dir_path,
+            config_dir_path=config_dir_path,
-            data_dir_path=os.getcwd(),
+            data_dir_path=data_dir_path,
             server_name=server_name,
             generate_secrets=False,
         )
@@ -430,8 +450,24 @@ class Config(object):
             )
         return config
 
-    def parse_config_dict(self, config_dict):
+    def parse_config_dict(self, config_dict, config_dir_path, data_dir_path):
-        self.invoke_all("read_config", config_dict)
+        """Read the information from the config dict into this Config object.
+
+        Args:
+            config_dict (dict): Configuration data, as read from the yaml
+
+            config_dir_path (str): The path where the config files are kept. Used to
+                create filenames for things like the log config and the signing key.
+
+            data_dir_path (str): The path where the data files are kept. Used to create
+                filenames for things like the database and media store.
+        """
+        self.invoke_all(
+            "read_config",
+            config_dict,
+            config_dir_path=config_dir_path,
+            data_dir_path=data_dir_path,
+        )
 
     def generate_missing_files(self, config_dict):
         self.invoke_all("generate_files", config_dict)

synapse/config/api.py

@@ -18,7 +18,7 @@ from ._base import Config
 
 
 class ApiConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.room_invite_state_types = config.get(
             "room_invite_state_types",
             [

synapse/config/appservice.py

@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 
 class AppServiceConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.app_service_config_files = config.get("app_service_config_files", [])
         self.notify_appservices = config.get("notify_appservices", True)
         self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)

synapse/config/captcha.py

@@ -16,7 +16,7 @@ from ._base import Config
 
 
 class CaptchaConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.recaptcha_private_key = config.get("recaptcha_private_key")
         self.recaptcha_public_key = config.get("recaptcha_public_key")
         self.enable_registration_captcha = config.get(

synapse/config/cas.py

@@ -22,7 +22,7 @@ class CasConfig(Config):
     cas_server_url: URL of CAS server
     """
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         cas_config = config.get("cas_config", None)
         if cas_config:
             self.cas_enabled = cas_config.get("enabled", True)

synapse/config/consent_config.py

@@ -84,7 +84,7 @@ class ConsentConfig(Config):
         self.user_consent_at_registration = False
         self.user_consent_policy_name = "Privacy Policy"
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         consent_config = config.get("user_consent")
         if consent_config is None:
             return

synapse/config/database.py

@@ -18,7 +18,7 @@ from ._base import Config
 
 
 class DatabaseConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
 
         self.database_config = config.get("database")

synapse/config/emailconfig.py

@@ -27,7 +27,7 @@ from ._base import Config, ConfigError
 
 
 class EmailConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         # TODO: We should separate better the email configuration from the notification
         # and account validity config.
 

synapse/config/groups.py

@@ -17,7 +17,7 @@ from ._base import Config
 
 
 class GroupsConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.enable_group_creation = config.get("enable_group_creation", False)
         self.group_creation_prefix = config.get("group_creation_prefix", "")
 

synapse/config/jwt_config.py

@@ -23,7 +23,7 @@ MISSING_JWT = """Missing jwt library. This is required for jwt login.
 
 
 class JWTConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         jwt_config = config.get("jwt_config", None)
         if jwt_config:
             self.jwt_enabled = jwt_config.get("enabled", False)

synapse/config/key.py

@@ -65,7 +65,7 @@ class TrustedKeyServer(object):
 
 
 class KeyConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         # the signing key can be specified inline or in a separate file
         if "signing_key" in config:
             self.signing_key = read_signing_keys([config["signing_key"]])

synapse/config/logger.py

@@ -74,7 +74,7 @@ root:
 
 
 class LoggingConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.verbosity = config.get("verbose", 0)
         self.no_redirect_stdio = config.get("no_redirect_stdio", False)
         self.log_config = self.abspath(config.get("log_config"))

synapse/config/metrics.py

@@ -21,7 +21,7 @@ MISSING_SENTRY = """Missing sentry-sdk library. This is required to enable sentr
 
 
 class MetricsConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.enable_metrics = config.get("enable_metrics", False)
         self.report_stats = config.get("report_stats", None)
         self.metrics_port = config.get("metrics_port")

synapse/config/password.py

@@ -20,7 +20,7 @@ class PasswordConfig(Config):
     """Password login configuration
     """
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         password_config = config.get("password_config", {})
         if password_config is None:
             password_config = {}

synapse/config/password_auth_providers.py

@@ -21,7 +21,7 @@ LDAP_PROVIDER = "ldap_auth_provider.LdapAuthProvider"
 
 
 class PasswordAuthProviderConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.password_providers = []
         providers = []
 

synapse/config/push.py

@@ -18,7 +18,7 @@ from ._base import Config
 
 
 class PushConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         push_config = config.get("push", {})
         self.push_include_content = push_config.get("include_content", True)
 

synapse/config/ratelimiting.py

@@ -36,7 +36,7 @@ class FederationRateLimitConfig(object):
 
 
 class RatelimitConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
 
         # Load the new-style messages config if it exists. Otherwise fall back
         # to the old method.

synapse/config/registration.py

@@ -46,7 +46,7 @@ class AccountValidityConfig(Config):
 
 
 class RegistrationConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.enable_registration = bool(
             strtobool(str(config.get("enable_registration", False)))
         )

synapse/config/repository.py

@@ -86,7 +86,7 @@ def parse_thumbnail_requirements(thumbnail_sizes):
 
 
 class ContentRepositoryConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))

synapse/config/room_directory.py

@@ -19,7 +19,7 @@ from ._base import Config, ConfigError
 
 
 class RoomDirectoryConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.enable_room_list_search = config.get("enable_room_list_search", True)
 
         alias_creation_rules = config.get("alias_creation_rules")

synapse/config/saml2_config.py

@@ -17,7 +17,7 @@ from ._base import Config, ConfigError
 
 
 class SAML2Config(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.saml2_enabled = False
 
         saml2_config = config.get("saml2_config")

synapse/config/server.py

@@ -40,7 +40,7 @@ DEFAULT_ROOM_VERSION = "4"
 
 
 class ServerConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.server_name = config["server_name"]
         self.server_context = config.get("server_context", None)
 

synapse/config/server_notices_config.py

@@ -66,7 +66,7 @@ class ServerNoticesConfig(Config):
         self.server_notices_mxid_avatar_url = None
         self.server_notices_room_name = None
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         c = config.get("server_notices")
         if c is None:
             return

synapse/config/spam_checker.py

@@ -19,7 +19,7 @@ from ._base import Config
 
 
 class SpamCheckerConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.spam_checker = None
 
         provider = config.get("spam_checker", None)

synapse/config/stats.py

@@ -25,7 +25,7 @@ class StatsConfig(Config):
     Configuration for the behaviour of synapse's stats engine
     """
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.stats_enabled = True
         self.stats_bucket_size = 86400
         self.stats_retention = sys.maxsize

synapse/config/third_party_event_rules.py

@@ -19,7 +19,7 @@ from ._base import Config
 
 
 class ThirdPartyRulesConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.third_party_event_rules = None
 
         provider = config.get("third_party_event_rules", None)

synapse/config/tls.py

@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class TlsConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
 
         acme_config = config.get("acme", None)
         if acme_config is None:

synapse/config/user_directory.py

@@ -21,7 +21,7 @@ class UserDirectoryConfig(Config):
     Configuration for the behaviour of the /user_directory API
     """
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.user_directory_search_enabled = True
         self.user_directory_search_all_users = False
         user_directory_config = config.get("user_directory", None)

synapse/config/voip.py

@@ -16,7 +16,7 @@ from ._base import Config
 
 
 class VoipConfig(Config):
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.turn_uris = config.get("turn_uris", [])
         self.turn_shared_secret = config.get("turn_shared_secret")
         self.turn_username = config.get("turn_username")

synapse/config/workers.py

@@ -21,7 +21,7 @@ class WorkerConfig(Config):
     They have their own pid_file and listener configuration. They use the
     replication_url to talk to the main synapse process."""
 
-    def read_config(self, config):
+    def read_config(self, config, **kwargs):
         self.worker_app = config.get("worker_app")
 
         # Canonicalise worker_app so that master always has None

tests/config/test_tls.py

@@ -65,7 +65,7 @@ s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg=
         }
 
         t = TestConfig()
-        t.read_config(config)
+        t.read_config(config, config_dir_path="", data_dir_path="")
         t.read_certificate_from_disk(require_cert_and_key=False)
 
         warnings = self.flushWarnings()

tests/http/federation/test_matrix_federation_agent.py

@@ -78,7 +78,7 @@ class MatrixFederationAgentTests(TestCase):
         # config_dict["trusted_key_servers"] = []
 
         self._config = config = HomeServerConfig()
-        config.parse_config_dict(config_dict)
+        config.parse_config_dict(config_dict, "", "")
 
         self.agent = MatrixFederationAgent(
             reactor=self.reactor,

tests/unittest.py

@@ -342,7 +342,7 @@ class HomeserverTestCase(TestCase):
 
         # Parse the config from a config dict into a HomeServerConfig
         config_obj = HomeServerConfig()
-        config_obj.parse_config_dict(config)
+        config_obj.parse_config_dict(config, "", "")
         kwargs["config"] = config_obj
 
         hs = setup_test_homeserver(self.addCleanup, *args, **kwargs)

tests/utils.py

@@ -182,7 +182,7 @@ def default_config(name, parse=False):
 
     if parse:
         config = HomeServerConfig()
-        config.parse_config_dict(config_dict)
+        config.parse_config_dict(config_dict, "", "")
         return config
 
     return config_dict