From 9f3c0a8556a82960c795b8dc41a4666e0adebfef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Jul 2019 17:55:26 +0100 Subject: [PATCH 01/12] Add basic admin cmd app --- synapse/app/_base.py | 17 +++- synapse/app/admin_cmd.py | 198 +++++++++++++++++++++++++++++++++++++++ synapse/config/_base.py | 48 +++++++++- 3 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 synapse/app/admin_cmd.py diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d50a9840d..200978a58 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -48,7 +48,7 @@ def register_sighup(func): _sighup_callbacks.append(func) -def start_worker_reactor(appname, config): +def start_worker_reactor(appname, config, run_command=reactor.run): """ Run the reactor in the main process Daemonizes if necessary, and then configures some resources, before starting @@ -57,6 +57,7 @@ def start_worker_reactor(appname, config): Args: appname (str): application name which will be sent to syslog config (synapse.config.Config): config object + run_command (Callable[]): callable that actually runs the reactor """ logger = logging.getLogger(config.worker_app) @@ -69,11 +70,19 @@ def start_worker_reactor(appname, config): daemonize=config.worker_daemonize, print_pidfile=config.print_pidfile, logger=logger, + run_command=run_command, ) def start_reactor( - appname, soft_file_limit, gc_thresholds, pid_file, daemonize, print_pidfile, logger + appname, + soft_file_limit, + gc_thresholds, + pid_file, + daemonize, + print_pidfile, + logger, + run_command=reactor.run, ): """ Run the reactor in the main process @@ -88,6 +97,7 @@ def start_reactor( daemonize (bool): true to run the reactor in a background process print_pidfile (bool): whether to print the pid file, if daemonize is True logger (logging.Logger): logger instance to pass to Daemonize + run_command (Callable[]): callable that actually runs the reactor """ install_dns_limiter(reactor) @@ -103,7 +113,8 @@ def start_reactor( change_resource_limit(soft_file_limit) if gc_thresholds: gc.set_threshold(*gc_thresholds) - reactor.run() + + run_command() if daemonize: if print_pidfile: diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py new file mode 100644 index 000000000..bd73c47ae --- /dev/null +++ b/synapse/app/admin_cmd.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import sys + +from twisted.internet import defer, task + +import synapse +from synapse.app import _base +from synapse.config._base import ConfigError +from synapse.config.homeserver import HomeServerConfig +from synapse.config.logger import setup_logging +from synapse.handlers.admin import FileExfiltrationWriter +from synapse.replication.slave.storage._base import BaseSlavedStore +from synapse.replication.slave.storage.account_data import SlavedAccountDataStore +from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore +from synapse.replication.slave.storage.client_ips import SlavedClientIpStore +from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore +from synapse.replication.slave.storage.devices import SlavedDeviceStore +from synapse.replication.slave.storage.events import SlavedEventStore +from synapse.replication.slave.storage.filtering import SlavedFilteringStore +from synapse.replication.slave.storage.groups import SlavedGroupServerStore +from synapse.replication.slave.storage.presence import SlavedPresenceStore +from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore +from synapse.replication.slave.storage.receipts import SlavedReceiptsStore +from synapse.replication.slave.storage.registration import SlavedRegistrationStore +from synapse.replication.slave.storage.room import RoomStore +from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.server import HomeServer +from synapse.storage.engines import create_engine +from synapse.util.logcontext import LoggingContext +from synapse.util.versionstring import get_version_string + +logger = logging.getLogger("synapse.app.admin_cmd") + + +class AdminCmdSlavedStore( + SlavedReceiptsStore, + SlavedAccountDataStore, + SlavedApplicationServiceStore, + SlavedRegistrationStore, + SlavedFilteringStore, + SlavedPresenceStore, + SlavedGroupServerStore, + SlavedDeviceInboxStore, + SlavedDeviceStore, + SlavedPushRuleStore, + SlavedEventStore, + SlavedClientIpStore, + RoomStore, + BaseSlavedStore, +): + pass + + +class AdminCmdServer(HomeServer): + DATASTORE_CLASS = AdminCmdSlavedStore + + def _listen_http(self, listener_config): + pass + + def start_listening(self, listeners): + pass + + def build_tcp_replication(self): + return AdminCmdReplicationHandler(self) + + +class AdminCmdReplicationHandler(ReplicationClientHandler): + @defer.inlineCallbacks + def on_rdata(self, stream_name, token, rows): + pass + + def get_streams_to_replicate(self): + return {} + + +@defer.inlineCallbacks +def export_data_command(hs, user_id, directory): + """Export data for a user. + + Args: + user_id (str) + directory (str|None): Directory to write output to. Will create a temp + directory if not specified. + """ + + res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( + user_id, FileExfiltrationWriter(user_id, directory=directory) + ) + print(res) + + +def start(config_options): + parser = HomeServerConfig.create_argument_parser("Synapse Admin Command") + + subparser = parser.add_subparsers( + title="Admin Commands", + description="Choose and admin command to perform.", + required=True, + dest="command", + metavar="", + help="The admin command to perform.", + ) + export_data_parser = subparser.add_parser( + "export-data", help="Export all data for a user" + ) + export_data_parser.add_argument("user_id", help="User to extra data from") + export_data_parser.add_argument( + "--output-directory", + action="store", + metavar="DIRECTORY", + required=False, + help="The directory to store the exported data in. Must be emtpy. Defaults" + " to creating a temp directory.", + ) + + try: + config, args = HomeServerConfig.load_config_with_parser(parser, config_options) + except ConfigError as e: + sys.stderr.write("\n" + str(e) + "\n") + sys.exit(1) + + if config.worker_app is not None: + assert config.worker_app == "synapse.app.admin_cmd" + + # Update the config with some basic overrides so that don't have to specify + # a full worker config. + config.worker_app = "synapse.app.admin_cmd" + + if ( + not config.worker_daemonize + and not config.worker_log_file + and not config.worker_log_config + ): + # Since we're meant to be run as a "command" let's not redirect stdio + # unless we've actually set log config. + config.no_redirect_stdio = True + + # Explicitly disable background processes + config.update_user_directory = False + config.start_pushers = False + config.send_federation = False + + setup_logging(config, use_worker_options=True) + + synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts + + database_engine = create_engine(config.database_config) + + ss = AdminCmdServer( + config.server_name, + db_config=config.database_config, + config=config, + version_string="Synapse/" + get_version_string(synapse), + database_engine=database_engine, + ) + + ss.setup() + + if args.command == "export-data": + command = lambda: export_data_command(ss, args.user_id, args.output_directory) + else: + # This shouldn't happen. + raise ConfigError("Unknown admin command %s" % (args.command,)) + + # We use task.react as the basic run command as it correctly handles tearing + # down the reactor when the deferreds resolve and setting the return value. + # We also make sure that `_base.start` gets run before we actually run the + # command. + + @defer.inlineCallbacks + def run(_reactor): + with LoggingContext("command"): + yield _base.start(ss, []) + yield command() + + _base.start_worker_reactor( + "synapse-admin-cmd", config, run_command=lambda: task.react(run) + ) + + +if __name__ == "__main__": + with LoggingContext("main"): + start(sys.argv[1:]) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 965478d8d..14d3f7c1f 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -201,6 +201,26 @@ class Config(object): Returns: Config object. """ + config_parser = cls.create_argument_parser(description) + obj, _ = cls.load_config_with_parser(config_parser, argv) + + return obj + + @classmethod + def create_argument_parser(cls, description): + """Create an ArgumentParser instance with all the config flags. + + Doesn't support config-file-generation: used by the worker apps. + + Used for workers where we want to add extra flags/subcommands. + + Args: + description (str): App description + + Returns: + ArgumentParser + """ + config_parser = argparse.ArgumentParser(description=description) config_parser.add_argument( "-c", @@ -219,9 +239,31 @@ class Config(object): " Defaults to the directory containing the last config file", ) - obj = cls() + # We can only invoke `add_arguments` on an actual object, but + # `add_arguments` should be side effect free so this is probably fine. + cls().invoke_all("add_arguments", config_parser) - obj.invoke_all("add_arguments", config_parser) + return config_parser + + @classmethod + def load_config_with_parser(cls, config_parser, argv): + """Parse the commandline and config files with the given parser + + Doesn't support config-file-generation: used by the worker apps. + + Used for workers where we want to add extra flags/subcommands. + + Args: + conifg_parser (ArgumentParser) + argv (list[str]) + + Returns: + tuple[HomeServerConfig, argparse.Namespace]: Returns the parsed + config object and the parsed argparse.Namespace object from + `config_parser.parse_args(..)` + """ + + obj = cls() config_args = config_parser.parse_args(argv) @@ -244,7 +286,7 @@ class Config(object): obj.invoke_all("read_arguments", config_args) - return obj + return obj, config_args @classmethod def load_or_generate_config(cls, description, argv): From 10fe904d88ae19d4214a249d9b24a99eb7edb618 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jul 2019 17:21:27 +0100 Subject: [PATCH 02/12] Newsfile --- changelog.d/5597.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5597.feature diff --git a/changelog.d/5597.feature b/changelog.d/5597.feature new file mode 100644 index 000000000..6f9274888 --- /dev/null +++ b/changelog.d/5597.feature @@ -0,0 +1 @@ +Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. From 823e13ddf49b28eb0dab5d7be3921acc92fd0e44 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:15:34 +0100 Subject: [PATCH 03/12] Change add_arguments to be a static method --- synapse/config/_base.py | 32 +++++++++++++++++++++++++++++++- synapse/config/database.py | 3 ++- synapse/config/logger.py | 3 ++- synapse/config/registration.py | 3 ++- synapse/config/server.py | 3 ++- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 14d3f7c1f..e588f8298 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -137,12 +137,42 @@ class Config(object): return file_stream.read() def invoke_all(self, name, *args, **kargs): + """Invoke all instance methods with the given name and arguments in the + class's MRO. + + Args: + name (str): Name of function to invoke + *args + **kwargs + + Returns: + list: The list of the return values from each method called + """ results = [] for cls in type(self).mro(): if name in cls.__dict__: results.append(getattr(cls, name)(self, *args, **kargs)) return results + @classmethod + def invoke_all_static(cls, name, *args, **kargs): + """Invoke all static methods with the given name and arguments in the + class's MRO. + + Args: + name (str): Name of function to invoke + *args + **kwargs + + Returns: + list: The list of the return values from each method called + """ + results = [] + for c in cls.mro(): + if name in c.__dict__: + results.append(getattr(c, name)(*args, **kargs)) + return results + def generate_config( self, config_dir_path, @@ -241,7 +271,7 @@ class Config(object): # We can only invoke `add_arguments` on an actual object, but # `add_arguments` should be side effect free so this is probably fine. - cls().invoke_all("add_arguments", config_parser) + cls.invoke_all_static("add_arguments", config_parser) return config_parser diff --git a/synapse/config/database.py b/synapse/config/database.py index bcb2089dd..746a6cd1f 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -69,7 +69,8 @@ class DatabaseConfig(Config): if database_path is not None: self.database_config["args"]["database"] = database_path - def add_arguments(self, parser): + @staticmethod + def add_arguments(parser): db_group = parser.add_argument_group("database") db_group.add_argument( "-d", diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 931aec41c..52cf69122 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -103,7 +103,8 @@ class LoggingConfig(Config): if args.log_file is not None: self.log_file = args.log_file - def add_arguments(cls, parser): + @staticmethod + def add_arguments(parser): logging_group = parser.add_argument_group("logging") logging_group.add_argument( "-v", diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 4a59e6ec9..ee5885251 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -222,7 +222,8 @@ class RegistrationConfig(Config): % locals() ) - def add_arguments(self, parser): + @staticmethod + def add_arguments(parser): reg_group = parser.add_argument_group("registration") reg_group.add_argument( "--enable-registration", diff --git a/synapse/config/server.py b/synapse/config/server.py index 2a74dea2e..080d0630b 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -639,7 +639,8 @@ class ServerConfig(Config): if args.print_pidfile is not None: self.print_pidfile = args.print_pidfile - def add_arguments(self, parser): + @staticmethod + def add_arguments(parser): server_group = parser.add_argument_group("server") server_group.add_argument( "-D", From 37b524f9718b0faeaaac0dccab62c9a5e270c4a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:19:57 +0100 Subject: [PATCH 04/12] Fix up comments --- synapse/app/admin_cmd.py | 4 ++-- synapse/config/_base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index bd73c47ae..e618a6243 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2016 OpenMarket Ltd +# Copyright 2019 Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -109,7 +109,7 @@ def start(config_options): subparser = parser.add_subparsers( title="Admin Commands", - description="Choose and admin command to perform.", + description="Choose an admin command to perform.", required=True, dest="command", metavar="", diff --git a/synapse/config/_base.py b/synapse/config/_base.py index e588f8298..74a7980eb 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -284,7 +284,7 @@ class Config(object): Used for workers where we want to add extra flags/subcommands. Args: - conifg_parser (ArgumentParser) + config_parser (ArgumentParser) argv (list[str]) Returns: From fdefb9e29a7c28e5b218fc8d9745a2ec58bc3d27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:43:25 +0100 Subject: [PATCH 05/12] Move creation of ArgumentParser to caller --- synapse/app/admin_cmd.py | 4 +++- synapse/config/_base.py | 15 +++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index e618a6243..c4d752593 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import argparse import logging import sys @@ -105,7 +106,8 @@ def export_data_command(hs, user_id, directory): def start(config_options): - parser = HomeServerConfig.create_argument_parser("Synapse Admin Command") + parser = argparse.ArgumentParser(description="Synapse Admin Command") + HomeServerConfig.add_arguments_to_parser(parser) subparser = parser.add_subparsers( title="Admin Commands", diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 74a7980eb..8c3acff03 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -231,27 +231,24 @@ class Config(object): Returns: Config object. """ - config_parser = cls.create_argument_parser(description) + config_parser = argparse.ArgumentParser(description=description) + cls.add_arguments_to_parser(config_parser) obj, _ = cls.load_config_with_parser(config_parser, argv) return obj @classmethod - def create_argument_parser(cls, description): - """Create an ArgumentParser instance with all the config flags. + def add_arguments_to_parser(cls, config_parser): + """Adds all the config flags to an ArgumentParser. Doesn't support config-file-generation: used by the worker apps. Used for workers where we want to add extra flags/subcommands. Args: - description (str): App description - - Returns: - ArgumentParser + config_parser (ArgumentParser): App description """ - config_parser = argparse.ArgumentParser(description=description) config_parser.add_argument( "-c", "--config-path", @@ -273,8 +270,6 @@ class Config(object): # `add_arguments` should be side effect free so this is probably fine. cls.invoke_all_static("add_arguments", config_parser) - return config_parser - @classmethod def load_config_with_parser(cls, config_parser, argv): """Parse the commandline and config files with the given parser From c8f35d8d38c91b3493bd34363096a6e26756d8d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:49:18 +0100 Subject: [PATCH 06/12] Use set_defaults(func=) style --- synapse/app/admin_cmd.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index c4d752593..611a196e5 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -90,15 +90,17 @@ class AdminCmdReplicationHandler(ReplicationClientHandler): @defer.inlineCallbacks -def export_data_command(hs, user_id, directory): +def export_data_command(hs, args): """Export data for a user. Args: - user_id (str) - directory (str|None): Directory to write output to. Will create a temp - directory if not specified. + hs (HomeServer) + args (argparse.Namespace) """ + user_id = args.user_id + directory = args.output_directory + res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( user_id, FileExfiltrationWriter(user_id, directory=directory) ) @@ -129,6 +131,7 @@ def start(config_options): help="The directory to store the exported data in. Must be emtpy. Defaults" " to creating a temp directory.", ) + export_data_parser.set_defaults(func=export_data_command) try: config, args = HomeServerConfig.load_config_with_parser(parser, config_options) @@ -173,12 +176,6 @@ def start(config_options): ss.setup() - if args.command == "export-data": - command = lambda: export_data_command(ss, args.user_id, args.output_directory) - else: - # This shouldn't happen. - raise ConfigError("Unknown admin command %s" % (args.command,)) - # We use task.react as the basic run command as it correctly handles tearing # down the reactor when the deferreds resolve and setting the return value. # We also make sure that `_base.start` gets run before we actually run the @@ -188,7 +185,7 @@ def start(config_options): def run(_reactor): with LoggingContext("command"): yield _base.start(ss, []) - yield command() + yield args.func(ss, args) _base.start_worker_reactor( "synapse-admin-cmd", config, run_command=lambda: task.react(run) From 1b2067f53d772de1cadca22bdee176b9509d5b6f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 14:15:22 +0100 Subject: [PATCH 07/12] Add FileExfiltrationWriter --- synapse/app/admin_cmd.py | 70 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 611a196e5..13281d0af 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -15,7 +15,11 @@ # limitations under the License. import argparse import logging +import os import sys +import tempfile + +from canonicaljson import json from twisted.internet import defer, task @@ -24,7 +28,7 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.handlers.admin import FileExfiltrationWriter +from synapse.handlers.admin import ExfiltrationWriter from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -107,6 +111,70 @@ def export_data_command(hs, args): print(res) +class FileExfiltrationWriter(ExfiltrationWriter): + """An ExfiltrationWriter that writes the users data to a directory. + Returns the directory location on completion. + + Note: This writes to disk on the main reactor thread. + + Args: + user_id (str): The user whose data is being exfiltrated. + directory (str|None): The directory to write the data to, if None then + will write to a temporary directory. + """ + + def __init__(self, user_id, directory=None): + self.user_id = user_id + + if directory: + self.base_directory = directory + else: + self.base_directory = tempfile.mkdtemp( + prefix="synapse-exfiltrate__%s__" % (user_id,) + ) + + os.makedirs(self.base_directory, exist_ok=True) + if list(os.listdir(self.base_directory)): + raise Exception("Directory must be empty") + + def write_events(self, room_id, events): + room_directory = os.path.join(self.base_directory, "rooms", room_id) + os.makedirs(room_directory, exist_ok=True) + events_file = os.path.join(room_directory, "events") + + with open(events_file, "a") as f: + for event in events: + print(json.dumps(event.get_pdu_json()), file=f) + + def write_state(self, room_id, event_id, state): + room_directory = os.path.join(self.base_directory, "rooms", room_id) + state_directory = os.path.join(room_directory, "state") + os.makedirs(state_directory, exist_ok=True) + + event_file = os.path.join(state_directory, event_id) + + with open(event_file, "a") as f: + for event in state.values(): + print(json.dumps(event.get_pdu_json()), file=f) + + def write_invite(self, room_id, event, state): + self.write_events(room_id, [event]) + + # We write the invite state somewhere else as they aren't full events + # and are only a subset of the state at the event. + room_directory = os.path.join(self.base_directory, "rooms", room_id) + os.makedirs(room_directory, exist_ok=True) + + invite_state = os.path.join(room_directory, "invite_state") + + with open(invite_state, "a") as f: + for event in state.values(): + print(json.dumps(event), file=f) + + def finished(self): + return self.base_directory + + def start(config_options): parser = argparse.ArgumentParser(description="Synapse Admin Command") HomeServerConfig.add_arguments_to_parser(parser) From eca4f5ac730d9ecf29d25888a40a7264b6041a54 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 14:17:28 +0100 Subject: [PATCH 08/12] s/exfiltrate_user_data/export_user_data/ --- synapse/app/admin_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 13281d0af..3ff19f18e 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -105,7 +105,7 @@ def export_data_command(hs, args): user_id = args.user_id directory = args.output_directory - res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( + res = yield hs.get_handlers().admin_handler.export_user_data( user_id, FileExfiltrationWriter(user_id, directory=directory) ) print(res) From 03cc8c4b5d187129904dd76a525f111424c31de0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 14:25:05 +0100 Subject: [PATCH 09/12] Fix invoking add_argument from homeserver.py --- synapse/config/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 8c3acff03..ba1025f86 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -468,7 +468,7 @@ class Config(object): formatter_class=argparse.RawDescriptionHelpFormatter, ) - obj.invoke_all("add_arguments", parser) + obj.invoke_all_static("add_arguments", parser) args = parser.parse_args(remaining_args) config_dict = read_config_files(config_files) From d0d479c1af78af2204a7ed5aee1e187225a5af90 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jul 2019 09:52:56 +0100 Subject: [PATCH 10/12] Fix typo in synapse/app/admin_cmd.py Co-Authored-By: Aaron Raimist --- synapse/app/admin_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 3ff19f18e..68802ca75 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -196,7 +196,7 @@ def start(config_options): action="store", metavar="DIRECTORY", required=False, - help="The directory to store the exported data in. Must be emtpy. Defaults" + help="The directory to store the exported data in. Must be empty. Defaults" " to creating a temp directory.", ) export_data_parser.set_defaults(func=export_data_command) From f44354e17f11c2a5949861052db1f49d8b67233b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jul 2019 11:39:13 +0100 Subject: [PATCH 11/12] Clean up arg name and remove lying comment --- synapse/config/_base.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index ba1025f86..6ce5cd07f 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -266,12 +266,10 @@ class Config(object): " Defaults to the directory containing the last config file", ) - # We can only invoke `add_arguments` on an actual object, but - # `add_arguments` should be side effect free so this is probably fine. cls.invoke_all_static("add_arguments", config_parser) @classmethod - def load_config_with_parser(cls, config_parser, argv): + def load_config_with_parser(cls, parser, argv): """Parse the commandline and config files with the given parser Doesn't support config-file-generation: used by the worker apps. @@ -279,23 +277,23 @@ class Config(object): Used for workers where we want to add extra flags/subcommands. Args: - config_parser (ArgumentParser) + parser (ArgumentParser) argv (list[str]) Returns: tuple[HomeServerConfig, argparse.Namespace]: Returns the parsed config object and the parsed argparse.Namespace object from - `config_parser.parse_args(..)` + `parser.parse_args(..)` """ obj = cls() - config_args = config_parser.parse_args(argv) + config_args = parser.parse_args(argv) config_files = find_config_files(search_paths=config_args.config_path) if not config_files: - config_parser.error("Must supply a config file.") + parser.error("Must supply a config file.") if config_args.keys_directory: config_dir_path = config_args.keys_directory From 5ed7853bb018b58e99c6dd3ca91905b877968494 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jul 2019 11:45:57 +0100 Subject: [PATCH 12/12] Remove pointless description --- synapse/app/admin_cmd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 68802ca75..1fd52a552 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -181,7 +181,6 @@ def start(config_options): subparser = parser.add_subparsers( title="Admin Commands", - description="Choose an admin command to perform.", required=True, dest="command", metavar="",