Merge pull request #6511 from matrix-org/erikj/remove_db_config_from_apps

Move database config from apps into HomeServer object
This commit is contained in:
Erik Johnston 2019-12-12 10:37:56 +00:00 committed by GitHub
commit b8e4b39b69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 27 additions and 102 deletions

1
changelog.d/6511.misc Normal file
View File

@ -0,0 +1 @@
Move database config from apps into HomeServer object.

View File

@ -26,7 +26,6 @@ from synapse.config.homeserver import HomeServerConfig
from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage import DataStore from synapse.storage import DataStore
from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database from synapse.storage.prepare_database import prepare_database
logger = logging.getLogger("update_database") logger = logging.getLogger("update_database")
@ -35,21 +34,11 @@ logger = logging.getLogger("update_database")
class MockHomeserver(HomeServer): class MockHomeserver(HomeServer):
DATASTORE_CLASS = DataStore DATASTORE_CLASS = DataStore
def __init__(self, config, database_engine, db_conn, **kwargs): def __init__(self, config, **kwargs):
super(MockHomeserver, self).__init__( super(MockHomeserver, self).__init__(
config.server_name, config.server_name, reactor=reactor, config=config, **kwargs
reactor=reactor,
config=config,
database_engine=database_engine,
**kwargs
) )
self.database_engine = database_engine
self.db_conn = db_conn
def get_db_conn(self):
return self.db_conn
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -85,24 +74,14 @@ if __name__ == "__main__":
config = HomeServerConfig() config = HomeServerConfig()
config.parse_config_dict(hs_config, "", "") config.parse_config_dict(hs_config, "", "")
# Create the database engine and a connection to it. # Instantiate and initialise the homeserver object.
database_engine = create_engine(config.database_config) hs = MockHomeserver(config)
db_conn = database_engine.module.connect(
**{
k: v
for k, v in config.database_config.get("args", {}).items()
if not k.startswith("cp_")
}
)
db_conn = hs.get_db_conn()
# Update the database to the latest schema. # Update the database to the latest schema.
prepare_database(db_conn, database_engine, config=config) prepare_database(db_conn, hs.database_engine, config=config)
db_conn.commit() db_conn.commit()
# Instantiate and initialise the homeserver object.
hs = MockHomeserver(
config, database_engine, db_conn, db_config=config.database_config,
)
# setup instantiates the store within the homeserver object. # setup instantiates the store within the homeserver object.
hs.setup() hs.setup()
store = hs.get_datastore() store = hs.get_datastore()

View File

@ -45,7 +45,6 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto
from synapse.replication.slave.storage.room import RoomStore from synapse.replication.slave.storage.room import RoomStore
from synapse.replication.tcp.client import ReplicationClientHandler from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.engines import create_engine
from synapse.util.logcontext import LoggingContext from synapse.util.logcontext import LoggingContext
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -229,14 +228,10 @@ def start(config_options):
synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = AdminCmdServer( ss = AdminCmdServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -34,7 +34,6 @@ from synapse.replication.slave.storage.events import SlavedEventStore
from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore
from synapse.replication.tcp.client import ReplicationClientHandler from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -143,8 +142,6 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
if config.notify_appservices: if config.notify_appservices:
sys.stderr.write( sys.stderr.write(
"\nThe appservices must be disabled in the main synapse process" "\nThe appservices must be disabled in the main synapse process"
@ -159,10 +156,8 @@ def start(config_options):
ps = AppserviceServer( ps = AppserviceServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ps, config, use_worker_options=True) setup_logging(ps, config, use_worker_options=True)

View File

@ -62,7 +62,6 @@ from synapse.rest.client.v2_alpha.keys import KeyChangesServlet, KeyQueryServlet
from synapse.rest.client.v2_alpha.register import RegisterRestServlet from synapse.rest.client.v2_alpha.register import RegisterRestServlet
from synapse.rest.client.versions import VersionsRestServlet from synapse.rest.client.versions import VersionsRestServlet
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -181,14 +180,10 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = ClientReaderServer( ss = ClientReaderServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -57,7 +57,6 @@ from synapse.rest.client.v1.room import (
) )
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.data_stores.main.user_directory import UserDirectoryStore from synapse.storage.data_stores.main.user_directory import UserDirectoryStore
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -180,14 +179,10 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = EventCreatorServer( ss = EventCreatorServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -46,7 +46,6 @@ from synapse.replication.slave.storage.transactions import SlavedTransactionStor
from synapse.replication.tcp.client import ReplicationClientHandler from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.key.v2 import KeyApiV2Resource
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -162,14 +161,10 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = FederationReaderServer( ss = FederationReaderServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -41,7 +41,6 @@ from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.replication.tcp.streams._base import ReceiptsStream from synapse.replication.tcp.streams._base import ReceiptsStream
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.database import Database from synapse.storage.database import Database
from synapse.storage.engines import create_engine
from synapse.types import ReadReceipt from synapse.types import ReadReceipt
from synapse.util.async_helpers import Linearizer from synapse.util.async_helpers import Linearizer
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
@ -174,8 +173,6 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
if config.send_federation: if config.send_federation:
sys.stderr.write( sys.stderr.write(
"\nThe send_federation must be disabled in the main synapse process" "\nThe send_federation must be disabled in the main synapse process"
@ -190,10 +187,8 @@ def start(config_options):
ss = FederationSenderServer( ss = FederationSenderServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -39,7 +39,6 @@ from synapse.replication.slave.storage.registration import SlavedRegistrationSto
from synapse.replication.tcp.client import ReplicationClientHandler from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.rest.client.v2_alpha._base import client_patterns from synapse.rest.client.v2_alpha._base import client_patterns
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -234,14 +233,10 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = FrontendProxyServer( ss = FrontendProxyServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -69,7 +69,7 @@ from synapse.rest.media.v0.content_repository import ContentRepoResource
from synapse.rest.well_known import WellKnownResource from synapse.rest.well_known import WellKnownResource
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage import DataStore from synapse.storage import DataStore
from synapse.storage.engines import IncorrectDatabaseSetup, create_engine from synapse.storage.engines import IncorrectDatabaseSetup
from synapse.storage.prepare_database import UpgradeDatabaseException from synapse.storage.prepare_database import UpgradeDatabaseException
from synapse.util.caches import CACHE_SIZE_FACTOR from synapse.util.caches import CACHE_SIZE_FACTOR
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
@ -328,15 +328,10 @@ def setup(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
hs = SynapseHomeServer( hs = SynapseHomeServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
synapse.config.logger.setup_logging(hs, config, use_worker_options=False) synapse.config.logger.setup_logging(hs, config, use_worker_options=False)

View File

@ -40,7 +40,6 @@ from synapse.rest.admin import register_servlets_for_media_repo
from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v0.content_repository import ContentRepoResource
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.data_stores.main.media_repository import MediaRepositoryStore from synapse.storage.data_stores.main.media_repository import MediaRepositoryStore
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -157,14 +156,10 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = MediaRepositoryServer( ss = MediaRepositoryServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -37,7 +37,6 @@ from synapse.replication.slave.storage.room import RoomStore
from synapse.replication.tcp.client import ReplicationClientHandler from synapse.replication.tcp.client import ReplicationClientHandler
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage import DataStore from synapse.storage import DataStore
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.versionstring import get_version_string from synapse.util.versionstring import get_version_string
@ -203,14 +202,10 @@ def start(config_options):
# Force the pushers to start since they will be disabled in the main config # Force the pushers to start since they will be disabled in the main config
config.start_pushers = True config.start_pushers = True
database_engine = create_engine(config.database_config)
ps = PusherServer( ps = PusherServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ps, config, use_worker_options=True) setup_logging(ps, config, use_worker_options=True)

View File

@ -55,7 +55,6 @@ from synapse.rest.client.v1.room import RoomInitialSyncRestServlet
from synapse.rest.client.v2_alpha import sync from synapse.rest.client.v2_alpha import sync
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.data_stores.main.presence import UserPresenceState from synapse.storage.data_stores.main.presence import UserPresenceState
from synapse.storage.engines import create_engine
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
from synapse.util.stringutils import random_string from synapse.util.stringutils import random_string
@ -437,14 +436,10 @@ def start(config_options):
synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
ss = SynchrotronServer( ss = SynchrotronServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
application_service_handler=SynchrotronApplicationService(), application_service_handler=SynchrotronApplicationService(),
) )

View File

@ -44,7 +44,6 @@ from synapse.rest.client.v2_alpha import user_directory
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage.data_stores.main.user_directory import UserDirectoryStore from synapse.storage.data_stores.main.user_directory import UserDirectoryStore
from synapse.storage.database import Database from synapse.storage.database import Database
from synapse.storage.engines import create_engine
from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.caches.stream_change_cache import StreamChangeCache
from synapse.util.httpresourcetree import create_resource_tree from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.manhole import manhole from synapse.util.manhole import manhole
@ -200,8 +199,6 @@ def start(config_options):
events.USE_FROZEN_DICTS = config.use_frozen_dicts events.USE_FROZEN_DICTS = config.use_frozen_dicts
database_engine = create_engine(config.database_config)
if config.update_user_directory: if config.update_user_directory:
sys.stderr.write( sys.stderr.write(
"\nThe update_user_directory must be disabled in the main synapse process" "\nThe update_user_directory must be disabled in the main synapse process"
@ -216,10 +213,8 @@ def start(config_options):
ss = UserDirectoryServer( ss = UserDirectoryServer(
config.server_name, config.server_name,
db_config=config.database_config,
config=config, config=config,
version_string="Synapse/" + get_version_string(synapse), version_string="Synapse/" + get_version_string(synapse),
database_engine=database_engine,
) )
setup_logging(ss, config, use_worker_options=True) setup_logging(ss, config, use_worker_options=True)

View File

@ -34,6 +34,7 @@ from synapse.api.filtering import Filtering
from synapse.api.ratelimiting import Ratelimiter from synapse.api.ratelimiting import Ratelimiter
from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.api import ApplicationServiceApi
from synapse.appservice.scheduler import ApplicationServiceScheduler from synapse.appservice.scheduler import ApplicationServiceScheduler
from synapse.config.homeserver import HomeServerConfig
from synapse.crypto import context_factory from synapse.crypto import context_factory
from synapse.crypto.keyring import Keyring from synapse.crypto.keyring import Keyring
from synapse.events.builder import EventBuilderFactory from synapse.events.builder import EventBuilderFactory
@ -97,6 +98,7 @@ from synapse.server_notices.worker_server_notices_sender import (
) )
from synapse.state import StateHandler, StateResolutionHandler from synapse.state import StateHandler, StateResolutionHandler
from synapse.storage import DataStores, Storage from synapse.storage import DataStores, Storage
from synapse.storage.engines import create_engine
from synapse.streams.events import EventSources from synapse.streams.events import EventSources
from synapse.util import Clock from synapse.util import Clock
from synapse.util.distributor import Distributor from synapse.util.distributor import Distributor
@ -209,16 +211,18 @@ class HomeServer(object):
# instantiated during setup() for future return by get_datastore() # instantiated during setup() for future return by get_datastore()
DATASTORE_CLASS = abc.abstractproperty() DATASTORE_CLASS = abc.abstractproperty()
def __init__(self, hostname, reactor=None, **kwargs): def __init__(self, hostname: str, config: HomeServerConfig, reactor=None, **kwargs):
""" """
Args: Args:
hostname : The hostname for the server. hostname : The hostname for the server.
config: The full config for the homeserver.
""" """
if not reactor: if not reactor:
from twisted.internet import reactor from twisted.internet import reactor
self._reactor = reactor self._reactor = reactor
self.hostname = hostname self.hostname = hostname
self.config = config
self._building = {} self._building = {}
self._listening_services = [] self._listening_services = []
self.start_time = None self.start_time = None
@ -229,6 +233,12 @@ class HomeServer(object):
self.admin_redaction_ratelimiter = Ratelimiter() self.admin_redaction_ratelimiter = Ratelimiter()
self.registration_ratelimiter = Ratelimiter() self.registration_ratelimiter = Ratelimiter()
self.database_engine = create_engine(config.database_config)
config.database_config.setdefault("args", {})[
"cp_openfun"
] = self.database_engine.on_new_connection
self.db_config = config.database_config
self.datastores = None self.datastores = None
# Other kwargs are explicit dependencies # Other kwargs are explicit dependencies

View File

@ -17,18 +17,15 @@ from synapse.api.errors import SynapseError
from synapse.types import GroupID, RoomAlias, UserID, map_username_to_mxid_localpart from synapse.types import GroupID, RoomAlias, UserID, map_username_to_mxid_localpart
from tests import unittest from tests import unittest
from tests.utils import TestHomeServer
mock_homeserver = TestHomeServer(hostname="my.domain")
class UserIDTestCase(unittest.TestCase): class UserIDTestCase(unittest.HomeserverTestCase):
def test_parse(self): def test_parse(self):
user = UserID.from_string("@1234abcd:my.domain") user = UserID.from_string("@1234abcd:test")
self.assertEquals("1234abcd", user.localpart) self.assertEquals("1234abcd", user.localpart)
self.assertEquals("my.domain", user.domain) self.assertEquals("test", user.domain)
self.assertEquals(True, mock_homeserver.is_mine(user)) self.assertEquals(True, self.hs.is_mine(user))
def test_pase_empty(self): def test_pase_empty(self):
with self.assertRaises(SynapseError): with self.assertRaises(SynapseError):
@ -48,13 +45,13 @@ class UserIDTestCase(unittest.TestCase):
self.assertTrue(userA != userB) self.assertTrue(userA != userB)
class RoomAliasTestCase(unittest.TestCase): class RoomAliasTestCase(unittest.HomeserverTestCase):
def test_parse(self): def test_parse(self):
room = RoomAlias.from_string("#channel:my.domain") room = RoomAlias.from_string("#channel:test")
self.assertEquals("channel", room.localpart) self.assertEquals("channel", room.localpart)
self.assertEquals("my.domain", room.domain) self.assertEquals("test", room.domain)
self.assertEquals(True, mock_homeserver.is_mine(room)) self.assertEquals(True, self.hs.is_mine(room))
def test_build(self): def test_build(self):
room = RoomAlias("channel", "my.domain") room = RoomAlias("channel", "my.domain")

View File

@ -260,9 +260,7 @@ def setup_test_homeserver(
hs = homeserverToUse( hs = homeserverToUse(
name, name,
config=config, config=config,
db_config=config.database_config,
version_string="Synapse/tests", version_string="Synapse/tests",
database_engine=db_engine,
tls_server_context_factory=Mock(), tls_server_context_factory=Mock(),
tls_client_options_factory=Mock(), tls_client_options_factory=Mock(),
reactor=reactor, reactor=reactor,