Add experimental 'databases' config (#6580)

This commit is contained in:
Erik Johnston 2020-01-06 14:44:01 +00:00 committed by GitHub
parent ab4b4ee6a7
commit 9f6c1befbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 64 additions and 13 deletions

1
changelog.d/6580.feature Normal file
View File

@ -0,0 +1 @@
Add experimental config option to specify multiple databases.

View File

@ -15,7 +15,6 @@
import logging import logging
import os import os
from textwrap import indent from textwrap import indent
from typing import List
import yaml import yaml
@ -30,16 +29,13 @@ class DatabaseConnectionConfig:
Args: Args:
name: A label for the database, used for logging. name: A label for the database, used for logging.
db_config: The config for a particular database, as per `database` db_config: The config for a particular database, as per `database`
section of main config. Has two fields: `name` for database section of main config. Has three fields: `name` for database
module name, and `args` for the args to give to the database module name, `args` for the args to give to the database
connector. connector, and optional `data_stores` that is a list of stores to
data_stores: The list of data stores that should be provisioned on the provision on this database (defaulting to all).
database. Defaults to all data stores.
""" """
def __init__( def __init__(self, name: str, db_config: dict):
self, name: str, db_config: dict, data_stores: List[str] = ["main", "state"]
):
if db_config["name"] not in ("sqlite3", "psycopg2"): if db_config["name"] not in ("sqlite3", "psycopg2"):
raise ConfigError("Unsupported database type %r" % (db_config["name"],)) raise ConfigError("Unsupported database type %r" % (db_config["name"],))
@ -48,6 +44,10 @@ class DatabaseConnectionConfig:
{"cp_min": 1, "cp_max": 1, "check_same_thread": False} {"cp_min": 1, "cp_max": 1, "check_same_thread": False}
) )
data_stores = db_config.get("data_stores")
if data_stores is None:
data_stores = ["main", "state"]
self.name = name self.name = name
self.config = db_config self.config = db_config
self.data_stores = data_stores self.data_stores = data_stores
@ -59,8 +59,37 @@ class DatabaseConfig(Config):
def read_config(self, config, **kwargs): def read_config(self, config, **kwargs):
self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K")) self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
# We *experimentally* support specifying multiple databases via the
# `databases` key. This is a map from a label to database config in the
# same format as the `database` config option, plus an extra
# `data_stores` key to specify which data store goes where. For example:
#
# databases:
# master:
# name: psycopg2
# data_stores: ["main"]
# args: {}
# state:
# name: psycopg2
# data_stores: ["state"]
# args: {}
multi_database_config = config.get("databases")
database_config = config.get("database") database_config = config.get("database")
if multi_database_config and database_config:
raise ConfigError("Can't specify both 'database' and 'datbases' in config")
if multi_database_config:
if config.get("database_path"):
raise ConfigError("Can't specify 'database_path' with 'databases'")
self.databases = [
DatabaseConnectionConfig(name, db_conf)
for name, db_conf in multi_database_config.items()
]
else:
if database_config is None: if database_config is None:
database_config = {"name": "sqlite3", "args": {}} database_config = {"name": "sqlite3", "args": {}}

View File

@ -37,6 +37,8 @@ class DataStores(object):
# store. # store.
self.databases = [] self.databases = []
self.main = None
self.state = None
for database_config in hs.config.database.databases: for database_config in hs.config.database.databases:
db_name = database_config.name db_name = database_config.name
@ -54,10 +56,22 @@ class DataStores(object):
if "main" in database_config.data_stores: if "main" in database_config.data_stores:
logger.info("Starting 'main' data store") logger.info("Starting 'main' data store")
# Sanity check we don't try and configure the main store on
# multiple databases.
if self.main:
raise Exception("'main' data store already configured")
self.main = main_store_class(database, db_conn, hs) self.main = main_store_class(database, db_conn, hs)
if "state" in database_config.data_stores: if "state" in database_config.data_stores:
logger.info("Starting 'state' data store") logger.info("Starting 'state' data store")
# Sanity check we don't try and configure the state store on
# multiple databases.
if self.state:
raise Exception("'state' data store already configured")
self.state = StateGroupDataStore(database, db_conn, hs) self.state = StateGroupDataStore(database, db_conn, hs)
db_conn.commit() db_conn.commit()
@ -65,3 +79,10 @@ class DataStores(object):
self.databases.append(database) self.databases.append(database)
logger.info("Database %r prepared", db_name) logger.info("Database %r prepared", db_name)
# Sanity check that we have actually configured all the required stores.
if not self.main:
raise Exception("No 'main' data store configured")
if not self.state:
raise Exception("No 'main' data store configured")