Improve database configuration docs (#6988)

Attempts to clarify the sample config for databases, and add some stuff about
tcp keepalives to `postgres.md`.
This commit is contained in:
Richard van der Hoff 2020-03-20 16:24:22 +01:00 committed by GitHub
parent fdb1344716
commit c165c1233b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 132 additions and 71 deletions

1
changelog.d/6988.doc Normal file
View File

@ -0,0 +1 @@
Improve the documentation for database configuration.

View File

@ -105,7 +105,8 @@ of free memory the database host has available.
When you are ready to start using PostgreSQL, edit the `database` When you are ready to start using PostgreSQL, edit the `database`
section in your config file to match the following lines: section in your config file to match the following lines:
database: ```yaml
database:
name: psycopg2 name: psycopg2
args: args:
user: <user> user: <user>
@ -114,10 +115,31 @@ section in your config file to match the following lines:
host: <host> host: <host>
cp_min: 5 cp_min: 5
cp_max: 10 cp_max: 10
```
All key, values in `args` are passed to the `psycopg2.connect(..)` All key, values in `args` are passed to the `psycopg2.connect(..)`
function, except keys beginning with `cp_`, which are consumed by the function, except keys beginning with `cp_`, which are consumed by the
twisted adbapi connection pool. twisted adbapi connection pool. See the [libpq
documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS)
for a list of options which can be passed.
You should consider tuning the `args.keepalives_*` options if there is any danger of
the connection between your homeserver and database dropping, otherwise Synapse
may block for an extended period while it waits for a response from the
database server. Example values might be:
```yaml
# seconds of inactivity after which TCP should send a keepalive message to the server
keepalives_idle: 10
# the number of seconds after which a TCP keepalive message that is not
# acknowledged by the server should be retransmitted
keepalives_interval: 10
# the number of TCP keepalives that can be lost before the client's connection
# to the server is considered dead
keepalives_count: 3
```
## Porting from SQLite ## Porting from SQLite

View File

@ -578,13 +578,46 @@ acme:
## Database ## ## Database ##
# The 'database' setting defines the database that synapse uses to store all of
# its data.
#
# 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
# 'psycopg2' (for PostgreSQL).
#
# 'args' gives options which are passed through to the database engine,
# except for options starting 'cp_', which are used to configure the Twisted
# connection pool. For a reference to valid arguments, see:
# * for sqlite: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
# * for postgres: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
# * for the connection pool: https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__
#
#
# Example SQLite configuration:
#
#database:
# name: sqlite3
# args:
# database: /path/to/homeserver.db
#
#
# Example Postgres configuration:
#
#database:
# name: psycopg2
# args:
# user: synapse
# password: secretpassword
# database: synapse
# host: localhost
# cp_min: 5
# cp_max: 10
#
# For more information on using Synapse with Postgres, see `docs/postgres.md`.
#
database: database:
# The database engine name name: sqlite3
name: "sqlite3"
# Arguments to pass to the engine
args: args:
# Path to the database database: DATADIR/homeserver.db
database: "DATADIR/homeserver.db"
# Number of events to cache in memory. # Number of events to cache in memory.
# #

View File

@ -294,7 +294,6 @@ class RootConfig(object):
report_stats=None, report_stats=None,
open_private_ports=False, open_private_ports=False,
listeners=None, listeners=None,
database_conf=None,
tls_certificate_path=None, tls_certificate_path=None,
tls_private_key_path=None, tls_private_key_path=None,
acme_domain=None, acme_domain=None,
@ -367,7 +366,6 @@ class RootConfig(object):
report_stats=report_stats, report_stats=report_stats,
open_private_ports=open_private_ports, open_private_ports=open_private_ports,
listeners=listeners, listeners=listeners,
database_conf=database_conf,
tls_certificate_path=tls_certificate_path, tls_certificate_path=tls_certificate_path,
tls_private_key_path=tls_private_key_path, tls_private_key_path=tls_private_key_path,
acme_domain=acme_domain, acme_domain=acme_domain,

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd # Copyright 2014-2016 OpenMarket Ltd
# Copyright 2020 The Matrix.org Foundation C.I.C.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -14,14 +15,60 @@
# limitations under the License. # limitations under the License.
import logging import logging
import os import os
from textwrap import indent
import yaml
from synapse.config._base import Config, ConfigError from synapse.config._base import Config, ConfigError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
DEFAULT_CONFIG = """\
## Database ##
# The 'database' setting defines the database that synapse uses to store all of
# its data.
#
# 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or
# 'psycopg2' (for PostgreSQL).
#
# 'args' gives options which are passed through to the database engine,
# except for options starting 'cp_', which are used to configure the Twisted
# connection pool. For a reference to valid arguments, see:
# * for sqlite: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
# * for postgres: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
# * for the connection pool: https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__
#
#
# Example SQLite configuration:
#
#database:
# name: sqlite3
# args:
# database: /path/to/homeserver.db
#
#
# Example Postgres configuration:
#
#database:
# name: psycopg2
# args:
# user: synapse
# password: secretpassword
# database: synapse
# host: localhost
# cp_min: 5
# cp_max: 10
#
# For more information on using Synapse with Postgres, see `docs/postgres.md`.
#
database:
name: sqlite3
args:
database: %(database_path)s
# Number of events to cache in memory.
#
#event_cache_size: 10K
"""
class DatabaseConnectionConfig: class DatabaseConnectionConfig:
"""Contains the connection config for a particular database. """Contains the connection config for a particular database.
@ -36,10 +83,12 @@ class DatabaseConnectionConfig:
""" """
def __init__(self, name: str, db_config: dict): def __init__(self, name: str, db_config: dict):
if db_config["name"] not in ("sqlite3", "psycopg2"): db_engine = db_config.get("name", "sqlite3")
raise ConfigError("Unsupported database type %r" % (db_config["name"],))
if db_config["name"] == "sqlite3": if db_engine not in ("sqlite3", "psycopg2"):
raise ConfigError("Unsupported database type %r" % (db_engine,))
if db_engine == "sqlite3":
db_config.setdefault("args", {}).update( db_config.setdefault("args", {}).update(
{"cp_min": 1, "cp_max": 1, "check_same_thread": False} {"cp_min": 1, "cp_max": 1, "check_same_thread": False}
) )
@ -97,34 +146,10 @@ class DatabaseConfig(Config):
self.set_databasepath(config.get("database_path")) self.set_databasepath(config.get("database_path"))
def generate_config_section(self, data_dir_path, database_conf, **kwargs): def generate_config_section(self, data_dir_path, **kwargs):
if not database_conf: return DEFAULT_CONFIG % {
database_path = os.path.join(data_dir_path, "homeserver.db") "database_path": os.path.join(data_dir_path, "homeserver.db")
database_conf = ( }
"""# The database engine name
name: "sqlite3"
# Arguments to pass to the engine
args:
# Path to the database
database: "%(database_path)s"
"""
% locals()
)
else:
database_conf = indent(yaml.dump(database_conf), " " * 10).lstrip()
return (
"""\
## Database ##
database:
%(database_conf)s
# Number of events to cache in memory.
#
#event_cache_size: 10K
"""
% locals()
)
def read_arguments(self, args): def read_arguments(self, args):
self.set_databasepath(args.database_path) self.set_databasepath(args.database_path)

View File

@ -21,9 +21,9 @@ from tests import unittest
class DatabaseConfigTestCase(unittest.TestCase): class DatabaseConfigTestCase(unittest.TestCase):
def test_database_configured_correctly_no_database_conf_param(self): def test_database_configured_correctly(self):
conf = yaml.safe_load( conf = yaml.safe_load(
DatabaseConfig().generate_config_section("/data_dir_path", None) DatabaseConfig().generate_config_section(data_dir_path="/data_dir_path")
) )
expected_database_conf = { expected_database_conf = {
@ -32,21 +32,3 @@ class DatabaseConfigTestCase(unittest.TestCase):
} }
self.assertEqual(conf["database"], expected_database_conf) self.assertEqual(conf["database"], expected_database_conf)
def test_database_configured_correctly_database_conf_param(self):
database_conf = {
"name": "my super fast datastore",
"args": {
"user": "matrix",
"password": "synapse_database_password",
"host": "synapse_database_host",
"database": "matrix",
},
}
conf = yaml.safe_load(
DatabaseConfig().generate_config_section("/data_dir_path", database_conf)
)
self.assertEqual(conf["database"], database_conf)