disallow-untyped-defs in docker and stubs directories (#12528)

This commit is contained in:
David Robertson 2022-04-25 13:32:35 +01:00 committed by GitHub
parent 185da8f0f2
commit 8bac3e0435
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 52 additions and 38 deletions

1
changelog.d/12528.misc Normal file
View File

@ -0,0 +1 @@
Add type hints so `docker` and `stubs` directories pass `mypy --disallow-untyped-defs`.

View File

@ -29,7 +29,7 @@
import os import os
import subprocess import subprocess
import sys import sys
from typing import Any, Dict, Mapping, Set from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
import jinja2 import jinja2
import yaml import yaml
@ -201,7 +201,7 @@ upstream {upstream_worker_type} {{
# Utility functions # Utility functions
def log(txt: str): def log(txt: str) -> None:
"""Log something to the stdout. """Log something to the stdout.
Args: Args:
@ -210,7 +210,7 @@ def log(txt: str):
print(txt) print(txt)
def error(txt: str): def error(txt: str) -> NoReturn:
"""Log something and exit with an error code. """Log something and exit with an error code.
Args: Args:
@ -220,7 +220,7 @@ def error(txt: str):
sys.exit(2) sys.exit(2)
def convert(src: str, dst: str, **template_vars): def convert(src: str, dst: str, **template_vars: object) -> None:
"""Generate a file from a template """Generate a file from a template
Args: Args:
@ -290,7 +290,7 @@ def add_sharding_to_shared_config(
shared_config.setdefault("media_instance_running_background_jobs", worker_name) shared_config.setdefault("media_instance_running_background_jobs", worker_name)
def generate_base_homeserver_config(): def generate_base_homeserver_config() -> None:
"""Starts Synapse and generates a basic homeserver config, which will later be """Starts Synapse and generates a basic homeserver config, which will later be
modified for worker support. modified for worker support.
@ -302,12 +302,14 @@ def generate_base_homeserver_config():
subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"]) subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
def generate_worker_files(environ, config_path: str, data_dir: str): def generate_worker_files(
environ: Mapping[str, str], config_path: str, data_dir: str
) -> None:
"""Read the desired list of workers from environment variables and generate """Read the desired list of workers from environment variables and generate
shared homeserver, nginx and supervisord configs. shared homeserver, nginx and supervisord configs.
Args: Args:
environ: _Environ[str] environ: os.environ instance.
config_path: The location of the generated Synapse main worker config file. config_path: The location of the generated Synapse main worker config file.
data_dir: The location of the synapse data directory. Where log and data_dir: The location of the synapse data directory. Where log and
user-facing config files live. user-facing config files live.
@ -369,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
nginx_locations = {} nginx_locations = {}
# Read the desired worker configuration from the environment # Read the desired worker configuration from the environment
worker_types = environ.get("SYNAPSE_WORKER_TYPES") worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
if worker_types is None: if worker_types_env is None:
# No workers, just the main process # No workers, just the main process
worker_types = [] worker_types = []
else: else:
# Split type names by comma # Split type names by comma
worker_types = worker_types.split(",") worker_types = worker_types_env.split(",")
# Create the worker configuration directory if it doesn't already exist # Create the worker configuration directory if it doesn't already exist
os.makedirs("/conf/workers", exist_ok=True) os.makedirs("/conf/workers", exist_ok=True)
@ -547,7 +549,7 @@ def generate_worker_log_config(
return log_config_filepath return log_config_filepath
def main(args, environ): def main(args: List[str], environ: MutableMapping[str, str]) -> None:
config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")

View File

@ -6,27 +6,28 @@ import os
import platform import platform
import subprocess import subprocess
import sys import sys
from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional
import jinja2 import jinja2
# Utility functions # Utility functions
def log(txt): def log(txt: str) -> None:
print(txt, file=sys.stderr) print(txt, file=sys.stderr)
def error(txt): def error(txt: str) -> NoReturn:
log(txt) log(txt)
sys.exit(2) sys.exit(2)
def convert(src, dst, environ): def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
"""Generate a file from a template """Generate a file from a template
Args: Args:
src (str): path to input file src: path to input file
dst (str): path to file to write dst: path to file to write
environ (dict): environment dictionary, for replacement mappings. environ: environment dictionary, for replacement mappings.
""" """
with open(src) as infile: with open(src) as infile:
template = infile.read() template = infile.read()
@ -35,25 +36,30 @@ def convert(src, dst, environ):
outfile.write(rendered) outfile.write(rendered)
def generate_config_from_template(config_dir, config_path, environ, ownership): def generate_config_from_template(
config_dir: str,
config_path: str,
os_environ: Mapping[str, str],
ownership: Optional[str],
) -> None:
"""Generate a homeserver.yaml from environment variables """Generate a homeserver.yaml from environment variables
Args: Args:
config_dir (str): where to put generated config files config_dir: where to put generated config files
config_path (str): where to put the main config file config_path: where to put the main config file
environ (dict): environment dictionary os_environ: environment mapping
ownership (str|None): "<user>:<group>" string which will be used to set ownership: "<user>:<group>" string which will be used to set
ownership of the generated configs. If None, ownership will not change. ownership of the generated configs. If None, ownership will not change.
""" """
for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"): for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
if v not in environ: if v not in os_environ:
error( error(
"Environment variable '%s' is mandatory when generating a config file." "Environment variable '%s' is mandatory when generating a config file."
% (v,) % (v,)
) )
# populate some params from data files (if they exist, else create new ones) # populate some params from data files (if they exist, else create new ones)
environ = environ.copy() environ: Dict[str, Any] = dict(os_environ)
secrets = { secrets = {
"registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
"macaroon": "SYNAPSE_MACAROON_SECRET_KEY", "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
subprocess.check_output(args) subprocess.check_output(args)
def run_generate_config(environ, ownership): def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
"""Run synapse with a --generate-config param to generate a template config file """Run synapse with a --generate-config param to generate a template config file
Args: Args:
environ (dict): env var dict environ: env vars from `os.enrivon`.
ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change. ownership: "userid:groupid" arg for chmod. If None, ownership will not change.
Never returns. Never returns.
""" """
@ -178,7 +184,7 @@ def run_generate_config(environ, ownership):
os.execv(sys.executable, args) os.execv(sys.executable, args)
def main(args, environ): def main(args: List[str], environ: MutableMapping[str, str]) -> None:
mode = args[1] if len(args) > 1 else "run" mode = args[1] if len(args) > 1 else "run"
# if we were given an explicit user to switch to, do so # if we were given an explicit user to switch to, do so

View File

@ -103,7 +103,7 @@ class SortedDict(Dict[_KT, _VT]):
self, self,
start: Optional[int] = ..., start: Optional[int] = ...,
stop: Optional[int] = ..., stop: Optional[int] = ...,
reverse=bool, reverse: bool = ...,
) -> Iterator[_KT]: ... ) -> Iterator[_KT]: ...
def bisect_left(self, value: _KT) -> int: ... def bisect_left(self, value: _KT) -> int: ...
def bisect_right(self, value: _KT) -> int: ... def bisect_right(self, value: _KT) -> int: ...

View File

@ -81,7 +81,7 @@ class SortedList(MutableSequence[_T]):
self, self,
start: Optional[int] = ..., start: Optional[int] = ...,
stop: Optional[int] = ..., stop: Optional[int] = ...,
reverse=bool, reverse: bool = ...,
) -> Iterator[_T]: ... ) -> Iterator[_T]: ...
def _islice( def _islice(
self, self,
@ -153,14 +153,14 @@ class SortedKeyList(SortedList[_T]):
maximum: Optional[int] = ..., maximum: Optional[int] = ...,
inclusive: Tuple[bool, bool] = ..., inclusive: Tuple[bool, bool] = ...,
reverse: bool = ..., reverse: bool = ...,
): ... ) -> Iterator[_T]: ...
def irange_key( def irange_key(
self, self,
min_key: Optional[Any] = ..., min_key: Optional[Any] = ...,
max_key: Optional[Any] = ..., max_key: Optional[Any] = ...,
inclusive: Tuple[bool, bool] = ..., inclusive: Tuple[bool, bool] = ...,
reserve: bool = ..., reserve: bool = ...,
): ... ) -> Iterator[_T]: ...
def bisect_left(self, value: _T) -> int: ... def bisect_left(self, value: _T) -> int: ...
def bisect_right(self, value: _T) -> int: ... def bisect_right(self, value: _T) -> int: ...
def bisect(self, value: _T) -> int: ... def bisect(self, value: _T) -> int: ...

View File

@ -103,7 +103,7 @@ class SortedSet(MutableSet[_T], Sequence[_T]):
self, self,
start: Optional[int] = ..., start: Optional[int] = ...,
stop: Optional[int] = ..., stop: Optional[int] = ...,
reverse=bool, reverse: bool = ...,
) -> Iterator[_T]: ... ) -> Iterator[_T]: ...
def irange( def irange(
self, self,

View File

@ -18,6 +18,8 @@ from typing import Any, List, Optional, Type, Union
from twisted.internet import protocol from twisted.internet import protocol
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
from twisted.internet.interfaces import IAddress
from twisted.python.failure import Failure
class RedisProtocol(protocol.Protocol): class RedisProtocol(protocol.Protocol):
def publish(self, channel: str, message: bytes) -> "Deferred[None]": ... def publish(self, channel: str, message: bytes) -> "Deferred[None]": ...
@ -34,11 +36,14 @@ class RedisProtocol(protocol.Protocol):
def get(self, key: str) -> "Deferred[Any]": ... def get(self, key: str) -> "Deferred[Any]": ...
class SubscriberProtocol(RedisProtocol): class SubscriberProtocol(RedisProtocol):
def __init__(self, *args, **kwargs): ... def __init__(self, *args: object, **kwargs: object): ...
password: Optional[str] password: Optional[str]
def subscribe(self, channels: Union[str, List[str]]): ... def subscribe(self, channels: Union[str, List[str]]) -> "Deferred[None]": ...
def connectionMade(self): ... def connectionMade(self) -> None: ...
def connectionLost(self, reason): ... # type-ignore: twisted.internet.protocol.Protocol provides a default argument for
# `reason`. txredisapi's LineReceiver Protocol doesn't. But that's fine: it's what's
# actually specified in twisted.internet.interfaces.IProtocol.
def connectionLost(self, reason: Failure) -> None: ... # type: ignore[override]
def lazyConnection( def lazyConnection(
host: str = ..., host: str = ...,
@ -74,7 +79,7 @@ class RedisFactory(protocol.ReconnectingClientFactory):
replyTimeout: Optional[int] = None, replyTimeout: Optional[int] = None,
convertNumbers: Optional[int] = True, convertNumbers: Optional[int] = True,
): ... ): ...
def buildProtocol(self, addr) -> RedisProtocol: ... def buildProtocol(self, addr: IAddress) -> RedisProtocol: ...
class SubscriberFactory(RedisFactory): class SubscriberFactory(RedisFactory):
def __init__(self) -> None: ... def __init__(self) -> None: ...