From 2607b3e1816341b3b8534077bd5d3a4daf3a3d15 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 13:35:20 +0100 Subject: [PATCH] Update mypy to 0.950 and fix complaints (#12650) --- changelog.d/12650.misc | 1 + poetry.lock | 64 ++++++++++--------- pyproject.toml | 4 +- stubs/sortedcontainers/sorteddict.pyi | 19 ++++-- synapse/appservice/api.py | 3 +- synapse/config/appservice.py | 3 +- synapse/events/presence_router.py | 4 +- synapse/handlers/message.py | 3 +- synapse/metrics/background_process_metrics.py | 44 ++++++++++--- tests/storage/test_monthly_active_users.py | 10 +-- 10 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 changelog.d/12650.misc diff --git a/changelog.d/12650.misc b/changelog.d/12650.misc new file mode 100644 index 000000000..07bb4ce5a --- /dev/null +++ b/changelog.d/12650.misc @@ -0,0 +1 @@ +Update to mypy 0.950. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 89e78576b..564ba7ec0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -572,7 +572,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.931" +version = "0.950" description = "Optional static typing for Python" category = "dev" optional = false @@ -580,13 +580,14 @@ python-versions = ">=3.6" [package.dependencies] mypy-extensions = ">=0.4.3" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] [[package]] name = "mypy-extensions" @@ -598,14 +599,14 @@ python-versions = "*" [[package]] name = "mypy-zope" -version = "0.3.5" +version = "0.3.7" description = "Plugin for mypy to support zope interfaces" category = "dev" optional = false python-versions = "*" [package.dependencies] -mypy = "0.931" +mypy = "0.950" "zope.interface" = "*" "zope.schema" = "*" @@ -1020,7 +1021,7 @@ jeepney = ">=0.6" [[package]] name = "sentry-sdk" -version = "1.5.7" +version = "1.5.11" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = true @@ -1562,7 +1563,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "2bda1a7cfc8cc02832b4a7d16bf7e1615cb05e0639bdb30688aadf692d851942" +content-hash = "f24699464828ac1a63f1034b4a18c841ef585737b9a802fd8311836444f1d702" [metadata.files] attrs = [ @@ -2089,34 +2090,37 @@ msgpack = [ {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, ] mypy = [ - {file = "mypy-0.931-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a"}, - {file = "mypy-0.931-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00"}, - {file = "mypy-0.931-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714"}, - {file = "mypy-0.931-cp310-cp310-win_amd64.whl", hash = "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc"}, - {file = "mypy-0.931-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d"}, - {file = "mypy-0.931-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d"}, - {file = "mypy-0.931-cp36-cp36m-win_amd64.whl", hash = "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c"}, - {file = "mypy-0.931-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0"}, - {file = "mypy-0.931-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05"}, - {file = "mypy-0.931-cp37-cp37m-win_amd64.whl", hash = "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7"}, - {file = "mypy-0.931-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0"}, - {file = "mypy-0.931-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069"}, - {file = "mypy-0.931-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799"}, - {file = "mypy-0.931-cp38-cp38-win_amd64.whl", hash = "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a"}, - {file = "mypy-0.931-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166"}, - {file = "mypy-0.931-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266"}, - {file = "mypy-0.931-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd"}, - {file = "mypy-0.931-cp39-cp39-win_amd64.whl", hash = "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697"}, - {file = "mypy-0.931-py3-none-any.whl", hash = "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d"}, - {file = "mypy-0.931.tar.gz", hash = "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, + {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, + {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, + {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, + {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, + {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, + {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, + {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, + {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, + {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, + {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, + {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, + {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, + {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, + {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, + {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, + {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, + {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] mypy-zope = [ - {file = "mypy-zope-0.3.5.tar.gz", hash = "sha256:489e7da1c2af887f2cfe3496995fc247f296512b495b57817edddda9d22308f3"}, - {file = "mypy_zope-0.3.5-py3-none-any.whl", hash = "sha256:3bd0cc9a3e5933b02931af4b214ba32a4f4ff98adb30c979ce733857db91a18b"}, + {file = "mypy-zope-0.3.7.tar.gz", hash = "sha256:9da171e78e8ef7ac8922c86af1a62f1b7f3244f121020bd94a2246bc3f33c605"}, + {file = "mypy_zope-0.3.7-py3-none-any.whl", hash = "sha256:9c7637d066e4d1bafa0651abc091c752009769098043b236446e6725be2bc9c2"}, ] netaddr = [ {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, @@ -2386,8 +2390,8 @@ secretstorage = [ {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] sentry-sdk = [ - {file = "sentry-sdk-1.5.7.tar.gz", hash = "sha256:aa52da941c56b5a76fd838f8e9e92a850bf893a9eb1e33ffce6c21431d07ee30"}, - {file = "sentry_sdk-1.5.7-py2.py3-none-any.whl", hash = "sha256:411a8495bd18cf13038e5749e4710beb4efa53da6351f67b4c2f307c2d9b6d49"}, + {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, + {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, ] service-identity = [ {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"}, diff --git a/pyproject.toml b/pyproject.toml index 446895711..877f19708 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,8 +248,8 @@ flake8-bugbear = "==21.3.2" flake8 = "*" # Typechecking -mypy = "==0.931" -mypy-zope = "==0.3.5" +mypy = "*" +mypy-zope = "*" types-bleach = ">=4.1.0" types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 3a4f9c307..7c399ab38 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -85,12 +85,19 @@ class SortedDict(Dict[_KT, _VT]): def popitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def peekitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def setdefault(self, key: _KT, default: Optional[_VT] = ...) -> _VT: ... - @overload - def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... - @overload - def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... - @overload - def update(self, **kwargs: _VT) -> None: ... + # Mypy now reports the first overload as an error, because typeshed widened the type + # of `__map` to its internal `_typeshed.SupportsKeysAndGetItem` type in + # https://github.com/python/typeshed/pull/6653 + # Since sorteddicts don't change the signature of `update` from that of `dict`, we + # let the stubs for `update` inherit from the stubs for `dict`. (I suspect we could + # do the same for many othe methods.) We leave the stubs commented to better track + # how this file has evolved from the original stubs. + # @overload + # def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... + # @overload + # def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... + # @overload + # def update(self, **kwargs: _VT) -> None: ... def __reduce__( self, ) -> Tuple[ diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index adc6b074d..d19f8dd99 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,7 @@ import urllib.parse from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple from prometheus_client import Counter +from typing_extensions import TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException @@ -66,7 +67,7 @@ def _is_valid_3pe_metadata(info: JsonDict) -> bool: return True -def _is_valid_3pe_result(r: JsonDict, field: str) -> bool: +def _is_valid_3pe_result(r: object, field: str) -> TypeGuard[JsonDict]: if not isinstance(r, dict): return False diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index b13cc6bb6..24498e794 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -55,7 +55,8 @@ def load_appservices( ) -> List[ApplicationService]: """Returns a list of Application Services from the config files.""" if not isinstance(config_files, list): - logger.warning("Expected %s to be a list of AS config files.", config_files) + # type-ignore: this function gets arbitrary json value; we do use this path. + logger.warning("Expected %s to be a list of AS config files.", config_files) # type: ignore[unreachable] return [] # Dicts of value -> filename diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index a58f313e8..98555c8c0 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -147,7 +147,9 @@ class PresenceRouter: # run all the callbacks for get_users_for_states and combine the results for callback in self._get_users_for_states_callbacks: try: - result = await callback(state_updates) + # Note: result is an object here, because we don't trust modules to + # return the types they're supposed to. + result: object = await callback(state_updates) except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 95a89ac01..c28b792e6 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1427,7 +1427,7 @@ class EventCreationHandler: # Validate a newly added alias or newly added alt_aliases. original_alias = None - original_alt_aliases: List[str] = [] + original_alt_aliases: object = [] original_event_id = event.unsigned.get("replaces_state") if original_event_id: @@ -1455,6 +1455,7 @@ class EventCreationHandler: # If the old version of alt_aliases is of an unknown form, # completely replace it. if not isinstance(original_alt_aliases, (list, tuple)): + # TODO: check that the original_alt_aliases' entries are all strings original_alt_aliases = [] # Check that each alias is currently valid. diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index f61396bb7..298809742 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -28,11 +28,11 @@ from typing import ( Type, TypeVar, Union, - cast, ) from prometheus_client import Metric from prometheus_client.core import REGISTRY, Counter, Gauge +from typing_extensions import ParamSpec from twisted.internet import defer @@ -256,24 +256,48 @@ def run_as_background_process( return defer.ensureDeferred(run()) -F = TypeVar("F", bound=Callable[..., Awaitable[Optional[Any]]]) +P = ParamSpec("P") -def wrap_as_background_process(desc: str) -> Callable[[F], F]: - """Decorator that wraps a function that gets called as a background - process. +def wrap_as_background_process( + desc: str, +) -> Callable[ + [Callable[P, Awaitable[Optional[R]]]], + Callable[P, "defer.Deferred[Optional[R]]"], +]: + """Decorator that wraps an asynchronous function `func`, returning a synchronous + decorated function. Calling the decorated version runs `func` as a background + process, forwarding all arguments verbatim. - Equivalent to calling the function with `run_as_background_process` + That is, + + @wrap_as_background_process + def func(*args): ... + func(1, 2, third=3) + + is equivalent to: + + def func(*args): ... + run_as_background_process(func, 1, 2, third=3) + + The former can be convenient if `func` needs to be run as a background process in + multiple places. """ - def wrap_as_background_process_inner(func: F) -> F: + def wrap_as_background_process_inner( + func: Callable[P, Awaitable[Optional[R]]] + ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrap_as_background_process_inner_2( - *args: Any, **kwargs: Any + *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - return run_as_background_process(desc, func, *args, **kwargs) + # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. + # Argument 4 to "run_as_background_process" has incompatible type + # "**P.kwargs"; expected "bool" + # See https://github.com/python/mypy/issues/8862 + return run_as_background_process(desc, func, *args, **kwargs) # type: ignore[arg-type] - return cast(F, wrap_as_background_process_inner_2) + return wrap_as_background_process_inner_2 return wrap_as_background_process_inner diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 0fbf46567..4c29ad79b 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -222,10 +222,11 @@ class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase): self.store.user_add_threepid(user, "email", email, now, now) ) - d = self.store.db_pool.runInteraction( - "initialise", self.store._initialise_reserved_users, threepids + self.get_success( + self.store.db_pool.runInteraction( + "initialise", self.store._initialise_reserved_users, threepids + ) ) - self.get_success(d) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, initial_users) @@ -233,8 +234,7 @@ class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase): users = self.get_success(self.store.get_registered_reserved_users()) self.assertEqual(len(users), reserved_user_number) - d = self.store.reap_monthly_active_users() - self.get_success(d) + self.get_success(self.store.reap_monthly_active_users()) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, self.hs.config.server.max_mau_value)