mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2024-12-25 09:29:27 -05:00
Use importlib.metadata to read requirements (#12088)
* Pull runtime dep checks into their own module * Reimplement `check_requirements` using `importlib` I've tried to make this clearer. We start by working out which of Synapse's requirements we need to be installed here and now. I was surprised that there wasn't an easier way to see which packages were installed by a given extra. I've pulled out the error messages into functions that deal with "is this for an extra or not". And I've rearranged the loop over two different sets of requirements into one loop with a "must be instaled" flag. I hope you agree that this is clearer. * Test cases
This commit is contained in:
parent
4d6b6c17c8
commit
313581e4e9
1
changelog.d/12088.misc
Normal file
1
changelog.d/12088.misc
Normal file
@ -0,0 +1 @@
|
|||||||
|
Inspect application dependencies using `importlib.metadata` or its backport.
|
@ -15,13 +15,13 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
from typing import Container
|
from typing import Container
|
||||||
|
|
||||||
from synapse import python_dependencies # noqa: E402
|
from synapse.util import check_dependencies
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
python_dependencies.check_requirements()
|
check_dependencies.check_requirements()
|
||||||
except python_dependencies.DependencyException as e:
|
except check_dependencies.DependencyException as e:
|
||||||
sys.stderr.writelines(
|
sys.stderr.writelines(
|
||||||
e.message # noqa: B306, DependencyException.message is a property
|
e.message # noqa: B306, DependencyException.message is a property
|
||||||
)
|
)
|
||||||
|
@ -59,7 +59,6 @@ from synapse.http.server import (
|
|||||||
from synapse.http.site import SynapseSite
|
from synapse.http.site import SynapseSite
|
||||||
from synapse.logging.context import LoggingContext
|
from synapse.logging.context import LoggingContext
|
||||||
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
|
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
|
||||||
from synapse.python_dependencies import check_requirements
|
|
||||||
from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
|
from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
|
||||||
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
|
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
|
||||||
from synapse.rest import ClientRestResource
|
from synapse.rest import ClientRestResource
|
||||||
@ -70,6 +69,7 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree
|
|||||||
from synapse.rest.well_known import well_known_resource
|
from synapse.rest.well_known import well_known_resource
|
||||||
from synapse.server import HomeServer
|
from synapse.server import HomeServer
|
||||||
from synapse.storage import DataStore
|
from synapse.storage import DataStore
|
||||||
|
from synapse.util.check_dependencies import check_requirements
|
||||||
from synapse.util.httpresourcetree import create_resource_tree
|
from synapse.util.httpresourcetree import create_resource_tree
|
||||||
from synapse.util.module_loader import load_module
|
from synapse.util.module_loader import load_module
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ from typing import Callable, Dict, Optional
|
|||||||
|
|
||||||
import attr
|
import attr
|
||||||
|
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
from synapse.util.check_dependencies import DependencyException, check_requirements
|
||||||
|
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
|
|
||||||
|
@ -15,7 +15,7 @@
|
|||||||
|
|
||||||
import attr
|
import attr
|
||||||
|
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
from synapse.util.check_dependencies import DependencyException, check_requirements
|
||||||
|
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
|
|
||||||
|
@ -20,11 +20,11 @@ import attr
|
|||||||
|
|
||||||
from synapse.config._util import validate_config
|
from synapse.config._util import validate_config
|
||||||
from synapse.config.sso import SsoAttributeRequirement
|
from synapse.config.sso import SsoAttributeRequirement
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
|
||||||
from synapse.types import JsonDict
|
from synapse.types import JsonDict
|
||||||
from synapse.util.module_loader import load_module
|
from synapse.util.module_loader import load_module
|
||||||
from synapse.util.stringutils import parse_and_validate_mxc_uri
|
from synapse.util.stringutils import parse_and_validate_mxc_uri
|
||||||
|
|
||||||
|
from ..util.check_dependencies import DependencyException, check_requirements
|
||||||
from ._base import Config, ConfigError, read_file
|
from ._base import Config, ConfigError, read_file
|
||||||
|
|
||||||
DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
|
DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
from synapse.config._base import Config
|
from synapse.config._base import Config
|
||||||
from synapse.python_dependencies import check_requirements
|
from synapse.util.check_dependencies import check_requirements
|
||||||
|
|
||||||
|
|
||||||
class RedisConfig(Config):
|
class RedisConfig(Config):
|
||||||
|
@ -20,8 +20,8 @@ from urllib.request import getproxies_environment # type: ignore
|
|||||||
import attr
|
import attr
|
||||||
|
|
||||||
from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
|
from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
|
||||||
from synapse.types import JsonDict
|
from synapse.types import JsonDict
|
||||||
|
from synapse.util.check_dependencies import DependencyException, check_requirements
|
||||||
from synapse.util.module_loader import load_module
|
from synapse.util.module_loader import load_module
|
||||||
|
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
|
@ -17,8 +17,8 @@ import logging
|
|||||||
from typing import Any, List, Set
|
from typing import Any, List, Set
|
||||||
|
|
||||||
from synapse.config.sso import SsoAttributeRequirement
|
from synapse.config.sso import SsoAttributeRequirement
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
|
||||||
from synapse.types import JsonDict
|
from synapse.types import JsonDict
|
||||||
|
from synapse.util.check_dependencies import DependencyException, check_requirements
|
||||||
from synapse.util.module_loader import load_module, load_python_module
|
from synapse.util.module_loader import load_module, load_python_module
|
||||||
|
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
from typing import Set
|
from typing import Set
|
||||||
|
|
||||||
from synapse.python_dependencies import DependencyException, check_requirements
|
from synapse.util.check_dependencies import DependencyException, check_requirements
|
||||||
|
|
||||||
from ._base import Config, ConfigError
|
from ._base import Config, ConfigError
|
||||||
|
|
||||||
|
@ -17,14 +17,7 @@
|
|||||||
|
|
||||||
import itertools
|
import itertools
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Set
|
from typing import Set
|
||||||
|
|
||||||
from pkg_resources import (
|
|
||||||
DistributionNotFound,
|
|
||||||
Requirement,
|
|
||||||
VersionConflict,
|
|
||||||
get_provider,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -90,6 +83,8 @@ REQUIREMENTS = [
|
|||||||
# ijson 3.1.4 fixes a bug with "." in property names
|
# ijson 3.1.4 fixes a bug with "." in property names
|
||||||
"ijson>=3.1.4",
|
"ijson>=3.1.4",
|
||||||
"matrix-common~=1.1.0",
|
"matrix-common~=1.1.0",
|
||||||
|
# For runtime introspection of our dependencies
|
||||||
|
"packaging~=21.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
CONDITIONAL_REQUIREMENTS = {
|
CONDITIONAL_REQUIREMENTS = {
|
||||||
@ -144,102 +139,6 @@ def list_requirements():
|
|||||||
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
|
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
|
||||||
|
|
||||||
|
|
||||||
class DependencyException(Exception):
|
|
||||||
@property
|
|
||||||
def message(self):
|
|
||||||
return "\n".join(
|
|
||||||
[
|
|
||||||
"Missing Requirements: %s" % (", ".join(self.dependencies),),
|
|
||||||
"To install run:",
|
|
||||||
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def dependencies(self):
|
|
||||||
for i in self.args[0]:
|
|
||||||
yield '"' + i + '"'
|
|
||||||
|
|
||||||
|
|
||||||
def check_requirements(for_feature=None):
|
|
||||||
deps_needed = []
|
|
||||||
errors = []
|
|
||||||
|
|
||||||
if for_feature:
|
|
||||||
reqs = CONDITIONAL_REQUIREMENTS[for_feature]
|
|
||||||
else:
|
|
||||||
reqs = REQUIREMENTS
|
|
||||||
|
|
||||||
for dependency in reqs:
|
|
||||||
try:
|
|
||||||
_check_requirement(dependency)
|
|
||||||
except VersionConflict as e:
|
|
||||||
deps_needed.append(dependency)
|
|
||||||
errors.append(
|
|
||||||
"Needed %s, got %s==%s"
|
|
||||||
% (
|
|
||||||
dependency,
|
|
||||||
e.dist.project_name, # type: ignore[attr-defined] # noqa
|
|
||||||
e.dist.version, # type: ignore[attr-defined] # noqa
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except DistributionNotFound:
|
|
||||||
deps_needed.append(dependency)
|
|
||||||
if for_feature:
|
|
||||||
errors.append(
|
|
||||||
"Needed %s for the '%s' feature but it was not installed"
|
|
||||||
% (dependency, for_feature)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
errors.append("Needed %s but it was not installed" % (dependency,))
|
|
||||||
|
|
||||||
if not for_feature:
|
|
||||||
# Check the optional dependencies are up to date. We allow them to not be
|
|
||||||
# installed.
|
|
||||||
OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
|
|
||||||
|
|
||||||
for dependency in OPTS:
|
|
||||||
try:
|
|
||||||
_check_requirement(dependency)
|
|
||||||
except VersionConflict as e:
|
|
||||||
deps_needed.append(dependency)
|
|
||||||
errors.append(
|
|
||||||
"Needed optional %s, got %s==%s"
|
|
||||||
% (
|
|
||||||
dependency,
|
|
||||||
e.dist.project_name, # type: ignore[attr-defined] # noqa
|
|
||||||
e.dist.version, # type: ignore[attr-defined] # noqa
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except DistributionNotFound:
|
|
||||||
# If it's not found, we don't care
|
|
||||||
pass
|
|
||||||
|
|
||||||
if deps_needed:
|
|
||||||
for err in errors:
|
|
||||||
logging.error(err)
|
|
||||||
|
|
||||||
raise DependencyException(deps_needed)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_requirement(dependency_string):
|
|
||||||
"""Parses a dependency string, and checks if the specified requirement is installed
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
VersionConflict if the requirement is installed, but with the the wrong version
|
|
||||||
DistributionNotFound if nothing is found to provide the requirement
|
|
||||||
"""
|
|
||||||
req = Requirement.parse(dependency_string)
|
|
||||||
|
|
||||||
# first check if the markers specify that this requirement needs installing
|
|
||||||
if req.marker is not None and not req.marker.evaluate():
|
|
||||||
# not required for this environment
|
|
||||||
return
|
|
||||||
|
|
||||||
get_provider(req)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
127
synapse/util/check_dependencies.py
Normal file
127
synapse/util/check_dependencies.py
Normal file
@ -0,0 +1,127 @@
|
|||||||
|
import logging
|
||||||
|
from typing import Iterable, NamedTuple, Optional
|
||||||
|
|
||||||
|
from packaging.requirements import Requirement
|
||||||
|
|
||||||
|
DISTRIBUTION_NAME = "matrix-synapse"
|
||||||
|
|
||||||
|
try:
|
||||||
|
from importlib import metadata
|
||||||
|
except ImportError:
|
||||||
|
import importlib_metadata as metadata # type: ignore[no-redef]
|
||||||
|
|
||||||
|
|
||||||
|
class DependencyException(Exception):
|
||||||
|
@property
|
||||||
|
def message(self) -> str:
|
||||||
|
return "\n".join(
|
||||||
|
[
|
||||||
|
"Missing Requirements: %s" % (", ".join(self.dependencies),),
|
||||||
|
"To install run:",
|
||||||
|
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dependencies(self) -> Iterable[str]:
|
||||||
|
for i in self.args[0]:
|
||||||
|
yield '"' + i + '"'
|
||||||
|
|
||||||
|
|
||||||
|
EXTRAS = set(metadata.metadata(DISTRIBUTION_NAME).get_all("Provides-Extra"))
|
||||||
|
|
||||||
|
|
||||||
|
class Dependency(NamedTuple):
|
||||||
|
requirement: Requirement
|
||||||
|
must_be_installed: bool
|
||||||
|
|
||||||
|
|
||||||
|
def _generic_dependencies() -> Iterable[Dependency]:
|
||||||
|
"""Yield pairs (requirement, must_be_installed)."""
|
||||||
|
requirements = metadata.requires(DISTRIBUTION_NAME)
|
||||||
|
assert requirements is not None
|
||||||
|
for raw_requirement in requirements:
|
||||||
|
req = Requirement(raw_requirement)
|
||||||
|
# https://packaging.pypa.io/en/latest/markers.html#usage notes that
|
||||||
|
# > Evaluating an extra marker with no environment is an error
|
||||||
|
# so we pass in a dummy empty extra value here.
|
||||||
|
must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
|
||||||
|
yield Dependency(req, must_be_installed)
|
||||||
|
|
||||||
|
|
||||||
|
def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
|
||||||
|
"""Yield additional dependencies needed for a given `extra`."""
|
||||||
|
requirements = metadata.requires(DISTRIBUTION_NAME)
|
||||||
|
assert requirements is not None
|
||||||
|
for raw_requirement in requirements:
|
||||||
|
req = Requirement(raw_requirement)
|
||||||
|
# Exclude mandatory deps by only selecting deps needed with this extra.
|
||||||
|
if (
|
||||||
|
req.marker is not None
|
||||||
|
and req.marker.evaluate({"extra": extra})
|
||||||
|
and not req.marker.evaluate({"extra": ""})
|
||||||
|
):
|
||||||
|
yield Dependency(req, True)
|
||||||
|
|
||||||
|
|
||||||
|
def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
|
||||||
|
if extra:
|
||||||
|
return f"Need {requirement.name} for {extra}, but it is not installed"
|
||||||
|
else:
|
||||||
|
return f"Need {requirement.name}, but it is not installed"
|
||||||
|
|
||||||
|
|
||||||
|
def _incorrect_version(
|
||||||
|
requirement: Requirement, got: str, extra: Optional[str] = None
|
||||||
|
) -> str:
|
||||||
|
if extra:
|
||||||
|
return f"Need {requirement} for {extra}, but got {requirement.name}=={got}"
|
||||||
|
else:
|
||||||
|
return f"Need {requirement}, but got {requirement.name}=={got}"
|
||||||
|
|
||||||
|
|
||||||
|
def check_requirements(extra: Optional[str] = None) -> None:
|
||||||
|
"""Check Synapse's dependencies are present and correctly versioned.
|
||||||
|
|
||||||
|
If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
|
||||||
|
`pip install matrix-synapse[saml2]`).
|
||||||
|
|
||||||
|
If `extra` is None, this function checks that
|
||||||
|
- all mandatory dependencies are installed and correctly versioned, and
|
||||||
|
- each optional dependency that's installed is correctly versioned.
|
||||||
|
|
||||||
|
If `extra` is not None, this function checks that
|
||||||
|
- the dependencies needed for that extra are installed and correctly versioned.
|
||||||
|
|
||||||
|
:raises DependencyException: if a dependency is missing or incorrectly versioned.
|
||||||
|
:raises ValueError: if this extra does not exist.
|
||||||
|
"""
|
||||||
|
# First work out which dependencies are required, and which are optional.
|
||||||
|
if extra is None:
|
||||||
|
dependencies = _generic_dependencies()
|
||||||
|
elif extra in EXTRAS:
|
||||||
|
dependencies = _dependencies_for_extra(extra)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Synapse does not provide the feature '{extra}'")
|
||||||
|
|
||||||
|
deps_unfulfilled = []
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
for (requirement, must_be_installed) in dependencies:
|
||||||
|
try:
|
||||||
|
dist: metadata.Distribution = metadata.distribution(requirement.name)
|
||||||
|
except metadata.PackageNotFoundError:
|
||||||
|
if must_be_installed:
|
||||||
|
deps_unfulfilled.append(requirement.name)
|
||||||
|
errors.append(_not_installed(requirement, extra))
|
||||||
|
else:
|
||||||
|
if not requirement.specifier.contains(dist.version):
|
||||||
|
deps_unfulfilled.append(requirement.name)
|
||||||
|
errors.append(_incorrect_version(requirement, dist.version, extra))
|
||||||
|
|
||||||
|
if deps_unfulfilled:
|
||||||
|
for err in errors:
|
||||||
|
logging.error(err)
|
||||||
|
|
||||||
|
raise DependencyException(deps_unfulfilled)
|
95
tests/util/test_check_dependencies.py
Normal file
95
tests/util/test_check_dependencies.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
from contextlib import contextmanager
|
||||||
|
from typing import Generator, Optional
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from synapse.util.check_dependencies import (
|
||||||
|
DependencyException,
|
||||||
|
check_requirements,
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
from tests.unittest import TestCase
|
||||||
|
|
||||||
|
|
||||||
|
class DummyDistribution(metadata.Distribution):
|
||||||
|
def __init__(self, version: str):
|
||||||
|
self._version = version
|
||||||
|
|
||||||
|
@property
|
||||||
|
def version(self):
|
||||||
|
return self._version
|
||||||
|
|
||||||
|
def locate_file(self, path):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def read_text(self, filename):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
old = DummyDistribution("0.1.2")
|
||||||
|
new = DummyDistribution("1.2.3")
|
||||||
|
|
||||||
|
# could probably use stdlib TestCase --- no need for twisted here
|
||||||
|
|
||||||
|
|
||||||
|
class TestDependencyChecker(TestCase):
|
||||||
|
@contextmanager
|
||||||
|
def mock_installed_package(
|
||||||
|
self, distribution: Optional[DummyDistribution]
|
||||||
|
) -> Generator[None, None, None]:
|
||||||
|
"""Pretend that looking up any distribution yields the given `distribution`."""
|
||||||
|
|
||||||
|
def mock_distribution(name: str):
|
||||||
|
if distribution is None:
|
||||||
|
raise metadata.PackageNotFoundError
|
||||||
|
else:
|
||||||
|
return distribution
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"synapse.util.check_dependencies.metadata.distribution",
|
||||||
|
mock_distribution,
|
||||||
|
):
|
||||||
|
yield
|
||||||
|
|
||||||
|
def test_mandatory_dependency(self) -> None:
|
||||||
|
"""Complain if a required package is missing or old."""
|
||||||
|
with patch(
|
||||||
|
"synapse.util.check_dependencies.metadata.requires",
|
||||||
|
return_value=["dummypkg >= 1"],
|
||||||
|
):
|
||||||
|
with self.mock_installed_package(None):
|
||||||
|
self.assertRaises(DependencyException, check_requirements)
|
||||||
|
with self.mock_installed_package(old):
|
||||||
|
self.assertRaises(DependencyException, check_requirements)
|
||||||
|
with self.mock_installed_package(new):
|
||||||
|
# should not raise
|
||||||
|
check_requirements()
|
||||||
|
|
||||||
|
def test_generic_check_of_optional_dependency(self) -> None:
|
||||||
|
"""Complain if an optional package is old."""
|
||||||
|
with patch(
|
||||||
|
"synapse.util.check_dependencies.metadata.requires",
|
||||||
|
return_value=["dummypkg >= 1; extra == 'cool-extra'"],
|
||||||
|
):
|
||||||
|
with self.mock_installed_package(None):
|
||||||
|
# should not raise
|
||||||
|
check_requirements()
|
||||||
|
with self.mock_installed_package(old):
|
||||||
|
self.assertRaises(DependencyException, check_requirements)
|
||||||
|
with self.mock_installed_package(new):
|
||||||
|
# should not raise
|
||||||
|
check_requirements()
|
||||||
|
|
||||||
|
def test_check_for_extra_dependencies(self) -> None:
|
||||||
|
"""Complain if a package required for an extra is missing or old."""
|
||||||
|
with patch(
|
||||||
|
"synapse.util.check_dependencies.metadata.requires",
|
||||||
|
return_value=["dummypkg >= 1; extra == 'cool-extra'"],
|
||||||
|
), patch("synapse.util.check_dependencies.EXTRAS", {"cool-extra"}):
|
||||||
|
with self.mock_installed_package(None):
|
||||||
|
self.assertRaises(DependencyException, check_requirements, "cool-extra")
|
||||||
|
with self.mock_installed_package(old):
|
||||||
|
self.assertRaises(DependencyException, check_requirements, "cool-extra")
|
||||||
|
with self.mock_installed_package(new):
|
||||||
|
# should not raise
|
||||||
|
check_requirements()
|
Loading…
Reference in New Issue
Block a user