mirror of
https://git.anonymousland.org/anonymousland/synapse-product.git
synced 2025-05-03 16:04:49 -04:00
Make the dependencies more like a standard Python project and hook up the optional dependencies to setuptools (#4298)
This commit is contained in:
Amber Brown
GitHub
parent
c8d32caba3
commit
c26f49a664
7 changed files with 120 additions and 167 deletions
|
|
@ -15,175 +15,121 @@
|
|||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from pkg_resources import DistributionNotFound, VersionConflict, get_distribution
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# this dict maps from python package name to a list of modules we expect it to
|
||||
# provide.
|
||||
|
||||
# REQUIREMENTS is a simple list of requirement specifiers[1], and must be
|
||||
# installed. It is passed to setup() as install_requires in setup.py.
|
||||
#
|
||||
# the key is a "requirement specifier", as used as a parameter to `pip
|
||||
# install`[1], or an `install_requires` argument to `setuptools.setup` [2].
|
||||
#
|
||||
# the value is a sequence of strings; each entry should be the name of the
|
||||
# python module, optionally followed by a version assertion which can be either
|
||||
# ">=<ver>" or "==<ver>".
|
||||
# CONDITIONAL_REQUIREMENTS is the optional dependencies, represented as a dict
|
||||
# of lists. The dict key is the optional dependency name and can be passed to
|
||||
# pip when installing. The list is a series of requirement specifiers[1] to be
|
||||
# installed when that optional dependency requirement is specified. It is passed
|
||||
# to setup() as extras_require in setup.py
|
||||
#
|
||||
# [1] https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers.
|
||||
# [2] https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-dependencies
|
||||
REQUIREMENTS = {
|
||||
"jsonschema>=2.5.1": ["jsonschema>=2.5.1"],
|
||||
"frozendict>=1": ["frozendict"],
|
||||
"unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"],
|
||||
"canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"],
|
||||
"signedjson>=1.0.0": ["signedjson>=1.0.0"],
|
||||
"pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"],
|
||||
"service_identity>=16.0.0": ["service_identity>=16.0.0"],
|
||||
"Twisted>=17.1.0": ["twisted>=17.1.0"],
|
||||
"treq>=15.1": ["treq>=15.1"],
|
||||
|
||||
REQUIREMENTS = [
|
||||
"jsonschema>=2.5.1",
|
||||
"frozendict>=1",
|
||||
"unpaddedbase64>=1.1.0",
|
||||
"canonicaljson>=1.1.3",
|
||||
"signedjson>=1.0.0",
|
||||
"pynacl>=1.2.1",
|
||||
"service_identity>=16.0.0",
|
||||
"Twisted>=17.1.0",
|
||||
"treq>=15.1",
|
||||
# Twisted has required pyopenssl 16.0 since about Twisted 16.6.
|
||||
"pyopenssl>=16.0.0": ["OpenSSL>=16.0.0"],
|
||||
|
||||
"pyyaml>=3.11": ["yaml"],
|
||||
"pyasn1>=0.1.9": ["pyasn1"],
|
||||
"pyasn1-modules>=0.0.7": ["pyasn1_modules"],
|
||||
"daemonize>=2.3.1": ["daemonize"],
|
||||
"bcrypt>=3.1.0": ["bcrypt>=3.1.0"],
|
||||
"pillow>=3.1.2": ["PIL"],
|
||||
"sortedcontainers>=1.4.4": ["sortedcontainers"],
|
||||
"psutil>=2.0.0": ["psutil>=2.0.0"],
|
||||
"pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
|
||||
"msgpack-python>=0.4.2": ["msgpack"],
|
||||
"phonenumbers>=8.2.0": ["phonenumbers"],
|
||||
"six>=1.10": ["six"],
|
||||
|
||||
"pyopenssl>=16.0.0",
|
||||
"pyyaml>=3.11",
|
||||
"pyasn1>=0.1.9",
|
||||
"pyasn1-modules>=0.0.7",
|
||||
"daemonize>=2.3.1",
|
||||
"bcrypt>=3.1.0",
|
||||
"pillow>=3.1.2",
|
||||
"sortedcontainers>=1.4.4",
|
||||
"psutil>=2.0.0",
|
||||
"pymacaroons-pynacl>=0.9.3",
|
||||
"msgpack-python>=0.4.2",
|
||||
"phonenumbers>=8.2.0",
|
||||
"six>=1.10",
|
||||
# prometheus_client 0.4.0 changed the format of counter metrics
|
||||
# (cf https://github.com/matrix-org/synapse/issues/4001)
|
||||
"prometheus_client>=0.0.18,<0.4.0": ["prometheus_client"],
|
||||
|
||||
"prometheus_client>=0.0.18,<0.4.0",
|
||||
# we use attr.s(slots), which arrived in 16.0.0
|
||||
"attrs>=16.0.0": ["attr>=16.0.0"],
|
||||
"netaddr>=0.7.18": ["netaddr"],
|
||||
}
|
||||
"attrs>=16.0.0",
|
||||
"netaddr>=0.7.18",
|
||||
]
|
||||
|
||||
CONDITIONAL_REQUIREMENTS = {
|
||||
"email.enable_notifs": {
|
||||
"Jinja2>=2.8": ["Jinja2>=2.8"],
|
||||
"bleach>=1.4.2": ["bleach>=1.4.2"],
|
||||
},
|
||||
"matrix-synapse-ldap3": {
|
||||
"matrix-synapse-ldap3>=0.1": ["ldap_auth_provider"],
|
||||
},
|
||||
"postgres": {
|
||||
"psycopg2>=2.6": ["psycopg2"]
|
||||
},
|
||||
"saml2": {
|
||||
"pysaml2>=4.5.0": ["saml2"],
|
||||
},
|
||||
"email.enable_notifs": ["Jinja2>=2.8", "bleach>=1.4.2"],
|
||||
"matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
|
||||
"postgres": ["psycopg2>=2.6"],
|
||||
"saml2": ["pysaml2>=4.5.0"],
|
||||
"url_preview": ["lxml>=3.5.0"],
|
||||
"test": ["mock>=2.0"],
|
||||
}
|
||||
|
||||
|
||||
def requirements(config=None, include_conditional=False):
|
||||
reqs = REQUIREMENTS.copy()
|
||||
if include_conditional:
|
||||
for _, req in CONDITIONAL_REQUIREMENTS.items():
|
||||
reqs.update(req)
|
||||
return reqs
|
||||
|
||||
|
||||
def github_link(project, version, egg):
|
||||
return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg)
|
||||
|
||||
|
||||
DEPENDENCY_LINKS = {
|
||||
}
|
||||
|
||||
|
||||
class MissingRequirementError(Exception):
|
||||
def __init__(self, message, module_name, dependency):
|
||||
super(MissingRequirementError, self).__init__(message)
|
||||
self.module_name = module_name
|
||||
self.dependency = dependency
|
||||
|
||||
|
||||
def check_requirements(config=None):
|
||||
"""Checks that all the modules needed by synapse have been correctly
|
||||
installed and are at the correct version"""
|
||||
for dependency, module_requirements in (
|
||||
requirements(config, include_conditional=False).items()):
|
||||
for module_requirement in module_requirements:
|
||||
if ">=" in module_requirement:
|
||||
module_name, required_version = module_requirement.split(">=")
|
||||
version_test = ">="
|
||||
elif "==" in module_requirement:
|
||||
module_name, required_version = module_requirement.split("==")
|
||||
version_test = "=="
|
||||
else:
|
||||
module_name = module_requirement
|
||||
version_test = None
|
||||
|
||||
try:
|
||||
module = __import__(module_name)
|
||||
except ImportError:
|
||||
logging.exception(
|
||||
"Can't import %r which is part of %r",
|
||||
module_name, dependency
|
||||
)
|
||||
raise MissingRequirementError(
|
||||
"Can't import %r which is part of %r"
|
||||
% (module_name, dependency), module_name, dependency
|
||||
)
|
||||
version = getattr(module, "__version__", None)
|
||||
file_path = getattr(module, "__file__", None)
|
||||
logger.info(
|
||||
"Using %r version %r from %r to satisfy %r",
|
||||
module_name, version, file_path, dependency
|
||||
)
|
||||
|
||||
if version_test == ">=":
|
||||
if version is None:
|
||||
raise MissingRequirementError(
|
||||
"Version of %r isn't set as __version__ of module %r"
|
||||
% (dependency, module_name), module_name, dependency
|
||||
)
|
||||
if LooseVersion(version) < LooseVersion(required_version):
|
||||
raise MissingRequirementError(
|
||||
"Version of %r in %r is too old. %r < %r"
|
||||
% (dependency, file_path, version, required_version),
|
||||
module_name, dependency
|
||||
)
|
||||
elif version_test == "==":
|
||||
if version is None:
|
||||
raise MissingRequirementError(
|
||||
"Version of %r isn't set as __version__ of module %r"
|
||||
% (dependency, module_name), module_name, dependency
|
||||
)
|
||||
if LooseVersion(version) != LooseVersion(required_version):
|
||||
raise MissingRequirementError(
|
||||
"Unexpected version of %r in %r. %r != %r"
|
||||
% (dependency, file_path, version, required_version),
|
||||
module_name, dependency
|
||||
)
|
||||
|
||||
|
||||
def list_requirements():
|
||||
result = []
|
||||
linked = []
|
||||
for link in DEPENDENCY_LINKS.values():
|
||||
egg = link.split("#egg=")[1]
|
||||
linked.append(egg.split('-')[0])
|
||||
result.append(link)
|
||||
for requirement in requirements(include_conditional=True):
|
||||
is_linked = False
|
||||
for link in linked:
|
||||
if requirement.replace('-', '_').startswith(link):
|
||||
is_linked = True
|
||||
if not is_linked:
|
||||
result.append(requirement)
|
||||
return result
|
||||
deps = set(REQUIREMENTS)
|
||||
for opt in CONDITIONAL_REQUIREMENTS.values():
|
||||
deps = set(opt) | deps
|
||||
|
||||
return list(deps)
|
||||
|
||||
|
||||
class DependencyException(Exception):
|
||||
@property
|
||||
def dependencies(self):
|
||||
for i in self.args[0]:
|
||||
yield '"' + i + '"'
|
||||
|
||||
|
||||
def check_requirements(_get_distribution=get_distribution):
|
||||
|
||||
deps_needed = []
|
||||
errors = []
|
||||
|
||||
# Check the base dependencies exist -- they all must be installed.
|
||||
for dependency in REQUIREMENTS:
|
||||
try:
|
||||
_get_distribution(dependency)
|
||||
except VersionConflict as e:
|
||||
deps_needed.append(dependency)
|
||||
errors.append(
|
||||
"Needed %s, got %s==%s"
|
||||
% (dependency, e.dist.project_name, e.dist.version)
|
||||
)
|
||||
except DistributionNotFound:
|
||||
deps_needed.append(dependency)
|
||||
errors.append("Needed %s but it was not installed" % (dependency,))
|
||||
|
||||
# Check the optional dependencies are up to date. We allow them to not be
|
||||
# installed.
|
||||
OPTS = sum(CONDITIONAL_REQUIREMENTS.values(), [])
|
||||
|
||||
for dependency in OPTS:
|
||||
try:
|
||||
_get_distribution(dependency)
|
||||
except VersionConflict:
|
||||
deps_needed.append(dependency)
|
||||
errors.append("Needed %s but it was not installed" % (dependency,))
|
||||
except DistributionNotFound:
|
||||
# If it's not found, we don't care
|
||||
pass
|
||||
|
||||
if deps_needed:
|
||||
for e in errors:
|
||||
logging.exception(e)
|
||||
|
||||
raise DependencyException(deps_needed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.stdout.writelines(req + "\n" for req in list_requirements())
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue