Allow configuration of the oEmbed URLs. (#10714)

This adds configuration options (under an `oembed` section) to
configure which URLs are matched to use oEmbed for URL
previews.
This commit is contained in:
Patrick Cloke 2021-08-31 18:37:07 -04:00 committed by GitHub
parent 287918e2d4
commit e2481dbe93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 458 additions and 247 deletions

View File

@ -0,0 +1 @@
Allow configuration of the oEmbed URLs used for URL previews.

View File

@ -1075,6 +1075,27 @@ url_preview_accept_language:
# - en # - en
# oEmbed allows for easier embedding content from a website. It can be
# used for generating URLs previews of services which support it.
#
oembed:
# A default list of oEmbed providers is included with Synapse.
#
# Uncomment the following to disable using these default oEmbed URLs.
# Defaults to 'false'.
#
#disable_default_providers: true
# Additional files with oEmbed configuration (each should be in the
# form of providers.json).
#
# By default, this list is empty (so only the default providers.json
# is used).
#
#additional_providers:
# - oembed/my_providers.json
## Captcha ## ## Captcha ##
# See docs/CAPTCHA_SETUP.md for full details of configuring this. # See docs/CAPTCHA_SETUP.md for full details of configuring this.

View File

@ -30,6 +30,7 @@ from .key import KeyConfig
from .logger import LoggingConfig from .logger import LoggingConfig
from .metrics import MetricsConfig from .metrics import MetricsConfig
from .modules import ModulesConfig from .modules import ModulesConfig
from .oembed import OembedConfig
from .oidc import OIDCConfig from .oidc import OIDCConfig
from .password_auth_providers import PasswordAuthProviderConfig from .password_auth_providers import PasswordAuthProviderConfig
from .push import PushConfig from .push import PushConfig
@ -65,6 +66,7 @@ class HomeServerConfig(RootConfig):
LoggingConfig, LoggingConfig,
RatelimitConfig, RatelimitConfig,
ContentRepositoryConfig, ContentRepositoryConfig,
OembedConfig,
CaptchaConfig, CaptchaConfig,
VoipConfig, VoipConfig,
RegistrationConfig, RegistrationConfig,

180
synapse/config/oembed.py Normal file
View File

@ -0,0 +1,180 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from typing import Any, Dict, Iterable, List, Pattern
from urllib import parse as urlparse
import attr
import pkg_resources
from synapse.types import JsonDict
from ._base import Config, ConfigError
from ._util import validate_config
@attr.s(slots=True, frozen=True, auto_attribs=True)
class OEmbedEndpointConfig:
# The API endpoint to fetch.
api_endpoint: str
# The patterns to match.
url_patterns: List[Pattern]
class OembedConfig(Config):
"""oEmbed Configuration"""
section = "oembed"
def read_config(self, config, **kwargs):
oembed_config: Dict[str, Any] = config.get("oembed") or {}
# A list of patterns which will be used.
self.oembed_patterns: List[OEmbedEndpointConfig] = list(
self._parse_and_validate_providers(oembed_config)
)
def _parse_and_validate_providers(
self, oembed_config: dict
) -> Iterable[OEmbedEndpointConfig]:
"""Extract and parse the oEmbed providers from the given JSON file.
Returns a generator which yields the OidcProviderConfig objects
"""
# Whether to use the packaged providers.json file.
if not oembed_config.get("disable_default_providers") or False:
providers = json.load(
pkg_resources.resource_stream("synapse", "res/providers.json")
)
yield from self._parse_and_validate_provider(
providers, config_path=("oembed",)
)
# The JSON files which includes additional provider information.
for i, file in enumerate(oembed_config.get("additional_providers") or []):
# TODO Error checking.
with open(file) as f:
providers = json.load(f)
yield from self._parse_and_validate_provider(
providers,
config_path=(
"oembed",
"additional_providers",
f"<item {i}>",
),
)
def _parse_and_validate_provider(
self, providers: List[JsonDict], config_path: Iterable[str]
) -> Iterable[OEmbedEndpointConfig]:
# Ensure it is the proper form.
validate_config(
_OEMBED_PROVIDER_SCHEMA,
providers,
config_path=config_path,
)
# Parse it and yield each result.
for provider in providers:
# Each provider might have multiple API endpoints, each which
# might have multiple patterns to match.
for endpoint in provider["endpoints"]:
api_endpoint = endpoint["url"]
patterns = [
self._glob_to_pattern(glob, config_path)
for glob in endpoint["schemes"]
]
yield OEmbedEndpointConfig(api_endpoint, patterns)
def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern:
"""
Convert the glob into a sane regular expression to match against. The
rules followed will be slightly different for the domain portion vs.
the rest.
1. The scheme must be one of HTTP / HTTPS (and have no globs).
2. The domain can have globs, but we limit it to characters that can
reasonably be a domain part.
TODO: This does not attempt to handle Unicode domain names.
TODO: The domain should not allow wildcard TLDs.
3. Other parts allow a glob to be any one, or more, characters.
"""
results = urlparse.urlparse(glob)
# Ensure the scheme does not have wildcards (and is a sane scheme).
if results.scheme not in {"http", "https"}:
raise ConfigError(f"Insecure oEmbed scheme: {results.scheme}", config_path)
pattern = urlparse.urlunparse(
[
results.scheme,
re.escape(results.netloc).replace("\\*", "[a-zA-Z0-9_-]+"),
]
+ [re.escape(part).replace("\\*", ".+") for part in results[2:]]
)
return re.compile(pattern)
def generate_config_section(self, **kwargs):
return """\
# oEmbed allows for easier embedding content from a website. It can be
# used for generating URLs previews of services which support it.
#
oembed:
# A default list of oEmbed providers is included with Synapse.
#
# Uncomment the following to disable using these default oEmbed URLs.
# Defaults to 'false'.
#
#disable_default_providers: true
# Additional files with oEmbed configuration (each should be in the
# form of providers.json).
#
# By default, this list is empty (so only the default providers.json
# is used).
#
#additional_providers:
# - oembed/my_providers.json
"""
_OEMBED_PROVIDER_SCHEMA = {
"type": "array",
"items": {
"type": "object",
"properties": {
"provider_name": {"type": "string"},
"provider_url": {"type": "string"},
"endpoints": {
"type": "array",
"items": {
"type": "object",
"properties": {
"schemes": {
"type": "array",
"items": {"type": "string"},
},
"url": {"type": "string"},
"formats": {"type": "array", "items": {"type": "string"}},
"discovery": {"type": "boolean"},
},
"required": ["schemes", "url"],
},
},
},
"required": ["provider_name", "provider_url", "endpoints"],
},
}

View File

@ -0,0 +1,17 @@
[
{
"provider_name": "Twitter",
"provider_url": "http://www.twitter.com/",
"endpoints": [
{
"schemes": [
"https://twitter.com/*/status/*",
"https://*.twitter.com/*/status/*",
"https://twitter.com/*/moments/*",
"https://*.twitter.com/*/moments/*"
],
"url": "https://publish.twitter.com/oembed"
}
]
}
]

View File

@ -0,0 +1,135 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import TYPE_CHECKING, Optional
import attr
from synapse.http.client import SimpleHttpClient
if TYPE_CHECKING:
from synapse.server import HomeServer
logger = logging.getLogger(__name__)
@attr.s(slots=True, auto_attribs=True)
class OEmbedResult:
# Either HTML content or URL must be provided.
html: Optional[str]
url: Optional[str]
title: Optional[str]
# Number of seconds to cache the content.
cache_age: int
class OEmbedError(Exception):
"""An error occurred processing the oEmbed object."""
class OEmbedProvider:
"""
A helper for accessing oEmbed content.
It can be used to check if a URL should be accessed via oEmbed and for
requesting/parsing oEmbed content.
"""
def __init__(self, hs: "HomeServer", client: SimpleHttpClient):
self._oembed_patterns = {}
for oembed_endpoint in hs.config.oembed.oembed_patterns:
for pattern in oembed_endpoint.url_patterns:
self._oembed_patterns[pattern] = oembed_endpoint.api_endpoint
self._client = client
def get_oembed_url(self, url: str) -> Optional[str]:
"""
Check whether the URL should be downloaded as oEmbed content instead.
Args:
url: The URL to check.
Returns:
A URL to use instead or None if the original URL should be used.
"""
for url_pattern, endpoint in self._oembed_patterns.items():
if url_pattern.fullmatch(url):
return endpoint
# No match.
return None
async def get_oembed_content(self, endpoint: str, url: str) -> OEmbedResult:
"""
Request content from an oEmbed endpoint.
Args:
endpoint: The oEmbed API endpoint.
url: The URL to pass to the API.
Returns:
An object representing the metadata returned.
Raises:
OEmbedError if fetching or parsing of the oEmbed information fails.
"""
try:
logger.debug("Trying to get oEmbed content for url '%s'", url)
result = await self._client.get_json(
endpoint,
# TODO Specify max height / width.
# Note that only the JSON format is supported.
args={"url": url},
)
# Ensure there's a version of 1.0.
if result.get("version") != "1.0":
raise OEmbedError("Invalid version: %s" % (result.get("version"),))
oembed_type = result.get("type")
# Ensure the cache age is None or an int.
cache_age = result.get("cache_age")
if cache_age:
cache_age = int(cache_age)
oembed_result = OEmbedResult(None, None, result.get("title"), cache_age)
# HTML content.
if oembed_type == "rich":
oembed_result.html = result.get("html")
return oembed_result
if oembed_type == "photo":
oembed_result.url = result.get("url")
return oembed_result
# TODO Handle link and video types.
if "thumbnail_url" in result:
oembed_result.url = result.get("thumbnail_url")
return oembed_result
raise OEmbedError("Incompatible oEmbed information.")
except OEmbedError as e:
# Trap OEmbedErrors first so we can directly re-raise them.
logger.warning("Error parsing oEmbed metadata from %s: %r", url, e)
raise
except Exception as e:
# Trap any exception and let the code follow as usual.
# FIXME: pass through 404s and other error messages nicely
logger.warning("Error downloading oEmbed metadata from %s: %r", url, e)
raise OEmbedError() from e

View File

@ -25,8 +25,6 @@ import traceback
from typing import TYPE_CHECKING, Any, Dict, Generator, Iterable, Optional, Union from typing import TYPE_CHECKING, Any, Dict, Generator, Iterable, Optional, Union
from urllib import parse as urlparse from urllib import parse as urlparse
import attr
from twisted.internet.error import DNSLookupError from twisted.internet.error import DNSLookupError
from twisted.web.server import Request from twisted.web.server import Request
@ -43,6 +41,7 @@ from synapse.logging.context import make_deferred_yieldable, run_in_background
from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.rest.media.v1._base import get_filename_from_headers from synapse.rest.media.v1._base import get_filename_from_headers
from synapse.rest.media.v1.media_storage import MediaStorage from synapse.rest.media.v1.media_storage import MediaStorage
from synapse.rest.media.v1.oembed import OEmbedError, OEmbedProvider
from synapse.util import json_encoder from synapse.util import json_encoder
from synapse.util.async_helpers import ObservableDeferred from synapse.util.async_helpers import ObservableDeferred
from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.caches.expiringcache import ExpiringCache
@ -71,63 +70,6 @@ OG_TAG_VALUE_MAXLEN = 1000
ONE_HOUR = 60 * 60 * 1000 ONE_HOUR = 60 * 60 * 1000
# A map of globs to API endpoints.
_oembed_globs = {
# Twitter.
"https://publish.twitter.com/oembed": [
"https://twitter.com/*/status/*",
"https://*.twitter.com/*/status/*",
"https://twitter.com/*/moments/*",
"https://*.twitter.com/*/moments/*",
# Include the HTTP versions too.
"http://twitter.com/*/status/*",
"http://*.twitter.com/*/status/*",
"http://twitter.com/*/moments/*",
"http://*.twitter.com/*/moments/*",
],
}
# Convert the globs to regular expressions.
_oembed_patterns = {}
for endpoint, globs in _oembed_globs.items():
for glob in globs:
# Convert the glob into a sane regular expression to match against. The
# rules followed will be slightly different for the domain portion vs.
# the rest.
#
# 1. The scheme must be one of HTTP / HTTPS (and have no globs).
# 2. The domain can have globs, but we limit it to characters that can
# reasonably be a domain part.
# TODO: This does not attempt to handle Unicode domain names.
# 3. Other parts allow a glob to be any one, or more, characters.
results = urlparse.urlparse(glob)
# Ensure the scheme does not have wildcards (and is a sane scheme).
if results.scheme not in {"http", "https"}:
raise ValueError("Insecure oEmbed glob scheme: %s" % (results.scheme,))
pattern = urlparse.urlunparse(
[
results.scheme,
re.escape(results.netloc).replace("\\*", "[a-zA-Z0-9_-]+"),
]
+ [re.escape(part).replace("\\*", ".+") for part in results[2:]]
)
_oembed_patterns[re.compile(pattern)] = endpoint
@attr.s(slots=True)
class OEmbedResult:
# Either HTML content or URL must be provided.
html = attr.ib(type=Optional[str])
url = attr.ib(type=Optional[str])
title = attr.ib(type=Optional[str])
# Number of seconds to cache the content.
cache_age = attr.ib(type=int)
class OEmbedError(Exception):
"""An error occurred processing the oEmbed object."""
class PreviewUrlResource(DirectServeJsonResource): class PreviewUrlResource(DirectServeJsonResource):
isLeaf = True isLeaf = True
@ -157,6 +99,8 @@ class PreviewUrlResource(DirectServeJsonResource):
self.primary_base_path = media_repo.primary_base_path self.primary_base_path = media_repo.primary_base_path
self.media_storage = media_storage self.media_storage = media_storage
self._oembed = OEmbedProvider(hs, self.client)
# We run the background jobs if we're the instance specified (or no # We run the background jobs if we're the instance specified (or no
# instance is specified, where we assume there is only one instance # instance is specified, where we assume there is only one instance
# serving media). # serving media).
@ -367,87 +311,6 @@ class PreviewUrlResource(DirectServeJsonResource):
return jsonog.encode("utf8") return jsonog.encode("utf8")
def _get_oembed_url(self, url: str) -> Optional[str]:
"""
Check whether the URL should be downloaded as oEmbed content instead.
Args:
url: The URL to check.
Returns:
A URL to use instead or None if the original URL should be used.
"""
for url_pattern, endpoint in _oembed_patterns.items():
if url_pattern.fullmatch(url):
return endpoint
# No match.
return None
async def _get_oembed_content(self, endpoint: str, url: str) -> OEmbedResult:
"""
Request content from an oEmbed endpoint.
Args:
endpoint: The oEmbed API endpoint.
url: The URL to pass to the API.
Returns:
An object representing the metadata returned.
Raises:
OEmbedError if fetching or parsing of the oEmbed information fails.
"""
try:
logger.debug("Trying to get oEmbed content for url '%s'", url)
result = await self.client.get_json(
endpoint,
# TODO Specify max height / width.
# Note that only the JSON format is supported.
args={"url": url},
)
# Ensure there's a version of 1.0.
if result.get("version") != "1.0":
raise OEmbedError("Invalid version: %s" % (result.get("version"),))
oembed_type = result.get("type")
# Ensure the cache age is None or an int.
cache_age = result.get("cache_age")
if cache_age:
cache_age = int(cache_age)
oembed_result = OEmbedResult(None, None, result.get("title"), cache_age)
# HTML content.
if oembed_type == "rich":
oembed_result.html = result.get("html")
return oembed_result
if oembed_type == "photo":
oembed_result.url = result.get("url")
return oembed_result
# TODO Handle link and video types.
if "thumbnail_url" in result:
oembed_result.url = result.get("thumbnail_url")
return oembed_result
raise OEmbedError("Incompatible oEmbed information.")
except OEmbedError as e:
# Trap OEmbedErrors first so we can directly re-raise them.
logger.warning("Error parsing oEmbed metadata from %s: %r", url, e)
raise
except Exception as e:
# Trap any exception and let the code follow as usual.
# FIXME: pass through 404s and other error messages nicely
logger.warning("Error downloading oEmbed metadata from %s: %r", url, e)
raise OEmbedError() from e
async def _download_url(self, url: str, user: str) -> Dict[str, Any]: async def _download_url(self, url: str, user: str) -> Dict[str, Any]:
# TODO: we should probably honour robots.txt... except in practice # TODO: we should probably honour robots.txt... except in practice
# we're most likely being explicitly triggered by a human rather than a # we're most likely being explicitly triggered by a human rather than a
@ -459,11 +322,11 @@ class PreviewUrlResource(DirectServeJsonResource):
# If this URL can be accessed via oEmbed, use that instead. # If this URL can be accessed via oEmbed, use that instead.
url_to_download: Optional[str] = url url_to_download: Optional[str] = url
oembed_url = self._get_oembed_url(url) oembed_url = self._oembed.get_oembed_url(url)
if oembed_url: if oembed_url:
# The result might be a new URL to download, or it might be HTML content. # The result might be a new URL to download, or it might be HTML content.
try: try:
oembed_result = await self._get_oembed_content(oembed_url, url) oembed_result = await self._oembed.get_oembed_content(oembed_url, url)
if oembed_result.url: if oembed_result.url:
url_to_download = oembed_result.url url_to_download = oembed_result.url
elif oembed_result.html: elif oembed_result.html:

View File

@ -14,13 +14,14 @@
import json import json
import os import os
import re import re
from unittest.mock import patch
from twisted.internet._resolver import HostResolution from twisted.internet._resolver import HostResolution
from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.address import IPv4Address, IPv6Address
from twisted.internet.error import DNSLookupError from twisted.internet.error import DNSLookupError
from twisted.test.proto_helpers import AccumulatingProtocol from twisted.test.proto_helpers import AccumulatingProtocol
from synapse.config.oembed import OEmbedEndpointConfig
from tests import unittest from tests import unittest
from tests.server import FakeTransport from tests.server import FakeTransport
@ -81,6 +82,19 @@ class URLPreviewTests(unittest.HomeserverTestCase):
hs = self.setup_test_homeserver(config=config) hs = self.setup_test_homeserver(config=config)
# After the hs is created, modify the parsed oEmbed config (to avoid
# messing with files).
#
# Note that HTTP URLs are used to avoid having to deal with TLS in tests.
hs.config.oembed.oembed_patterns = [
OEmbedEndpointConfig(
api_endpoint="http://publish.twitter.com/oembed",
url_patterns=[
re.compile(r"http://twitter\.com/.+/status/.+"),
],
)
]
return hs return hs
def prepare(self, reactor, clock, hs): def prepare(self, reactor, clock, hs):
@ -544,17 +558,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
def test_oembed_photo(self): def test_oembed_photo(self):
"""Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL.""" """Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL."""
# Route the HTTP version to an HTTP endpoint so that the tests work.
with patch.dict(
"synapse.rest.media.v1.preview_url_resource._oembed_patterns",
{
re.compile(
r"http://twitter\.com/.+/status/.+"
): "http://publish.twitter.com/oembed",
},
clear=True,
):
self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")] self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")] self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
@ -617,17 +620,6 @@ class URLPreviewTests(unittest.HomeserverTestCase):
def test_oembed_rich(self): def test_oembed_rich(self):
"""Test an oEmbed endpoint which returns HTML content via the 'rich' type.""" """Test an oEmbed endpoint which returns HTML content via the 'rich' type."""
# Route the HTTP version to an HTTP endpoint so that the tests work.
with patch.dict(
"synapse.rest.media.v1.preview_url_resource._oembed_patterns",
{
re.compile(
r"http://twitter\.com/.+/status/.+"
): "http://publish.twitter.com/oembed",
},
clear=True,
):
self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")] self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
result = { result = {