Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Allow configuration of the oEmbed URLs. #10714

Merged
merged 8 commits into from
Aug 31, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/10714.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allow configuration of the oEmbed URLs used for URL previews.
21 changes: 21 additions & 0 deletions docs/sample_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1075,6 +1075,27 @@ url_preview_accept_language:
# - en


# oEmbed allows for easier embedding content from a website. It can be
# used for generating URLs previews of services which support it.
#
oembed:
# A default list of oEmbed providers is included with Synapse.
#
# Uncomment the following to disable using these default oEmbed URLs.
# Defaults to 'false'.
#
#disable_default_providers: true

# Additional files with oEmbed configuration (each should be in the
# form of providers.json).
#
# By default, this list is empty (so only the default providers.json
# is used).
#
#additional_providers:
# - oembed/my_providers.json


## Captcha ##
# See docs/CAPTCHA_SETUP.md for full details of configuring this.

Expand Down
2 changes: 2 additions & 0 deletions synapse/config/homeserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from .logger import LoggingConfig
from .metrics import MetricsConfig
from .modules import ModulesConfig
from .oembed import OembedConfig
from .oidc import OIDCConfig
from .password_auth_providers import PasswordAuthProviderConfig
from .push import PushConfig
Expand Down Expand Up @@ -65,6 +66,7 @@ class HomeServerConfig(RootConfig):
LoggingConfig,
RatelimitConfig,
ContentRepositoryConfig,
OembedConfig,
CaptchaConfig,
VoipConfig,
RegistrationConfig,
Expand Down
180 changes: 180 additions & 0 deletions synapse/config/oembed.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from typing import Any, Dict, Iterable, List, Pattern
from urllib import parse as urlparse

import attr
import pkg_resources

from synapse.types import JsonDict

from ._base import Config, ConfigError
from ._util import validate_config


@attr.s(slots=True, frozen=True, auto_attribs=True)
class OEmbedEndpointConfig:
# The API endpoint to fetch.
api_endpoint: str
# The patterns to match.
url_patterns: List[Pattern]


class OembedConfig(Config):
"""oEmbed Configuration"""

section = "oembed"

def read_config(self, config, **kwargs):
oembed_config: Dict[str, Any] = config.get("oembed") or {}

# A list of patterns which will be used.
self.oembed_patterns: List[OEmbedEndpointConfig] = list(
self._parse_and_validate_providers(oembed_config)
)

def _parse_and_validate_providers(
self, oembed_config: dict
) -> Iterable[OEmbedEndpointConfig]:
"""Extract and parse the oEmbed providers from the given JSON file.

Returns a generator which yields the OidcProviderConfig objects
"""
# Whether to use the packaged providers.json file.
if not oembed_config.get("disable_default_providers") or False:
providers = json.load(
pkg_resources.resource_stream("synapse", "res/providers.json")
)
yield from self._parse_and_validate_provider(
providers, config_path=("oembed",)
)

# The JSON files which includes additional provider information.
for i, file in enumerate(oembed_config.get("additional_providers") or []):
# TODO Error checking.
with open(file) as f:
providers = json.load(f)

yield from self._parse_and_validate_provider(
providers,
config_path=(
"oembed",
"additional_providers",
f"<item {i}>",
),
)

def _parse_and_validate_provider(
self, providers: List[JsonDict], config_path: Iterable[str]
) -> Iterable[OEmbedEndpointConfig]:
# Ensure it is the proper form.
validate_config(
_OEMBED_PROVIDER_SCHEMA,
providers,
config_path=config_path,
)

# Parse it and yield each result.
for provider in providers:
# Each provider might have multiple API endpoints, each which
# might have multiple patterns to match.
for endpoint in provider["endpoints"]:
api_endpoint = endpoint["url"]
patterns = [
self._glob_to_pattern(glob, config_path)
for glob in endpoint["schemes"]
]
yield OEmbedEndpointConfig(api_endpoint, patterns)

def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern:
"""
Convert the glob into a sane regular expression to match against. The
rules followed will be slightly different for the domain portion vs.
the rest.

1. The scheme must be one of HTTP / HTTPS (and have no globs).
2. The domain can have globs, but we limit it to characters that can
reasonably be a domain part.
TODO: This does not attempt to handle Unicode domain names.
TODO: The domain should not allow wildcard TLDs.
3. Other parts allow a glob to be any one, or more, characters.
"""
results = urlparse.urlparse(glob)

# Ensure the scheme does not have wildcards (and is a sane scheme).
if results.scheme not in {"http", "https"}:
raise ConfigError(f"Insecure oEmbed scheme: {results.scheme}", config_path)

pattern = urlparse.urlunparse(
[
results.scheme,
re.escape(results.netloc).replace("\\*", "[a-zA-Z0-9_-]+"),
]
+ [re.escape(part).replace("\\*", ".+") for part in results[2:]]
)
return re.compile(pattern)

def generate_config_section(self, **kwargs):
return """\
# oEmbed allows for easier embedding content from a website. It can be
# used for generating URLs previews of services which support it.
#
oembed:
clokep marked this conversation as resolved.
Show resolved Hide resolved
# A default list of oEmbed providers is included with Synapse.
#
# Uncomment the following to disable using these default oEmbed URLs.
# Defaults to 'false'.
#
#disable_default_providers: true

# Additional files with oEmbed configuration (each should be in the
# form of providers.json).
#
# By default, this list is empty (so only the default providers.json
# is used).
#
#additional_providers:
# - oembed/my_providers.json
"""


_OEMBED_PROVIDER_SCHEMA = {
"type": "array",
"items": {
"type": "object",
"properties": {
"provider_name": {"type": "string"},
"provider_url": {"type": "string"},
"endpoints": {
"type": "array",
"items": {
"type": "object",
"properties": {
"schemes": {
"type": "array",
"items": {"type": "string"},
},
"url": {"type": "string"},
"formats": {"type": "array", "items": {"type": "string"}},
"discovery": {"type": "boolean"},
},
"required": ["schemes", "url"],
},
},
},
"required": ["provider_name", "provider_url", "endpoints"],
},
}
17 changes: 17 additions & 0 deletions synapse/res/providers.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
[
{
"provider_name": "Twitter",
"provider_url": "http://www.twitter.com/",
"endpoints": [
{
"schemes": [
"https://twitter.com/*/status/*",
"https://*.twitter.com/*/status/*",
"https://twitter.com/*/moments/*",
"https://*.twitter.com/*/moments/*"
],
"url": "https://publish.twitter.com/oembed"
}
]
}
]
135 changes: 135 additions & 0 deletions synapse/rest/media/v1/oembed.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import TYPE_CHECKING, Optional

import attr

from synapse.http.client import SimpleHttpClient

if TYPE_CHECKING:
from synapse.server import HomeServer

logger = logging.getLogger(__name__)


@attr.s(slots=True, auto_attribs=True)
class OEmbedResult:
# Either HTML content or URL must be provided.
html: Optional[str]
url: Optional[str]
title: Optional[str]
# Number of seconds to cache the content.
cache_age: int


class OEmbedError(Exception):
"""An error occurred processing the oEmbed object."""


class OEmbedProvider:
clokep marked this conversation as resolved.
Show resolved Hide resolved
"""
A helper for accessing oEmbed content.

It can be used to check if a URL should be accessed via oEmbed and for
requesting/parsing oEmbed content.
"""

def __init__(self, hs: "HomeServer", client: SimpleHttpClient):
self._oembed_patterns = {}
for oembed_endpoint in hs.config.oembed.oembed_patterns:
for pattern in oembed_endpoint.url_patterns:
self._oembed_patterns[pattern] = oembed_endpoint.api_endpoint
self._client = client

def get_oembed_url(self, url: str) -> Optional[str]:
"""
Check whether the URL should be downloaded as oEmbed content instead.

Args:
url: The URL to check.

Returns:
A URL to use instead or None if the original URL should be used.
"""
for url_pattern, endpoint in self._oembed_patterns.items():
if url_pattern.fullmatch(url):
return endpoint

# No match.
return None

async def get_oembed_content(self, endpoint: str, url: str) -> OEmbedResult:
"""
Request content from an oEmbed endpoint.

Args:
endpoint: The oEmbed API endpoint.
url: The URL to pass to the API.

Returns:
An object representing the metadata returned.

Raises:
OEmbedError if fetching or parsing of the oEmbed information fails.
"""
try:
logger.debug("Trying to get oEmbed content for url '%s'", url)
result = await self._client.get_json(
endpoint,
# TODO Specify max height / width.
# Note that only the JSON format is supported.
args={"url": url},
)

# Ensure there's a version of 1.0.
if result.get("version") != "1.0":
raise OEmbedError("Invalid version: %s" % (result.get("version"),))

oembed_type = result.get("type")

# Ensure the cache age is None or an int.
cache_age = result.get("cache_age")
if cache_age:
cache_age = int(cache_age)

oembed_result = OEmbedResult(None, None, result.get("title"), cache_age)

# HTML content.
if oembed_type == "rich":
oembed_result.html = result.get("html")
return oembed_result

if oembed_type == "photo":
oembed_result.url = result.get("url")
return oembed_result

# TODO Handle link and video types.

if "thumbnail_url" in result:
oembed_result.url = result.get("thumbnail_url")
return oembed_result

raise OEmbedError("Incompatible oEmbed information.")

except OEmbedError as e:
# Trap OEmbedErrors first so we can directly re-raise them.
logger.warning("Error parsing oEmbed metadata from %s: %r", url, e)
raise

except Exception as e:
# Trap any exception and let the code follow as usual.
# FIXME: pass through 404s and other error messages nicely
logger.warning("Error downloading oEmbed metadata from %s: %r", url, e)
raise OEmbedError() from e
Loading