Merge branch 'develop' into rav/enforce_report_api

This commit is contained in:
Richard van der Hoff 2018-07-12 09:56:28 +01:00
commit 482d17b58b
370 changed files with 5309 additions and 2679 deletions

View file

@ -104,7 +104,7 @@ class HttpTransactionCache(object):
def _cleanup(self):
now = self.clock.time_msec()
for key in self.transactions.keys():
for key in list(self.transactions):
ts = self.transactions[key][1]
if now > (ts + CLEANUP_PERIOD_MS): # after cleanup period
del self.transactions[key]

View file

@ -14,17 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from six.moves import http_client
from twisted.internet import defer
from synapse.api.constants import Membership
from synapse.api.errors import AuthError, SynapseError, Codes, NotFoundError
from synapse.types import UserID, create_requester
from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import UserID, create_requester
from .base import ClientV1RestServlet, client_path_patterns
import logging
logger = logging.getLogger(__name__)
@ -169,16 +171,12 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
yield self.store.find_first_stream_ordering_after_ts(ts)
)
room_event_after_stream_ordering = (
r = (
yield self.store.get_room_event_after_stream_ordering(
room_id, stream_ordering,
)
)
if room_event_after_stream_ordering:
token = yield self.store.get_topological_token_for_event(
room_event_after_stream_ordering,
)
else:
if not r:
logger.warn(
"[purge] purging events not possible: No event found "
"(received_ts %i => stream_ordering %i)",
@ -189,8 +187,10 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
"there is no event to be purged",
errcode=Codes.NOT_FOUND,
)
(stream, topo, _event_id) = r
token = "t%d-%d" % (topo, stream)
logger.info(
"[purge] purging up to token %d (received_ts %i => "
"[purge] purging up to token %s (received_ts %i => "
"stream_ordering %i)",
token, ts, stream_ordering,
)
@ -249,6 +249,15 @@ class DeactivateAccountRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def on_POST(self, request, target_user_id):
body = parse_json_object_from_request(request, allow_empty_body=True)
erase = body.get("erase", False)
if not isinstance(erase, bool):
raise SynapseError(
http_client.BAD_REQUEST,
"Param 'erase' must be a boolean, if given",
Codes.BAD_JSON,
)
UserID.from_string(target_user_id)
requester = yield self.auth.get_user_by_req(request)
is_admin = yield self.auth.is_server_admin(requester.user)
@ -256,7 +265,9 @@ class DeactivateAccountRestServlet(ClientV1RestServlet):
if not is_admin:
raise AuthError(403, "You are not a server admin")
yield self._deactivate_account_handler.deactivate_account(target_user_id)
yield self._deactivate_account_handler.deactivate_account(
target_user_id, erase,
)
defer.returnValue((200, {}))

View file

@ -16,14 +16,12 @@
"""This module contains base REST classes for constructing client v1 servlets.
"""
from synapse.http.servlet import RestServlet
from synapse.api.urls import CLIENT_PREFIX
from synapse.rest.client.transactions import HttpTransactionCache
import logging
import re
import logging
from synapse.api.urls import CLIENT_PREFIX
from synapse.http.servlet import RestServlet
from synapse.rest.client.transactions import HttpTransactionCache
logger = logging.getLogger(__name__)

View file

@ -14,16 +14,15 @@
# limitations under the License.
from twisted.internet import defer
from synapse.api.errors import AuthError, SynapseError, Codes
from synapse.types import RoomAlias
from synapse.http.servlet import parse_json_object_from_request
from .base import ClientV1RestServlet, client_path_patterns
import logging
from twisted.internet import defer
from synapse.api.errors import AuthError, Codes, SynapseError
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import RoomAlias
from .base import ClientV1RestServlet, client_path_patterns
logger = logging.getLogger(__name__)

View file

@ -14,15 +14,15 @@
# limitations under the License.
"""This module contains REST servlets to do with event streaming, /events."""
import logging
from twisted.internet import defer
from synapse.api.errors import SynapseError
from synapse.streams.config import PaginationConfig
from .base import ClientV1RestServlet, client_path_patterns
from synapse.events.utils import serialize_event
from synapse.streams.config import PaginationConfig
import logging
from .base import ClientV1RestServlet, client_path_patterns
logger = logging.getLogger(__name__)

View file

@ -16,6 +16,7 @@
from twisted.internet import defer
from synapse.streams.config import PaginationConfig
from .base import ClientV1RestServlet, client_path_patterns

View file

@ -13,30 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer
import logging
import urllib
import xml.etree.ElementTree as ET
from synapse.api.errors import SynapseError, LoginError, Codes
from synapse.types import UserID
from six.moves.urllib import parse as urlparse
from canonicaljson import json
from saml2 import BINDING_HTTP_POST, config
from saml2.client import Saml2Client
from twisted.internet import defer
from twisted.web.client import PartialDownloadError
from synapse.api.errors import Codes, LoginError, SynapseError
from synapse.http.server import finish_request
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import UserID
from synapse.util.msisdn import phone_number_to_msisdn
from .base import ClientV1RestServlet, client_path_patterns
import simplejson as json
import urllib
from six.moves.urllib import parse as urlparse
import logging
from saml2 import BINDING_HTTP_POST
from saml2 import config
from saml2.client import Saml2Client
import xml.etree.ElementTree as ET
from twisted.web.client import PartialDownloadError
logger = logging.getLogger(__name__)

View file

@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.api.auth import get_access_token_from_request
@ -20,9 +22,6 @@ from synapse.api.errors import AuthError
from .base import ClientV1RestServlet, client_path_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -15,17 +15,18 @@
""" This module contains REST servlets to do with presence: /presence/<paths>
"""
from twisted.internet import defer
from synapse.api.errors import SynapseError, AuthError
from synapse.types import UserID
from synapse.handlers.presence import format_user_presence_state
from synapse.http.servlet import parse_json_object_from_request
from .base import ClientV1RestServlet, client_path_patterns
import logging
from six import string_types
import logging
from twisted.internet import defer
from synapse.api.errors import AuthError, SynapseError
from synapse.handlers.presence import format_user_presence_state
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import UserID
from .base import ClientV1RestServlet, client_path_patterns
logger = logging.getLogger(__name__)

View file

@ -16,9 +16,10 @@
""" This module contains REST servlets to do with profile: /profile/<paths> """
from twisted.internet import defer
from .base import ClientV1RestServlet, client_path_patterns
from synapse.types import UserID
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import UserID
from .base import ClientV1RestServlet, client_path_patterns
class ProfileDisplaynameRestServlet(ClientV1RestServlet):

View file

@ -16,16 +16,18 @@
from twisted.internet import defer
from synapse.api.errors import (
SynapseError, UnrecognizedRequestError, NotFoundError, StoreError
NotFoundError,
StoreError,
SynapseError,
UnrecognizedRequestError,
)
from .base import ClientV1RestServlet, client_path_patterns
from synapse.storage.push_rule import (
InconsistentRuleException, RuleNotFoundException
)
from synapse.push.clientformat import format_push_rules_for_user
from synapse.push.baserules import BASE_RULE_IDS
from synapse.push.rulekinds import PRIORITY_CLASS_MAP
from synapse.http.servlet import parse_json_value_from_request
from synapse.push.baserules import BASE_RULE_IDS
from synapse.push.clientformat import format_push_rules_for_user
from synapse.push.rulekinds import PRIORITY_CLASS_MAP
from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
from .base import ClientV1RestServlet, client_path_patterns
class PushRuleRestServlet(ClientV1RestServlet):

View file

@ -13,20 +13,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.api.errors import SynapseError, Codes
from synapse.push import PusherConfigException
from synapse.http.servlet import (
parse_json_object_from_request, parse_string, RestServlet
)
from synapse.api.errors import Codes, StoreError, SynapseError
from synapse.http.server import finish_request
from synapse.api.errors import StoreError
from synapse.http.servlet import (
RestServlet,
parse_json_object_from_request,
parse_string,
)
from synapse.push import PusherConfigException
from .base import ClientV1RestServlet, client_path_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -14,23 +14,22 @@
# limitations under the License.
"""This module contains REST servlets to do with registration: /register"""
import hmac
import logging
from hashlib import sha1
from six import string_types
from twisted.internet import defer
from synapse.api.errors import SynapseError, Codes
from synapse.api.constants import LoginType
from synapse.api.auth import get_access_token_from_request
from .base import ClientV1RestServlet, client_path_patterns
import synapse.util.stringutils as stringutils
from synapse.api.auth import get_access_token_from_request
from synapse.api.constants import LoginType
from synapse.api.errors import Codes, SynapseError
from synapse.http.servlet import parse_json_object_from_request
from synapse.types import create_requester
from synapse.util.async import run_on_reactor
from hashlib import sha1
import hmac
import logging
from six import string_types
from .base import ClientV1RestServlet, client_path_patterns
logger = logging.getLogger(__name__)
@ -272,7 +271,6 @@ class RegisterRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def _do_password(self, request, register_json, session):
yield run_on_reactor()
if (self.hs.config.enable_registration_captcha and
not session[LoginType.RECAPTCHA]):
# captcha should've been done by this stage!
@ -333,8 +331,6 @@ class RegisterRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def _do_shared_secret(self, request, register_json, session):
yield run_on_reactor()
if not isinstance(register_json.get("mac", None), string_types):
raise SynapseError(400, "Expected mac.")
if not isinstance(register_json.get("user", None), string_types):
@ -423,8 +419,6 @@ class CreateUserRestServlet(ClientV1RestServlet):
@defer.inlineCallbacks
def _do_create(self, requester, user_json):
yield run_on_reactor()
if "localpart" not in user_json:
raise SynapseError(400, "Expected 'localpart' key.")

View file

@ -15,23 +15,27 @@
# limitations under the License.
""" This module contains REST servlets to do with rooms: /rooms/<paths> """
from twisted.internet import defer
from .base import ClientV1RestServlet, client_path_patterns
from synapse.api.errors import SynapseError, Codes, AuthError
from synapse.streams.config import PaginationConfig
from synapse.api.constants import EventTypes, Membership
from synapse.api.filtering import Filter
from synapse.types import UserID, RoomID, RoomAlias, ThirdPartyInstanceID
from synapse.events.utils import serialize_event, format_event_for_client_v2
from synapse.http.servlet import (
parse_json_object_from_request, parse_string, parse_integer
)
import logging
from six.moves.urllib import parse as urlparse
import logging
import simplejson as json
from canonicaljson import json
from twisted.internet import defer
from synapse.api.constants import EventTypes, Membership
from synapse.api.errors import AuthError, Codes, SynapseError
from synapse.api.filtering import Filter
from synapse.events.utils import format_event_for_client_v2, serialize_event
from synapse.http.servlet import (
parse_integer,
parse_json_object_from_request,
parse_string,
)
from synapse.streams.config import PaginationConfig
from synapse.types import RoomAlias, RoomID, ThirdPartyInstanceID, UserID
from .base import ClientV1RestServlet, client_path_patterns
logger = logging.getLogger(__name__)

View file

@ -13,16 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
import hmac
from twisted.internet import defer
from .base import ClientV1RestServlet, client_path_patterns
import hmac
import hashlib
import base64
class VoipRestServlet(ClientV1RestServlet):
PATTERNS = client_path_patterns("/voip/turnServer$")

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
# Copyright 2017 Vector Creations Ltd
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,18 +16,21 @@
# limitations under the License.
import logging
from six.moves import http_client
from twisted.internet import defer
from synapse.api.auth import has_access_token
from synapse.api.constants import LoginType
from synapse.api.errors import Codes, SynapseError
from synapse.http.servlet import (
RestServlet, assert_params_in_request,
RestServlet,
assert_params_in_request,
parse_json_object_from_request,
)
from synapse.util.async import run_on_reactor
from synapse.util.msisdn import phone_number_to_msisdn
from synapse.util.threepids import check_3pid_allowed
from ._base import client_v2_patterns, interactive_auth_handler
logger = logging.getLogger(__name__)
@ -187,13 +191,20 @@ class DeactivateAccountRestServlet(RestServlet):
@defer.inlineCallbacks
def on_POST(self, request):
body = parse_json_object_from_request(request)
erase = body.get("erase", False)
if not isinstance(erase, bool):
raise SynapseError(
http_client.BAD_REQUEST,
"Param 'erase' must be a boolean, if given",
Codes.BAD_JSON,
)
requester = yield self.auth.get_user_by_req(request)
# allow ASes to dectivate their own users
if requester.app_service:
yield self._deactivate_account_handler.deactivate_account(
requester.user.to_string()
requester.user.to_string(), erase,
)
defer.returnValue((200, {}))
@ -201,7 +212,7 @@ class DeactivateAccountRestServlet(RestServlet):
requester, body, self.hs.get_ip_from_request(request),
)
yield self._deactivate_account_handler.deactivate_account(
requester.user.to_string(),
requester.user.to_string(), erase,
)
defer.returnValue((200, {}))
@ -300,8 +311,6 @@ class ThreepidRestServlet(RestServlet):
@defer.inlineCallbacks
def on_GET(self, request):
yield run_on_reactor()
requester = yield self.auth.get_user_by_req(request)
threepids = yield self.datastore.user_get_threepids(
@ -312,8 +321,6 @@ class ThreepidRestServlet(RestServlet):
@defer.inlineCallbacks
def on_POST(self, request):
yield run_on_reactor()
body = parse_json_object_from_request(request)
threePidCreds = body.get('threePidCreds')
@ -365,8 +372,6 @@ class ThreepidDeleteRestServlet(RestServlet):
@defer.inlineCallbacks
def on_POST(self, request):
yield run_on_reactor()
body = parse_json_object_from_request(request)
required = ['medium', 'address']
@ -381,9 +386,16 @@ class ThreepidDeleteRestServlet(RestServlet):
requester = yield self.auth.get_user_by_req(request)
user_id = requester.user.to_string()
yield self.auth_handler.delete_threepid(
user_id, body['medium'], body['address']
)
try:
yield self.auth_handler.delete_threepid(
user_id, body['medium'], body['address']
)
except Exception:
# NB. This endpoint should succeed if there is nothing to
# delete, so it should only throw if something is wrong
# that we ought to care about.
logger.exception("Failed to remove threepid")
raise SynapseError(500, "Failed to remove threepid")
defer.returnValue((200, {}))

View file

@ -13,14 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ._base import client_v2_patterns
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.api.errors import AuthError, SynapseError
import logging
from twisted.internet import defer
import logging
from synapse.api.errors import AuthError, SynapseError
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.api.constants import LoginType
@ -23,9 +25,6 @@ from synapse.http.servlet import RestServlet
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)
RECAPTCHA_TEMPLATE = """

View file

@ -19,6 +19,7 @@ from twisted.internet import defer
from synapse.api import errors
from synapse.http import servlet
from ._base import client_v2_patterns, interactive_auth_handler
logger = logging.getLogger(__name__)

View file

@ -13,17 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.api.errors import AuthError, SynapseError, StoreError, Codes
from synapse.api.errors import AuthError, Codes, StoreError, SynapseError
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.types import UserID
from ._base import client_v2_patterns
from ._base import set_timeline_upper_limit
import logging
from ._base import client_v2_patterns, set_timeline_upper_limit
logger = logging.getLogger(__name__)

View file

@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.http.servlet import RestServlet, parse_json_object_from_request
@ -21,8 +23,6 @@ from synapse.types import GroupID
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -19,10 +19,13 @@ from twisted.internet import defer
from synapse.api.errors import SynapseError
from synapse.http.servlet import (
RestServlet, parse_json_object_from_request, parse_integer
RestServlet,
parse_integer,
parse_json_object_from_request,
parse_string,
)
from synapse.http.servlet import parse_string
from synapse.types import StreamToken
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -13,19 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.http.servlet import (
RestServlet, parse_string, parse_integer
)
from synapse.events.utils import (
serialize_event, format_event_for_client_v2_without_room_id,
format_event_for_client_v2_without_room_id,
serialize_event,
)
from synapse.http.servlet import RestServlet, parse_integer, parse_string
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -14,15 +14,15 @@
# limitations under the License.
from ._base import client_v2_patterns
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.api.errors import AuthError
from synapse.util.stringutils import random_string
import logging
from twisted.internet import defer
import logging
from synapse.api.errors import AuthError
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.util.stringutils import random_string
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -13,14 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -13,15 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from twisted.internet import defer
from synapse.api.errors import SynapseError
from synapse.http.servlet import RestServlet
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -14,30 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import hmac
import logging
from hashlib import sha1
from six import string_types
from twisted.internet import defer
import synapse
import synapse.types
from synapse.api.auth import get_access_token_from_request, has_access_token
from synapse.api.constants import LoginType
from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError
from synapse.http.servlet import (
RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string
RestServlet,
assert_params_in_request,
parse_json_object_from_request,
parse_string,
)
from synapse.util.msisdn import phone_number_to_msisdn
from synapse.util.ratelimitutils import FederationRateLimiter
from synapse.util.threepids import check_3pid_allowed
from ._base import client_v2_patterns, interactive_auth_handler
import logging
import hmac
from hashlib import sha1
from synapse.util.async import run_on_reactor
from synapse.util.ratelimitutils import FederationRateLimiter
from six import string_types
# We ought to be using hmac.compare_digest() but on older pythons it doesn't
# exist. It's a _really minor_ security flaw to use plain string comparison
# because the timing attack is so obscured by all the other code here it's
@ -191,8 +192,6 @@ class RegisterRestServlet(RestServlet):
@interactive_auth_handler
@defer.inlineCallbacks
def on_POST(self, request):
yield run_on_reactor()
body = parse_json_object_from_request(request)
kind = "user"

View file

@ -13,21 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from six import string_types
from six.moves import http_client
from twisted.internet import defer
from synapse.api.errors import SynapseError, Codes
from synapse.api.errors import Codes, SynapseError
from synapse.http.servlet import (
RestServlet, parse_json_object_from_request,
RestServlet,
assert_params_in_request,
parse_json_object_from_request,
)
from ._base import client_v2_patterns
import logging
logger = logging.getLogger(__name__)

View file

@ -13,27 +13,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer
from synapse.http.servlet import (
RestServlet, parse_string, parse_integer, parse_boolean
)
from synapse.handlers.presence import format_user_presence_state
from synapse.handlers.sync import SyncConfig
from synapse.types import StreamToken
from synapse.events.utils import (
serialize_event, format_event_for_client_v2_without_room_id,
)
from synapse.api.filtering import FilterCollection, DEFAULT_FILTER_COLLECTION
from synapse.api.errors import SynapseError
from synapse.api.constants import PresenceState
from ._base import client_v2_patterns
from ._base import set_timeline_upper_limit
import itertools
import logging
import simplejson as json
from canonicaljson import json
from twisted.internet import defer
from synapse.api.constants import PresenceState
from synapse.api.errors import SynapseError
from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection
from synapse.events.utils import (
format_event_for_client_v2_without_room_id,
serialize_event,
)
from synapse.handlers.presence import format_user_presence_state
from synapse.handlers.sync import SyncConfig
from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string
from synapse.types import StreamToken
from ._base import client_v2_patterns, set_timeline_upper_limit
logger = logging.getLogger(__name__)

View file

@ -13,14 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from ._base import client_v2_patterns
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from synapse.api.errors import AuthError
import logging
from twisted.internet import defer
import logging
from synapse.api.errors import AuthError
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -20,6 +20,7 @@ from twisted.internet import defer
from synapse.api.constants import ThirdPartyEntityKind
from synapse.http.servlet import RestServlet
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -19,6 +19,7 @@ from twisted.internet import defer
from synapse.api.errors import SynapseError
from synapse.http.servlet import RestServlet, parse_json_object_from_request
from ._base import client_v2_patterns
logger = logging.getLogger(__name__)

View file

@ -13,11 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.http.servlet import RestServlet
import logging
import re
from synapse.http.servlet import RestServlet
logger = logging.getLogger(__name__)