From 6a4b650d8ad3e6c095020cac3861e430d643d53d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:22:23 +0100 Subject: [PATCH 01/46] Attempt to validate macaroons A couple of weird caveats: * If we can't validate your macaroon, we fall back to checking that your access token is in the DB, and ignoring the failure * Even if we can validate your macaroon, we still have to hit the DB to get the access token ID, which we pretend is a device ID all over the codebase. This mostly adds the interesting code, and points out the two pieces we need to delete (and necessary conditions) in order to fix the above caveats. --- synapse/api/auth.py | 104 ++++++++++++++++-- tests/api/test_auth.py | 142 ++++++++++++++++++++++++- tests/rest/client/v1/test_presence.py | 8 +- tests/rest/client/v1/test_rooms.py | 28 ++--- tests/rest/client/v1/test_typing.py | 4 +- tests/rest/client/v1/utils.py | 3 - tests/rest/client/v2_alpha/__init__.py | 4 +- 7 files changed, 257 insertions(+), 36 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452c..f8ea1e2c6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -23,6 +23,7 @@ from synapse.util.logutils import log_function from synapse.types import UserID import logging +import pymacaroons logger = logging.getLogger(__name__) @@ -40,6 +41,12 @@ class Auth(object): self.store = hs.get_datastore() self.state = hs.get_state_handler() self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 + self._KNOWN_CAVEAT_PREFIXES = set([ + "gen = ", + "type = ", + "time < ", + "user_id = ", + ]) def check(self, event, auth_events): """ Checks if this event is correctly authed. @@ -359,8 +366,8 @@ class Auth(object): except KeyError: pass # normal users won't have the user_id query parameter set. - user_info = yield self.get_user_by_access_token(access_token) - user = user_info["user"] + user_info = yield self._get_user_by_access_token(access_token) + user_id = user_info["user_id"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -368,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user and access_token and ip_addr: + if user_id and access_token and ip_addr: self.store.insert_client_ip( - user=user, + user=user_id, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user.to_string() + request.authenticated_entity = user_id.to_string() - defer.returnValue((user, token_id,)) + defer.returnValue((user_id, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -386,7 +393,7 @@ class Auth(object): ) @defer.inlineCallbacks - def get_user_by_access_token(self, token): + def _get_user_by_access_token(self, token): """ Get a registered user's ID. Args: @@ -396,6 +403,86 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ + try: + ret = yield self._get_user_from_macaroon(token) + except AuthError: + # TODO(daniel): Remove this fallback when all existing access tokens + # have been re-issued as macaroons. + ret = yield self._look_up_user_by_access_token(token) + defer.returnValue(ret) + + @defer.inlineCallbacks + def _get_user_from_macaroon(self, macaroon_str): + try: + macaroon = pymacaroons.Macaroon.deserialize(macaroon_str) + self._validate_macaroon(macaroon) + + user_prefix = "user_id = " + for caveat in macaroon.caveats: + if caveat.caveat_id.startswith(user_prefix): + user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + # This codepath exists so that we can actually return a + # token ID, because we use token IDs in place of device + # identifiers throughout the codebase. + # TODO(daniel): Remove this fallback when device IDs are + # properly implemented. + ret = yield self._look_up_user_by_access_token(macaroon_str) + if ret["user_id"] != user_id: + logger.error( + "Macaroon user (%s) != DB user (%s)", + user_id, + ret["user_id"] + ) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, + "User mismatch in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + defer.returnValue(ret) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "No user caveat in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError): + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Invalid macaroon passed.", + errcode=Codes.UNKNOWN_TOKEN + ) + + def _validate_macaroon(self, macaroon): + v = pymacaroons.Verifier() + v.satisfy_exact("gen = 1") + v.satisfy_exact("type = access") + v.satisfy_general(lambda c: c.startswith("user_id = ")) + v.satisfy_general(self._verify_expiry) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + v = pymacaroons.Verifier() + v.satisfy_general(self._verify_recognizes_caveats) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + def _verify_expiry(self, caveat): + prefix = "time < " + if not caveat.startswith(prefix): + return False + # TODO(daniel): Enable expiry check when clients actually know how to + # refresh tokens. (And remember to enable the tests) + return True + expiry = int(caveat[len(prefix):]) + now = self.hs.get_clock().time_msec() + return now < expiry + + def _verify_recognizes_caveats(self, caveat): + first_space = caveat.find(" ") + if first_space < 0: + return False + second_space = caveat.find(" ", first_space + 1) + if second_space < 0: + return False + return caveat[:second_space + 1] in self._KNOWN_CAVEAT_PREFIXES + + @defer.inlineCallbacks + def _look_up_user_by_access_token(self, token): ret = yield self.store.get_user_by_access_token(token) if not ret: raise AuthError( @@ -403,10 +490,9 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user": UserID.from_string(ret.get("name")), + "user_id": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } - defer.returnValue(user_info) @defer.inlineCallbacks diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 22fc80433..1ba85d6f8 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,22 +14,27 @@ # limitations under the License. from tests import unittest from twisted.internet import defer +from twisted.trial.unittest import FailTest from mock import Mock from synapse.api.auth import Auth from synapse.api.errors import AuthError +from synapse.types import UserID +from tests.utils import setup_test_homeserver + +import pymacaroons class AuthTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.state_handler = Mock() self.store = Mock() - self.hs = Mock() + self.hs = yield setup_test_homeserver(handlers=None) self.hs.get_datastore = Mock(return_value=self.store) - self.hs.get_state_handler = Mock(return_value=self.state_handler) self.auth = Auth(self.hs) self.test_user = "@foo:bar" @@ -133,3 +138,136 @@ class AuthTestCase(unittest.TestCase): request.requestHeaders.getRawHeaders = Mock(return_value=[""]) d = self.auth.get_user_by_req(request) self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_from_macaroon(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) + user_id = user_info["user_id"] + self.assertEqual(UserID.from_string(user), user_id) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_user_db_mismatch(self): + self.store.get_user_by_access_token = Mock( + return_value={"name": "@percy:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("User mismatch", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_missing_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("No user caveat", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_wrong_key(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key + "wrong") + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_unknown_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("cunning > fox") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_expired(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + self.todo = (FailTest, "Token expiry isn't currently enabled",) + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("time < 1") # ms + + self.hs.clock.now = 5000 # seconds + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 91547bdd0..d8d1416f5 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,11 +72,11 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } @@ -169,7 +169,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token presence.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02..be1d52f72 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,10 +56,10 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -441,10 +441,10 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -519,10 +519,10 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -610,11 +610,11 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -715,10 +715,10 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -840,10 +840,10 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -935,10 +935,10 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 1c4519406..da6fc975f 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,11 +63,11 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index c472d5304..85096a032 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -37,9 +37,6 @@ class RestTestCase(unittest.TestCase): self.mock_resource = None self.auth_user_id = None - def mock_get_user_by_access_token(self, token=None): - return self.auth_user_id - @defer.inlineCallbacks def create_room_as(self, room_creator, is_public=True, tok=None): temp_id = self.auth_user_id diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index ef972a53a..7d0f77a3e 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,10 +45,10 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.USER_ID), + "user_id": UserID.from_string(self.USER_ID), "token_id": 1, } - hs.get_auth().get_user_by_access_token = _get_user_by_access_token + hs.get_auth()._get_user_by_access_token = _get_user_by_access_token for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) From 81450fded8c4d2a0f4a914251cc2d11a366efdbd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:56:01 +0100 Subject: [PATCH 02/46] Turn TODO into thing which actually will fail --- tests/api/test_auth.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1ba85d6f8..2e2d0c428 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,7 +14,6 @@ # limitations under the License. from tests import unittest from twisted.internet import defer -from twisted.trial.unittest import FailTest from mock import Mock @@ -251,7 +250,6 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - self.todo = (FailTest, "Token expiry isn't currently enabled",) self.store.get_user_by_access_token = Mock( return_value={"name": "@baldrick:matrix.org"} ) @@ -267,7 +265,12 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("time < 1") # ms self.hs.clock.now = 5000 # seconds - with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) - self.assertEqual(401, cm.exception.code) - self.assertIn("Invalid macaroon", cm.exception.msg) + + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # TODO(daniel): Turn on the check that we validate expiration, when we + # validate expiration (and remove the above line, which will start + # throwing). + # with self.assertRaises(AuthError) as cm: + # yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # self.assertEqual(401, cm.exception.code) + # self.assertIn("Invalid macaroon", cm.exception.msg) From e255c2c32ff85db03abbf2dac184b2949f481cfb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Sep 2015 12:41:16 +0100 Subject: [PATCH 03/46] s/user_id/user/g for consistency --- synapse/api/auth.py | 20 ++++++++++---------- tests/api/test_auth.py | 8 ++++---- tests/rest/client/v1/test_presence.py | 4 ++-- tests/rest/client/v1/test_rooms.py | 14 +++++++------- tests/rest/client/v1/test_typing.py | 2 +- tests/rest/client/v2_alpha/__init__.py | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f8ea1e2c6..0a77a76cb 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -367,7 +367,7 @@ class Auth(object): pass # normal users won't have the user_id query parameter set. user_info = yield self._get_user_by_access_token(access_token) - user_id = user_info["user_id"] + user = user_info["user"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -375,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user_id and access_token and ip_addr: + if user and access_token and ip_addr: self.store.insert_client_ip( - user=user_id, + user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user_id.to_string() + request.authenticated_entity = user.to_string() - defer.returnValue((user_id, token_id,)) + defer.returnValue((user, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -420,18 +420,18 @@ class Auth(object): user_prefix = "user_id = " for caveat in macaroon.caveats: if caveat.caveat_id.startswith(user_prefix): - user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + user = UserID.from_string(caveat.caveat_id[len(user_prefix):]) # This codepath exists so that we can actually return a # token ID, because we use token IDs in place of device # identifiers throughout the codebase. # TODO(daniel): Remove this fallback when device IDs are # properly implemented. ret = yield self._look_up_user_by_access_token(macaroon_str) - if ret["user_id"] != user_id: + if ret["user"] != user: logger.error( "Macaroon user (%s) != DB user (%s)", - user_id, - ret["user_id"] + user, + ret["user"] ) raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, @@ -490,7 +490,7 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user_id": UserID.from_string(ret.get("name")), + "user": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 2e2d0c428..c96273480 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -146,17 +146,17 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - user = "@baldrick:matrix.org" + user_id = "@baldrick:matrix.org" macaroon = pymacaroons.Macaroon( location=self.hs.config.server_name, identifier="key", key=self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("type = access") - macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) - user_id = user_info["user_id"] - self.assertEqual(UserID.from_string(user), user_id) + user = user_info["user"] + self.assertEqual(UserID.from_string(user_id), user) @defer.inlineCallbacks def test_get_user_from_macaroon_user_db_mismatch(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index d8d1416f5..2ee3da0b3 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index be1d52f72..9fb2bfb31 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,7 +56,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -441,7 +441,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -519,7 +519,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -610,7 +610,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } @@ -715,7 +715,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -840,7 +840,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -935,7 +935,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index da6fc975f..6395ce79d 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,7 +63,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 7d0f77a3e..f45570a1c 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,7 +45,7 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.USER_ID), + "user": UserID.from_string(self.USER_ID), "token_id": 1, } hs.get_auth()._get_user_by_access_token = _get_user_by_access_token From ab9e01809def16dca19d6391ab1234eaa423b182 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:15:19 +0100 Subject: [PATCH 04/46] Check room exists when authenticating an event, by asserting they reference a creation event --- synapse/api/auth.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452c..410f4c11e 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -65,6 +65,14 @@ class Auth(object): # FIXME return True + creation_event = auth_events.get((EventTypes.Create, ""), None) + + if not creation_event: + raise SynapseError( + 400, + "Room %r does not exist" % (event.room_id,) + ) + # FIXME: Temp hack if event.type == EventTypes.Aliases: return True From 00149c063b8f81548bd3eefd3e497acc03512d35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Sep 2015 15:42:03 +0100 Subject: [PATCH 05/46] Fix tests --- synapse/api/auth.py | 2 +- tests/test_state.py | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 410f4c11e..df7fb6aab 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -69,7 +69,7 @@ class Auth(object): if not creation_event: raise SynapseError( - 400, + 403, "Room %r does not exist" % (event.room_id,) ) diff --git a/tests/test_state.py b/tests/test_state.py index 584535875..55f37c521 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -204,8 +204,8 @@ class StateTestCase(unittest.TestCase): nodes={ "START": DictObj( type=EventTypes.Create, - state_key="creator", - content={"membership": "@user_id:example.com"}, + state_key="", + content={"creator": "@user_id:example.com"}, depth=1, ), "A": DictObj( @@ -259,8 +259,8 @@ class StateTestCase(unittest.TestCase): nodes={ "START": DictObj( type=EventTypes.Create, - state_key="creator", - content={"membership": "@user_id:example.com"}, + state_key="", + content={"creator": "@user_id:example.com"}, depth=1, ), "A": DictObj( @@ -432,13 +432,19 @@ class StateTestCase(unittest.TestCase): def test_resolve_message_conflict(self): event = create_event(type="test_message", name="event") + creation = create_event( + type=EventTypes.Create, state_key="" + ) + old_state_1 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test1", state_key="2"), create_event(type="test2", state_key=""), ] old_state_2 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test3", state_key="2"), create_event(type="test4", state_key=""), @@ -446,7 +452,7 @@ class StateTestCase(unittest.TestCase): context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(len(context.current_state), 5) + self.assertEqual(len(context.current_state), 6) self.assertIsNone(context.state_group) @@ -454,13 +460,19 @@ class StateTestCase(unittest.TestCase): def test_resolve_state_conflict(self): event = create_event(type="test4", state_key="", name="event") + creation = create_event( + type=EventTypes.Create, state_key="" + ) + old_state_1 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test1", state_key="2"), create_event(type="test2", state_key=""), ] old_state_2 = [ + creation, create_event(type="test1", state_key="1"), create_event(type="test3", state_key="2"), create_event(type="test4", state_key=""), @@ -468,7 +480,7 @@ class StateTestCase(unittest.TestCase): context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(len(context.current_state), 5) + self.assertEqual(len(context.current_state), 6) self.assertIsNone(context.state_group) @@ -484,36 +496,45 @@ class StateTestCase(unittest.TestCase): } ) + creation = create_event( + type=EventTypes.Create, state_key="", + content={"creator": "@foo:bar"} + ) + old_state_1 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=1), ] old_state_2 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=2), ] context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(old_state_2[1], context.current_state[("test1", "1")]) + self.assertEqual(old_state_2[2], context.current_state[("test1", "1")]) # Reverse the depth to make sure we are actually using the depths # during state resolution. old_state_1 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=2), ] old_state_2 = [ + creation, member_event, create_event(type="test1", state_key="1", depth=1), ] context = yield self._get_context(event, old_state_1, old_state_2) - self.assertEqual(old_state_1[1], context.current_state[("test1", "1")]) + self.assertEqual(old_state_1[2], context.current_state[("test1", "1")]) def _get_context(self, event, old_state_1, old_state_2): group_name_1 = "group_name_1" From 3cdfd37d95484b9ebca012a8169b8d6613385656 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:47:26 +0100 Subject: [PATCH 06/46] Print an example "pip install" line for a missing requirement --- synapse/app/homeserver.py | 17 +++++++++++++++-- synapse/python_dependencies.py | 17 +++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc629914..2a85932b8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,10 +16,23 @@ import sys sys.dont_write_bytecode = True -from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS +from synapse.python_dependencies import ( + check_requirements, DEPENDENCY_LINKS, MissingRequirementError +) if __name__ == '__main__': - check_requirements() + try: + check_requirements() + except MissingRequirementError as e: + message = "\n".join([ + "Missing Requirement: %s" % (e.message,), + "To install run:", + " pip install --upgrade --force \"%s\"" % (e.dependency,), + "", + ]) + sys.stderr.writelines(message) + sys.exit(1) + from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 795ef2718..387183b50 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -60,7 +60,10 @@ DEPENDENCY_LINKS = { class MissingRequirementError(Exception): - pass + def __init__(self, message, module_name, dependency): + super(MissingRequirementError, self).__init__(message) + self.module_name = module_name + self.dependency = dependency def check_requirements(config=None): @@ -88,7 +91,7 @@ def check_requirements(config=None): ) raise MissingRequirementError( "Can't import %r which is part of %r" - % (module_name, dependency) + % (module_name, dependency), module_name, dependency ) version = getattr(module, "__version__", None) file_path = getattr(module, "__file__", None) @@ -101,23 +104,25 @@ def check_requirements(config=None): if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) < LooseVersion(required_version): raise MissingRequirementError( "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) elif version_test == "==": if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) != LooseVersion(required_version): raise MissingRequirementError( "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) From 8bab7abddda956b4131d8e4ad04b623c93df02ad Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:51:10 +0100 Subject: [PATCH 07/46] Add nacl.bindings to the list of modules checked. Re-arrange import order to check packages after the packages they depend on --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 387183b50..e95316720 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,18 +18,18 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { + "frozendict>=0.4": ["frozendict"], "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "Twisted>=15.1.0": ["twisted>=15.1.0"], + "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "Twisted>=15.1.0": ["twisted>=15.1.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], - "pynacl>=0.3.0": ["nacl>=0.3.0"], "daemonize": ["daemonize"], "py-bcrypt": ["bcrypt"], - "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], "ujson": ["ujson"], From 2ff439cff7dd666beeaaeae647a34031e285331e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 11:01:48 +0100 Subject: [PATCH 08/46] Bump version/changelog --- CHANGES.rst | 5 +++++ synapse/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ec10516f..a0a12ca4a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.10.0-r1 (2015-09-08) +========================================== + +* Fix bug with python packaging + Changes in synapse v0.10.0 (2015-09-03) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index d85bb3dce..5b5ff6c3f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0" +__version__ = "0.10.0-r1" From 417485eefaff86206b5f961102f882b3fbe44651 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:14:54 +0100 Subject: [PATCH 09/46] Include the event_id and stream_ordering of membership events when looking up which rooms a user is in --- synapse/storage/roommember.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8eee2dfbc..cd9eefbd9 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) RoomsForUser = namedtuple( "RoomsForUser", - ("room_id", "sender", "membership") + ("room_id", "sender", "membership", "event_id", "stream_ordering") ) @@ -141,9 +141,11 @@ class RoomMemberStore(SQLBaseStore): args.extend(membership_list) sql = ( - "SELECT m.room_id, m.sender, m.membership" + "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" " FROM room_memberships as m" " INNER JOIN current_state_events as c" + " ON e.event_id = c.event_id " + " INNER JOIN events as e " " ON m.event_id = c.event_id " " AND m.room_id = c.room_id " " AND m.user_id = c.state_key" From dd42bb78d088c719aaa288b892fe58caa5850deb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:16:09 +0100 Subject: [PATCH 10/46] Include rooms that a user has left in an initialSync. Include the state and messages at the point they left the room --- synapse/handlers/message.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 23b779ad7..5447c97e8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -316,7 +316,9 @@ class MessageHandler(BaseHandler): """ room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=user_id, - membership_list=[Membership.INVITE, Membership.JOIN] + membership_list=[ + Membership.INVITE, Membership.JOIN, Membership.LEAVE + ] ) user = UserID.from_string(user_id) @@ -358,19 +360,32 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) - if event.membership != Membership.JOIN: + if event.membership not in (Membership.JOIN, Membership.LEAVE): return try: + if event.membership == Membership.JOIN: + room_end_token = now_token.room_key + deferred_room_state = self.state_handler.get_current_state( + event.room_id + ) + else: + room_end_token = "s%d" % (event.stream_ordering,) + deferred_room_state = self.store.get_state_for_events( + event.room_id, [event.event_id], None + ) + deferred_room_state.addCallback( + lambda states: states[event.event_id] + ) + + (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( event.room_id, limit=limit, - end_token=now_token.room_key, - ), - self.state_handler.get_current_state( - event.room_id + end_token=room_end_token, ), + deferred_room_state, ] ).addErrback(unwrapFirstError) From e530208e68a2d0c37732855e4ed8f22a9f84d334 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 09:57:49 +0100 Subject: [PATCH 11/46] Change default history visibility for private rooms --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b5..bb3d42828 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -39,7 +39,7 @@ class RoomCreationHandler(BaseHandler): PRESETS_DICT = { RoomCreationPreset.PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, - "history_visibility": "invited", + "history_visibility": "shared", "original_invitees_have_ops": False, }, RoomCreationPreset.PUBLIC_CHAT: { From 81a93ddcc8798568276582ed9c7a63bc64dc5bc0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 12:02:07 +0100 Subject: [PATCH 12/46] Allow configuration to ignore invalid SSL certs This will be useful for sytest, and sytest only, hence the aggressive config key name. --- synapse/app/homeserver.py | 8 ++++---- synapse/config/tls.py | 4 ++++ synapse/crypto/keyring.py | 4 ++-- synapse/handlers/auth.py | 3 +-- synapse/http/client.py | 25 +++++++++++++++++++++++-- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 14 ++++++++++++++ 7 files changed, 50 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc629914..ba76ee362 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,6 +15,7 @@ # limitations under the License. import sys + sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS @@ -221,7 +222,7 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, ), - self.tls_context_factory, + self.tls_server_context_factory, interface=bind_address ) else: @@ -365,7 +366,6 @@ def setup(config_options): Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. - should_run (bool): Whether to start the reactor. Returns: HomeServer @@ -388,7 +388,7 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - tls_context_factory = context_factory.ServerContextFactory(config) + tls_server_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection @@ -396,7 +396,7 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, db_config=config.database_config, - tls_context_factory=tls_context_factory, + tls_server_context_factory=tls_server_context_factory, config=config, content_addr=config.content_addr, version_string=version_string, diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 4751d39bc..472cf7ac4 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,6 +42,10 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + self.use_insecure_ssl_client = config.get( + "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" + "though_it_is_woefully_insecure_because_i_hate_my_users", False) + def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe5..e98a625fe 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -463,7 +463,7 @@ class Keyring(object): continue (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory, + server_name, self.hs.tls_server_context_factory, path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), )).encode("ascii"), @@ -597,7 +597,7 @@ class Keyring(object): # Try to fetch the key from the remote server. (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + server_name, self.hs.tls_server_context_factory ) # Check the response. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 59f687e0f..793b3fcd8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,7 +19,6 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes -from synapse.http.client import SimpleHttpClient from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -187,7 +186,7 @@ class AuthHandler(BaseHandler): # TODO: get this from the homeserver rather than creating a new one for # each request try: - client = SimpleHttpClient(self.hs) + client = self.hs.get_simple_http_client() resp_body = yield client.post_urlencoded_get_json( self.hs.config.recaptcha_siteverify_api, args={ diff --git a/synapse/http/client.py b/synapse/http/client.py index 4b8fd3d3a..da77c8b0a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from OpenSSL import SSL +from OpenSSL.SSL import VERIFY_NONE from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn @@ -19,7 +21,7 @@ import synapse.metrics from canonicaljson import encode_canonical_json -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, HTTPConnectionPool, @@ -59,7 +61,12 @@ class SimpleHttpClient(object): # 'like a browser' pool = HTTPConnectionPool(reactor) pool.maxPersistentPerHost = 10 - self.agent = Agent(reactor, pool=pool) + self.agent = Agent( + reactor, + pool=pool, + connectTimeout=15, + contextFactory=hs.get_http_client_context_factory() + ) self.version_string = hs.version_string def request(self, method, uri, *args, **kwargs): @@ -252,3 +259,17 @@ def _print_ex(e): _print_ex(ex) else: logger.exception(e) + + +class WoefullyInsecureContextFactory(ssl.ContextFactory): + """ + Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + + Do not use this unless you really, really hate your users.""" + + def __init__(self): + self._context = SSL.Context(SSL.SSLv23_METHOD) + self._context.set_verify(VERIFY_NONE, lambda *_: None) + + def getContext(self, hostname, port): + return self._context diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1c9e55278..b50a0c445 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -57,14 +57,14 @@ incoming_responses_counter = metrics.register_counter( class MatrixFederationEndpointFactory(object): def __init__(self, hs): - self.tls_context_factory = hs.tls_context_factory + self.tls_server_context_factory = hs.tls_server_context_factory def endpointForURI(self, uri): destination = uri.netloc return matrix_federation_endpoint( reactor, destination, timeout=10, - ssl_context_factory=self.tls_context_factory + ssl_context_factory=self.tls_server_context_factory ) diff --git a/synapse/server.py b/synapse/server.py index 4d1fb1cbf..656e534df 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -19,7 +19,9 @@ # partial one for unit test mocking. # Imports required for the default HomeServer() implementation +from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication +from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -87,6 +89,8 @@ class BaseHomeServer(object): 'pusherpool', 'event_builder_factory', 'filtering', + 'http_client_context_factory', + 'simple_http_client', ] def __init__(self, hostname, **kwargs): @@ -174,6 +178,16 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_http_client_context_factory(self): + config = self.get_config() + return ( + WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + else BrowserLikePolicyForHTTPS() + ) + + def build_simple_http_client(self): + return SimpleHttpClient(self) + def build_v1auth(self): orf = Auth(self) # Matrix spec makes no reference to what HTTP status code is returned, From 6485f03d91a5f96da28f9dcc8e9ebc3adb213f6f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:05:00 +0100 Subject: [PATCH 13/46] Fix random formatting --- synapse/app/homeserver.py | 1 - synapse/http/client.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ba76ee362..8e60304e2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,7 +15,6 @@ # limitations under the License. import sys - sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS diff --git a/synapse/http/client.py b/synapse/http/client.py index da77c8b0a..815a83872 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -265,7 +265,8 @@ class WoefullyInsecureContextFactory(ssl.ContextFactory): """ Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. - Do not use this unless you really, really hate your users.""" + Do not use this unless you really, really hate your users. + """ def __init__(self): self._context = SSL.Context(SSL.SSLv23_METHOD) From 89ae0166ded093be2343409cfe42f475dea83139 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 13:25:22 +0100 Subject: [PATCH 14/46] Allow room initialSync for users that have left the room, returning a snapshot of how the room was when they left it --- synapse/api/auth.py | 49 +++++++++++++++++++++ synapse/handlers/message.py | 85 +++++++++++++++++++++++++++++++++---- synapse/storage/stream.py | 15 +++++++ 3 files changed, 140 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0c0d67856..9b614a12b 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -104,6 +104,20 @@ class Auth(object): @defer.inlineCallbacks def check_joined_room(self, room_id, user_id, current_state=None): + """Check if the user is currently joined in the room + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user is not in the room. + Returns: + A deferred membership event for the user if the user is in + the room. + """ if current_state: member = current_state.get( (EventTypes.Member, user_id), @@ -119,6 +133,41 @@ class Auth(object): self._check_joined_room(member, user_id, room_id) defer.returnValue(member) + @defer.inlineCallbacks + def check_user_was_in_room(self, room_id, user_id, current_state=None): + """Check if the user was in the room at some point. + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user was never in the room. + Returns: + A deferred membership event for the user if the user was in + the room. + """ + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + + if not member: + raise AuthError(403, "User %s not in room %s" % ( + user_id, room_id + )) + + defer.returnValue(member) + @defer.inlineCallbacks def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5447c97e8..fc9a23433 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util import unwrapFirstError from synapse.util.logcontext import PreserveLoggingContext -from synapse.types import UserID, RoomStreamToken +from synapse.types import UserID, RoomStreamToken, StreamToken from ._base import BaseHandler @@ -377,7 +377,6 @@ class MessageHandler(BaseHandler): lambda states: states[event.event_id] ) - (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( @@ -434,13 +433,83 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - current_state = yield self.state.get_current_state( - room_id=room_id, + """Capture the a snapshot of a room. If user is currently a member of + the room this will be what is currently in the room. If the user left + the room this will be what was in the room when they left. + + Args: + user_id(str): The user to get a snapshot for. + room_id(str): The room to get a snapshot of. + pagin_config(synapse.api.streams.PaginationConfig): The pagination + config used to determine how many messages to return. + Raises: + AuthError if the user wasn't in the room. + Returns: + A JSON object with the snapshot of the room. + """ + + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + result = yield self._room_initial_sync_joined( + user_id, room_id, pagin_config, member_event + ) + elif member_event.membership == Membership.LEAVE: + result = yield self._room_initial_sync_parted( + user_id, room_id, pagin_config, member_event + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _room_initial_sync_parted(self, user_id, room_id, pagin_config, + member_event): + room_state = yield self.store.get_state_for_events( + member_event.room_id, [member_event.event_id], None ) - yield self.auth.check_joined_room( - room_id, user_id, - current_state=current_state + room_state = room_state[member_event.event_id] + + limit = pagin_config.limit if pagin_config else None + if limit is None: + limit = 10 + + stream_token = yield self.store.get_stream_token_for_event( + member_event.event_id + ) + + messages, token = yield self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=stream_token + ) + + messages = yield self._filter_events_for_client( + user_id, room_id, messages + ) + + start_token = StreamToken(token[0], 0, 0, 0) + end_token = StreamToken(token[1], 0, 0, 0) + + time_now = self.clock.time_msec() + + defer.returnValue({ + "membership": member_event.membership, + "room_id": room_id, + "messages": { + "chunk": [serialize_event(m, time_now) for m in messages], + "start": start_token.to_string(), + "end": end_token.to_string(), + }, + "state": [serialize_event(s, time_now) for s in room_state.values()], + "presence": [], + "receipts": [], + }) + + @defer.inlineCallbacks + def _room_initial_sync_joined(self, user_id, room_id, pagin_config, + member_event): + current_state = yield self.state.get_current_state( + room_id=room_id, ) # TODO(paul): I wish I was called with user objects not user_id @@ -454,8 +523,6 @@ class MessageHandler(BaseHandler): for x in current_state.values() ] - member_event = current_state.get((EventTypes.Member, user_id,)) - now_token = yield self.hs.get_event_sources().get_current_token() limit = pagin_config.limit if pagin_config else None diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d7fe423f5..0abfa86cd 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -379,6 +379,21 @@ class StreamStore(SQLBaseStore): ) defer.returnValue("t%d-%d" % (topo, token)) + def get_stream_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "s%d" stream token. + """ + return self._simple_select_one_onecol( + table="events", + keyvalues={"event_id": event_id}, + retcol="stream_ordering", + ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + def _get_max_topological_txn(self, txn): txn.execute( "SELECT MAX(topological_ordering) FROM events" From ddfe30ba835da4357670f2a2a39386b8b8e65b60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:26:23 +0100 Subject: [PATCH 15/46] Better document the intent of the insecure SSL setting --- synapse/config/tls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 472cf7ac4..35ff13f4b 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,9 +42,13 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + # This config option applies to non-federation HTTP clients + # (e.g. for talking to recaptcha, identity servers, and such) + # It should never be used in production, and is intended for + # use only when running tests. self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" - "though_it_is_woefully_insecure_because_i_hate_my_users", False) + "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" + "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) From 1d579df66475c342e0bf5fed338808bdbfd03c94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 14:12:24 +0100 Subject: [PATCH 16/46] Allow rooms/{roomId}/state for a room that has been left --- synapse/api/auth.py | 3 ++- synapse/handlers/message.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9b614a12b..d8bb64a4a 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -160,8 +160,9 @@ class Auth(object): event_type=EventTypes.Member, state_key=user_id ) + membership = member.membership if member else None - if not member: + if membership not in (Membership.JOIN, Membership.LEAVE): raise AuthError(403, "User %s not in room %s" % ( user_id, room_id )) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index fc9a23433..171e9d72a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -277,7 +277,9 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_state_events(self, user_id, room_id): - """Retrieve all state events for a given room. + """Retrieve all state events for a given room. If the user is + joined to the room then return the current state. If the user has + left the room return the state events from when they left. Args: user_id(str): The user requesting state events. @@ -285,13 +287,19 @@ class MessageHandler(BaseHandler): Returns: A list of dicts representing state events. [{}, {}, {}] """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + room_state = yield self.state_handler.get_current_state(room_id) + elif member_event.membership == Membership.LEAVE: + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], None + ) + room_state = room_state[member_event.event_id] - # TODO: This is duplicating logic from snapshot_all_rooms - current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() defer.returnValue( - [serialize_event(c, now) for c in current_state.values()] + [serialize_event(c, now) for c in room_state.values()] ) @defer.inlineCallbacks From bc8b25eb56bf4fcec3546c2ea28741189a519da5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 15:42:16 +0100 Subject: [PATCH 17/46] Allow users that have left the room to view the member list from the point they left --- synapse/handlers/room.py | 36 ------------------------------ synapse/rest/client/v1/room.py | 18 ++++++++++----- tests/rest/client/v1/test_rooms.py | 4 ++-- 3 files changed, 15 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b5..0ff816d53 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,7 +25,6 @@ from synapse.api.constants import ( from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor -from synapse.events.utils import serialize_event from collections import OrderedDict import logging @@ -342,41 +341,6 @@ class RoomMemberHandler(BaseHandler): if remotedomains is not None: remotedomains.add(member.domain) - @defer.inlineCallbacks - def get_room_members_as_pagination_chunk(self, room_id=None, user_id=None, - limit=0, start_tok=None, - end_tok=None): - """Retrieve a list of room members in the room. - - Args: - room_id (str): The room to get the member list for. - user_id (str): The ID of the user making the request. - limit (int): The max number of members to return. - start_tok (str): Optional. The start token if known. - end_tok (str): Optional. The end token if known. - Returns: - dict: A Pagination streamable dict. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, user_id) - - member_list = yield self.store.get_room_members(room_id=room_id) - time_now = self.clock.time_msec() - event_list = [ - serialize_event(entry, time_now) - for entry in member_list - ] - chunk_data = { - "start": "START", # FIXME (erikj): START is no longer valid - "end": "END", - "chunk": event_list - } - # TODO honor Pagination stream params - # TODO snapshot this list to return on subsequent requests when - # paginating - defer.returnValue(chunk_data) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c9c27dd5a..f4558b95a 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -290,12 +290,18 @@ class RoomMemberListRestServlet(ClientV1RestServlet): def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) user, _ = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_member_handler - members = yield handler.get_room_members_as_pagination_chunk( + handler = self.handlers.message_handler + events = yield handler.get_state_events( room_id=room_id, - user_id=user.to_string()) + user_id=user.to_string(), + ) - for event in members["chunk"]: + chunk = [] + + for event in events: + if event["type"] != EventTypes.Member: + continue + chunk.append(event) # FIXME: should probably be state_key here, not user_id target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it @@ -308,7 +314,9 @@ class RoomMemberListRestServlet(ClientV1RestServlet): except: pass - defer.returnValue((200, members)) + defer.returnValue((200, { + "chunk": chunk + })) # TODO: Needs unit testing diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02..d50cfe429 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -492,9 +492,9 @@ class RoomsMemberListTestCase(RestTestCase): self.assertEquals(200, code, msg=str(response)) yield self.leave(room=room_id, user=self.user_id) - # can no longer see list, you've left. + # can see old list once left (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) class RoomsCreateTestCase(RestTestCase): From 3c166a24c591afdc851de3c6c754c90471b1b0a9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 16:05:09 +0100 Subject: [PATCH 18/46] Remove undocumented and unimplemented 'feedback' parameter from the Client-Server API --- synapse/api/constants.py | 11 ----------- synapse/handlers/message.py | 21 +++------------------ synapse/handlers/room.py | 1 - synapse/rest/client/v1/initial_sync.py | 2 -- synapse/rest/client/v1/room.py | 2 -- synapse/storage/stream.py | 10 ++-------- 6 files changed, 5 insertions(+), 42 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 1423986c1..338566439 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -27,16 +27,6 @@ class Membership(object): LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) -class Feedback(object): - - """Represents the types of feedback a user can send in response to a - message.""" - - DELIVERED = u"delivered" - READ = u"read" - LIST = (DELIVERED, READ) - - class PresenceState(object): """Represents the presence state of a user.""" OFFLINE = u"offline" @@ -73,7 +63,6 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" - Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 171e9d72a..72ebac047 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -71,7 +71,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False, as_client_event=True): + as_client_event=True): """Get messages in a room. Args: @@ -79,7 +79,6 @@ class MessageHandler(BaseHandler): room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. - feedback (bool): True to get compressed feedback with the messages as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results @@ -264,17 +263,6 @@ class MessageHandler(BaseHandler): ) defer.returnValue(data) - @defer.inlineCallbacks - def get_feedback(self, event_id): - # yield self.auth.check_joined_room(room_id, user_id) - - # Pull out the feedback from the db - fb = yield self.store.get_feedback(event_id) - - if fb: - defer.returnValue(fb) - defer.returnValue(None) - @defer.inlineCallbacks def get_state_events(self, user_id, room_id): """Retrieve all state events for a given room. If the user is @@ -303,8 +291,7 @@ class MessageHandler(BaseHandler): ) @defer.inlineCallbacks - def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False, as_client_event=True): + def snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -314,7 +301,6 @@ class MessageHandler(BaseHandler): user_id (str): The ID of the user making the request. pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. - feedback (bool): True to get feedback along with these messages. as_client_event (bool): True to get events in client-server format. Returns: A list of dicts with "room_id" and "membership" keys for all rooms @@ -439,8 +425,7 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) @defer.inlineCallbacks - def room_initial_sync(self, user_id, room_id, pagin_config=None, - feedback=False): + def room_initial_sync(self, user_id, room_id, pagin_config=None): """Capture the a snapshot of a room. If user is currently a member of the room this will be what is currently in the room. If the user left the room this will be what was in the room when they left. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0ff816d53..243623190 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -610,7 +610,6 @@ class RoomEventSource(object): to_key=config.to_key, direction=config.direction, limit=config.limit, - with_feedback=True ) defer.returnValue((events, next_key)) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 4ea4da653..bac68cc29 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -26,14 +26,12 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): user, _ = yield self.auth.get_user_by_req(request) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f4558b95a..23871f161 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -329,14 +329,12 @@ class RoomMessageListRestServlet(ClientV1RestServlet): pagination_config = PaginationConfig.from_request( request, default_limit=10, ) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 0abfa86cd..5763c462a 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -159,9 +159,7 @@ class StreamStore(SQLBaseStore): @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, - limit=0, with_feedback=False): - # TODO (erikj): Handle compressed feedback - + limit=0): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " " INNER JOIN current_state_events as c" @@ -227,10 +225,7 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def paginate_room_events(self, room_id, from_key, to_key=None, - direction='b', limit=-1, - with_feedback=False): - # TODO (erikj): Handle compressed feedback - + direction='b', limit=-1): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -302,7 +297,6 @@ class StreamStore(SQLBaseStore): @cachedInlineCallbacks(num_args=4) def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None): - # TODO (erikj): Handle compressed feedback end_token = RoomStreamToken.parse_stream_token(end_token) From 09cb5c7d33c32e2cbf5a5b6f6f0e2780338491d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 17:31:09 +0100 Subject: [PATCH 19/46] Allow users that have left a room to get the messages that happend in the room before they left --- synapse/handlers/message.py | 31 +++++++++++++++++++++++++++---- synapse/storage/stream.py | 19 ++++++++++++++++++- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 72ebac047..db89491b4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -83,21 +83,44 @@ class MessageHandler(BaseHandler): Returns: dict: Pagination API results """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) data_source = self.hs.get_event_sources().sources["room"] - if not pagin_config.from_token: + if pagin_config.from_token: + room_token = pagin_config.from_token.room_key + else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) + room_token = pagin_config.from_token.room_key - room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) + room_token = RoomStreamToken.parse(room_token) if room_token.topological is None: raise SynapseError(400, "Invalid token") + pagin_config.from_token = pagin_config.from_token.copy_and_replace( + "room_key", str(room_token) + ) + + source_config = pagin_config.get_source_config("room") + + if member_event.membership == Membership.LEAVE: + # If they have left the room then clamp the token to be before + # they left the room + leave_token = yield self.store.get_topological_token_for_event( + member_event.event_id + ) + leave_token = RoomStreamToken.parse(leave_token) + if leave_token.topological < room_token.topological: + source_config.from_key = str(leave_token) + + if source_config.direction == "f": + if source_config.to_key is None: + source_config.to_key = str(leave_token) + yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) @@ -105,7 +128,7 @@ class MessageHandler(BaseHandler): user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( - user, pagin_config.get_source_config("room"), room_id + user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5763c462a..3cab06fde 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -386,7 +386,24 @@ class StreamStore(SQLBaseStore): table="events", keyvalues={"event_id": event_id}, retcol="stream_ordering", - ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + ).addCallback(lambda row: "s%d" % (row,)) + + def get_topological_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "t%d-%d" topological token. + """ + return self._simple_select_one( + table="events", + keyvalues={"event_id": event_id}, + retcols=("stream_ordering", "topological_ordering"), + ).addCallback(lambda row: "t%d-%d" % ( + row["topological_ordering"], row["stream_ordering"],) + ) def _get_max_topological_txn(self, txn): txn.execute( From e1eb1f3fb959468ddacb46e0766ce0c2ce938778 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 17:02:39 +0100 Subject: [PATCH 20/46] Various bug fixes to crypto.keyring --- synapse/crypto/keyring.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index aa74d4d0c..644c7b14a 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -162,7 +162,9 @@ class Keyring(object): def remove_deferreds(res, server_name, group_id): server_to_gids[server_name].discard(group_id) if not server_to_gids[server_name]: - server_to_deferred.pop(server_name).callback(None) + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) return res for g_id, deferred in deferreds.items(): @@ -200,8 +202,15 @@ class Keyring(object): else: break - for server_name, deferred in server_to_deferred: - self.key_downloads[server_name] = ObservableDeferred(deferred) + for server_name, deferred in server_to_deferred.items(): + d = ObservableDeferred(deferred) + self.key_downloads[server_name] = d + + def rm(r, server_name): + self.key_downloads.pop(server_name, None) + return r + + d.addBoth(rm, server_name) def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred): """Takes a dict of KeyGroups and tries to find at least one key for @@ -220,9 +229,8 @@ class Keyring(object): merged_results = {} missing_keys = { - group.server_name: key_id + group.server_name: set(group.key_ids) for group in group_id_to_group.values() - for key_id in group.key_ids } for fn in key_fetch_fns: @@ -279,16 +287,15 @@ class Keyring(object): def get_keys_from_store(self, server_name_and_key_ids): res = yield defer.gatherResults( [ - self.store.get_server_verify_keys(server_name, key_ids) + self.store.get_server_verify_keys( + server_name, key_ids + ).addCallback(lambda ks, server: (server, ks), server_name) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, ).addErrback(unwrapFirstError) - defer.returnValue(dict(zip( - [server_name for server_name, _ in server_name_and_key_ids], - res - ))) + defer.returnValue(dict(res)) @defer.inlineCallbacks def get_keys_from_perspectives(self, server_name_and_key_ids): From 3f604816555040a501874c8638d861b66dcd960b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Sep 2015 09:58:32 +0100 Subject: [PATCH 21/46] Bump version and change log --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a0a12ca4a..1f6b712cb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.10.0-r2 (2015-09-10) +========================================== + +* Fix bug where we always fetched remote server signing keys instead of using + ones in our cache. + Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 5b5ff6c3f..d62294e6b 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0-r1" +__version__ = "0.10.0-r2" From ca32c7a0657c056a4d503b92e4120b880639663f Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Sep 2015 10:33:48 +0100 Subject: [PATCH 22/46] Fix adding threepids to an existing account --- synapse/rest/client/v2_alpha/account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 522a312c9..6281e2d02 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -96,6 +96,7 @@ class ThreepidRestServlet(RestServlet): self.hs = hs self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler @defer.inlineCallbacks def on_GET(self, request): From 4ba8189b744ab942322b7da5136292d3c4bb4e58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Sep 2015 10:45:22 +0100 Subject: [PATCH 23/46] Bump change log --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1f6b712cb..cab5e495c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,7 @@ Changes in synapse v0.10.0-r2 (2015-09-10) * Fix bug where we always fetched remote server signing keys instead of using ones in our cache. +* Fix adding threepids to an existing account. Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== From e2054ce21a04f3d741293f50b283c01bbe2b0591 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Sep 2015 15:06:47 +0100 Subject: [PATCH 24/46] Allow users to GET individual state events for rooms that they have left --- synapse/handlers/message.py | 20 +++++++++++++------- tests/rest/client/v1/test_rooms.py | 10 +++++----- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index db89491b4..5d18aaacf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError, SynapseError +from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -277,13 +277,19 @@ class MessageHandler(BaseHandler): Raises: SynapseError if something went wrong. """ - have_joined = yield self.auth.check_joined_room(room_id, user_id) - if not have_joined: - raise RoomError(403, "User not in room.") + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + data = yield self.state_handler.get_current_state( + room_id, event_type, state_key + ) + elif member_event.membership == Membership.LEAVE: + key = (event_type, state_key) + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], [key] + ) + data = room_state[member_event.event_id].get(key) - data = yield self.state_handler.get_current_state( - room_id, event_type, state_key - ) defer.returnValue(data) @defer.inlineCallbacks diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index d50cfe429..ed0ac8d5c 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -239,7 +239,7 @@ class RoomPermissionsTestCase(RestTestCase): "PUT", topic_path, topic_content) self.assertEquals(403, code, msg=str(response)) (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) # get topic in PUBLIC room, not joined, expect 403 (code, response) = yield self.mock_resource.trigger_get( @@ -301,11 +301,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, private room + left - # expect all 403s + # expect all 200s yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_membership_public_room_perms(self): @@ -326,11 +326,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, public room + left - # expect 403. + # expect 200. yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_invited_permissions(self): From dffc9c4ae0eea0616cc017c7f858f8a923202075 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 14 Sep 2015 14:41:37 +0100 Subject: [PATCH 25/46] Drop unused index --- .../storage/schema/delta/23/drop_state_index.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 synapse/storage/schema/delta/23/drop_state_index.sql diff --git a/synapse/storage/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/delta/23/drop_state_index.sql new file mode 100644 index 000000000..07d0ea5cb --- /dev/null +++ b/synapse/storage/schema/delta/23/drop_state_index.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DROP INDEX IF EXISTS state_groups_state_tuple; From 9cd5b9a802f4c38f716a507a40188557103993f3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 14 Sep 2015 19:03:53 +0100 Subject: [PATCH 26/46] Hacky attempt at catching SIGHUP and rotating the logfile around --- synapse/config/logger.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index fa542623b..daca698d0 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -21,6 +21,7 @@ import logging.config import yaml from string import Template import os +import signal DEFAULT_LOG_CONFIG = Template(""" @@ -142,6 +143,19 @@ class LoggingConfig(Config): handler = logging.handlers.RotatingFileHandler( self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 ) + + def sighup(signum, stack): + logger.info("Closing log file due to SIGHUP") + handler.doRollover() + logger.info("Opened new log file due to SIGHUP") + + # TODO(paul): obviously this is a terrible mechanism for + # stealing SIGHUP, because it means no other part of synapse + # can use it instead. If we want to catch SIGHUP anywhere + # else as well, I'd suggest we find a nicer way to broadcast + # it around. + if getattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, sighup) else: handler = logging.StreamHandler() handler.setFormatter(formatter) From 3bcbabc9fb5446e74a675352e22963d528189957 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:46:22 +0100 Subject: [PATCH 27/46] Rename context factory Mjark is officially no fun. --- synapse/http/client.py | 6 +++--- synapse/server.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 815a83872..0933388c0 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -261,11 +261,11 @@ def _print_ex(e): logger.exception(e) -class WoefullyInsecureContextFactory(ssl.ContextFactory): +class InsecureInterceptableContextFactory(ssl.ContextFactory): """ - Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. - Do not use this unless you really, really hate your users. + Do not use this since it allows an attacker to intercept your communications. """ def __init__(self): diff --git a/synapse/server.py b/synapse/server.py index 656e534df..d96c5a573 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,7 +21,7 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication -from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory +from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -181,7 +181,7 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() if config.use_insecure_ssl_client else BrowserLikePolicyForHTTPS() ) From d4af08a167cb5351110036c35bdfc267242d8131 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:50:13 +0100 Subject: [PATCH 28/46] Use shorter config key name --- synapse/config/tls.py | 6 +++--- synapse/server.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 35ff13f4b..e6023a718 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -46,9 +46,9 @@ class TlsConfig(Config): # (e.g. for talking to recaptcha, identity servers, and such) # It should never be used in production, and is intended for # use only when running tests. - self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" - "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) + self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( + "use_insecure_ssl_client_just_for_testing_do_not_use" + ) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/server.py b/synapse/server.py index d96c5a573..8424798b1 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -181,7 +181,8 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - InsecureInterceptableContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() + if config.use_insecure_ssl_client_just_for_testing_do_not_use else BrowserLikePolicyForHTTPS() ) From 3b05b67c89bacfa729f14ad7c6de80b32d87cc98 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Sep 2015 16:34:42 +0100 Subject: [PATCH 29/46] When updating a stored event from outlier to non-outlier, remember to update the extremeties --- synapse/storage/event_federation.py | 3 +++ synapse/storage/events.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index dda3027b6..27284cd08 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -303,6 +303,9 @@ class EventFederationStore(SQLBaseStore): ], ) + self._update_extremeties(txn, events) + + def _update_extremeties(self, txn, events): events_by_room = {} for ev in events: events_by_room.setdefault(ev.room_id, []).append(ev) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e3eabab13..8774b3b38 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -275,6 +275,8 @@ class EventsStore(SQLBaseStore): (False, event.event_id,) ) + self._update_extremeties(txn, [event]) + events_and_contexts = filter( lambda ec: ec[0] not in to_remove, events_and_contexts From 8148c48f11845af1747dc9447e47f9cce70c615f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Sep 2015 16:54:43 +0100 Subject: [PATCH 30/46] "Comments" --- synapse/storage/event_federation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 27284cd08..7cb314dee 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -306,6 +306,12 @@ class EventFederationStore(SQLBaseStore): self._update_extremeties(txn, events) def _update_extremeties(self, txn, events): + """Updates the event_*_extremities tables based on the new/updated + events being persisted. + + This is called for new events *and* for events that were outliers, but + are are now being persisted as non-outliers. + """ events_by_room = {} for ev in events: events_by_room.setdefault(ev.room_id, []).append(ev) From 0b1a55c60a7b10a987583dc2c231e8b1cfd29f30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Sep 2015 09:55:44 +0100 Subject: [PATCH 31/46] Update changelog --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cab5e495c..f1d2c7a76 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,11 @@ -Changes in synapse v0.10.0-r2 (2015-09-10) +Changes in synapse v0.10.0-r2 (2015-09-16) ========================================== * Fix bug where we always fetched remote server signing keys instead of using ones in our cache. * Fix adding threepids to an existing account. +* Fix bug with invinting over federation where remote server was already in + the room. (PR #281, SYN-392) Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== From ffe8cf7e59bab7c9da98534f27b1e486cb642223 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 11:51:25 +0100 Subject: [PATCH 32/46] Fix bug where we sometimes didn't fetch all the keys requested for a server. --- synapse/crypto/keyring.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 1b1b31c5c..8b6a59866 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -228,10 +228,9 @@ class Keyring(object): def do_iterations(): merged_results = {} - missing_keys = { - group.server_name: set(group.key_ids) - for group in group_id_to_group.values() - } + missing_keys = {} + for group in group_id_to_group.values(): + missing_keys.setdefault(group.server_name, set()).union(group.key_ids) for fn in key_fetch_fns: results = yield fn(missing_keys.items()) From b105996fc16ac264457c5a9f8e00c6a8263142d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 17 Sep 2015 10:28:36 +0100 Subject: [PATCH 33/46] Remove run_on_reactor --- synapse/state.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 1fe4d066b..ed36f844c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError @@ -119,8 +118,6 @@ class StateHandler(object): Returns: an EventContext """ - yield run_on_reactor() - context = EventContext() if outlier: From 8e3bbc9bd01b8d0bd75e28d9c8862e555384e124 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 13:47:44 +0100 Subject: [PATCH 34/46] Clarify which event is returned by check_user_was_in_room --- synapse/api/auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d8bb64a4a..49a068afb 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -146,8 +146,9 @@ class Auth(object): Raises: AuthError if the user was never in the room. Returns: - A deferred membership event for the user if the user was in - the room. + A deferred membership event for the user if the user was in the + room. This will be the join event if they are currently joined to + the room. This will be the leave event if they have left the room. """ if current_state: member = current_state.get( From 1e101ed4a45bad68f4865229e0c8d617fed67065 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:13:10 +0100 Subject: [PATCH 35/46] Clamp the "to" token for /rooms/{roomId}/messages to when the user left the room. There isn't a way for the client to learn a valid "to" token for a room that they have left in the C-S API but that doesn't stop a client making one up. --- synapse/handlers/message.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5d18aaacf..bca592f5d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -120,6 +120,10 @@ class MessageHandler(BaseHandler): if source_config.direction == "f": if source_config.to_key is None: source_config.to_key = str(leave_token) + else: + to_token = RoomStreamToken.parse(source_config.to_key) + if leave_token.topological < to_token.topological: + source_config.to_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological From 0c162859897cb0cae5e501109e3d51d0e861e194 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:17:16 +0100 Subject: [PATCH 36/46] Add explicit "elif event.membership == Membership.LEAVE" for clarity --- synapse/handlers/message.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bca592f5d..5593a8d4f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -389,13 +389,14 @@ class MessageHandler(BaseHandler): if event.membership not in (Membership.JOIN, Membership.LEAVE): return + try: if event.membership == Membership.JOIN: room_end_token = now_token.room_key deferred_room_state = self.state_handler.get_current_state( event.room_id ) - else: + elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( event.room_id, [event.event_id], None From 95c304e3f93bc1113c2b4ac64d85c9fdeb7120b9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:18:47 +0100 Subject: [PATCH 37/46] Fix doc string to point at the right class --- synapse/handlers/message.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5593a8d4f..bb7328672 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -467,8 +467,9 @@ class MessageHandler(BaseHandler): Args: user_id(str): The user to get a snapshot for. room_id(str): The room to get a snapshot of. - pagin_config(synapse.api.streams.PaginationConfig): The pagination - config used to determine how many messages to return. + pagin_config(synapse.streams.config.PaginationConfig): + The pagination config used to determine how many messages to + return. Raises: AuthError if the user wasn't in the room. Returns: From 49c0a0b5c4c2385fdf6755b1a5e1a3f0b04ef503 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:21:03 +0100 Subject: [PATCH 38/46] Clarify that room_initial_sync returns a python dict --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bb7328672..bda8eb5f3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -473,7 +473,7 @@ class MessageHandler(BaseHandler): Raises: AuthError if the user wasn't in the room. Returns: - A JSON object with the snapshot of the room. + A JSON serialisable dict with the snapshot of the room. """ member_event = yield self.auth.check_user_was_in_room(room_id, user_id) From 7213588083dd9a721b0cd623fe22b308f25f19a5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 12:57:40 +0100 Subject: [PATCH 39/46] Implement configurable stats reporting SYN-287 This requires that HS owners either opt in or out of stats reporting. When --generate-config is passed, --report-stats must be specified If an already-generated config is used, and doesn't have the report_stats key, it is requested to be set. --- synapse/app/homeserver.py | 35 +++++- synapse/app/synctl.py | 12 +- synapse/config/_base.py | 45 ++++++- synapse/config/appservice.py | 2 +- synapse/config/captcha.py | 2 +- synapse/config/database.py | 2 +- synapse/config/key.py | 2 +- synapse/config/logger.py | 2 +- synapse/config/metrics.py | 8 +- synapse/config/ratelimiting.py | 2 +- synapse/config/registration.py | 2 +- synapse/config/repository.py | 2 +- synapse/config/saml2.py | 2 +- synapse/config/server.py | 2 +- synapse/config/tls.py | 2 +- synapse/config/voip.py | 2 +- synapse/storage/__init__.py | 20 ++- synapse/storage/events.py | 58 ++++++++- synapse/storage/registration.py | 12 ++ .../schema/delta/24/stats_reporting.sql | 22 ++++ tests/storage/event_injector.py | 81 ++++++++++++ tests/storage/test_events.py | 116 ++++++++++++++++++ tests/storage/test_room.py | 2 +- tests/storage/test_stream.py | 68 +++------- 24 files changed, 425 insertions(+), 78 deletions(-) create mode 100644 synapse/storage/schema/delta/24/stats_reporting.sql create mode 100644 tests/storage/event_injector.py create mode 100644 tests/storage/test_events.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c0a4a00..b4429bd4f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -42,7 +42,7 @@ from synapse.storage import ( from synapse.server import HomeServer -from twisted.internet import reactor +from twisted.internet import reactor, task, defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper @@ -677,6 +677,39 @@ def run(hs): ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) + start_time = hs.get_clock().time() + + @defer.inlineCallbacks + def phone_stats_home(): + now = int(hs.get_clock().time()) + uptime = int(now - start_time) + if uptime < 0: + uptime = 0 + + stats = {} + stats["homeserver"] = hs.config.server_name + stats["timestamp"] = now + stats["uptime_seconds"] = uptime + stats["total_users"] = yield hs.get_datastore().count_all_users() + + all_rooms = yield hs.get_datastore().get_rooms(False) + stats["total_room_count"] = len(all_rooms) + + stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() + daily_messages = yield hs.get_datastore().count_daily_messages() + if daily_messages is not None: + stats["daily_messages"] = daily_messages + + logger.info("Reporting stats to matrix.org: %s" % (stats,)) + hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + + if hs.config.report_stats: + phone_home_task = task.LoopingCall(phone_stats_home) + phone_home_task.start(60 * 60 * 24, now=False) + def in_thread(): with LoggingContext("run"): change_resource_limit(hs.config.soft_file_limit) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 1f7d543c3..6bcc43759 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -25,6 +25,7 @@ SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" GREEN = "\x1b[1;32m" +RED = "\x1b[1;31m" NORMAL = "\x1b[m" if not os.path.exists(CONFIGFILE): @@ -45,8 +46,15 @@ def start(): print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", CONFIGFILE]) - subprocess.check_call(args) - print GREEN + "started" + NORMAL + try: + subprocess.check_call(args) + print GREEN + "started" + NORMAL + except subprocess.CalledProcessError as e: + print ( + RED + + "error starting (exit code: %d); see above for logs" % e.returncode + + NORMAL + ) def stop(): diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 8a75c4873..b9983f72a 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -26,6 +26,16 @@ class ConfigError(Exception): class Config(object): + stats_reporting_begging_spiel = ( + "We would really appreciate it if you could help our project out by " + "reporting anonymized usage statistics from your homeserver. Only very " + "basic aggregate data (e.g. number of users) will be reported, but it " + "helps us to track the growth of the Matrix community, and helps us to " + "make Matrix a success, as well as to convince other networks that they " + "should peer with us.\n" + "Thank you." + ) + @staticmethod def parse_size(value): if isinstance(value, int) or isinstance(value, long): @@ -111,11 +121,14 @@ class Config(object): results.append(getattr(cls, name)(self, *args, **kargs)) return results - def generate_config(self, config_dir_path, server_name): + def generate_config(self, config_dir_path, server_name, report_stats=None): default_config = "# vim:ft=yaml\n" default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( - "default_config", config_dir_path, server_name + "default_config", + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=report_stats, )) config = yaml.load(default_config) @@ -139,6 +152,12 @@ class Config(object): action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "--report-stats", + action="store", + help="Stuff", + choices=["yes", "no"] + ) config_parser.add_argument( "--generate-keys", action="store_true", @@ -189,6 +208,11 @@ class Config(object): config_files.append(config_path) if config_args.generate_config: + if config_args.report_stats is None: + config_parser.error( + "Please specify either --report-stats=yes or --report-stats=no\n\n" + + cls.stats_reporting_begging_spiel + ) if not config_files: config_parser.error( "Must supply a config file.\nA config file can be automatically" @@ -211,7 +235,9 @@ class Config(object): os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: config_bytes, config = obj.generate_config( - config_dir_path, server_name + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=(config_args.report_stats == "yes"), ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) @@ -261,9 +287,20 @@ class Config(object): specified_config.update(yaml_config) server_name = specified_config["server_name"] - _, config = obj.generate_config(config_dir_path, server_name) + _, config = obj.generate_config( + config_dir_path=config_dir_path, + server_name=server_name + ) config.pop("log_config") config.update(specified_config) + if "report_stats" not in config: + sys.stderr.write( + "Please opt in or out of reporting anonymized homeserver usage " + "statistics, by setting the report_stats key in your config file " + " ( " + config_path + " ) " + + "to either True or False.\n\n" + + Config.stats_reporting_begging_spiel + "\n") + sys.exit(1) if generate_keys: obj.invoke_all("generate_files", config) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 38f41933b..b8d301995 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -20,7 +20,7 @@ class AppServiceConfig(Config): def read_config(self, config): self.app_service_config_files = config.get("app_service_config_files", []) - def default_config(cls, config_dir_path, server_name): + def default_config(cls, **kwargs): return """\ # A list of application service config file to use app_service_config_files: [] diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 15a132b4e..dd92fcd0d 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -24,7 +24,7 @@ class CaptchaConfig(Config): self.captcha_bypass_secret = config.get("captcha_bypass_secret") self.recaptcha_siteverify_api = config["recaptcha_siteverify_api"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Captcha ## diff --git a/synapse/config/database.py b/synapse/config/database.py index f0611e888..baeda8f30 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -45,7 +45,7 @@ class DatabaseConfig(Config): self.set_databasepath(config.get("database_path")) - def default_config(self, config, config_dir_path): + def default_config(self, **kwargs): database_path = self.abspath("homeserver.db") return """\ # Database configuration diff --git a/synapse/config/key.py b/synapse/config/key.py index 23ac8a3fc..2c187065e 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -40,7 +40,7 @@ class KeyConfig(Config): config["perspectives"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) return """\ ## Signing Keys ## diff --git a/synapse/config/logger.py b/synapse/config/logger.py index daca698d0..bd0c17c86 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -70,7 +70,7 @@ class LoggingConfig(Config): self.log_config = self.abspath(config.get("log_config")) self.log_file = self.abspath(config.get("log_file")) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): log_file = self.abspath("homeserver.log") log_config = self.abspath( os.path.join(config_dir_path, server_name + ".log.config") diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index ae5a69152..825fec9a3 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -19,13 +19,15 @@ from ._base import Config class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] + self.report_stats = config.get("report_stats", None) self.metrics_port = config.get("metrics_port") self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") - def default_config(self, config_dir_path, server_name): - return """\ + def default_config(self, report_stats=None, **kwargs): + suffix = "" if report_stats is None else "report_stats: %(report_stats)s\n" + return ("""\ ## Metrics ### # Enable collection and rendering of performance metrics enable_metrics: False - """ + """ + suffix) % locals() diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 76d9970e5..611b598ec 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -27,7 +27,7 @@ class RatelimitConfig(Config): self.federation_rc_reject_limit = config["federation_rc_reject_limit"] self.federation_rc_concurrent = config["federation_rc_concurrent"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Ratelimiting ## diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 62de4b399..fa98eced3 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -34,7 +34,7 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") - def default_config(self, config_dir, server_name): + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) macaroon_secret_key = random_string_with_symbols(50) return """\ diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 64644b9a7..2fcf87244 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -60,7 +60,7 @@ class ContentRepositoryConfig(Config): config["thumbnail_sizes"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): media_store = self.default_path("media_store") uploads_path = self.default_path("uploads") return """ diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index 153203687..4c6133cf2 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -41,7 +41,7 @@ class SAML2Config(Config): self.saml2_config_path = None self.saml2_idp_redirect_url = None - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable SAML2 for registration and login. Uses pysaml2 # config_path: Path to the sp_conf.py configuration file diff --git a/synapse/config/server.py b/synapse/config/server.py index a03e55c22..4d12d4985 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -117,7 +117,7 @@ class ServerConfig(Config): self.content_addr = content_addr - def default_config(self, config_dir_path, server_name): + def default_config(self, server_name, **kwargs): if ":" in server_name: bind_port = int(server_name.split(":")[1]) unsecure_port = bind_port - 400 diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e6023a718..0ac269829 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -50,7 +50,7 @@ class TlsConfig(Config): "use_insecure_ssl_client_just_for_testing_do_not_use" ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" diff --git a/synapse/config/voip.py b/synapse/config/voip.py index a1707223d..a093354cc 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -22,7 +22,7 @@ class VoipConfig(Config): self.turn_shared_secret = config["turn_shared_secret"] self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"]) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Turn ## diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 77cb1dbd8..b64c90d63 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 23 +SCHEMA_VERSION = 24 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -126,6 +126,24 @@ class DataStore(RoomMemberStore, RoomStore, lock=False, ) + @defer.inlineCallbacks + def count_daily_users(self): + def _count_users(txn): + txn.execute( + "SELECT COUNT(DISTINCT user_id) AS users" + " FROM user_ips" + " WHERE last_seen > ?", + # This is close enough to a day for our purposes. + (int(self._clock.time_msec()) - (1000 * 60 * 60 * 24),) + ) + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) + def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0a477e312..2b51db994 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer, reactor @@ -28,6 +27,7 @@ from canonicaljson import encode_canonical_json from contextlib import contextmanager import logging +import math import ujson as json logger = logging.getLogger(__name__) @@ -905,3 +905,59 @@ class EventsStore(SQLBaseStore): txn.execute(sql, (event.event_id,)) result = txn.fetchone() return result[0] if result else None + + @defer.inlineCallbacks + def count_daily_messages(self): + def _count_messages(txn): + now = self.hs.get_clock().time() + + txn.execute( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + last_reported = self.cursor_to_dict(txn) + + txn.execute( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + now_reporting = self.cursor_to_dict(txn) + if not now_reporting: + return None + now_reporting = now_reporting[0]["stream_ordering"] + + txn.execute("DELETE FROM stats_reporting") + txn.execute( + "INSERT INTO stats_reporting" + " (reported_stream_token, reported_time)" + " VALUES (?, ?)", + (now_reporting, now,) + ) + + if not last_reported: + return None + + # Close enough to correct for our purposes. + yesterday = (now - 24 * 60 * 60) + if math.fabs(yesterday - last_reported[0]["reported_time"]) > 60 * 60: + return None + + txn.execute( + "SELECT COUNT(*) as messages" + " FROM events NATURAL JOIN event_json" + " WHERE json like '%m.room.message%'" + " AND stream_ordering > ?" + " AND stream_ordering <= ?", + ( + last_reported[0]["reported_stream_token"], + now_reporting, + ) + ) + rows = self.cursor_to_dict(txn) + if not rows: + return None + return rows[0]["messages"] + + ret = yield self.runInteraction("count_messages", _count_messages) + defer.returnValue(ret) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c9ceb132a..6d7623765 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -289,3 +289,15 @@ class RegistrationStore(SQLBaseStore): if ret: defer.returnValue(ret['user_id']) defer.returnValue(None) + + @defer.inlineCallbacks + def count_all_users(self): + def _count_users(txn): + txn.execute("SELECT COUNT(*) AS users FROM users") + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) diff --git a/synapse/storage/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/delta/24/stats_reporting.sql new file mode 100644 index 000000000..e9165d291 --- /dev/null +++ b/synapse/storage/schema/delta/24/stats_reporting.sql @@ -0,0 +1,22 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Should only ever contain one row +CREATE TABLE IF NOT EXISTS stats_reporting( + -- The stream ordering token which was most recently reported as stats + reported_stream_token INTEGER, + -- The time (seconds since epoch) stats were most recently reported + reported_time BIGINT +); diff --git a/tests/storage/event_injector.py b/tests/storage/event_injector.py new file mode 100644 index 000000000..42bd8928b --- /dev/null +++ b/tests/storage/event_injector.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID, RoomID + +from tests.utils import setup_test_homeserver + +from mock import Mock + + +class EventInjector: + def __init__(self, hs): + self.hs = hs + self.store = hs.get_datastore() + self.message_handler = hs.get_handlers().message_handler + self.event_builder_factory = hs.get_event_builder_factory() + + @defer.inlineCallbacks + def create_room(self, room): + builder = self.event_builder_factory.new({ + "type": EventTypes.Create, + "room_id": room.to_string(), + "content": {}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + @defer.inlineCallbacks + def inject_room_member(self, room, user, membership): + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"membership": membership}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + defer.returnValue(event) + + @defer.inlineCallbacks + def inject_message(self, room, user, body): + builder = self.event_builder_factory.new({ + "type": EventTypes.Message, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"body": body, "msgtype": u"message"}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py new file mode 100644 index 000000000..313013009 --- /dev/null +++ b/tests/storage/test_events.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid +from mock.mock import Mock +from synapse.types import RoomID, UserID + +from tests import unittest +from twisted.internet import defer +from tests.storage.event_injector import EventInjector + +from tests.utils import setup_test_homeserver + + +class EventsStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + self.hs = yield setup_test_homeserver( + resource_for_federation=Mock(), + http_client=None, + ) + self.store = self.hs.get_datastore() + self.db_pool = self.hs.get_db_pool() + self.message_handler = self.hs.get_handlers().message_handler + self.event_injector = EventInjector(self.hs) + + @defer.inlineCallbacks + def test_count_daily_messages(self): + self.db_pool.runQuery("DELETE FROM stats_reporting") + + self.hs.clock.now = 100 + + # Never reported before, and nothing which could be reported + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + count = yield self.db_pool.runQuery("SELECT COUNT(*) FROM stats_reporting") + self.assertEqual([(0,)], count) + + # Create something to report + room = RoomID.from_string("!abc123:test") + user = UserID.from_string("@raccoonlover:test") + yield self.event_injector.create_room(room) + + self.base_event = yield self._get_last_stream_token() + + yield self.event_injector.inject_message(room, user, "Raccoons are really cute") + + # Never reported before, something could be reported, but isn't because + # it isn't old enough. + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(1, self.hs.clock.now) + + # Already reported yesterday, two new events from today. + yield self.event_injector.inject_message(room, user, "Yeah they are!") + yield self.event_injector.inject_message(room, user, "Incredibly!") + self.hs.clock.now += 60 * 60 * 24 + count = yield self.store.count_daily_messages() + self.assertEqual(2, count) # 2 since yesterday + self._assert_stats_reporting(3, self.hs.clock.now) # 3 ever + + # Last reported too recently. + yield self.event_injector.inject_message(room, user, "Who could disagree?") + self.hs.clock.now += 60 * 60 * 22 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(4, self.hs.clock.now) + + # Last reported too long ago + yield self.event_injector.inject_message(room, user, "No one.") + self.hs.clock.now += 60 * 60 * 26 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(5, self.hs.clock.now) + + # And now let's actually report something + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + # A little over 24 hours is fine :) + self.hs.clock.now += (60 * 60 * 24) + 50 + count = yield self.store.count_daily_messages() + self.assertEqual(3, count) + self._assert_stats_reporting(8, self.hs.clock.now) + + @defer.inlineCallbacks + def _get_last_stream_token(self): + rows = yield self.db_pool.runQuery( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + if not rows: + defer.returnValue(0) + else: + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _assert_stats_reporting(self, messages, time): + rows = yield self.db_pool.runQuery( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + self.assertEqual([(self.base_event + messages, time,)], rows) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index ab7625a3c..caffce64e 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -85,7 +85,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): # Room events need the full datastore, for persist_event() and # get_room_state() self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory(); + self.event_factory = hs.get_event_factory() self.room = RoomID.from_string("!abcde:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 0c9b89d76..a658a789a 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID +from tests.storage.event_injector import EventInjector from tests.utils import setup_test_homeserver @@ -36,6 +37,7 @@ class StreamStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_builder_factory = hs.get_event_builder_factory() + self.event_injector = EventInjector(hs) self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler @@ -45,60 +47,20 @@ class StreamStoreTestCase(unittest.TestCase): self.room1 = RoomID.from_string("!abc123:test") self.room2 = RoomID.from_string("!xyx987:test") - self.depth = 1 - - @defer.inlineCallbacks - def inject_room_member(self, room, user, membership): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Member, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"membership": membership}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - - defer.returnValue(event) - - @defer.inlineCallbacks - def inject_message(self, room, user, body): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Message, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"body": body, "msgtype": u"message"}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - @defer.inlineCallbacks def test_event_stream_get_other(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -125,17 +87,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_get_own(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -162,22 +124,22 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_join_leave(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Then bob leaves again. - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.LEAVE ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -193,17 +155,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_prev_content(self): - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) - event1 = yield self.inject_room_member( + event1 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) start = yield self.store.get_room_events_max_id() - event2 = yield self.inject_room_member( + event2 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN, ) From 6d7f291b93c363ecee7a3f1e5f5e2cdf81a0fbae Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:13:07 +0100 Subject: [PATCH 40/46] Front-load spaces --- synapse/config/_base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index b9983f72a..ceef309af 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,13 +27,13 @@ class ConfigError(Exception): class Config(object): stats_reporting_begging_spiel = ( - "We would really appreciate it if you could help our project out by " - "reporting anonymized usage statistics from your homeserver. Only very " - "basic aggregate data (e.g. number of users) will be reported, but it " - "helps us to track the growth of the Matrix community, and helps us to " - "make Matrix a success, as well as to convince other networks that they " - "should peer with us.\n" - "Thank you." + "We would really appreciate it if you could help our project out by" + " reporting anonymized usage statistics from your homeserver. Only very" + " basic aggregate data (e.g. number of users) will be reported, but it" + " helps us to track the growth of the Matrix community, and helps us to" + " make Matrix a success, as well as to convince other networks that they" + " should peer with us." + "\nThank you." ) @staticmethod From eb011cd99ba03f40f4ed7a023b64f93dfa2cbdc9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:29:36 +0100 Subject: [PATCH 41/46] Add docstring --- synapse/storage/events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2b51db994..46df6b4d6 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -908,6 +908,12 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def count_daily_messages(self): + """ + Returns an estimate of the number of messages sent in the last day. + + If it has been significantly less or more than one day since the last + call to this function, it will return None. + """ def _count_messages(txn): now = self.hs.get_clock().time() From b6e0303c833bb37891294046598d48cafe2b1c9e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:34:29 +0100 Subject: [PATCH 42/46] Catch stats-reporting errors --- synapse/app/homeserver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4429bd4f..21840e4a2 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -701,10 +701,13 @@ def run(hs): stats["daily_messages"] = daily_messages logger.info("Reporting stats to matrix.org: %s" % (stats,)) - hs.get_simple_http_client().put_json( - "https://matrix.org/report-usage-stats/push", - stats - ) + try: + yield hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + except Exception as e: + logger.warn("Error reporting stats: %s", e) if hs.config.report_stats: phone_home_task = task.LoopingCall(phone_stats_home) From 6d59ffe1ce9a821d50d491f97bf05950198f6f53 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:47:40 +0100 Subject: [PATCH 43/46] Add some docstrings --- synapse/storage/__init__.py | 3 +++ synapse/storage/registration.py | 1 + 2 files changed, 4 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b64c90d63..340e59afc 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -128,6 +128,9 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks def count_daily_users(self): + """ + Counts the number of users who used this homeserver in the last 24 hours. + """ def _count_users(txn): txn.execute( "SELECT COUNT(DISTINCT user_id) AS users" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 6d7623765..b454dd5b3 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -292,6 +292,7 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def count_all_users(self): + """Counts all users registered on the homeserver.""" def _count_users(txn): txn.execute("SELECT COUNT(*) AS users FROM users") rows = self.cursor_to_dict(txn) From b35baf6f3c5e9f24e9af241eb2423c4b94dd5a14 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 15:13:10 +0100 Subject: [PATCH 44/46] Define __repr__ methods for StreamConfig and PaginationConfig So that they can be used with "%r" log formats. --- synapse/streams/config.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2ec7c5403..167bfe0de 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -34,6 +34,11 @@ class SourcePaginationConfig(object): self.direction = 'f' if direction == 'f' else 'b' self.limit = int(limit) if limit is not None else None + def __repr__(self): + return ( + "StreamConfig(from_key=%r, to_key=%r, direction=%r, limit=%r)" + ) % (self.from_key, self.to_key, self.direction, self.limit) + class PaginationConfig(object): @@ -94,10 +99,10 @@ class PaginationConfig(object): logger.exception("Failed to create pagination config") raise SynapseError(400, "Invalid request.") - def __str__(self): + def __repr__(self): return ( - "" + "PaginationConfig(from_tok=%r, to_tok=%r," + " direction=%r, limit=%r)" ) % (self.from_token, self.to_token, self.direction, self.limit) def get_source_config(self, source_name): From 8a519ac76d646c991c4f77973a157ac7bea25665 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Sep 2015 09:55:24 +0100 Subject: [PATCH 45/46] Fix demo/start.sh to work with --report-stats --- demo/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/start.sh b/demo/start.sh index 572dbfab0..a90561488 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -25,6 +25,7 @@ for port in 8080 8081 8082; do --generate-config \ -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ + --report-stats no # Check script parameters if [ $# -eq 1 ]; then From cf1100887b454535e25dfeb67d649c2a4673eab7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 17:34:02 +0100 Subject: [PATCH 46/46] Fix order of ON constraints in _get_rooms_for_user_where_membership_is_txn --- synapse/storage/roommember.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index cd9eefbd9..41c939efb 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -142,12 +142,12 @@ class RoomMemberStore(SQLBaseStore): sql = ( "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" - " FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON e.event_id = c.event_id " - " INNER JOIN events as e " - " ON m.event_id = c.event_id " - " AND m.room_id = c.room_id " + " FROM current_state_events as c" + " INNER JOIN room_memberships as m" + " ON m.event_id = c.event_id" + " INNER JOIN events as e" + " ON e.event_id = c.event_id" + " AND m.room_id = c.room_id" " AND m.user_id = c.state_key" " WHERE %s" ) % (where_clause,)