From 9550ba94f2bbb7170a975d1e83a81ef700690e59 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 17:31:39 +0000 Subject: [PATCH 01/91] Mention that we should pull in new deps before running upgrade script --- UPGRADE.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/UPGRADE.rst b/UPGRADE.rst index a602a9f3e..9618ad2d5 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,6 +1,10 @@ Upgrading to v0.6.0 =================== +To pull in new dependencies, run:: + + python setup.py develop --user + This update includes a change to the database schema. To upgrade you first need to upgrade the database by running:: From 5b39cfff6981905d500398e6b937003ecb9f0362 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 18:25:24 +0000 Subject: [PATCH 02/91] Don't assume an event exists --- synapse/storage/_base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6dc857c4a..e0d97f440 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -488,11 +488,13 @@ class SQLBaseStore(object): ev.unsigned["redacted_because"] = because if get_prev_content and "replaces_state" in ev.unsigned: - ev.unsigned["prev_content"] = self._get_event_txn( + prev = self._get_event_txn( txn, ev.unsigned["replaces_state"], get_prev_content=False, - ).get_dict()["content"] + ) + if prev: + ev.unsigned["prev_content"] = prev.get_dict()["content"] return ev From 52f99243ab0eef93558ddc95b744c548241057ac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 18:33:50 +0000 Subject: [PATCH 03/91] Use is_outlier() so that we don't get AttributeError --- synapse/federation/replication.py | 6 +++--- synapse/state.py | 2 +- synapse/storage/__init__.py | 4 +--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9f8aadccc..ec9b6e246 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -562,8 +562,8 @@ class ReplicationLayer(object): already_seen = ( existing and ( - not existing.internal_metadata.outlier - or pdu.internal_metadata.outlier + not existing.internal_metadata.is_outlier() + or pdu.internal_metadata.is_outlier() ) ) if already_seen: @@ -604,7 +604,7 @@ class ReplicationLayer(object): # ) # Get missing pdus if necessary. - if not pdu.internal_metadata.outlier: + if not pdu.internal_metadata.is_outlier(): # We only backfill backwards to the min depth. min_depth = yield self.handler.get_min_depth_for_context( pdu.room_id diff --git a/synapse/state.py b/synapse/state.py index 99f873b6e..580053d3f 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -79,7 +79,7 @@ class StateHandler(object): defer.returnValue(False) return - if hasattr(event, "outlier") and event.outlier: + if event.is_outlier(): event.state_group = None event.old_state_events = None event.state_events = None diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2a683b25f..e236bf495 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -143,9 +143,7 @@ class DataStore(RoomMemberStore, RoomStore, elif event.type == EventTypes.Redaction: self._store_redaction(txn, event) - outlier = False - if hasattr(event.internal_metadata, "outlier"): - outlier = event.internal_metadata.outlier + outlier = event.internal_metadata.is_outlier() event_dict = { k: v From 2d7716d4d06285eec6d7e4e0c7109be3b278c127 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 18:41:48 +0000 Subject: [PATCH 04/91] Make error messages slightly more helpful --- synapse/handlers/federation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4aec3563a..fadb48fde 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -163,7 +163,7 @@ class FederationHandler(BaseHandler): yield self._handle_new_event(e, fetch_missing=False) except: logger.exception( - "Failed to parse auth event %s", + "Failed to handle auth event %s", e.event_id, ) @@ -183,7 +183,7 @@ class FederationHandler(BaseHandler): yield self._handle_new_event(e) except: logger.exception( - "Failed to parse state event %s", + "Failed to handle state event %s", e.event_id, ) @@ -394,7 +394,7 @@ class FederationHandler(BaseHandler): yield self._handle_new_event(e, fetch_missing=False) except: logger.exception( - "Failed to parse auth event %s", + "Failed to handle auth event %s", e.event_id, ) @@ -408,7 +408,7 @@ class FederationHandler(BaseHandler): ) except: logger.exception( - "Failed to parse state event %s", + "Failed to handle state event %s", e.event_id, ) @@ -713,7 +713,7 @@ class FederationHandler(BaseHandler): event.event_id, e_id, known_ids, ) # FIXME: How does raising AuthError work with federation? - raise AuthError(403, "Auth events are stale") + raise AuthError(403, "Cannot find auth event") context.auth_events[(e.type, e.state_key)] = e From 96779d2490f64507148ae07a3f4e88f048185565 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 18:57:36 +0000 Subject: [PATCH 05/91] Fix bug where we did not send the full auth chain to people that joined over federation --- synapse/handlers/federation.py | 7 +++++-- synapse/storage/event_federation.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 925eb5376..692c2d8a7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -285,7 +285,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_event_auth(self, event_id): - auth = yield self.store.get_auth_chain(event_id) + auth = yield self.store.get_auth_chain([event_id]) for event in auth: event.signatures.update( @@ -494,7 +494,10 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu) - auth_chain = yield self.store.get_auth_chain(event.event_id) + state_ids = [e.event_id for e in event.state_events.values()] + auth_chain = yield self.store.get_auth_chain(set( + [event.event_id] + state_ids + )) defer.returnValue({ "state": event.state_events.values(), diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 6c559f8f6..0ff9a23ee 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -32,15 +32,15 @@ class EventFederationStore(SQLBaseStore): and backfilling from another server respectively. """ - def get_auth_chain(self, event_id): + def get_auth_chain(self, event_ids): return self.runInteraction( "get_auth_chain", self._get_auth_chain_txn, - event_id + event_ids ) - def _get_auth_chain_txn(self, txn, event_id): - results = self._get_auth_chain_ids_txn(txn, event_id) + def _get_auth_chain_txn(self, txn, event_ids): + results = self._get_auth_chain_ids_txn(txn, event_ids) sql = "SELECT * FROM events WHERE event_id = ?" rows = [] @@ -50,21 +50,21 @@ class EventFederationStore(SQLBaseStore): return self._parse_events_txn(txn, rows) - def get_auth_chain_ids(self, event_id): + def get_auth_chain_ids(self, event_ids): return self.runInteraction( "get_auth_chain_ids", self._get_auth_chain_ids_txn, - event_id + event_ids ) - def _get_auth_chain_ids_txn(self, txn, event_id): + def _get_auth_chain_ids_txn(self, txn, event_ids): results = set() base_sql = ( "SELECT auth_id FROM event_auth WHERE %s" ) - front = set([event_id]) + front = set(event_ids) while front: sql = base_sql % ( " OR ".join(["event_id=?"] * len(front)), From 3defd5b3ee39f0e528632bbab431e2f8bac5c1fd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 19:07:20 +0000 Subject: [PATCH 06/91] Add FIXME --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 692c2d8a7..f52eb1205 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -179,6 +179,7 @@ class FederationHandler(BaseHandler): context=event.room_id, event_id=event.event_id, ) + # FIXME: Get auth chain for these state events current_state = state From 21cab3a7ec0bd9052df8ab1d70e66c9d0af82055 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 19:16:15 +0000 Subject: [PATCH 07/91] Fix where we pulled in event.state_events from hotfixes branch --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cd9e655f9..b76dcd98e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -529,7 +529,7 @@ class FederationHandler(BaseHandler): yield self.replication_layer.send_pdu(new_pdu, destinations) - state_ids = [e.event_id for e in event.state_events.values()] + state_ids = [e.event_id for e in context.current_state.values()] auth_chain = yield self.store.get_auth_chain(set( [event.event_id] + state_ids )) From dec5b62339705401fd280b423fbc8cec5add0ca8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 19:16:41 +0000 Subject: [PATCH 08/91] Use _get_events_txn instead of _parse_events_txn --- synapse/storage/event_federation.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7a6009c9e..fb2eb2171 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -42,13 +42,7 @@ class EventFederationStore(SQLBaseStore): def _get_auth_chain_txn(self, txn, event_ids): results = self._get_auth_chain_ids_txn(txn, event_ids) - sql = "SELECT * FROM events WHERE event_id = ?" - rows = [] - for ev_id in results: - c = txn.execute(sql, (ev_id,)) - rows.extend(self.cursor_to_dict(c)) - - return self._parse_events_txn(txn, rows) + return self._get_events_txn(txn, results) def get_auth_chain_ids(self, event_ids): return self.runInteraction( From f3788e3c7881de25c7d699bb9940b3cbd4dc3682 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Dec 2014 23:37:08 +0000 Subject: [PATCH 09/91] Test some ideas that might help performance a bit --- synapse/handlers/federation.py | 4 ++-- synapse/storage/_base.py | 34 ++++++++++++++++++++++++++++------ synapse/storage/roommember.py | 8 +------- synapse/storage/state.py | 9 +-------- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b76dcd98e..2f6036145 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -91,7 +91,7 @@ class FederationHandler(BaseHandler): yield run_on_reactor() - yield self.replication_layer.send_pdu(event, destinations) + self.replication_layer.send_pdu(event, destinations) @log_function @defer.inlineCallbacks @@ -527,7 +527,7 @@ class FederationHandler(BaseHandler): event.signatures, ) - yield self.replication_layer.send_pdu(new_pdu, destinations) + self.replication_layer.send_pdu(new_pdu, destinations) state_ids = [e.event_id for e in context.current_state.values()] auth_chain = yield self.store.get_auth_chain(set( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e0d97f440..a6e2e0e2e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -440,14 +440,29 @@ class SQLBaseStore(object): ) def _get_events_txn(self, txn, event_ids): - events = [] - for e_id in event_ids: - ev = self._get_event_txn(txn, e_id) + if not event_ids: + return [] - if ev: - events.append(ev) + if len(event_ids) > 50: + events = [] + n = 50 + for e_ids in [event_ids[i:i + n] for i in range(0, len(event_ids), n)]: + events.extend(self._get_events_txn(txn, e_ids)) + return events - return events + where_clause = " OR ".join(["e.event_id = ?" for _ in event_ids]) + + sql = ( + "SELECT internal_metadata, json, r.event_id FROM event_json as e " + "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "WHERE %s" + ) % (where_clause,) + + txn.execute(sql, event_ids) + + res = txn.fetchall() + + return [self._get_event_from_row_txn(txn, *r) for r in res] def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=True): @@ -467,6 +482,13 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + + def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, + check_redacted=True, get_prev_content=True): d = json.loads(js) internal_metadata = json.loads(internal_metadata) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 05b275663..4e416c50b 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -183,20 +183,14 @@ class RoomMemberStore(SQLBaseStore): ) def _get_members_query_txn(self, txn, where_clause, where_values): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "SELECT e.* FROM events as e " "INNER JOIN room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " "WHERE %(where)s " ) % { - "redacted": del_sql, "where": where_clause, } diff --git a/synapse/storage/state.py b/synapse/storage/state.py index afe3e5ede..ab8090971 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -62,14 +62,7 @@ class StateStore(SQLBaseStore): keyvalues={"state_group": group}, retcol="event_id", ) - state = [] - for state_id in state_ids: - s = self._get_events_txn( - txn, - [state_id], - ) - if s: - state.extend(s) + state = self._get_events_txn(txn, state_ids) res[group] = state From dea5d4b03b4236f20926f31ff8e865a9a8c48e90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 18 Dec 2014 11:29:46 +0000 Subject: [PATCH 10/91] Don't yield on sending the event accross federation. --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b76dcd98e..2f6036145 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -91,7 +91,7 @@ class FederationHandler(BaseHandler): yield run_on_reactor() - yield self.replication_layer.send_pdu(event, destinations) + self.replication_layer.send_pdu(event, destinations) @log_function @defer.inlineCallbacks @@ -527,7 +527,7 @@ class FederationHandler(BaseHandler): event.signatures, ) - yield self.replication_layer.send_pdu(new_pdu, destinations) + self.replication_layer.send_pdu(new_pdu, destinations) state_ids = [e.event_id for e in context.current_state.values()] auth_chain = yield self.store.get_auth_chain(set( From dbe77ec79a450f4e7b69d6de247864aad8928ebf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 18 Dec 2014 17:47:00 +0000 Subject: [PATCH 11/91] Replace distributor deferred list, with a simple for loop until I understand why the former breaks and the latter doesn't --- synapse/util/distributor.py | 12 ++++++------ tests/test_distributor.py | 27 +++++++++++++++++++-------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 701ccdb78..6e69296d6 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -115,10 +115,10 @@ class Signal(object): failure.value, failure.getTracebackObject())) if not self.suppress_failures: - raise failure + failure.raiseException() deferreds.append(d.addErrback(eb)) - - result = yield defer.DeferredList( - deferreds, fireOnOneErrback=not self.suppress_failures - ) - defer.returnValue(result) + results = [] + for deferred in deferreds: + result = yield deferred + results.append(results) + defer.returnValue(results) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 39c5b8dff..6a0095d85 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from tests import unittest +from . import unittest from twisted.internet import defer from mock import Mock, patch from synapse.util.distributor import Distributor +from synapse.util.async import run_on_reactor class DistributorTestCase(unittest.TestCase): @@ -26,6 +27,7 @@ class DistributorTestCase(unittest.TestCase): def setUp(self): self.dist = Distributor() + @defer.inlineCallbacks def test_signal_dispatch(self): self.dist.declare("alert") @@ -33,10 +35,11 @@ class DistributorTestCase(unittest.TestCase): self.dist.observe("alert", observer) d = self.dist.fire("alert", 1, 2, 3) - + yield d self.assertTrue(d.called) observer.assert_called_with(1, 2, 3) + @defer.inlineCallbacks def test_signal_dispatch_deferred(self): self.dist.declare("whine") @@ -50,8 +53,10 @@ class DistributorTestCase(unittest.TestCase): self.assertFalse(d_outer.called) d_inner.callback(None) + yield d_outer self.assertTrue(d_outer.called) + @defer.inlineCallbacks def test_signal_catch(self): self.dist.declare("alarm") @@ -65,6 +70,7 @@ class DistributorTestCase(unittest.TestCase): spec=["warning"] ) as mock_logger: d = self.dist.fire("alarm", "Go") + yield d self.assertTrue(d.called) observers[0].assert_called_once("Go") @@ -81,23 +87,28 @@ class DistributorTestCase(unittest.TestCase): self.dist.declare("whail") - observer = Mock() - observer.return_value = defer.fail( - Exception("Oopsie") - ) + class MyException(Exception): + pass + + @defer.inlineCallbacks + def observer(): + yield run_on_reactor() + raise MyException("Oopsie") self.dist.observe("whail", observer) d = self.dist.fire("whail") - yield self.assertFailure(d, Exception) + yield self.assertFailure(d, MyException) + self.dist.suppress_failures = True + @defer.inlineCallbacks def test_signal_prereg(self): observer = Mock() self.dist.observe("flare", observer) self.dist.declare("flare") - self.dist.fire("flare", 4, 5) + yield self.dist.fire("flare", 4, 5) observer.assert_called_with(4, 5) From 041ac476a53f7adaa436309ccbb85f269bbb47dd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 18 Dec 2014 18:47:13 +0000 Subject: [PATCH 12/91] Supply auth_chain along with current state in '/state/', fetch auth events from a remote server if we are missing some of them --- synapse/federation/replication.py | 29 ++++++++++----- synapse/handlers/federation.py | 55 +++++++++++++++++------------ synapse/util/distributor.py | 2 +- tests/federation/test_federation.py | 2 ++ 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index ec9b6e246..8abf67b1b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -256,31 +256,35 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function - def get_state_for_context(self, destination, context, event_id=None): + def get_state_for_context(self, destination, context, event_id): """Requests all of the `current` state PDUs for a given context from a remote home server. Args: destination (str): The remote homeserver to query for the state. context (str): The context we're interested in. + event_id (str): The id of the event we want the state at. Returns: Deferred: Results in a list of PDUs. """ - transaction_data = yield self.transport_layer.get_context_state( + result = yield self.transport_layer.get_context_state( destination, context, event_id=event_id, ) - transaction = Transaction(**transaction_data) pdus = [ - self.event_from_pdu_json(p, outlier=True) - for p in transaction.pdus + self.event_from_pdu_json(p, outlier=True) for p in result["pdus"] ] - defer.returnValue(pdus) + auth_chain = [ + self.event_from_pdu_json(p, outlier=True) + for p in result.get("auth_chain", []) + ] + + defer.returnValue((pdus, auth_chain)) @defer.inlineCallbacks @log_function @@ -383,10 +387,16 @@ class ReplicationLayer(object): context, event_id, ) + auth_chain = yield self.store.get_auth_chain( + [pdu.event_id for pdu in pdus] + ) else: raise NotImplementedError("Specify an event") - defer.returnValue((200, self._transaction_from_pdus(pdus).get_dict())) + defer.returnValue((200, { + "pdus": [pdu.get_pdu_json() for pdu in pdus], + "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain], + })) @defer.inlineCallbacks @log_function @@ -573,6 +583,8 @@ class ReplicationLayer(object): state = None + auth_chain = [] + # We need to make sure we have all the auth events. # for e_id, _ in pdu.auth_events: # exists = yield self._get_persisted_pdu( @@ -645,7 +657,7 @@ class ReplicationLayer(object): "_handle_new_pdu getting state for %s", pdu.room_id ) - state = yield self.get_state_for_context( + state, auth_chain = yield self.get_state_for_context( origin, pdu.room_id, pdu.event_id, ) @@ -655,6 +667,7 @@ class ReplicationLayer(object): pdu, backfilled=backfilled, state=state, + auth_chain=auth_chain, ) else: ret = None diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2f6036145..e23c5c219 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -95,7 +95,8 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def on_receive_pdu(self, origin, pdu, backfilled, state=None): + def on_receive_pdu(self, origin, pdu, backfilled, state=None, + auth_chain=None): """ Called by the ReplicationLayer when we have a new pdu. We need to do auth checks and put it through the StateHandler. """ @@ -150,35 +151,35 @@ class FederationHandler(BaseHandler): if not is_in_room and not event.internal_metadata.outlier: logger.debug("Got event for room we're not in.") - replication_layer = self.replication_layer - auth_chain = yield replication_layer.get_event_auth( - origin, - context=event.room_id, - event_id=event.event_id, - ) + replication = self.replication_layer + + if not state: + state, auth_chain = yield replication.get_state_for_context( + origin, context=event.room_id, event_id=event.event_id, + ) + + if not auth_chain: + auth_chain = yield replication.get_event_auth( + origin, + context=event.room_id, + event_id=event.event_id, + ) for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_missing=False) + yield self._handle_new_event(e, fetch_auth_from=origin) except: logger.exception( "Failed to handle auth event %s", e.event_id, ) - if not state: - state = yield replication_layer.get_state_for_context( - origin, - context=event.room_id, - event_id=event.event_id, - ) - # FIXME: Get auth chain for these state events - current_state = state if state: for e in state: + logging.info("A :) %r", e) e.internal_metadata.outlier = True try: yield self._handle_new_event(e) @@ -392,7 +393,7 @@ class FederationHandler(BaseHandler): for e in auth_chain: e.internal_metadata.outlier = True try: - yield self._handle_new_event(e, fetch_missing=False) + yield self._handle_new_event(e) except: logger.exception( "Failed to handle auth event %s", @@ -404,8 +405,7 @@ class FederationHandler(BaseHandler): e.internal_metadata.outlier = True try: yield self._handle_new_event( - e, - fetch_missing=True + e, fetch_auth_from=target_host ) except: logger.exception( @@ -682,7 +682,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _handle_new_event(self, event, state=None, backfilled=False, - current_state=None, fetch_missing=True): + current_state=None, fetch_auth_from=None): logger.debug( "_handle_new_event: Before annotate: %s, sigs: %s", @@ -703,11 +703,20 @@ class FederationHandler(BaseHandler): known_ids = set( [s.event_id for s in context.auth_events.values()] ) + for e_id, _ in event.auth_events: if e_id not in known_ids: - e = yield self.store.get_event( - e_id, allow_none=True, - ) + e = yield self.store.get_event(e_id, allow_none=True) + + if not e and fetch_auth_from is not None: + # Grab the auth_chain over federation if we are missing + # auth events. + auth_chain = yield self.replication_layer.get_event_auth( + fetch_auth_from, event.event_id, event.room_id + ) + for auth_event in auth_chain: + yield self._handle_new_event(auth_event) + e = yield self.store.get_event(e_id, allow_none=True) if not e: # TODO: Do some conflict res to make sure that we're diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 6e69296d6..6925ac96b 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -120,5 +120,5 @@ class Signal(object): results = [] for deferred in deferreds: result = yield deferred - results.append(results) + results.append(result) defer.returnValue(results) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 79ac1ce10..3e484cd30 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -52,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "get_received_txn_response", "set_received_txn_response", "get_destination_retry_timings", + "get_auth_chain", ]) self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) @@ -59,6 +60,7 @@ class FederationTestCase(unittest.TestCase): self.mock_persistence.get_destination_retry_timings.return_value = ( defer.succeed(DestinationsTable.EntryType("", 0, 0)) ) + self.mock_persistence.get_auth_chain.return_value = [] self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() From 5739e6c60684556cbdf1a792514a74de36b480b7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 11:43:46 +0000 Subject: [PATCH 13/91] s/user_id/sender/ --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8567d7409..4b8385c95 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -147,7 +147,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Member, "state_key": invitee, "room_id": room_id, - "user_id": user_id, + "sender": user_id, "content": {"membership": Membership.INVITE}, }) From 390e48a8b03784d7cdb49d792c7798b194471ed0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 12:05:26 +0000 Subject: [PATCH 14/91] SYN-203: Handle requests for thunbnails for images that are small --- synapse/media/v1/thumbnail_resource.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index e19620d45..5ddcf54b9 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -165,18 +165,27 @@ class ThumbnailResource(BaseMediaResource): aspect_quality, size_quality, type_quality, length_quality, info )) - return min(info_list)[-1] + if info_list: + return min(info_list)[-1] else: info_list = [] + info_list2 = [] for info in thumbnail_infos: t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] t_method = info["thumbnail_method"] + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] if t_method == "scale" and (t_w >= d_w or t_h >= d_h): - size_quality = abs((d_w - t_w) * (d_h - t_h)) - type_quality = desired_type != info["thumbnail_type"] - length_quality = info["thumbnail_length"] info_list.append(( size_quality, type_quality, length_quality, info )) - return min(info_list)[-1] + elif t_method == "scale": + info_list2.append(( + size_quality, type_quality, length_quality, info + )) + if info_list: + return min(info_list)[-1] + else: + return min(info_list2)[-1] From 5dbe820e9ac94247148eb57ce1c78f6a0c516004 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 12:16:26 +0000 Subject: [PATCH 15/91] Remove unneeded federation keys from events --- synapse/events/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 94f3f15f5..4849d3ce4 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -126,5 +126,8 @@ def serialize_event(hs, e): del d["prev_events"] del d["hashes"] del d["signatures"] + d.pop("depth", None) + d.pop("unsigned", None) + d.pop("origin", None) return d From 1e7f83b91dfdc68a9233dca5870d2511ec891bf0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 12:31:46 +0000 Subject: [PATCH 16/91] Set display name when joining via alias --- synapse/handlers/room.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4b8385c95..9644cd3d3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -390,6 +390,11 @@ class RoomMemberHandler(BaseHandler): host = hosts[0] + # If event doesn't include a display name, add one. + yield self.distributor.fire( + "collect_presencelike_data", joinee, content + ) + content.update({"membership": Membership.JOIN}) builder = self.event_builder_factory.new({ "type": EventTypes.Member, From 1e4a56c3a9589bb3d88e5ee07eb4dd1aafde3893 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 13:39:24 +0000 Subject: [PATCH 17/91] Bump web sdk version to 0.6.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index eb410aa6d..043cd044a 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.5.3b", + "matrix_angular_sdk==0.6.0", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -47,7 +47,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.0/#egg=matrix_angular_sdk-0.6.0", ], setup_requires=[ "setuptools_trial", From a999f0dec3b9ec12f8fe605c6d08d226c4d87ae8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 14:18:27 +0000 Subject: [PATCH 18/91] Don't ratelimit room create events --- synapse/handlers/directory.py | 2 +- synapse/handlers/message.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 66d3b533d..a907a66e1 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -155,4 +155,4 @@ class DirectoryHandler(BaseHandler): "room_id": room_id, "sender": user_id, "content": {"aliases": aliases}, - }) + }, ratelimit=False) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 49c0e9811..01a718354 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -106,7 +106,7 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def create_and_send_event(self, event_dict): + def create_and_send_event(self, event_dict, ratelimit=True): """ Given a dict from a client, create and handle a new event. Creates an FrozenEvent object, filling out auth_events, prev_events, @@ -123,7 +123,8 @@ class MessageHandler(BaseHandler): self.validator.validate_new(builder) - self.ratelimit(builder.user_id) + if ratelimit: + self.ratelimit(builder.user_id) # TODO(paul): Why does 'event' not have a 'user' object? user = self.hs.parse_userid(builder.user_id) assert self.hs.is_mine(user), "User must be our own: %s" % (user,) From f70e622d59e7b97c539ee03ffc02315b4d626b00 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 14:30:57 +0000 Subject: [PATCH 19/91] bump_presence_active_time when sending a message event --- synapse/handlers/message.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 01a718354..854b2c73c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -19,6 +19,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.events.validator import EventValidator +from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -153,6 +154,11 @@ class MessageHandler(BaseHandler): context=context, ) + if event.type == EventTypes.Message: + presence = self.hs.get_handlers().presence_handler + with PreserveLoggingContext(): + presence.bump_presence_active_time(user) + defer.returnValue(event) @defer.inlineCallbacks From 9c71d945d6072323df6d1e183efc4b6f7ad35237 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 15:16:48 +0000 Subject: [PATCH 20/91] Look for name, topic in the event content rather than the event itself when persisting room name and topic events --- synapse/storage/room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 2378d6594..c20abbfe4 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -135,26 +135,26 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) def _store_room_topic_txn(self, txn, event): - if hasattr(event, "topic"): + if hasattr(event, "content") and "topic" in event.content: self._simple_insert_txn( txn, "topics", { "event_id": event.event_id, "room_id": event.room_id, - "topic": event.topic, + "topic": event.content["topic"], } ) def _store_room_name_txn(self, txn, event): - if hasattr(event, "name"): + if hasattr(event, "content") and "name" in event.content: self._simple_insert_txn( txn, "room_names", { "event_id": event.event_id, "room_id": event.room_id, - "name": event.name, + "name": event.content["name"], } ) From efd27ff01b201c2e9f52cc90daccbda72709482e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 19 Dec 2014 15:31:27 +0000 Subject: [PATCH 21/91] Set a state_key for the topic and room name, otherwise they won't be treated as room state --- synapse/handlers/room.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9644cd3d3..deefc3c11 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -130,6 +130,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Name, "room_id": room_id, "sender": user_id, + "state_key": "", "content": {"name": name}, }) @@ -139,6 +140,7 @@ class RoomCreationHandler(BaseHandler): "type": EventTypes.Topic, "room_id": room_id, "sender": user_id, + "state_key": "", "content": {"topic": topic}, }) From 67a406a7540f21aae2fde325c66422cffa27fbfa Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 19 Dec 2014 17:36:33 +0000 Subject: [PATCH 22/91] Rate limit display names and avatar urls per request rather than per event. --- synapse/handlers/profile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 33a2c167e..3f11e2dcf 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -194,6 +194,8 @@ class ProfileHandler(BaseHandler): if not self.hs.is_mine(user): return + self.ratelimit(user.to_string()) + joins = yield self.store.get_rooms_for_user_where_membership_is( user.to_string(), [Membership.JOIN], @@ -214,5 +216,5 @@ class ProfileHandler(BaseHandler): "room_id": j.room_id, "state_key": j.state_key, "content": content, - "sender": j.state_key, - }) + "sender": j.state_key + }, ratelimit=False) From 2a5b53bc4a2c2ece46d40921707880d28164b876 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 19 Dec 2014 17:39:29 +0000 Subject: [PATCH 23/91] more changelogs --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 23bdac6a8..813ad364e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,12 @@ Changes in synapse 0.6.0 (2014-12-16) ===================================== * Add new API for media upload and download that supports thumbnailing. + * Replicate media uploads over multiple homeservers so media is always served + to clients from their local homeserver. This obsoletes the + --content-addr parameter and confusion over accessing content directly + from remote homeservers. + * Implement exponential backoff when retrying federation requests when + sending to remote homeservers which are offline. * Implement typing notifications. * Fix bugs where we sent events with invalid signatures due to bugs where we incorrectly persisted events. From 4640239d3480bcc5c37454d45d12a821a405c541 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 19 Dec 2014 17:49:39 +0000 Subject: [PATCH 24/91] Mock ratelimiter to make tests pass. --- tests/handlers/test_presencelike.py | 11 ++++++++--- tests/handlers/test_profile.py | 9 ++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/handlers/test_presencelike.py b/tests/handlers/test_presencelike.py index 532ecf0f2..0584e4c8b 100644 --- a/tests/handlers/test_presencelike.py +++ b/tests/handlers/test_presencelike.py @@ -19,7 +19,7 @@ presence and profiles; namely, the displayname and avatar_url.""" from tests import unittest from twisted.internet import defer -from mock import Mock, call, ANY +from mock import Mock, call, ANY, NonCallableMock from ..utils import MockClock, MockKey @@ -75,8 +75,13 @@ class PresenceProfilelikeDataTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, replication_layer=MockReplication(), - config=self.mock_config, - ) + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) hs.handlers = PresenceAndProfileHandlers(hs) self.datastore = hs.get_datastore() diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 1660e7e92..25b172aa5 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer -from mock import Mock +from mock import Mock, NonCallableMock from synapse.api.errors import AuthError from synapse.server import HomeServer @@ -59,7 +59,14 @@ class ProfileTestCase(unittest.TestCase): resource_for_federation=Mock(), replication_layer=self.mock_federation, config=self.mock_config, + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]) ) + + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + hs.handlers = ProfileHandlers(hs) self.store = hs.get_datastore() From 24b5d0185357a308c066bae45306a9c78625dba7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 22 Dec 2014 10:16:02 +0000 Subject: [PATCH 25/91] Include version in User-Agent and Server headers --- synapse/http/agent_name.py | 18 ++++++++++++++++++ synapse/http/client.py | 10 ++++++++-- synapse/http/matrixfederationclient.py | 4 +++- synapse/http/server.py | 9 ++++++--- 4 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 synapse/http/agent_name.py diff --git a/synapse/http/agent_name.py b/synapse/http/agent_name.py new file mode 100644 index 000000000..c98024b6a --- /dev/null +++ b/synapse/http/agent_name.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse import __version__ + +AGENT_NAME = ("Synapse/%s" % (__version__,)).encode("ascii") diff --git a/synapse/http/client.py b/synapse/http/client.py index 048a42890..11d6d9cb2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,6 +14,7 @@ # limitations under the License. +from synapse.http.agent_name import AGENT_NAME from twisted.internet import defer, reactor from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError @@ -51,7 +52,8 @@ class SimpleHttpClient(object): "POST", uri.encode("ascii"), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -86,6 +88,9 @@ class SimpleHttpClient(object): response = yield self.agent.request( "GET", uri.encode("ascii"), + headers=Headers({ + b"User-Agent": AGENT_NAME, + }) ) body = yield readBody(response) @@ -108,7 +113,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): url.encode("ascii"), bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ - "Content-Type": ["application/x-www-form-urlencoded"] + b"Content-Type": [b"application/x-www-form-urlencoded"], + b"User-Agent": AGENT_NAME, }) ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8f4db59c7..fc371155a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -20,6 +20,7 @@ from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers from twisted.web._newclient import ResponseDone +from synapse.http.agent_name import AGENT_NAME from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep from synapse.util.logcontext import PreserveLoggingContext @@ -71,6 +72,7 @@ class MatrixFederationHttpClient(object): requests. """ + def __init__(self, hs): self.hs = hs self.signing_key = hs.config.signing_key[0] @@ -83,7 +85,7 @@ class MatrixFederationHttpClient(object): query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ - headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"User-Agent"] = [AGENT_NAME] headers_dict[b"Host"] = [destination] url_bytes = urlparse.urlunparse( diff --git a/synapse/http/server.py b/synapse/http/server.py index f33859cf7..5765dffe3 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,14 +14,16 @@ # limitations under the License. -from syutil.jsonutil import ( - encode_canonical_json, encode_pretty_printed_json -) +from synapse.http.agent_name import AGENT_NAME from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) from synapse.util.logcontext import LoggingContext +from syutil.jsonutil import ( + encode_canonical_json, encode_pretty_printed_json +) + from twisted.internet import defer, reactor from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET @@ -230,6 +232,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") + request.setHeader(b"Server", AGENT_NAME) if send_cors: request.setHeader("Access-Control-Allow-Origin", "*") From d90e586c858dbd0fffafa41c4fe9efca3559359f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 24 Dec 2014 16:56:20 +0000 Subject: [PATCH 26/91] spell out that upgrading is just installing over the top --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index f5d2b0af3..87e0d7c63 100644 --- a/README.rst +++ b/README.rst @@ -239,6 +239,11 @@ Upgrading an existing homeserver IMPORTANT: Before upgrading an existing homeserver to a new version, please refer to UPGRADE.rst for any additional instructions. +Otherwise, simply re-install the new codebase over the current one - e.g. +by ``pip install --user --process-dependency-links +https://github.com/matrix-org/synapse/tarball/master`` +if using pip, or by ``git pull`` if running off a git working copy. + Setting up Federation ===================== From 1eb319806b9186d6ddfff3c4516799faa5efbd20 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 24 Dec 2014 16:56:32 +0000 Subject: [PATCH 27/91] clarify these instructions a media-repo specific --- docs/media_repository.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/media_repository.rst b/docs/media_repository.rst index e4a697404..1037b5be6 100644 --- a/docs/media_repository.rst +++ b/docs/media_repository.rst @@ -1,6 +1,8 @@ -Media Repository +Media Repository ================ +*Synapse implementation-specific details for the media repository* + The media repository is where attachments and avatar photos are stored. It stores attachment content and thumbnails for media uploaded by local users. It caches attachment content and thumbnails for media uploaded by remote users. From 407c299828c028d00d48f85f9bf3ba1105b94943 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 24 Dec 2014 17:50:42 +0000 Subject: [PATCH 28/91] improve error msg --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 140c99f18..c78b6f574 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -234,7 +234,7 @@ def setup(): except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" - "Have you followed any instructions in UPGRADES.rst?\n" + "Have you checked for version specific instructions in UPGRADES.rst?\n" ) sys.exit(1) From 0e93e01fcb9ca3d0769a52a19527e74724db58b2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 24 Dec 2014 19:45:28 +0000 Subject: [PATCH 29/91] spell out that VoIP needs TURN --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 87e0d7c63..92b94bcd7 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,9 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. +For reliable VoIP calls to be routed via this homeserver, you MUST configure +a TURN server. See docs/turn-howto.rst for details. + Troubleshooting Installation ---------------------------- From af61c295272d299414d609aa7f7e55c7b07189e8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 29 Dec 2014 13:54:05 +0000 Subject: [PATCH 30/91] Return the argument passed to the callback in a deferred callback, otherwise twisted will replace the deferred result with 'None' --- synapse/media/v1/base_resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 2f5440ab6..499be8cca 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -139,6 +139,7 @@ class BaseMediaResource(Resource): @download.addBoth def callback(media_info): del self.downloads[key] + return media_info return download @defer.inlineCallbacks From f0128f9600c59fbcb993bccbbbb32486009694d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 10:55:43 +0000 Subject: [PATCH 31/91] Add RoomMemberStore.get_users_in_room, so that we can get the list of joined users without having to retrieve the full events --- synapse/handlers/room.py | 13 +++++-------- synapse/storage/roommember.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index deefc3c11..5e5d95add 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -245,14 +245,12 @@ class RoomMemberHandler(BaseHandler): self.distributor.declare("user_left_room") @defer.inlineCallbacks - def get_room_members(self, room_id, membership=Membership.JOIN): + def get_room_members(self, room_id): hs = self.hs - memberships = yield self.store.get_room_members( - room_id=room_id, membership=membership - ) + users = yield self.store.get_users_in_room(room_id) - defer.returnValue([hs.parse_userid(m.user_id) for m in memberships]) + defer.returnValue([hs.parse_userid(u) for u in users]) @defer.inlineCallbacks def fetch_room_distributions_into(self, room_id, localusers=None, @@ -531,11 +529,10 @@ class RoomListHandler(BaseHandler): def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) for room in chunk: - joined_members = yield self.store.get_room_members( + joined_users = yield self.store.get_users_in_room( room_id=room["room_id"], - membership=Membership.JOIN ) - room["num_joined_members"] = len(joined_members) + room["num_joined_members"] = len(joined_users) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 4e416c50b..4921561fc 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -123,6 +123,19 @@ class RoomMemberStore(SQLBaseStore): else: return None + def get_users_in_room(self, room_id): + def f(txn): + sql = ( + "SELECT m.user_id FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE m.membership = ? AND m.room_id = ?" + ) + + txn.execute(sql, (Membership.JOIN, room_id)) + return [r[0] for r in txn.fetchall()] + return self.runInteraction("get_users_in_room", f) + def get_room_members(self, room_id, membership=None): """Retrieve the current room member list for a room. From d7e8ea67b374d3b006f7277de531302abc410e57 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 11:18:02 +0000 Subject: [PATCH 32/91] Reformat --- synapse/storage/_base.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6e2e0e2e..a30b0bc41 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -440,15 +440,16 @@ class SQLBaseStore(object): ) def _get_events_txn(self, txn, event_ids): - if not event_ids: - return [] + if not event_ids: + return [] - if len(event_ids) > 50: - events = [] - n = 50 - for e_ids in [event_ids[i:i + n] for i in range(0, len(event_ids), n)]: - events.extend(self._get_events_txn(txn, e_ids)) - return events + if len(event_ids) > 50: + events = [] + n = 50 + split = [event_ids[i:i + n] for i in range(0, len(event_ids), n)] + for e_ids in split: + events.extend(self._get_events_txn(txn, e_ids)) + return events where_clause = " OR ".join(["e.event_id = ?" for _ in event_ids]) @@ -482,13 +483,13 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res - return self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) + return self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=True): + check_redacted=True, get_prev_content=True): d = json.loads(js) internal_metadata = json.loads(internal_metadata) From 753126b8ccb56dc6539ce95758f3e87fe181064d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 11:18:12 +0000 Subject: [PATCH 33/91] Add some debug logging --- synapse/storage/state.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index ab8090971..9aeb0b406 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,10 @@ from ._base import SQLBaseStore +import logging + +logger = logging.getLogger(__name__) + class StateStore(SQLBaseStore): """ Keeps track of the state at a given event. @@ -54,6 +58,8 @@ class StateStore(SQLBaseStore): if group: groups.add(group) + logger.debug("Got groups: %s", groups) + res = {} for group in groups: state_ids = self._simple_select_onecol_txn( @@ -62,6 +68,12 @@ class StateStore(SQLBaseStore): keyvalues={"state_group": group}, retcol="event_id", ) + + logger.debug( + "Got %d events for group %s", + len(state_ids), group + ) + state = self._get_events_txn(txn, state_ids) res[group] = state From f4ea78e9e2578f0d0ba4345b3b45390e905438e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 11:24:18 +0000 Subject: [PATCH 34/91] More debug logging --- synapse/storage/_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a30b0bc41..de08c78ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -443,6 +443,8 @@ class SQLBaseStore(object): if not event_ids: return [] + logger.debug("_get_events_txn called with %d events", len(event_ids)) + if len(event_ids) > 50: events = [] n = 50 @@ -451,6 +453,8 @@ class SQLBaseStore(object): events.extend(self._get_events_txn(txn, e_ids)) return events + logger.debug("_get_events_txn Fetching %d events", len(event_ids)) + where_clause = " OR ".join(["e.event_id = ?" for _ in event_ids]) sql = ( From 3e26720e0574393dc8076b3d4099e16213ce2e6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 11:26:58 +0000 Subject: [PATCH 35/91] Temporarily turn off 'redacted_because' and 'prev_content' keys --- synapse/storage/_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index de08c78ed..d63655643 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -499,6 +499,8 @@ class SQLBaseStore(object): ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) + return ev + if check_redacted and redacted: ev = prune_event(ev) From 3c8c3bf3b781068637223d47d542d9d93a05a9b3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Jan 2015 11:32:36 +0000 Subject: [PATCH 36/91] SYN-229: Include Content-Length when downloading files --- synapse/media/v1/base_resource.py | 12 ++++++++++-- synapse/media/v1/download_resource.py | 12 +++++++++--- synapse/media/v1/thumbnail_resource.py | 6 ++++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 2f5440ab6..38e01970c 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -201,7 +201,8 @@ class BaseMediaResource(Resource): defer.returnValue(media_info) @defer.inlineCallbacks - def _respond_with_file(self, request, media_type, file_path): + def _respond_with_file(self, request, media_type, file_path, + file_size=None): logger.debug("Responding with %r", file_path) if os.path.isfile(file_path): @@ -215,13 +216,20 @@ class BaseMediaResource(Resource): request.setHeader( b"Cache-Control", b"public,max-age=86400,s-maxage=86400" ) + if file_size is None: + stat = os.stat(file_path) + file_size = stat.st_size + + request.setHeader( + b"Content-Length", b"%d" % (file_size,) + ) with open(file_path, "rb") as f: yield FileSender().beginFileTransfer(f, request) request.finish() else: - self._respond_404() + self._respond_404(request) def _get_thumbnail_requirements(self, media_type): if media_type == "image/jpeg": diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index f3a6804e0..8b5072ebb 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -46,23 +46,29 @@ class DownloadResource(BaseMediaResource): def _respond_local_file(self, request, media_id): media_info = yield self.store.get_local_media(media_id) if not media_info: - self._respond_404() + self._respond_404(request) return media_type = media_info["media_type"] + media_length = media_info["media_length"] file_path = self.filepaths.local_media_filepath(media_id) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] + media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) - yield self._respond_with_file(request, media_type, file_path) + yield self._respond_with_file( + request, media_type, file_path, media_length + ) diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index 5ddcf54b9..666764203 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -100,11 +100,12 @@ class ThumbnailResource(BaseMediaResource): t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] file_id = thumbnail_info["filesystem_id"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.remote_media_thumbnail( server_name, file_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, t_type, file_path) + yield self._respond_with_file(request, t_type, file_path, t_length) else: yield self._respond_default_thumbnail( request, media_info, width, height, method, m_type, @@ -139,11 +140,12 @@ class ThumbnailResource(BaseMediaResource): t_height = thumbnail_info["thumbnail_height"] t_type = thumbnail_info["thumbnail_type"] t_method = thumbnail_info["thumbnail_method"] + t_length = thumbnail_info["thumbnail_length"] file_path = self.filepaths.default_thumbnail( top_level_type, sub_type, t_width, t_height, t_type, t_method, ) - yield self.respond_with_file(request, t_type, file_path) + yield self.respond_with_file(request, t_type, file_path, t_length) def _select_thumbnail(self, desired_width, desired_height, desired_method, desired_type, thumbnail_infos): From 78edb47cc53e52504f2ceb8efa23ae1e50b66946 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 6 Jan 2015 11:43:04 +0000 Subject: [PATCH 37/91] SYN-208/SYN-228: Add runtime checks on startup to enforce that JPEG/PNG support is included when installing pillow. --- synapse/media/v1/__init__.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py index e69de29bb..2b1762dcd 100644 --- a/synapse/media/v1/__init__.py +++ b/synapse/media/v1/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +import PIL.Image + +# check for JPEG support. +try: + PIL.Image._getdecoder("rgb", "jpeg", None) +except IOError as e: + if str(e).startswith("decoder jpeg not available"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass + + +# check for PNG support. +try: + PIL.Image._getdecoder("rgb", "zip", None) +except IOError as e: + if str(e).startswith("decoder zip not available"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + ) +except Exception: + # any other exception is fine + pass \ No newline at end of file From 98933e3db6d43dcb3c8c21d0b65e2647bc3fb303 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 13:03:23 +0000 Subject: [PATCH 38/91] Only fetch prev_content when a client is streaming/paginating. Use transactions for event streams. --- synapse/storage/_base.py | 32 ++++++--- synapse/storage/stream.py | 142 ++++++++++++++++++-------------------- 2 files changed, 90 insertions(+), 84 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d63655643..9702ab4f4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -434,12 +434,15 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _get_events(self, event_ids): + def _get_events(self, event_ids, check_redacted=True, + get_prev_content=False): return self.runInteraction( - "_get_events", self._get_events_txn, event_ids + "_get_events", self._get_events_txn, event_ids, + check_redacted=check_redacted, get_prev_content=get_prev_content, ) - def _get_events_txn(self, txn, event_ids): + def _get_events_txn(self, txn, event_ids, check_redacted=True, + get_prev_content=False): if not event_ids: return [] @@ -450,7 +453,13 @@ class SQLBaseStore(object): n = 50 split = [event_ids[i:i + n] for i in range(0, len(event_ids), n)] for e_ids in split: - events.extend(self._get_events_txn(txn, e_ids)) + events.extend( + self._get_events_txn( + txn, e_ids, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + ) return events logger.debug("_get_events_txn Fetching %d events", len(event_ids)) @@ -467,10 +476,17 @@ class SQLBaseStore(object): res = txn.fetchall() - return [self._get_event_from_row_txn(txn, *r) for r in res] + return [ + self._get_event_from_row_txn( + txn, r[0], r[1], r[2], + check_redacted=check_redacted, + get_prev_content=get_prev_content, + ) + for r in res + ] def _get_event_txn(self, txn, event_id, check_redacted=True, - get_prev_content=True): + get_prev_content=False): sql = ( "SELECT internal_metadata, json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " @@ -493,14 +509,12 @@ class SQLBaseStore(object): ) def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=True): + check_redacted=True, get_prev_content=False): d = json.loads(js) internal_metadata = json.loads(internal_metadata) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) - return ev - if check_redacted and redacted: ev = prune_event(ev) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3405cb365..c51f48945 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -137,12 +137,12 @@ class StreamStore(SQLBaseStore): with_feedback=with_feedback, ) - @defer.inlineCallbacks @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, limit=0, with_feedback=False): # TODO (erikj): Handle compressed feedback + current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " @@ -157,11 +157,6 @@ class StreamStore(SQLBaseStore): "WHERE m.user_id = ? " ) - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - if limit: limit = max(limit, MAX_STREAM_SIZE) else: @@ -172,38 +167,42 @@ class StreamStore(SQLBaseStore): to_id = _parse_stream_token(to_key) if from_key == to_key: - defer.returnValue(([], to_key)) - return + return defer.succeed(([], to_key)) sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events AS e WHERE " + "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " ) % { - "redacted": del_sql, "current": current_room_membership_sql, "invites": membership_sql, "limit": limit } - rows = yield self._execute_and_decode( - sql, - user_id, user_id, from_id, to_id - ) + def f(txn): + txn.execute(sql, (user_id, user_id, from_id, to_id,)) - ret = yield self._parse_events(rows) + rows = self.cursor_to_dict(txn) - if rows: - key = "s%d" % max([r["stream_ordering"] for r in rows]) - else: - # Assume we didn't get anything because there was nothing to get. - key = to_key + ret = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) - defer.returnValue((ret, key)) + if rows: + key = "s%d" % max([r["stream_ordering"] for r in rows]) + else: + # Assume we didn't get anything because there was nothing to + # get. + key = to_key + + return ret, key + + return self.runInteraction("get_room_events_stream", f) - @defer.inlineCallbacks @log_function def paginate_room_events(self, room_id, from_key, to_key=None, direction='b', limit=-1, @@ -221,7 +220,9 @@ class StreamStore(SQLBaseStore): bounds = _get_token_bound(from_key, from_comp) if to_key: - bounds = "%s AND %s" % (bounds, _get_token_bound(to_key, to_comp)) + bounds = "%s AND %s" % ( + bounds, _get_token_bound(to_key, to_comp) + ) if int(limit) > 0: args.append(int(limit)) @@ -229,87 +230,78 @@ class StreamStore(SQLBaseStore): else: limit_str = "" - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events" + "SELECT * FROM events" " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { - "redacted": del_sql, "bounds": bounds, "order": order, "limit": limit_str } - rows = yield self._execute_and_decode( - sql, - *args - ) + def f(txn): + txn.execute(sql, args) - if rows: - topo = rows[-1]["topological_ordering"] - toke = rows[-1]["stream_ordering"] - if direction == 'b': - topo -= 1 - toke -= 1 - next_token = "t%s-%s" % (topo, toke) - else: - # TODO (erikj): We should work out what to do here instead. - next_token = to_key if to_key else from_key + rows = self.cursor_to_dict(txn) - events = yield self._parse_events(rows) + if rows: + topo = rows[-1]["topological_ordering"] + toke = rows[-1]["stream_ordering"] + if direction == 'b': + topo -= 1 + toke -= 1 + next_token = "t%s-%s" % (topo, toke) + else: + # TODO (erikj): We should work out what to do here instead. + next_token = to_key if to_key else from_key - defer.returnValue( - ( - events, - next_token + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True ) - ) - @defer.inlineCallbacks + return events, next_token, + + return self.runInteraction("paginate_room_events", f) + def get_recent_events_for_room(self, room_id, limit, end_token, with_feedback=False): # TODO (erikj): Handle compressed feedback - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = events.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT *, (%(redacted)s) AS redacted FROM events " + "SELECT * FROM events " "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " - ) % { - "redacted": del_sql, - } - - rows = yield self._execute_and_decode( - sql, - room_id, end_token, limit ) - rows.reverse() # As we selected with reverse ordering + def f(txn): + txn.execute(sql, (room_id, end_token, limit,)) - if rows: - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] - start_token = "t%s-%s" % (topo, toke) + rows = self.cursor_to_dict(txn) - token = (start_token, end_token) - else: - token = (end_token, end_token) + rows.reverse() # As we selected with reverse ordering - events = yield self._parse_events(rows) + if rows: + topo = rows[0]["topological_ordering"] + toke = rows[0]["stream_ordering"] + start_token = "t%s-%s" % (topo, toke) - ret = (events, token) + token = (start_token, end_token) + else: + token = (end_token, end_token) - defer.returnValue(ret) + events = self._get_events_txn( + txn, + [r["event_id"] for r in rows], + get_prev_content=True + ) + + return events, token + + return self.runInteraction("get_recent_events_for_room", f) def get_room_events_max_id(self): return self.runInteraction( From 773de09774399692cb43d56587ae77cbbee42d01 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Jan 2015 13:05:07 +0000 Subject: [PATCH 39/91] Set a content-length for JSON responses --- synapse/http/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/http/server.py b/synapse/http/server.py index 5765dffe3..de26afb48 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -233,6 +233,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") request.setHeader(b"Server", AGENT_NAME) + request.setHeader(b"Content-Length", b"%d" % len(json_bytes)) if send_cors: request.setHeader("Access-Control-Allow-Origin", "*") From 52d85190081044b9fbaf24869d652d3fe3c23e5d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 13:10:27 +0000 Subject: [PATCH 40/91] Don't do batching when getting events. --- synapse/storage/_base.py | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9702ab4f4..9687222e7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -446,43 +446,12 @@ class SQLBaseStore(object): if not event_ids: return [] - logger.debug("_get_events_txn called with %d events", len(event_ids)) - - if len(event_ids) > 50: - events = [] - n = 50 - split = [event_ids[i:i + n] for i in range(0, len(event_ids), n)] - for e_ids in split: - events.extend( - self._get_events_txn( - txn, e_ids, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - ) - ) - return events - - logger.debug("_get_events_txn Fetching %d events", len(event_ids)) - - where_clause = " OR ".join(["e.event_id = ?" for _ in event_ids]) - - sql = ( - "SELECT internal_metadata, json, r.event_id FROM event_json as e " - "LEFT JOIN redactions as r ON e.event_id = r.redacts " - "WHERE %s" - ) % (where_clause,) - - txn.execute(sql, event_ids) - - res = txn.fetchall() - return [ - self._get_event_from_row_txn( - txn, r[0], r[1], r[2], - check_redacted=check_redacted, - get_prev_content=get_prev_content, + self._get_event_txn( + txn, event_id, + check_redacted=check_redacted, get_prev_content=get_prev_content ) - for r in res + for event_id in event_ids ] def _get_event_txn(self, txn, event_id, check_redacted=True, From 12819d5082ac73adc309428770c9270ba378c6e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 13:12:30 +0000 Subject: [PATCH 41/91] Remove debug lines --- synapse/storage/state.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9aeb0b406..fd6f1e3b0 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -58,8 +58,6 @@ class StateStore(SQLBaseStore): if group: groups.add(group) - logger.debug("Got groups: %s", groups) - res = {} for group in groups: state_ids = self._simple_select_onecol_txn( @@ -69,11 +67,6 @@ class StateStore(SQLBaseStore): retcol="event_id", ) - logger.debug( - "Got %d events for group %s", - len(state_ids), group - ) - state = self._get_events_txn(txn, state_ids) res[group] = state From af1c7c7808d297711e4b76b862c41a5ec2ca3a9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 13:13:17 +0000 Subject: [PATCH 42/91] PEP8 --- synapse/storage/_base.py | 6 ++++-- synapse/storage/stream.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9687222e7..e799ac6c5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -449,7 +449,8 @@ class SQLBaseStore(object): return [ self._get_event_txn( txn, event_id, - check_redacted=check_redacted, get_prev_content=get_prev_content + check_redacted=check_redacted, + get_prev_content=get_prev_content ) for event_id in event_ids ] @@ -473,7 +474,8 @@ class SQLBaseStore(object): internal_metadata, js, redacted = res return self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, check_redacted=check_redacted, + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, get_prev_content=get_prev_content, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index c51f48945..bd3a411ea 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -142,7 +142,6 @@ class StreamStore(SQLBaseStore): limit=0, with_feedback=False): # TODO (erikj): Handle compressed feedback - current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " "INNER JOIN current_state_events as c ON m.event_id = c.event_id " From adb04b1e572d13b75541f4684aac3683e94d70b8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Jan 2015 13:21:39 +0000 Subject: [PATCH 43/91] Update copyright notices --- scripts/copyrighter.pl | 2 +- synapse/__init__.py | 2 +- synapse/api/__init__.py | 2 +- synapse/api/auth.py | 2 +- synapse/api/constants.py | 2 +- synapse/api/errors.py | 2 +- synapse/api/ratelimiting.py | 2 +- synapse/api/urls.py | 2 +- synapse/app/__init__.py | 2 +- synapse/app/homeserver.py | 2 +- synapse/app/synctl.py | 2 +- synapse/config/__init__.py | 2 +- synapse/config/_base.py | 2 +- synapse/config/captcha.py | 2 +- synapse/config/database.py | 2 +- synapse/config/email.py | 2 +- synapse/config/homeserver.py | 2 +- synapse/config/logger.py | 2 +- synapse/config/ratelimiting.py | 2 +- synapse/config/repository.py | 2 +- synapse/config/server.py | 2 +- synapse/config/tls.py | 2 +- synapse/config/voip.py | 2 +- synapse/crypto/__init__.py | 2 +- synapse/crypto/context_factory.py | 2 +- synapse/crypto/event_signing.py | 2 +- synapse/crypto/keyclient.py | 2 +- synapse/crypto/keyring.py | 2 +- synapse/events/__init__.py | 2 +- synapse/events/builder.py | 2 +- synapse/events/snapshot.py | 2 +- synapse/events/utils.py | 2 +- synapse/events/validator.py | 2 +- synapse/federation/__init__.py | 2 +- synapse/federation/persistence.py | 2 +- synapse/federation/replication.py | 2 +- synapse/federation/transport.py | 2 +- synapse/federation/units.py | 2 +- synapse/handlers/__init__.py | 2 +- synapse/handlers/_base.py | 2 +- synapse/handlers/admin.py | 2 +- synapse/handlers/directory.py | 2 +- synapse/handlers/events.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/handlers/login.py | 2 +- synapse/handlers/message.py | 2 +- synapse/handlers/presence.py | 2 +- synapse/handlers/profile.py | 2 +- synapse/handlers/register.py | 2 +- synapse/handlers/room.py | 2 +- synapse/handlers/typing.py | 2 +- synapse/http/__init__.py | 2 +- synapse/http/agent_name.py | 2 +- synapse/http/client.py | 2 +- synapse/http/endpoint.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/http/server.py | 4 ++-- synapse/http/server_key_resource.py | 2 +- synapse/media/v0/content_repository.py | 2 +- synapse/media/v1/__init__.py | 16 +++++++++++++++- synapse/media/v1/base_resource.py | 2 +- synapse/media/v1/download_resource.py | 2 +- synapse/media/v1/filepath.py | 2 +- synapse/media/v1/media_repository.py | 2 +- synapse/media/v1/thumbnail_resource.py | 2 +- synapse/media/v1/thumbnailer.py | 2 +- synapse/media/v1/upload_resource.py | 2 +- synapse/notifier.py | 2 +- synapse/rest/__init__.py | 2 +- synapse/rest/admin.py | 2 +- synapse/rest/base.py | 2 +- synapse/rest/directory.py | 2 +- synapse/rest/events.py | 2 +- synapse/rest/initial_sync.py | 2 +- synapse/rest/login.py | 2 +- synapse/rest/presence.py | 2 +- synapse/rest/profile.py | 2 +- synapse/rest/register.py | 2 +- synapse/rest/room.py | 2 +- synapse/rest/transactions.py | 2 +- synapse/rest/voip.py | 2 +- synapse/server.py | 2 +- synapse/state.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/directory.py | 2 +- synapse/storage/event_federation.py | 2 +- synapse/storage/feedback.py | 2 +- synapse/storage/keys.py | 2 +- synapse/storage/media_repository.py | 2 +- synapse/storage/presence.py | 2 +- synapse/storage/profile.py | 2 +- synapse/storage/registration.py | 2 +- synapse/storage/room.py | 2 +- synapse/storage/roommember.py | 2 +- synapse/storage/schema/delta/v2.sql | 2 +- synapse/storage/schema/delta/v3.sql | 2 +- synapse/storage/schema/delta/v4.sql | 14 ++++++++++++++ synapse/storage/schema/delta/v5.sql | 14 ++++++++++++++ synapse/storage/schema/delta/v6.sql | 2 +- synapse/storage/schema/delta/v8.sql | 2 +- synapse/storage/schema/delta/v9.sql | 2 +- synapse/storage/schema/event_edges.sql | 14 ++++++++++++++ synapse/storage/schema/event_signatures.sql | 2 +- synapse/storage/schema/im.sql | 2 +- synapse/storage/schema/keys.sql | 2 +- synapse/storage/schema/media_repository.sql | 2 +- synapse/storage/schema/presence.sql | 2 +- synapse/storage/schema/profiles.sql | 2 +- synapse/storage/schema/redactions.sql | 14 ++++++++++++++ synapse/storage/schema/room_aliases.sql | 2 +- synapse/storage/schema/state.sql | 2 +- synapse/storage/schema/transactions.sql | 2 +- synapse/storage/schema/users.sql | 2 +- synapse/storage/signatures.py | 2 +- synapse/storage/state.py | 2 +- synapse/storage/stream.py | 2 +- synapse/storage/transactions.py | 2 +- synapse/streams/__init__.py | 2 +- synapse/streams/config.py | 2 +- synapse/streams/events.py | 2 +- synapse/types.py | 2 +- synapse/util/__init__.py | 2 +- synapse/util/async.py | 2 +- synapse/util/distributor.py | 2 +- synapse/util/emailutils.py | 2 +- synapse/util/frozenutils.py | 2 +- synapse/util/jsonobject.py | 2 +- synapse/util/lockutils.py | 2 +- synapse/util/logcontext.py | 14 ++++++++++++++ synapse/util/logutils.py | 2 +- synapse/util/stringutils.py | 2 +- 132 files changed, 212 insertions(+), 128 deletions(-) diff --git a/scripts/copyrighter.pl b/scripts/copyrighter.pl index 7c03ef21f..a913d74c8 100755 --- a/scripts/copyrighter.pl +++ b/scripts/copyrighter.pl @@ -14,7 +14,7 @@ # limitations under the License. $copyright = < Date: Tue, 6 Jan 2015 14:04:43 +0000 Subject: [PATCH 44/91] Increase default maximum attachment size to 10M --- synapse/config/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 017374919..e1827f05e 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -37,7 +37,7 @@ class ContentRepositoryConfig(Config): super(ContentRepositoryConfig, cls).add_arguments(parser) db_group = parser.add_argument_group("content_repository") db_group.add_argument( - "--max-upload-size", default="1M" + "--max-upload-size", default="10M" ) db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") From 0529a7e2e9284f479f50160b48b4e4686076c06a Mon Sep 17 00:00:00 2001 From: Matrix Date: Tue, 6 Jan 2015 14:06:25 +0000 Subject: [PATCH 45/91] Add some logging for when we are sending transactions. --- synapse/federation/replication.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2aa1149d3..a4c29b484 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -730,6 +730,7 @@ class _TransactionQueue(object): destinations = set(destinations) destinations.discard(self.server_name) + destinations.discard("localhost") logger.debug("Sending to: %s", str(destinations)) @@ -814,6 +815,8 @@ class _TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) + logger.info("TX [%s] _attempt_new_transaction", destination) + if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -826,6 +829,9 @@ class _TransactionQueue(object): pending_edus = self.pending_edus_by_dest.pop(destination, []) pending_failures = self.pending_failures_by_dest.pop(destination, []) + if pending_pdus: + logger.info("TX [%s] len(pending_pdus_by_dest[dest]) = %d", destination, len(pending_pdus)) + if not pending_pdus and not pending_edus and not pending_failures: return From bc2ec808f431b41e8d39a155a9fd0b5177bccb45 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 6 Jan 2015 14:13:18 +0000 Subject: [PATCH 46/91] SYN-32 Use the ANTIALIAS resize method for thumbnailing images --- synapse/media/v1/thumbnailer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index 65468cf60..bc86efea8 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -48,7 +48,7 @@ class Thumbnailer(object): def scale(self, output_path, width, height, output_type): """Rescales the image to the given dimensions""" - scaled = self.image.resize((width, height), Image.BILINEAR) + scaled = self.image.resize((width, height), Image.ANTIALIAS) return self.save_image(scaled, output_type, output_path) def crop(self, output_path, width, height, output_type): @@ -65,7 +65,7 @@ class Thumbnailer(object): if width * self.height > height * self.width: scaled_height = (width * self.height) // self.width scaled_image = self.image.resize( - (width, scaled_height), Image.BILINEAR + (width, scaled_height), Image.ANTIALIAS ) crop_top = (scaled_height - height) // 2 crop_bottom = height + crop_top @@ -73,7 +73,7 @@ class Thumbnailer(object): else: scaled_width = (height * self.width) // self.height scaled_image = self.image.resize( - (scaled_width, height), Image.BILINEAR + (scaled_width, height), Image.ANTIALIAS ) crop_left = (scaled_width - width) // 2 crop_right = width + crop_left From 76ec154e95518fe91d430c76303feadf6c76dabc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 14:37:00 +0000 Subject: [PATCH 47/91] We don't need the full events for get_rooms_for_user_where_membership_is --- synapse/handlers/profile.py | 10 +++++----- synapse/storage/roommember.py | 36 ++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 3f11e2dcf..8d4d44150 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, AuthError, CodeMessageException -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.util.logcontext import PreserveLoggingContext from ._base import BaseHandler @@ -203,7 +203,7 @@ class ProfileHandler(BaseHandler): for j in joins: content = { - "membership": j.content["membership"], + "membership": Membership.JOIN, } yield self.distributor.fire( @@ -212,9 +212,9 @@ class ProfileHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event({ - "type": j.type, + "type": EventTypes.Member, "room_id": j.room_id, - "state_key": j.state_key, + "state_key": user.to_string(), "content": content, - "sender": j.state_key + "sender": user.to_string() }, ratelimit=False) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 4921561fc..c495fab91 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -15,6 +15,8 @@ from twisted.internet import defer +from collections import namedtuple + from ._base import SQLBaseStore from synapse.api.constants import Membership @@ -24,6 +26,12 @@ import logging logger = logging.getLogger(__name__) +RoomsForUser = namedtuple( + "RoomsForUser", + ("room_id", "sender", "membership") +) + + class RoomMemberStore(SQLBaseStore): def _store_room_member_txn(self, txn, event): @@ -163,19 +171,37 @@ class RoomMemberStore(SQLBaseStore): membership_list (list): A list of synapse.api.constants.Membership values which the user must be in. Returns: - A list of RoomMemberEvent objects + A list of dictionary objects, with room_id, membership and sender + defined. """ if not membership_list: return defer.succeed(None) - args = [user_id] - args.extend(membership_list) - where_clause = "user_id = ? AND (%s)" % ( " OR ".join(["membership = ?" for _ in membership_list]), ) - return self._get_members_query(where_clause, args) + args = [user_id] + args.extend(membership_list) + + def f(txn): + sql = ( + "SELECT m.room_id, m.sender, m.membership" + " FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %s" + ) % (where_clause,) + + txn.execute(sql, args) + return [ + RoomsForUser(**r) for r in self.cursor_to_dict(txn) + ] + + return self.runInteraction( + "get_rooms_for_user_where_membership_is", + f + ) def get_joined_hosts_for_room(self, room_id): return self._simple_select_onecol( From 96707ed7185fc5c1668c8a31b62c67bdf39ed777 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 14:44:27 +0000 Subject: [PATCH 48/91] Name 'user_rooms_intersect' transaction --- synapse/storage/roommember.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index c495fab91..11f8f7877 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -239,26 +239,28 @@ class RoomMemberStore(SQLBaseStore): results = self._parse_events_txn(txn, rows) return results - @defer.inlineCallbacks def user_rooms_intersect(self, user_id_list): """ Checks whether all the users whose IDs are given in a list share a room. """ - user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) - sql = ( - "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE m.membership = 'join' " - "AND (%(clause)s) " - # TODO(paul): We've got duplicate rows in the database somewhere - # so we have to DISTINCT m.user_id here - "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" - ) % {"clause": user_list_clause} + def interaction(txn): + user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list)) + sql = ( + "SELECT m.room_id FROM room_memberships as m " + "INNER JOIN current_state_events as c " + "ON m.event_id = c.event_id " + "WHERE m.membership = 'join' " + "AND (%(clause)s) " + # TODO(paul): We've got duplicate rows in the database somewhere + # so we have to DISTINCT m.user_id here + "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" + ) % {"clause": user_list_clause} - args = list(user_id_list) - args.append(len(user_id_list)) + args = list(user_id_list) + args.append(len(user_id_list)) - rows = yield self._execute(None, sql, *args) + txn.execute(sql, args) - defer.returnValue(len(rows) > 0) + return len(txn.fetchall()) > 0 + + return self.runInteraction("user_rooms_intersect", interaction) From 52b2c6c9c73d47a269756d8da57b4dcff54e0d21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 14:56:57 +0000 Subject: [PATCH 49/91] Don't include None's in _get_events_txn --- synapse/storage/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 728d1df8f..a0b7f943f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -446,7 +446,7 @@ class SQLBaseStore(object): if not event_ids: return [] - return [ + events = [ self._get_event_txn( txn, event_id, check_redacted=check_redacted, @@ -455,6 +455,8 @@ class SQLBaseStore(object): for event_id in event_ids ] + return [e for e in events if e] + def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False): sql = ( From 03a501456ca8815a7d6fd8ea84d9c2a1feba33cf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:22:28 +0000 Subject: [PATCH 50/91] Time how long calls to _get_destination_retry_timings take --- synapse/storage/transactions.py | 3 ++ synapse/util/logutils.py | 73 +++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e06ef3569..36ddf30d6 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -19,6 +19,8 @@ from collections import namedtuple from twisted.internet import defer +from synapse.util.logutils import time_function + import logging logger = logging.getLogger(__name__) @@ -228,6 +230,7 @@ class TransactionStore(SQLBaseStore): "get_destination_retry_timings", self._get_destination_retry_timings, destination) + @time_function def _get_destination_retry_timings(cls, txn, destination): query = DestinationsTable.select_statement("destination = ?") txn.execute(query, (destination,)) diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index 18ba405c4..c4dfb69c5 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -19,14 +19,37 @@ from functools import wraps import logging import inspect +import time + + +_TIME_FUNC_ID = 0 + + +def _log_debug_as_f(f, msg, msg_args): + name = f.__module__ + logger = logging.getLogger(name) + + if logger.isEnabledFor(logging.DEBUG): + lineno = f.func_code.co_firstlineno + pathname = f.func_code.co_filename + + record = logging.LogRecord( + name=name, + level=logging.DEBUG, + pathname=pathname, + lineno=lineno, + msg=msg, + args=msg_args, + exc_info=None + ) + + logger.handle(record) def log_function(f): """ Function decorator that logs every call to that function. """ func_name = f.__name__ - lineno = f.func_code.co_firstlineno - pathname = f.func_code.co_filename @wraps(f) def wrapped(*args, **kwargs): @@ -52,24 +75,50 @@ def log_function(f): "args": ", ".join(func_args) } - record = logging.LogRecord( - name=name, - level=level, - pathname=pathname, - lineno=lineno, - msg="Invoked '%(func_name)s' with args: %(args)s", - args=msg_args, - exc_info=None + _log_debug_as_f( + f, + "Invoked '%(func_name)s' with args: %(args)s", + msg_args ) - logger.handle(record) - return f(*args, **kwargs) wrapped.__name__ = func_name return wrapped +def time_function(f): + func_name = f.__name__ + + @wraps(f) + def wrapped(*args, **kwargs): + global _TIME_FUNC_ID + id = _TIME_FUNC_ID + _TIME_FUNC_ID += 1 + + start = time.clock() * 1000 + + try: + _log_debug_as_f( + f, + "[FUNC START] {%s-%d}", + (func_name, _TIME_FUNC_ID), + ) + + r = f(*args, **kwargs) + finally: + end = time.clock() * 1000 + _log_debug_as_f( + f, + "[FUNC END] {%s-%d} %f", + (func_name, _TIME_FUNC_ID, end-start,), + ) + + return r + + return wrapped + + def trace_function(f): func_name = f.__name__ linenum = f.func_code.co_firstlineno From 9bd07bed238337151ae79dc948f49cdf7141578c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:28:56 +0000 Subject: [PATCH 51/91] Actually time that function --- synapse/storage/transactions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 36ddf30d6..9d14f8930 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -200,6 +200,7 @@ class TransactionStore(SQLBaseStore): self._get_transactions_after, transaction_id, destination ) + @time_function def _get_transactions_after(cls, txn, transaction_id, destination): where = ( "destination = ? AND id > (select id FROM %s WHERE " From f6da237c353d598946a6c81260653203602800c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:40:38 +0000 Subject: [PATCH 52/91] Add index on transaction_id to sent_transcations --- synapse/storage/schema/transactions.sql | 1 + synapse/storage/transactions.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 86f530d82..2d30f99b0 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -42,6 +42,7 @@ CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destinatio CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( destination ); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9d14f8930..e06ef3569 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -19,8 +19,6 @@ from collections import namedtuple from twisted.internet import defer -from synapse.util.logutils import time_function - import logging logger = logging.getLogger(__name__) @@ -200,7 +198,6 @@ class TransactionStore(SQLBaseStore): self._get_transactions_after, transaction_id, destination ) - @time_function def _get_transactions_after(cls, txn, transaction_id, destination): where = ( "destination = ? AND id > (select id FROM %s WHERE " @@ -231,7 +228,6 @@ class TransactionStore(SQLBaseStore): "get_destination_retry_timings", self._get_destination_retry_timings, destination) - @time_function def _get_destination_retry_timings(cls, txn, destination): query = DestinationsTable.select_statement("destination = ?") txn.execute(query, (destination,)) From a01416cf217db5e442ff90715cbe1955b67b1efb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:42:18 +0000 Subject: [PATCH 53/91] Add delta and bump DB version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v11.sql | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v11.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8d482dece..4beb951b9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 10 +SCHEMA_VERSION = 11 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/v11.sql new file mode 100644 index 000000000..313592221 --- /dev/null +++ b/synapse/storage/schema/delta/v11.sql @@ -0,0 +1,16 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); \ No newline at end of file From 9e5545a6fac24e15a0493473548a669df85052b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:53:50 +0000 Subject: [PATCH 54/91] RoomsForUser now has sender instead of user_id --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 24f8c1bc8..7195de98b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -263,7 +263,7 @@ class MessageHandler(BaseHandler): } if event.membership == Membership.INVITE: - d["inviter"] = event.user_id + d["inviter"] = event.sender rooms_ret.append(d) From fd9a8db7ea93225af1774a604a97e84315cdccc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 15:59:31 +0000 Subject: [PATCH 55/91] Only fetch the columns we need. --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index a5e1c38f7..bedc3c6c5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -271,7 +271,7 @@ class StreamStore(SQLBaseStore): # TODO (erikj): Handle compressed feedback sql = ( - "SELECT * FROM events " + "SELECT stream_ordering, topological_ordering, event_id FROM events " "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " ) From d5ae67e67d90e745b8eb06661af60370c68f813b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 16:05:01 +0000 Subject: [PATCH 56/91] Fix typo where we used wrong var. --- synapse/util/logutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/util/logutils.py b/synapse/util/logutils.py index c4dfb69c5..fd9ac4d4d 100644 --- a/synapse/util/logutils.py +++ b/synapse/util/logutils.py @@ -102,7 +102,7 @@ def time_function(f): _log_debug_as_f( f, "[FUNC START] {%s-%d}", - (func_name, _TIME_FUNC_ID), + (func_name, id), ) r = f(*args, **kwargs) @@ -111,7 +111,7 @@ def time_function(f): _log_debug_as_f( f, "[FUNC END] {%s-%d} %f", - (func_name, _TIME_FUNC_ID, end-start,), + (func_name, id, end-start,), ) return r From 36a2a877e2920a20a679af96cbb1c4d041b89f96 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 16:34:26 +0000 Subject: [PATCH 57/91] Use time.time() instead of time.clock() --- synapse/storage/_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a0b7f943f..f660fc6ea 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -64,7 +64,7 @@ class LoggingTransaction(object): # Don't let logging failures stop SQL from working pass - start = time.clock() * 1000 + start = time.time() * 1000 try: return self.txn.execute( sql, *args, **kwargs @@ -73,7 +73,7 @@ class LoggingTransaction(object): logger.exception("[SQL FAIL] {%s}", self.name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) @@ -93,7 +93,7 @@ class SQLBaseStore(object): def inner_func(txn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) - start = time.clock() * 1000 + start = time.time() * 1000 txn_id = SQLBaseStore._TXN_ID # We don't really need these to be unique, so lets stop it from @@ -109,7 +109,7 @@ class SQLBaseStore(object): logger.exception("[TXN FAIL] {%s}", name) raise finally: - end = time.clock() * 1000 + end = time.time() * 1000 transaction_logger.debug( "[TXN END] {%s} %f", name, end - start From dfa05f0cd69779f21ae9b1085c40079440f87d3b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Jan 2015 18:51:03 +0000 Subject: [PATCH 58/91] Optimize FrozenEvent creation --- synapse/events/__init__.py | 20 +++++++++++++------- synapse/util/frozenutils.py | 5 +++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 5dca04d92..6030c5887 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -20,7 +20,7 @@ import copy class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = copy.deepcopy(internal_metadata_dict) + self.__dict__ = internal_metadata_dict def get_dict(self): return dict(self.__dict__) @@ -49,10 +49,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, internal_metadata_dict={}): - self.signatures = copy.deepcopy(signatures) - self.unsigned = copy.deepcopy(unsigned) + self.signatures = signatures + self.unsigned = unsigned - self._event_dict = copy.deepcopy(event_dict) + self._event_dict = event_dict self.internal_metadata = _EventInternalMetadata( internal_metadata_dict @@ -112,10 +112,16 @@ class EventBase(object): class FrozenEvent(EventBase): def __init__(self, event_dict, internal_metadata_dict={}): - event_dict = copy.deepcopy(event_dict) + event_dict = dict(event_dict) - signatures = copy.deepcopy(event_dict.pop("signatures", {})) - unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) + # Signatures is a dict of dicts, and this is faster than doing a + # copy.deepcopy + signatures = { + name: {sig_id: sig for sig_id, sig in sigs.items()} + for name, sigs in event_dict.pop("signatures", {}).items() + } + + unsigned = dict(event_dict.pop("unsigned", {})) frozen_dict = freeze(event_dict) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 061f79d79..a13a2015e 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -17,10 +17,11 @@ from frozendict import frozendict def freeze(o): - if isinstance(o, dict) or isinstance(o, frozendict): + t = type(o) + if t is dict: return frozendict({k: freeze(v) for k, v in o.items()}) - if isinstance(o, basestring): + if t is str or t is unicode: return o try: From a039e2544c5093f4f30147c063e97a2f31cbfe3a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jan 2015 09:48:03 +0000 Subject: [PATCH 59/91] Remove unused import --- synapse/events/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 6030c5887..4252e5ab5 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -15,8 +15,6 @@ from synapse.util.frozenutils import freeze, unfreeze -import copy - class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): From 89fc09c3d11b3574919a311dbf68816116873abb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jan 2015 13:56:56 +0000 Subject: [PATCH 60/91] Bump version and changelog --- CHANGES.rst | 8 ++++++++ VERSION | 2 +- synapse/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 813ad364e..3bd69367b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +Changes in synapse 0.6.1 (2015-01-07) +===================================== + + * Various optimization to improve performance of event sending and initial + sync. + * Media repository now includes a Content-Length header on media downloads. + * Improve quality of thumbnails by changing resizing algorithm. + Changes in synapse 0.6.0 (2014-12-16) ===================================== diff --git a/VERSION b/VERSION index a918a2aa1..ee6cdce3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1 diff --git a/synapse/__init__.py b/synapse/__init__.py index 06167e3c1..c3f1ac63b 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.0" +__version__ = "0.6.1" From 72d8d1265b8316878d8d61ba8a7781c4ac0764d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jan 2015 14:16:38 +0000 Subject: [PATCH 61/91] Improve change log --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3bd69367b..297ae914f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,8 @@ Changes in synapse 0.6.1 (2015-01-07) ===================================== - * Various optimization to improve performance of event sending and initial - sync. + * Major optimizations to improve performance of initial sync and event sending + in large rooms (by up to 10x) * Media repository now includes a Content-Length header on media downloads. * Improve quality of thumbnails by changing resizing algorithm. From 9b8e348b15faba1469a93c7daa009e27ee377bc0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 7 Jan 2015 15:08:22 +0000 Subject: [PATCH 62/91] *cough* --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index c3f1ac63b..9bfa09edf 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" This is a reference implementation of a synapse home server. +""" This is a reference implementation of a Matrix home server. """ __version__ = "0.6.1" From 9cb4f75d53d99634e79e791de22cb7de718248d6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Jan 2015 15:16:31 +0000 Subject: [PATCH 63/91] SYN-154: Better error messages when joining an unknown room by ID. The simple fix doesn't work here because room creation also involves unknown room IDs. The check relies on the presence of m.room.create for rooms being created, whereas bogus room IDs have no state events at all. --- synapse/api/auth.py | 11 ++++++++++- synapse/handlers/federation.py | 4 ++-- synapse/handlers/room.py | 8 +++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e31482cfa..8a3455ec5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -98,7 +98,16 @@ class Auth(object): defer.returnValue(member) @defer.inlineCallbacks - def check_host_in_room(self, room_id, host): + def check_host_in_room(self, room_id, host, context=None): + if context: + # XXX: check_host_in_room should really return True for a new + # room created by this home server. There are no m.room.member + # join events yet so we need to check for the m.room.create event + # instead. + if (u"m.room.create", u"") in context.auth_events: + defer.returnValue(True) + return + curr_state = yield self.state.get_current_state(room_id) for event in curr_state: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d26975a88..d0de6fd04 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -617,8 +617,8 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_backfill_request(self, origin, context, pdu_list, limit): - in_room = yield self.auth.check_host_in_room(context, origin) + def on_backfill_request(self, origin, room_id, pdu_list, limit): + in_room = yield self.auth.check_host_in_room(room_id, origin) if not in_room: raise AuthError(403, "Host not in room.") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 59719a1fa..3cb7e324f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -423,12 +423,13 @@ class RoomMemberHandler(BaseHandler): is_host_in_room = yield self.auth.check_host_in_room( event.room_id, - self.hs.hostname + self.hs.hostname, + context=context ) if is_host_in_room: should_do_dance = False - elif room_host: + elif room_host: # TODO: Shouldn't this be remote_room_host? should_do_dance = True else: # TODO(markjh): get prev_state from snapshot @@ -442,7 +443,8 @@ class RoomMemberHandler(BaseHandler): should_do_dance = not self.hs.is_mine(inviter) room_host = inviter.domain else: - should_do_dance = False + # return the same error as join_room_alias does + raise SynapseError(404, "No known servers") if should_do_dance: handler = self.hs.get_handlers().federation_handler From 4c68460392ef032b156b8d006f4aec5496ceedcb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Jan 2015 16:09:00 +0000 Subject: [PATCH 64/91] SYN-154: Tweak how the m.room.create check is done. Don't perform the check in auth.is_host_in_room but instead do it in _do_join and also assert that there are no m.room.members in the room before doing so. --- synapse/api/auth.py | 11 +---------- synapse/handlers/room.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 8a3455ec5..e31482cfa 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -98,16 +98,7 @@ class Auth(object): defer.returnValue(member) @defer.inlineCallbacks - def check_host_in_room(self, room_id, host, context=None): - if context: - # XXX: check_host_in_room should really return True for a new - # room created by this home server. There are no m.room.member - # join events yet so we need to check for the m.room.create event - # instead. - if (u"m.room.create", u"") in context.auth_events: - defer.returnValue(True) - return - + def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) for event in curr_state: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3cb7e324f..16c662829 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -423,9 +423,18 @@ class RoomMemberHandler(BaseHandler): is_host_in_room = yield self.auth.check_host_in_room( event.room_id, - self.hs.hostname, - context=context + self.hs.hostname ) + if not is_host_in_room: + # is *anyone* in the room? + room_member_keys = [ + v for (k,v) in context.current_state.keys() if k == "m.room.member" + ] + if len(room_member_keys) == 0: + # has the room been created so we can join it? + create_event = context.current_state.get(("m.room.create", "")) + if create_event: + is_host_in_room = True if is_host_in_room: should_do_dance = False From a09882de8378f143af79f97929bd1655cc7ac495 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Jan 2015 16:12:14 +0000 Subject: [PATCH 65/91] Update tests --- tests/rest/test_rooms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 84fd730af..8e65ff9a1 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -294,7 +294,7 @@ class RoomPermissionsTestCase(RestTestCase): # set [invite/join/left] of self, set [invite/join/left] of other, # expect all 403s for usr in [self.user_id, self.rmcreator_id]: - yield self.join(room=room, user=usr, expect_code=403) + yield self.join(room=room, user=usr, expect_code=404) yield self.leave(room=room, user=usr, expect_code=403) @defer.inlineCallbacks From 333836ff9205a53934cf0c412b75916740e407b5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Jan 2015 16:18:12 +0000 Subject: [PATCH 66/91] PEP8 and pyflakes warnings --- synapse/handlers/federation.py | 2 +- synapse/handlers/room.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d0de6fd04..195f7c618 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -623,7 +623,7 @@ class FederationHandler(BaseHandler): raise AuthError(403, "Host not in room.") events = yield self.store.get_backfill_events( - context, + room_id, pdu_list, limit ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 16c662829..6d0db18e5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -428,7 +428,9 @@ class RoomMemberHandler(BaseHandler): if not is_host_in_room: # is *anyone* in the room? room_member_keys = [ - v for (k,v) in context.current_state.keys() if k == "m.room.member" + v for (k, v) in context.current_state.keys() if ( + k == "m.room.member" + ) ] if len(room_member_keys) == 0: # has the room been created so we can join it? From 76e1565200dda04e4091be761c737042f9a15e67 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 7 Jan 2015 17:11:19 +0000 Subject: [PATCH 67/91] Change error message for missing pillow libs. --- synapse/media/v1/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py index 619999d26..d6c669057 100644 --- a/synapse/media/v1/__init__.py +++ b/synapse/media/v1/__init__.py @@ -22,7 +22,8 @@ except IOError as e: if str(e).startswith("decoder jpeg not available"): raise Exception( "FATAL: jpeg codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" ) except Exception: # any other exception is fine @@ -36,7 +37,8 @@ except IOError as e: if str(e).startswith("decoder zip not available"): raise Exception( "FATAL: zip codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip install -I pillow'" + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" ) except Exception: # any other exception is fine From 42507b0011a1285645206f5bd627809a8a6337e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jan 2015 17:25:28 +0000 Subject: [PATCH 68/91] Log server version on startup --- synapse/app/homeserver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5fec8da7c..fba43aa2b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -39,6 +39,8 @@ from synapse.util.logcontext import LoggingContext from daemonize import Daemonize import twisted.manhole.telnet +import synapse + import logging import os import re @@ -199,6 +201,7 @@ def setup(): config.setup_logging() logger.info("Server hostname: %s", config.server_name) + logger.info("Server version: %s", synapse.__version__) if re.search(":[0-9]+$", config.server_name): domain_with_port = config.server_name From c9d2cecac9727ac3be8ad8cab21dc78b9dffe7a2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Jan 2015 09:41:11 +0000 Subject: [PATCH 69/91] SYN-231: User agent header broken --- synapse/http/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index e5d4939e2..7793bab10 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -53,7 +53,7 @@ class SimpleHttpClient(object): uri.encode("ascii"), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": AGENT_NAME, + b"User-Agent": [AGENT_NAME], }), bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) @@ -89,7 +89,7 @@ class SimpleHttpClient(object): "GET", uri.encode("ascii"), headers=Headers({ - b"User-Agent": AGENT_NAME, + b"User-Agent": [AGENT_NAME], }) ) @@ -114,7 +114,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): bodyProducer=FileBodyProducer(StringIO(query_bytes)), headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], - b"User-Agent": AGENT_NAME, + b"User-Agent": [AGENT_NAME], }) ) From 5a0e687d5cfbb01d817a2ea5a795d3fee7ca5083 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Jan 2015 09:42:23 +0000 Subject: [PATCH 70/91] Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index ee6cdce3c..8b9dea59d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1 +0.6.1a diff --git a/synapse/__init__.py b/synapse/__init__.py index c3f1ac63b..2195ad317 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1" +__version__ = "0.6.1a" From d44dd47fbf7a2e4a9b253128e645ceb698ec274a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jan 2015 10:53:03 +0000 Subject: [PATCH 71/91] Add optional limit to graph script --- graph/graph2.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/graph/graph2.py b/graph/graph2.py index b9b8a562a..6b551d42e 100644 --- a/graph/graph2.py +++ b/graph/graph2.py @@ -23,14 +23,27 @@ import argparse from synapse.events import FrozenEvent -def make_graph(db_name, room_id, file_prefix): +def make_graph(db_name, room_id, file_prefix, limit): conn = sqlite3.connect(db_name) - c = conn.execute( - "SELECT json FROM event_json where room_id = ?", - (room_id,) + sql = ( + "SELECT json FROM event_json as j " + "INNER JOIN events as e ON e.event_id = j.event_id " + "WHERE j.room_id = ?" ) + args = [room_id] + + if limit: + sql += ( + " ORDER BY topological_ordering DESC, stream_ordering DESC " + "LIMIT ?" + ) + + args.append(limit) + + c = conn.execute(sql, args) + events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()] events.sort(key=lambda e: e.depth) @@ -128,11 +141,16 @@ if __name__ == "__main__": ) parser.add_argument( "-p", "--prefix", dest="prefix", - help="String to prefix output files with" + help="String to prefix output files with", + default="graph_output" + ) + parser.add_argument( + "-l", "--limit", + help="Only retrieve the last N events.", ) parser.add_argument('db') parser.add_argument('room') args = parser.parse_args() - make_graph(args.db, args.room, args.prefix) + make_graph(args.db, args.room, args.prefix, args.limit) From 5720ab59e03d6f5ab48c3be22e8957a8891ea56c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Jan 2015 13:57:29 +0000 Subject: [PATCH 72/91] Add 'raw' query parameter to expose the event graph and signatures to savvy clients. --- synapse/events/utils.py | 17 +++++++++-------- synapse/handlers/events.py | 7 +++++-- synapse/handlers/message.py | 6 ++++-- synapse/rest/events.py | 5 ++++- synapse/rest/initial_sync.py | 5 ++++- synapse/server.py | 4 ++-- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4f4914467..258dedb27 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,7 +89,7 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e): +def serialize_event(hs, e, remove_data=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e @@ -122,12 +122,13 @@ def serialize_event(hs, e): d["prev_content"] = e.unsigned["prev_content"] del d["unsigned"]["prev_content"] - del d["auth_events"] - del d["prev_events"] - del d["hashes"] - del d["signatures"] - d.pop("depth", None) - d.pop("unsigned", None) - d.pop("origin", None) + if remove_data: + del d["auth_events"] + del d["prev_events"] + del d["hashes"] + del d["signatures"] + d.pop("depth", None) + d.pop("unsigned", None) + d.pop("origin", None) return d diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 808219bd1..4e805606b 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -46,7 +46,8 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function - def get_stream(self, auth_user_id, pagin_config, timeout=0): + def get_stream(self, auth_user_id, pagin_config, timeout=0, + trim_events=True): auth_user = self.hs.parse_userid(auth_user_id) try: @@ -78,7 +79,9 @@ class EventStreamHandler(BaseHandler): auth_user, room_ids, pagin_config, timeout ) - chunks = [self.hs.serialize_event(e) for e in events] + chunks = [ + self.hs.serialize_event(e, trim_events) for e in events + ] chunk = { "chunk": chunks, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7195de98b..b2bbcfc6e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -211,7 +211,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False): + feedback=False, trim_events=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -280,7 +280,9 @@ class MessageHandler(BaseHandler): end_token = now_token.copy_and_replace("room_key", token[1]) d["messages"] = { - "chunk": [self.hs.serialize_event(m) for m in messages], + "chunk": [ + self.hs.serialize_event(m, trim_events) for m in messages + ], "start": start_token.to_string(), "end": end_token.to_string(), } diff --git a/synapse/rest/events.py b/synapse/rest/events.py index cf6d13f81..ac1a75a55 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -44,8 +44,11 @@ class EventStreamRestServlet(RestServlet): except ValueError: raise SynapseError(400, "timeout must be in milliseconds.") + trim_events = "raw" not in request.args + chunk = yield handler.get_stream( - auth_user.to_string(), pagin_config, timeout=timeout + auth_user.to_string(), pagin_config, timeout=timeout, + trim_events=trim_events ) except: logger.exception("Event stream failed") diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index a57158958..d2c0c63aa 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -27,12 +27,15 @@ class InitialSyncRestServlet(RestServlet): def on_GET(self, request): user = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args + trim_events = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback) + feedback=with_feedback, + trim_events=trim_events + ) defer.returnValue((200, content)) diff --git a/synapse/server.py b/synapse/server.py index c3bf46abb..88161107a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -149,8 +149,8 @@ class BaseHomeServer(object): object.""" return EventID.from_string(s) - def serialize_event(self, e): - return serialize_event(self, e) + def serialize_event(self, e, remove_data=True): + return serialize_event(self, e, remove_data) def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config From 5940ec993bf75d5d05885544e811da88703f1800 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jan 2015 13:59:29 +0000 Subject: [PATCH 73/91] Add missing continuation indent. --- synapse/handlers/message.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b2bbcfc6e..9b20e4f50 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -281,7 +281,8 @@ class MessageHandler(BaseHandler): d["messages"] = { "chunk": [ - self.hs.serialize_event(m, trim_events) for m in messages + self.hs.serialize_event(m, trim_events) + for m in messages ], "start": start_token.to_string(), "end": end_token.to_string(), From edb557b2ad98d3260caaba41ef2278b3eafc7e85 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Jan 2015 14:27:04 +0000 Subject: [PATCH 74/91] Return the raw federation event rather than adding extra keys for federation data. --- synapse/events/utils.py | 25 ++++++++++++++++--------- synapse/handlers/events.py | 4 ++-- synapse/handlers/message.py | 5 +++-- synapse/rest/events.py | 4 ++-- synapse/rest/initial_sync.py | 4 ++-- synapse/server.py | 4 ++-- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 258dedb27..4687d96f2 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -89,13 +89,21 @@ def prune_event(event): return type(event)(allowed_fields) -def serialize_event(hs, e, remove_data=True): +def serialize_event(hs, e, client_event=True): # FIXME(erikj): To handle the case of presence events and the like if not isinstance(e, EventBase): return e # Should this strip out None's? d = {k: v for k, v in e.get_dict().items()} + + if not client_event: + # set the age and keep all other keys + if "age_ts" in d["unsigned"]: + now = int(hs.get_clock().time_msec()) + d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + return d + if "age_ts" in d["unsigned"]: now = int(hs.get_clock().time_msec()) d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] @@ -122,13 +130,12 @@ def serialize_event(hs, e, remove_data=True): d["prev_content"] = e.unsigned["prev_content"] del d["unsigned"]["prev_content"] - if remove_data: - del d["auth_events"] - del d["prev_events"] - del d["hashes"] - del d["signatures"] - d.pop("depth", None) - d.pop("unsigned", None) - d.pop("origin", None) + del d["auth_events"] + del d["prev_events"] + del d["hashes"] + del d["signatures"] + d.pop("depth", None) + d.pop("unsigned", None) + d.pop("origin", None) return d diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 4e805606b..c9ade253d 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -47,7 +47,7 @@ class EventStreamHandler(BaseHandler): @defer.inlineCallbacks @log_function def get_stream(self, auth_user_id, pagin_config, timeout=0, - trim_events=True): + as_client_event=True): auth_user = self.hs.parse_userid(auth_user_id) try: @@ -80,7 +80,7 @@ class EventStreamHandler(BaseHandler): ) chunks = [ - self.hs.serialize_event(e, trim_events) for e in events + self.hs.serialize_event(e, as_client_event) for e in events ] chunk = { diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9b20e4f50..30f5a08b5 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -211,7 +211,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False, trim_events=True): + feedback=False, as_client_event=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -222,6 +222,7 @@ class MessageHandler(BaseHandler): pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. feedback (bool): True to get feedback along with these messages. + as_client_event (bool): True to get events in client-server format. Returns: A list of dicts with "room_id" and "membership" keys for all rooms the user is currently invited or joined in on. Rooms where the user @@ -281,7 +282,7 @@ class MessageHandler(BaseHandler): d["messages"] = { "chunk": [ - self.hs.serialize_event(m, trim_events) + self.hs.serialize_event(m, as_client_event) for m in messages ], "start": start_token.to_string(), diff --git a/synapse/rest/events.py b/synapse/rest/events.py index ac1a75a55..bedcb2bcc 100644 --- a/synapse/rest/events.py +++ b/synapse/rest/events.py @@ -44,11 +44,11 @@ class EventStreamRestServlet(RestServlet): except ValueError: raise SynapseError(400, "timeout must be in milliseconds.") - trim_events = "raw" not in request.args + as_client_event = "raw" not in request.args chunk = yield handler.get_stream( auth_user.to_string(), pagin_config, timeout=timeout, - trim_events=trim_events + as_client_event=as_client_event ) except: logger.exception("Event stream failed") diff --git a/synapse/rest/initial_sync.py b/synapse/rest/initial_sync.py index d2c0c63aa..b13d56b28 100644 --- a/synapse/rest/initial_sync.py +++ b/synapse/rest/initial_sync.py @@ -27,14 +27,14 @@ class InitialSyncRestServlet(RestServlet): def on_GET(self, request): user = yield self.auth.get_user_by_req(request) with_feedback = "feedback" in request.args - trim_events = "raw" not in request.args + as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, feedback=with_feedback, - trim_events=trim_events + as_client_event=as_client_event ) defer.returnValue((200, content)) diff --git a/synapse/server.py b/synapse/server.py index 88161107a..d861efd2f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -149,8 +149,8 @@ class BaseHomeServer(object): object.""" return EventID.from_string(s) - def serialize_event(self, e, remove_data=True): - return serialize_event(self, e, remove_data) + def serialize_event(self, e, as_client_event=True): + return serialize_event(self, e, as_client_event) def get_ip_from_request(self, request): # May be an X-Forwarding-For header depending on config From 379a653ae3e46bc27b8ad4bde9bb7c25d0e048f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jan 2015 14:32:53 +0000 Subject: [PATCH 75/91] Add better help message for --server-name config option. --- synapse/config/server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 4f73c8546..31e44cc85 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -47,8 +47,12 @@ class ServerConfig(Config): def add_arguments(cls, parser): super(ServerConfig, cls).add_arguments(parser) server_group = parser.add_argument_group("server") - server_group.add_argument("-H", "--server-name", default="localhost", - help="The name of the server") + server_group.add_argument( + "-H", "--server-name", default="localhost", + help="The domain name of the server, with optional explicit port. " + "This is used by remote servers to connect to this server, " + "e.g. matrix.org, localhost:8080, etc." + ) server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") server_group.add_argument("-p", "--bind-port", metavar="PORT", From b5924cae04e549b3e19addc9257b462627f3d334 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Jan 2015 14:36:33 +0000 Subject: [PATCH 76/91] Add raw query param for scrollback. --- synapse/handlers/message.py | 7 +++++-- synapse/rest/room.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 30f5a08b5..f2a2f1693 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -67,7 +67,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False): + feedback=False, as_client_event=True): """Get messages in a room. Args: @@ -76,6 +76,7 @@ class MessageHandler(BaseHandler): pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. feedback (bool): True to get compressed feedback with the messages + as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results """ @@ -99,7 +100,9 @@ class MessageHandler(BaseHandler): ) chunk = { - "chunk": [self.hs.serialize_event(e) for e in events], + "chunk": [ + self.hs.serialize_event(e, as_client_event) for e in events + ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), } diff --git a/synapse/rest/room.py b/synapse/rest/room.py index e40773758..caafa959e 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -314,12 +314,15 @@ class RoomMessageListRestServlet(RestServlet): request, default_limit=10, ) with_feedback = "feedback" in request.args + as_client_event = "raw" not in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback) + feedback=with_feedback, + as_client_event=as_client_event + ) defer.returnValue((200, msgs)) From 7f83613733bc39a14b4eaff78313047d0fc50739 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 8 Jan 2015 15:11:22 +0000 Subject: [PATCH 77/91] make our JPEG thumbnail quality less horrifically ugly --- synapse/media/v1/thumbnailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index bc86efea8..28404f2b7 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -82,7 +82,7 @@ class Thumbnailer(object): def save_image(self, output_image, output_type, output_path): output_bytes_io = BytesIO() - output_image.save(output_bytes_io, self.FORMATS[output_type]) + output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70) output_bytes = output_bytes_io.getvalue() with open(output_path, "wb") as output_file: output_file.write(output_bytes) From 9d0dcf2e3ca8b8c9cc8d87a451ed901f102dc2c6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Jan 2015 15:31:06 +0000 Subject: [PATCH 78/91] SYN-142: Rotate logs if logging to file. Fixed to a 4 file rotate with 100MB/file for now. --- synapse/config/logger.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 15383b318..f9568ebd2 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -66,7 +66,10 @@ class LoggingConfig(Config): formatter = logging.Formatter(log_format) if self.log_file: - handler = logging.FileHandler(self.log_file) + # TODO: Customisable file size / backup count + handler = logging.handlers.RotatingFileHandler( + self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 + ) else: handler = logging.StreamHandler() handler.setFormatter(formatter) From 63403aa7a57704cde86344b48390d16b1d74b035 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 8 Jan 2015 17:07:28 +0000 Subject: [PATCH 79/91] Check the existance and versions of necessary modules when starting synapse, log which modules are used --- synapse/app/homeserver.py | 5 +++ synapse/python_dependencies.py | 80 ++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 synapse/python_dependencies.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fba43aa2b..43b5c2614 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,6 +18,8 @@ from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer +from synapse.python_dependencies import check_requirements + from twisted.internet import reactor from twisted.enterprise import adbapi from twisted.web.resource import Resource @@ -200,6 +202,8 @@ def setup(): config.setup_logging() + check_requirements() + logger.info("Server hostname: %s", config.server_name) logger.info("Server version: %s", synapse.__version__) @@ -280,6 +284,7 @@ def run(): def main(): with LoggingContext("main"): + check_requirements() setup() diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py new file mode 100644 index 000000000..b1fae991e --- /dev/null +++ b/synapse/python_dependencies.py @@ -0,0 +1,80 @@ +import logging +from distutils.version import LooseVersion + +logger = logging.getLogger(__name__) + +REQUIREMENTS = { + "syutil==0.0.2": ["syutil"], + "matrix_angular_sdk==0.6.0": ["syweb==0.6.0"], + "Twisted>=14.0.0": ["twisted>=14.0.0"], + "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "pyopenssl>=0.14": ["OpenSSL>=0.14"], + "pyyaml": ["yaml"], + "pyasn1": ["pyasn1"], + "pynacl": ["nacl"], + "daemonize": ["daemonize"], + "py-bcrypt": ["bcrypt"], + "frozendict>=0.4": ["frozendict"], + "pillow": ["PIL"], +} + + +class MissingRequirementError(Exception): + pass + + +def check_requirements(): + """Checks that all the modules needed by synapse have been correctly + installed and are at the correct version""" + for dependency, module_requirements in REQUIREMENTS.items(): + for module_requirement in module_requirements: + if ">=" in module_requirement: + module_name, required_version = module_requirement.split(">=") + version_test = ">=" + elif "==" in module_requirement: + module_name, required_version = module_requirement.split("==") + version_test = "==" + else: + module_name = module_requirement + version_test = None + + try: + module = __import__(module_name) + except ImportError: + logging.exception( + "Can't import %r which is part of %r", + module_name, dependency + ) + raise MissingRequirementError( + "Can't import %r which is part of %r" + % (module_name, dependency) + ) + version = getattr(module, "__version__", None) + file_path = getattr(module, "__file__", None) + logger.info( + "Using %r version %r from %r to satisfy %r", + module_name, version, file_path, dependency + ) + + if version_test == ">=": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) < LooseVersion(required_version): + raise MissingRequirementError( + "Version of %r in %r is too old. %r < %r" + % (dependency, file_path, version, required_version) + ) + elif version_test == "==": + if version is None: + raise MissingRequirementError( + "Version of %r isn't set as __version__ of module %r" + % (dependency, module_name) + ) + if LooseVersion(version) != LooseVersion(required_version): + raise MissingRequirementError( + "Unexpected version of %r in %r. %r != %r" + % (dependency, file_path, version, required_version) + ) From 80e89772e2d531941bf4403ecf3d539557763985 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 8 Jan 2015 20:36:34 +0000 Subject: [PATCH 80/91] spell out that local libs may need to be explicitly given priority --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 92b94bcd7..2be201178 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,15 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into ``$HOME/.local/lib/`` on Linux or ``$HOME/Library/Python/2.7/lib/`` on OSX. +Your python may not give priority to locally installed libraries over system +libraries, in which case you must add your local packages to your python path:: + + $ # on Linux: + $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages + + $ # on OSX: + $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python2.7/site-packages + For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From 28db5dde4c37ec69449995de40c02b7f4c532746 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 8 Jan 2015 20:38:55 +0000 Subject: [PATCH 81/91] oops --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2be201178..326f1d9cc 100644 --- a/README.rst +++ b/README.rst @@ -115,7 +115,7 @@ libraries, in which case you must add your local packages to your python path:: $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages $ # on OSX: - $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python2.7/site-packages + $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From bfb198a6eb0d1c0b2c73e88b8420549f84ebd626 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 9 Jan 2015 18:14:05 +0000 Subject: [PATCH 82/91] don't clobber pythonpath --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 326f1d9cc..768da3df6 100644 --- a/README.rst +++ b/README.rst @@ -112,10 +112,10 @@ Your python may not give priority to locally installed libraries over system libraries, in which case you must add your local packages to your python path:: $ # on Linux: - $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages + $ export PYTHONPATH=$HOME/.local/lib/python2.7/site-packages:$PYTHONPATH $ # on OSX: - $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages + $ export PYTHONPATH=$HOME/Library/Python/2.7/lib/python/site-packages:$PYTHONPATH For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. From d8fcc4e00a05252d4402a834d4b8ef66784de62b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 12 Jan 2015 14:30:54 +0000 Subject: [PATCH 83/91] Add copyrighter script for sql --- scripts/copyrighter-sql.pl | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 scripts/copyrighter-sql.pl diff --git a/scripts/copyrighter-sql.pl b/scripts/copyrighter-sql.pl new file mode 100755 index 000000000..890e51e58 --- /dev/null +++ b/scripts/copyrighter-sql.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl -pi +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$copyright = < Date: Mon, 12 Jan 2015 17:38:30 +0000 Subject: [PATCH 84/91] SYN-178: Fix off by one. --- synapse/storage/stream.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index bedc3c6c5..744c821df 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -284,8 +284,12 @@ class StreamStore(SQLBaseStore): rows.reverse() # As we selected with reverse ordering if rows: - topo = rows[0]["topological_ordering"] - toke = rows[0]["stream_ordering"] + # XXX: Always subtract 1 since the start token always goes + # backwards (parity with paginate_room_events). It isn't + # obvious that this is correct; we should clarify the algorithm + # used here. + topo = rows[0]["topological_ordering"] - 1 + toke = rows[0]["stream_ordering"] - 1 start_token = "t%s-%s" % (topo, toke) token = (start_token, end_token) From 968dc988f9008b15348705c52992100dcabf206f Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 12 Jan 2015 18:01:33 +0000 Subject: [PATCH 85/91] Check that setting typing notification still works after explicit timeout - SYN-230 --- tests/handlers/test_typing.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 0d4b368a4..6a498b23a 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -352,3 +352,29 @@ class TypingNotificationsTestCase(unittest.TestCase): }}, ] ) + + # SYN-230 - see if we can still set after timeout + + yield self.handler.started_typing( + target_user=self.u_apple, + auth_user=self.u_apple, + room_id=self.room_id, + timeout=10000, + ) + + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), + ]) + self.on_new_user_event.reset_mock() + + self.assertEquals(self.event_source.get_current_key(), 3) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "content": { + "user_ids": [self.u_apple.to_string()], + }}, + ] + ) From db72a07ef52dd3a911978df9a13f23febdcc00ce Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 12 Jan 2015 18:16:27 +0000 Subject: [PATCH 86/91] Don't make @unittest.DEBUG print the huge amount of verbosity generated by the synapse.storage loggers --- tests/unittest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittest.py b/tests/unittest.py index a9c0e0554..fe26b7574 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -69,6 +69,8 @@ class TestCase(unittest.TestCase): return ret logging.getLogger().setLevel(level) + # Don't set SQL logging + logging.getLogger("synapse.storage").setLevel(old_level) return orig() def assertObjectHasAttributes(self, attrs, obj): From 67d8305aea65d52abe4ce1c40bf78fdab3dc6471 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 12 Jan 2015 18:22:00 +0000 Subject: [PATCH 87/91] Make typing notification timeouts print a (debug) logging message --- synapse/handlers/typing.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index ab698b36e..15039ff0d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -83,9 +83,15 @@ class TypingNotificationHandler(BaseHandler): if member in self._member_typing_timer: self.clock.cancel_call_later(self._member_typing_timer[member]) + def _cb(): + logger.debug( + "%s has timed out in %s", target_user.to_string(), room_id + ) + self._stopped_typing(member) + self._member_typing_until[member] = until self._member_typing_timer[member] = self.clock.call_later( - timeout / 1000, lambda: self._stopped_typing(member) + timeout / 1000, _cb ) if was_present: From 9c804bc3fd23a2bafe5d6f7368c90a7fba99bcf7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 12 Jan 2015 18:31:48 +0000 Subject: [PATCH 88/91] Check that setting typing notification still works after explicit timeout at REST layer - SYN-230 --- tests/rest/test_typing.py | 51 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/rest/test_typing.py b/tests/rest/test_typing.py index c550294d5..18138af1b 100644 --- a/tests/rest/test_typing.py +++ b/tests/rest/test_typing.py @@ -21,7 +21,7 @@ from twisted.internet import defer import synapse.rest.room from synapse.server import HomeServer -from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from ..utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey from .utils import RestTestCase from mock import Mock, NonCallableMock @@ -36,6 +36,8 @@ class RoomTypingTestCase(RestTestCase): @defer.inlineCallbacks def setUp(self): + self.clock = MockClock() + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.auth_user_id = self.user_id @@ -47,6 +49,7 @@ class RoomTypingTestCase(RestTestCase): hs = HomeServer( "red", + clock=self.clock, db_pool=db_pool, http_client=None, replication_layer=Mock(), @@ -77,6 +80,30 @@ class RoomTypingTestCase(RestTestCase): return defer.succeed(None) hs.get_datastore().insert_client_ip = _insert_client_ip + def get_room_members(room_id): + if room_id == self.room_id: + return defer.succeed([hs.parse_userid(self.user_id)]) + else: + return defer.succeed([]) + + @defer.inlineCallbacks + def fetch_room_distributions_into(room_id, localusers=None, + remotedomains=None, ignore_user=None): + + members = yield get_room_members(room_id) + for member in members: + if ignore_user is not None and member == ignore_user: + continue + + if hs.is_mine(member): + if localusers is not None: + localusers.add(member) + else: + if remotedomains is not None: + remotedomains.add(member.domain) + hs.get_handlers().room_member_handler.fetch_room_distributions_into = ( + fetch_room_distributions_into) + synapse.rest.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) @@ -113,3 +140,25 @@ class RoomTypingTestCase(RestTestCase): '{"typing": false}' ) self.assertEquals(200, code) + + @defer.inlineCallbacks + def test_typing_timeout(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 1) + + self.clock.advance_time(31); + + self.assertEquals(self.event_source.get_current_key(), 2) + + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 3) From 02ffbb20d00dbda213ba9321537ac12e347dcc35 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 12 Jan 2015 19:09:14 +0000 Subject: [PATCH 89/91] Use float rather than integer divisions to turn msec into sec - so timeouts under 1000msec will actually work --- synapse/handlers/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 15039ff0d..22ce7873d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -91,7 +91,7 @@ class TypingNotificationHandler(BaseHandler): self._member_typing_until[member] = until self._member_typing_timer[member] = self.clock.call_later( - timeout / 1000, _cb + timeout / 1000.0, _cb ) if was_present: From 39585bf5560e64bf6fca7d043cc3357b5dba59de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Jan 2015 09:57:32 +0000 Subject: [PATCH 90/91] Insert 'age' into top level when returning events to clients --- synapse/events/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4f4914467..4ad37188b 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -98,7 +98,7 @@ def serialize_event(hs, e): d = {k: v for k, v in e.get_dict().items()} if "age_ts" in d["unsigned"]: now = int(hs.get_clock().time_msec()) - d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + d["age"] = now - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] d["user_id"] = d.pop("sender", None) From 1d3d37937d680e460d1931dd36f36bf59c606561 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Jan 2015 09:59:47 +0000 Subject: [PATCH 91/91] Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 8b9dea59d..3b3e72317 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1a +0.6.1b diff --git a/synapse/__init__.py b/synapse/__init__.py index 2195ad317..e7e27b06e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.6.1a" +__version__ = "0.6.1b"