From 62b51b84520243d25b82a303ff44ff2f329c8d82 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 29 Apr 2016 12:00:51 +0100 Subject: [PATCH 1/6] Fix typo in event_auth servlet path --- synapse/federation/transport/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index d65a7893d..3e552b6c4 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -323,7 +323,7 @@ class FederationSendLeaveServlet(BaseFederationServlet): class FederationEventAuthServlet(BaseFederationServlet): - PATH = "/event_auth(?P[^/]*)/(?P[^/]*)" + PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" def on_GET(self, origin, content, query, context, event_id): return self.handler.on_event_auth(origin, context, event_id) From 984d4a2c0f59039a623b6a6f1945ff697f004c27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 11:28:10 +0100 Subject: [PATCH 2/6] Add /report endpoint --- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/report_event.py | 59 ++++++++++++++++++++ synapse/storage/prepare_database.py | 2 +- synapse/storage/room.py | 14 +++++ synapse/storage/schema/delta/32/reports.sql | 23 ++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/report_event.py create mode 100644 synapse/storage/schema/delta/32/reports.sql diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 6688fa8fa..e805cb911 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -44,6 +44,7 @@ from synapse.rest.client.v2_alpha import ( tokenrefresh, tags, account_data, + report_event, ) from synapse.http.server import JsonResource @@ -86,3 +87,4 @@ class ClientRestResource(JsonResource): tokenrefresh.register_servlets(hs, client_resource) tags.register_servlets(hs, client_resource) account_data.register_servlets(hs, client_resource) + report_event.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py new file mode 100644 index 000000000..412e5b190 --- /dev/null +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet, parse_json_object_from_request +from ._base import client_v2_patterns + +import logging + + +logger = logging.getLogger(__name__) + + +class ReportEventRestServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/rooms/(?P[^/]*)/report$" + ) + + def __init__(self, hs): + super(ReportEventRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def on_POST(self, request, room_id): + requester = yield self.auth.get_user_by_req(request) + user_id = requester.user.to_string() + + body = parse_json_object_from_request(request) + + event_id = body["event_id"] + + yield self.store.add_event_report( + room_id=room_id, + event_id=event_id, + user_id=user_id, + reason=body.get("reason"), + content=body, + ) + + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + ReportEventRestServlet(hs).register(http_server) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 57f14fd12..c8487c883 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 31 +SCHEMA_VERSION = 32 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 70aa64fb3..ceced7d51 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -23,6 +23,7 @@ from .engines import PostgresEngine, Sqlite3Engine import collections import logging +import ujson as json logger = logging.getLogger(__name__) @@ -221,3 +222,16 @@ class RoomStore(SQLBaseStore): aliases.extend(e.content['aliases']) defer.returnValue((name, aliases)) + + def add_event_report(self, room_id, event_id, user_id, reason, content): + return self._simple_insert( + table="event_reports", + values={ + "room_id": room_id, + "event_id": event_id, + "user_id": user_id, + "reason": reason, + "content": json.dumps(content), + }, + desc="add_event_report" + ) diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql new file mode 100644 index 000000000..06bf0d9b5 --- /dev/null +++ b/synapse/storage/schema/delta/32/reports.sql @@ -0,0 +1,23 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE event_reports( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + reason TEXT, + content TEXT +); From 8a04412fa1e837d733302038c64854ba0766efc0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 4 May 2016 12:19:04 +0100 Subject: [PATCH 3/6] starting point for doc on how log contexts are supposed to work --- docs/log_contexts.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/log_contexts.rst diff --git a/docs/log_contexts.rst b/docs/log_contexts.rst new file mode 100644 index 000000000..0046e171b --- /dev/null +++ b/docs/log_contexts.rst @@ -0,0 +1,10 @@ +What do I do about "Unexpected logging context" debug log-lines everywhere? + + The logging context lives in thread local storage + Sometimes it gets out of sync with what it should actually be, usually because something scheduled something to run on the reactor without preserving the logging context. + what is the impact of it getting out of sync? and how and when should we preserve log context? + The impact is that some of the CPU and database metrics will be under-reported, and some log lines will be mis-attributed. + It should happen auto-magically in all the APIs that do IO or otherwise defer to the reactor. + Mjark: the other place is if we branch, e.g. using defer.gatherResults + +Unanswered: how and when should we preserve log context? \ No newline at end of file From 5650e38e7de4cf89074ff84f4ecfbfcd81fa810d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 13:19:39 +0100 Subject: [PATCH 4/6] Move event_id to path --- synapse/rest/client/v2_alpha/report_event.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py index 412e5b190..9c1c9662c 100644 --- a/synapse/rest/client/v2_alpha/report_event.py +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class ReportEventRestServlet(RestServlet): PATTERNS = client_v2_patterns( - "/rooms/(?P[^/]*)/report$" + "/rooms/(?P[^/]*)/report/(?P[^/]*)$" ) def __init__(self, hs): @@ -36,14 +36,12 @@ class ReportEventRestServlet(RestServlet): self.store = hs.get_datastore() @defer.inlineCallbacks - def on_POST(self, request, room_id): + def on_POST(self, request, room_id, event_id): requester = yield self.auth.get_user_by_req(request) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - event_id = body["event_id"] - yield self.store.add_event_report( room_id=room_id, event_id=event_id, From 8e6a163f2762b3f62ae9b350c5050bc2318ec268 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 15:19:12 +0100 Subject: [PATCH 5/6] Add timestamp and auto incrementing ID --- synapse/rest/client/v2_alpha/report_event.py | 2 ++ synapse/storage/__init__.py | 1 + synapse/storage/room.py | 6 +++++- synapse/storage/schema/delta/32/reports.sql | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/report_event.py b/synapse/rest/client/v2_alpha/report_event.py index 9c1c9662c..8903e1240 100644 --- a/synapse/rest/client/v2_alpha/report_event.py +++ b/synapse/rest/client/v2_alpha/report_event.py @@ -33,6 +33,7 @@ class ReportEventRestServlet(RestServlet): super(ReportEventRestServlet, self).__init__() self.hs = hs self.auth = hs.get_auth() + self.clock = hs.get_clock() self.store = hs.get_datastore() @defer.inlineCallbacks @@ -48,6 +49,7 @@ class ReportEventRestServlet(RestServlet): user_id=user_id, reason=body.get("reason"), content=body, + received_ts=self.clock.time_msec(), ) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 045ae6c03..7122b0cbb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -114,6 +114,7 @@ class DataStore(RoomMemberStore, RoomStore, self._state_groups_id_gen = StreamIdGenerator(db_conn, "state_groups", "id") self._access_tokens_id_gen = IdGenerator(db_conn, "access_tokens", "id") self._refresh_tokens_id_gen = IdGenerator(db_conn, "refresh_tokens", "id") + self._event_reports_id_gen = IdGenerator(db_conn, "event_reports", "id") self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") self._push_rules_stream_id_gen = ChainedIdGenerator( diff --git a/synapse/storage/room.py b/synapse/storage/room.py index ceced7d51..26933e593 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -223,10 +223,14 @@ class RoomStore(SQLBaseStore): defer.returnValue((name, aliases)) - def add_event_report(self, room_id, event_id, user_id, reason, content): + def add_event_report(self, room_id, event_id, user_id, reason, content, + received_ts): + next_id = self._event_reports_id_gen.get_next() return self._simple_insert( table="event_reports", values={ + "id": next_id, + "received_ts": received_ts, "room_id": room_id, "event_id": event_id, "user_id": user_id, diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql index 06bf0d9b5..3f2502745 100644 --- a/synapse/storage/schema/delta/32/reports.sql +++ b/synapse/storage/schema/delta/32/reports.sql @@ -15,6 +15,8 @@ CREATE TABLE event_reports( + id BIGINT NOT NULL, + received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL, user_id TEXT NOT NULL, From fcd1eb642dc8b73e372b78bb788e3a17a5e40ace Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 4 May 2016 16:51:51 +0100 Subject: [PATCH 6/6] Add primary key --- synapse/storage/schema/delta/32/reports.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/32/reports.sql b/synapse/storage/schema/delta/32/reports.sql index 3f2502745..d13609776 100644 --- a/synapse/storage/schema/delta/32/reports.sql +++ b/synapse/storage/schema/delta/32/reports.sql @@ -15,7 +15,7 @@ CREATE TABLE event_reports( - id BIGINT NOT NULL, + id BIGINT NOT NULL PRIMARY KEY, received_ts BIGINT NOT NULL, room_id TEXT NOT NULL, event_id TEXT NOT NULL,