mirror of
https://mau.dev/maunium/synapse.git
synced 2024-08-22 02:05:24 +00:00
Censor redactions in DB after a month
This commit is contained in:
parent
92c1550f4a
commit
a4bf72c30c
@ -23,7 +23,7 @@ from functools import wraps
|
|||||||
from six import iteritems, text_type
|
from six import iteritems, text_type
|
||||||
from six.moves import range
|
from six.moves import range
|
||||||
|
|
||||||
from canonicaljson import json
|
from canonicaljson import encode_canonical_json, json
|
||||||
from prometheus_client import Counter, Histogram
|
from prometheus_client import Counter, Histogram
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
@ -33,6 +33,7 @@ from synapse.api.constants import EventTypes
|
|||||||
from synapse.api.errors import SynapseError
|
from synapse.api.errors import SynapseError
|
||||||
from synapse.events import EventBase # noqa: F401
|
from synapse.events import EventBase # noqa: F401
|
||||||
from synapse.events.snapshot import EventContext # noqa: F401
|
from synapse.events.snapshot import EventContext # noqa: F401
|
||||||
|
from synapse.events.utils import prune_event_dict
|
||||||
from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
|
from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
|
||||||
from synapse.logging.utils import log_function
|
from synapse.logging.utils import log_function
|
||||||
from synapse.metrics import BucketCollector
|
from synapse.metrics import BucketCollector
|
||||||
@ -262,6 +263,13 @@ class EventsStore(
|
|||||||
|
|
||||||
hs.get_clock().looping_call(read_forward_extremities, 60 * 60 * 1000)
|
hs.get_clock().looping_call(read_forward_extremities, 60 * 60 * 1000)
|
||||||
|
|
||||||
|
def _censor_redactions():
|
||||||
|
return run_as_background_process(
|
||||||
|
"_censor_redactions", self._censor_redactions
|
||||||
|
)
|
||||||
|
|
||||||
|
hs.get_clock().looping_call(_censor_redactions, 10 * 60 * 1000)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _read_forward_extremities(self):
|
def _read_forward_extremities(self):
|
||||||
def fetch(txn):
|
def fetch(txn):
|
||||||
@ -1548,6 +1556,84 @@ class EventsStore(
|
|||||||
(event.event_id, event.redacts),
|
(event.event_id, event.redacts),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _censor_redactions(self):
|
||||||
|
"""Censors all redactions older than a month that haven't been censored.
|
||||||
|
|
||||||
|
By censor we mean update the event_json table with the redacted event.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Deferred
|
||||||
|
"""
|
||||||
|
|
||||||
|
if self.stream_ordering_month_ago is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
max_pos = self.stream_ordering_month_ago
|
||||||
|
|
||||||
|
# We fetch all redactions that point to an event that we have that has
|
||||||
|
# a stream ordering from over a month ago, that we haven't yet censored
|
||||||
|
# in the DB.
|
||||||
|
sql = """
|
||||||
|
SELECT er.event_id, redacts FROM redactions
|
||||||
|
INNER JOIN events AS er USING (event_id)
|
||||||
|
INNER JOIN events AS eb ON (er.room_id = eb.room_id AND redacts = eb.event_id)
|
||||||
|
WHERE NOT have_censored
|
||||||
|
AND ? <= er.stream_ordering AND er.stream_ordering <= ?
|
||||||
|
ORDER BY er.stream_ordering ASC
|
||||||
|
LIMIT ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
rows = yield self._execute(
|
||||||
|
"_censor_redactions_fetch", None, sql, -max_pos, max_pos, 100
|
||||||
|
)
|
||||||
|
|
||||||
|
updates = []
|
||||||
|
|
||||||
|
for redaction_id, event_id in rows:
|
||||||
|
redaction_event = yield self.get_event(redaction_id, allow_none=True)
|
||||||
|
original_event = yield self.get_event(
|
||||||
|
event_id, allow_rejected=True, allow_none=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# The SQL above ensures that we have both the redaction and
|
||||||
|
# original event, so if the `get_event` calls return None it
|
||||||
|
# means that the redaction wasn't allowed. Either way we know that
|
||||||
|
# the result won't change so we mark the fact that we've checked.
|
||||||
|
if (
|
||||||
|
redaction_event
|
||||||
|
and original_event
|
||||||
|
and original_event.internal_metadata.is_redacted()
|
||||||
|
):
|
||||||
|
# Redaction was allowed
|
||||||
|
pruned_json = encode_canonical_json(
|
||||||
|
prune_event_dict(original_event.get_dict())
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Redaction wasn't allowed
|
||||||
|
pruned_json = None
|
||||||
|
|
||||||
|
updates.append((redaction_id, event_id, pruned_json))
|
||||||
|
|
||||||
|
def _update_censor_txn(txn):
|
||||||
|
for redaction_id, event_id, pruned_json in updates:
|
||||||
|
if pruned_json:
|
||||||
|
self._simple_update_one_txn(
|
||||||
|
txn,
|
||||||
|
table="event_json",
|
||||||
|
keyvalues={"event_id": event_id},
|
||||||
|
updatevalues={"json": pruned_json},
|
||||||
|
)
|
||||||
|
|
||||||
|
self._simple_update_one_txn(
|
||||||
|
txn,
|
||||||
|
table="redactions",
|
||||||
|
keyvalues={"event_id": redaction_id},
|
||||||
|
updatevalues={"have_censored": True},
|
||||||
|
)
|
||||||
|
|
||||||
|
yield self.runInteraction("_update_censor_txn", _update_censor_txn)
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def count_daily_messages(self):
|
def count_daily_messages(self):
|
||||||
"""
|
"""
|
||||||
|
17
synapse/storage/schema/delta/56/redaction_censor.sql
Normal file
17
synapse/storage/schema/delta/56/redaction_censor.sql
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
/* Copyright 2019 The Matrix.org Foundation C.I.C.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
ALTER TABLE redactions ADD COLUMN have_censored BOOL NOT NULL DEFAULT false;
|
||||||
|
CREATE INDEX redactions_have_censored ON redactions(event_id) WHERE not have_censored;
|
@ -17,6 +17,8 @@
|
|||||||
|
|
||||||
from mock import Mock
|
from mock import Mock
|
||||||
|
|
||||||
|
from canonicaljson import json
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
|
||||||
from synapse.api.constants import EventTypes, Membership
|
from synapse.api.constants import EventTypes, Membership
|
||||||
@ -286,3 +288,72 @@ class RedactionTestCase(unittest.HomeserverTestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
fetched.unsigned["redacted_because"].event_id, redaction_event_id2
|
fetched.unsigned["redacted_because"].event_id, redaction_event_id2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_redact_censor(self):
|
||||||
|
"""Test that a redacted event gets censored in the DB after a month
|
||||||
|
"""
|
||||||
|
|
||||||
|
self.get_success(
|
||||||
|
self.inject_room_member(self.room1, self.u_alice, Membership.JOIN)
|
||||||
|
)
|
||||||
|
|
||||||
|
msg_event = self.get_success(self.inject_message(self.room1, self.u_alice, "t"))
|
||||||
|
|
||||||
|
# Check event has not been redacted:
|
||||||
|
event = self.get_success(self.store.get_event(msg_event.event_id))
|
||||||
|
|
||||||
|
self.assertObjectHasAttributes(
|
||||||
|
{
|
||||||
|
"type": EventTypes.Message,
|
||||||
|
"user_id": self.u_alice.to_string(),
|
||||||
|
"content": {"body": "t", "msgtype": "message"},
|
||||||
|
},
|
||||||
|
event,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse("redacted_because" in event.unsigned)
|
||||||
|
|
||||||
|
# Redact event
|
||||||
|
reason = "Because I said so"
|
||||||
|
self.get_success(
|
||||||
|
self.inject_redaction(self.room1, msg_event.event_id, self.u_alice, reason)
|
||||||
|
)
|
||||||
|
|
||||||
|
event = self.get_success(self.store.get_event(msg_event.event_id))
|
||||||
|
|
||||||
|
self.assertTrue("redacted_because" in event.unsigned)
|
||||||
|
|
||||||
|
self.assertObjectHasAttributes(
|
||||||
|
{
|
||||||
|
"type": EventTypes.Message,
|
||||||
|
"user_id": self.u_alice.to_string(),
|
||||||
|
"content": {},
|
||||||
|
},
|
||||||
|
event,
|
||||||
|
)
|
||||||
|
|
||||||
|
event_json = self.get_success(
|
||||||
|
self.store._simple_select_one_onecol(
|
||||||
|
table="event_json",
|
||||||
|
keyvalues={"event_id": msg_event.event_id},
|
||||||
|
retcol="json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assert_dict(
|
||||||
|
{"content": {"body": "t", "msgtype": "message"}}, json.loads(event_json)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Advance by 30 days
|
||||||
|
self.reactor.advance(60 * 60 * 24 * 31)
|
||||||
|
self.reactor.advance(60 * 60 * 2)
|
||||||
|
|
||||||
|
event_json = self.get_success(
|
||||||
|
self.store._simple_select_one_onecol(
|
||||||
|
table="event_json",
|
||||||
|
keyvalues={"event_id": msg_event.event_id},
|
||||||
|
retcol="json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assert_dict({"content": {}}, json.loads(event_json))
|
||||||
|
Loading…
Reference in New Issue
Block a user