delete_local_events for purge_history

Add a flag which makes the purger delete local events
This commit is contained in:
Richard van der Hoff 2018-02-08 18:44:52 +00:00
parent e571aef06d
commit 74fcbf741b
5 changed files with 67 additions and 15 deletions

View File

@ -4,8 +4,6 @@ Purge History API
The purge history API allows server admins to purge historic events from their The purge history API allows server admins to purge historic events from their
database, reclaiming disk space. database, reclaiming disk space.
**NB!** This will not delete local events (locally sent messages content etc) from the database, but will remove lots of the metadata about them and does dramatically reduce the on disk space usage
Depending on the amount of history being purged a call to the API may take Depending on the amount of history being purged a call to the API may take
several minutes or longer. During this period users will not be able to several minutes or longer. During this period users will not be able to
paginate further back in the room from the point being purged from. paginate further back in the room from the point being purged from.
@ -15,3 +13,15 @@ The API is simply:
``POST /_matrix/client/r0/admin/purge_history/<room_id>/<event_id>`` ``POST /_matrix/client/r0/admin/purge_history/<room_id>/<event_id>``
including an ``access_token`` of a server admin. including an ``access_token`` of a server admin.
By default, events sent by local users are not deleted, as they may represent
the only copies of this content in existence. (Events sent by remote users are
deleted, and room state data before the cutoff is always removed).
To delete local events as well, set ``delete_local_events`` in the body:
.. code:: json
{
"delete_local_events": True,
}

View File

@ -63,7 +63,7 @@ class MessageHandler(BaseHandler):
self.spam_checker = hs.get_spam_checker() self.spam_checker = hs.get_spam_checker()
@defer.inlineCallbacks @defer.inlineCallbacks
def purge_history(self, room_id, event_id): def purge_history(self, room_id, event_id, delete_local_events=False):
event = yield self.store.get_event(event_id) event = yield self.store.get_event(event_id)
if event.room_id != room_id: if event.room_id != room_id:
@ -72,7 +72,7 @@ class MessageHandler(BaseHandler):
depth = event.depth depth = event.depth
with (yield self.pagination_lock.write(room_id)): with (yield self.pagination_lock.write(room_id)):
yield self.store.purge_history(room_id, depth) yield self.store.purge_history(room_id, depth, delete_local_events)
@defer.inlineCallbacks @defer.inlineCallbacks
def get_messages(self, requester, room_id=None, pagin_config=None, def get_messages(self, requester, room_id=None, pagin_config=None,

View File

@ -148,11 +148,13 @@ def parse_string_from_args(args, name, default=None, required=False,
return default return default
def parse_json_value_from_request(request): def parse_json_value_from_request(request, allow_empty_body=False):
"""Parse a JSON value from the body of a twisted HTTP request. """Parse a JSON value from the body of a twisted HTTP request.
Args: Args:
request: the twisted HTTP request. request: the twisted HTTP request.
allow_empty_body (bool): if True, an empty body will be accepted and
turned into None
Returns: Returns:
The JSON value. The JSON value.
@ -165,6 +167,9 @@ def parse_json_value_from_request(request):
except Exception: except Exception:
raise SynapseError(400, "Error reading JSON content.") raise SynapseError(400, "Error reading JSON content.")
if not content_bytes and allow_empty_body:
return None
try: try:
content = simplejson.loads(content_bytes) content = simplejson.loads(content_bytes)
except Exception as e: except Exception as e:
@ -174,17 +179,24 @@ def parse_json_value_from_request(request):
return content return content
def parse_json_object_from_request(request): def parse_json_object_from_request(request, allow_empty_body=False):
"""Parse a JSON object from the body of a twisted HTTP request. """Parse a JSON object from the body of a twisted HTTP request.
Args: Args:
request: the twisted HTTP request. request: the twisted HTTP request.
allow_empty_body (bool): if True, an empty body will be accepted and
turned into an empty dict.
Raises: Raises:
SynapseError if the request body couldn't be decoded as JSON or SynapseError if the request body couldn't be decoded as JSON or
if it wasn't a JSON object. if it wasn't a JSON object.
""" """
content = parse_json_value_from_request(request) content = parse_json_value_from_request(
request, allow_empty_body=allow_empty_body,
)
if allow_empty_body and content is None:
return {}
if type(content) != dict: if type(content) != dict:
message = "Content must be a JSON object." message = "Content must be a JSON object."

View File

@ -128,7 +128,16 @@ class PurgeHistoryRestServlet(ClientV1RestServlet):
if not is_admin: if not is_admin:
raise AuthError(403, "You are not a server admin") raise AuthError(403, "You are not a server admin")
yield self.handlers.message_handler.purge_history(room_id, event_id) body = parse_json_object_from_request(request, allow_empty_body=True)
delete_local_events = bool(
body.get("delete_local_history", False)
)
yield self.handlers.message_handler.purge_history(
room_id, event_id,
delete_local_events=delete_local_events,
)
defer.returnValue((200, {})) defer.returnValue((200, {}))

View File

@ -2031,16 +2031,32 @@ class EventsStore(SQLBaseStore):
) )
return self.runInteraction("get_all_new_events", get_all_new_events_txn) return self.runInteraction("get_all_new_events", get_all_new_events_txn)
def purge_history(self, room_id, topological_ordering): def purge_history(
self, room_id, topological_ordering, delete_local_events,
):
"""Deletes room history before a certain point """Deletes room history before a certain point
Args:
room_id (str):
topological_ordering (int):
minimum topo ordering to preserve
delete_local_events (bool):
if True, we will delete local events as well as remote ones
(instead of just marking them as outliers and deleting their
state groups).
""" """
return self.runInteraction( return self.runInteraction(
"purge_history", "purge_history",
self._purge_history_txn, room_id, topological_ordering self._purge_history_txn, room_id, topological_ordering,
delete_local_events,
) )
def _purge_history_txn(self, txn, room_id, topological_ordering): def _purge_history_txn(
self, txn, room_id, topological_ordering, delete_local_events,
):
# Tables that should be pruned: # Tables that should be pruned:
# event_auth # event_auth
# event_backward_extremities # event_backward_extremities
@ -2093,11 +2109,14 @@ class EventsStore(SQLBaseStore):
to_delete = [ to_delete = [
(event_id,) for event_id, state_key in event_rows (event_id,) for event_id, state_key in event_rows
if state_key is None and not self.hs.is_mine_id(event_id) if state_key is None and (
delete_local_events or not self.hs.is_mine_id(event_id)
)
] ]
logger.info( logger.info(
"[purge] found %i events before cutoff, of which %i are remote" "[purge] found %i events before cutoff, of which %i can be deleted",
" non-state events to delete", len(event_rows), len(to_delete)) len(event_rows), len(to_delete),
)
logger.info("[purge] Finding new backward extremities") logger.info("[purge] Finding new backward extremities")
@ -2273,7 +2292,9 @@ class EventsStore(SQLBaseStore):
" WHERE event_id = ?", " WHERE event_id = ?",
[ [
(True, event_id,) for event_id, state_key in event_rows (True, event_id,) for event_id, state_key in event_rows
if state_key is not None or self.hs.is_mine_id(event_id) if state_key is not None or (
not delete_local_events and self.hs.is_mine_id(event_id)
)
] ]
) )