diff --git a/src/Mjolnir.ts b/src/Mjolnir.ts index b134afd..8ce085d 100644 --- a/src/Mjolnir.ts +++ b/src/Mjolnir.ts @@ -34,8 +34,9 @@ import { logMessage } from "./LogProxy"; import ErrorCache, { ERROR_KIND_FATAL, ERROR_KIND_PERMISSION } from "./ErrorCache"; import { IProtection } from "./protections/IProtection"; import { PROTECTIONS } from "./protections/protections"; -import { AutomaticRedactionQueue } from "./queues/AutomaticRedactionQueue"; +import { UnlistedUserRedactionQueue } from "./queues/UnlistedUserRedactionQueue"; import { Healthz } from "./health/healthz"; +import { EventRedactionQueue, RedactUserInRoom } from "./queues/EventRedactionQueue"; export const STATE_NOT_STARTED = "not_started"; export const STATE_CHECKING_PERMISSIONS = "checking_permissions"; @@ -53,7 +54,16 @@ export class Mjolnir { private localpart: string; private currentState: string = STATE_NOT_STARTED; private protections: IProtection[] = []; - private redactionQueue = new AutomaticRedactionQueue(); + /** + * This is for users who are not listed on a watchlist, + * but have been flagged by the automatic spam detection as suispicous + */ + private unlistedUserRedactionQueue = new UnlistedUserRedactionQueue(); + /** + * This is a queue for redactions to process after mjolnir + * has finished applying ACL and bans when syncing. + */ + private eventRedactionQueue = new EventRedactionQueue(); private automaticRedactionReasons: MatrixGlob[] = []; private protectedJoinedRoomIds: string[] = []; private explicitlyProtectedRoomIds: string[] = []; @@ -138,8 +148,13 @@ export class Mjolnir { return this.protections; } - public get redactionHandler(): AutomaticRedactionQueue { - return this.redactionQueue; + /** + * Returns the handler to flag a user for redaction, removing any future messages that they send. + * Typically this is used by the flooding or image protection on users that have not been banned from a list yet. + * It cannot used to redact any previous messages the user has sent, in that cas you should use the `EventRedactionQueue`. + */ + public get unlistedUserRedactionHandler(): UnlistedUserRedactionQueue { + return this.unlistedUserRedactionQueue; } public get automaticRedactGlobs(): MatrixGlob[] { @@ -484,6 +499,10 @@ export class Mjolnir { return errors; } + /** + * Sync all the rooms with all the watched lists, banning and applying any changed ACLS. + * @param verbose Whether to report any errors to the management room. + */ public async syncLists(verbose = true) { for (const list of this.banLists) { await list.updateList(); @@ -493,8 +512,10 @@ export class Mjolnir { const aclErrors = await applyServerAcls(this.banLists, Object.keys(this.protectedRooms), this); const banErrors = await applyUserBans(this.banLists, Object.keys(this.protectedRooms), this); + const redactionErrors = await this.processRedactionQueue(); hadErrors = hadErrors || await this.printActionResult(aclErrors, "Errors updating server ACLs:"); hadErrors = hadErrors || await this.printActionResult(banErrors, "Errors updating member bans:"); + hadErrors = hadErrors || await this.printActionResult(redactionErrors, "Error updating redactions:"); if (!hadErrors && verbose) { const html = `Done updating rooms - no errors`; @@ -521,8 +542,10 @@ export class Mjolnir { const aclErrors = await applyServerAcls(this.banLists, Object.keys(this.protectedRooms), this); const banErrors = await applyUserBans(this.banLists, Object.keys(this.protectedRooms), this); + const redactionErrors = await this.processRedactionQueue(); hadErrors = hadErrors || await this.printActionResult(aclErrors, "Errors updating server ACLs:"); hadErrors = hadErrors || await this.printActionResult(banErrors, "Errors updating member bans:"); + hadErrors = hadErrors || await this.printActionResult(redactionErrors, "Error updating redactions:"); if (!hadErrors) { const html = `Done updating rooms - no errors`; @@ -575,7 +598,7 @@ export class Mjolnir { // Run the event handlers - we always run this after protections so that the protections // can flag the event for redaction. - await this.redactionQueue.handleEvent(roomId, event, this.client); + await this.unlistedUserRedactionHandler.handleEvent(roomId, event, this.client); if (event['type'] === 'm.room.power_levels' && event['state_key'] === '') { // power levels were updated - recheck permissions @@ -588,9 +611,14 @@ export class Mjolnir { } return; } else if (event['type'] === "m.room.member") { - // Only apply bans in the room we're looking at. - const errors = await applyUserBans(this.banLists, [roomId], this); - await this.printActionResult(errors); + // The reason we have to apply bans on each member change is because + // we cannot eagerly ban users (that is to ban them when they have never been a member) + // as they can be force joined to a room they might not have known existed. + // Only apply bans and then redactions in the room we are currently looking at. + const banErrors = await applyUserBans(this.banLists, [roomId], this); + const redactionErrors = await this.processRedactionQueue(roomId); + await this.printActionResult(banErrors); + await this.printActionResult(redactionErrors); } } } @@ -658,4 +686,20 @@ export class Mjolnir { message: message /* If `undefined`, we'll use Synapse's default message. */ }); } + + public queueRedactUserMessagesIn(userId: string, roomId: string) { + this.eventRedactionQueue.add(new RedactUserInRoom(userId, roomId)); + } + + /** + * Process all queued redactions, this is usually called at the end of the sync process, + * after all users have been banned and ACLs applied. + * If a redaction cannot be processed, the redaction is skipped and removed from the queue. + * We then carry on processing the next redactions. + * @param roomId Limit processing to one room only, otherwise process redactions for all rooms. + * @returns The list of errors encountered, for reporting to the management room. + */ + public async processRedactionQueue(roomId?: string): Promise { + return await this.eventRedactionQueue.process(this.client, roomId); + } } diff --git a/src/actions/ApplyBan.ts b/src/actions/ApplyBan.ts index 7731767..b7922ad 100644 --- a/src/actions/ApplyBan.ts +++ b/src/actions/ApplyBan.ts @@ -21,7 +21,6 @@ import config from "../config"; import { logMessage } from "../LogProxy"; import { LogLevel } from "matrix-bot-sdk"; import { ERROR_KIND_FATAL, ERROR_KIND_PERMISSION } from "../ErrorCache"; -import { redactUserMessagesIn } from "../utils"; /** * Applies the member bans represented by the ban lists to the provided rooms, returning the @@ -69,7 +68,7 @@ export async function applyUserBans(lists: BanList[], roomIds: string[], mjolnir if (!config.noop) { await mjolnir.client.banUser(member.userId, roomId, userRule.reason); if (mjolnir.automaticRedactGlobs.find(g => g.test(userRule.reason.toLowerCase()))) { - await redactUserMessagesIn(mjolnir.client, member.userId, [roomId]); + mjolnir.queueRedactUserMessagesIn(member.userId, roomId); } } else { await logMessage(LogLevel.WARN, "ApplyBan", `Tried to ban ${member.userId} in ${roomId} but Mjolnir is running in no-op mode`, roomId); diff --git a/src/protections/BasicFlooding.ts b/src/protections/BasicFlooding.ts index 92d6a86..017b568 100644 --- a/src/protections/BasicFlooding.ts +++ b/src/protections/BasicFlooding.ts @@ -65,7 +65,7 @@ export class BasicFlooding implements IProtection { } if (this.recentlyBanned.includes(event['sender'])) return; // already handled (will be redacted) - mjolnir.redactionHandler.addUser(event['sender']); + mjolnir.unlistedUserRedactionHandler.addUser(event['sender']); this.recentlyBanned.push(event['sender']); // flag to reduce spam // Redact all the things the user said too diff --git a/src/protections/FirstMessageIsImage.ts b/src/protections/FirstMessageIsImage.ts index c42a63c..89a7865 100644 --- a/src/protections/FirstMessageIsImage.ts +++ b/src/protections/FirstMessageIsImage.ts @@ -59,7 +59,7 @@ export class FirstMessageIsImage implements IProtection { } if (this.recentlyBanned.includes(event['sender'])) return; // already handled (will be redacted) - mjolnir.redactionHandler.addUser(event['sender']); + mjolnir.unlistedUserRedactionHandler.addUser(event['sender']); this.recentlyBanned.push(event['sender']); // flag to reduce spam // Redact the event diff --git a/src/queues/EventRedactionQueue.ts b/src/queues/EventRedactionQueue.ts new file mode 100644 index 0000000..f71b918 --- /dev/null +++ b/src/queues/EventRedactionQueue.ts @@ -0,0 +1,147 @@ +/* +Copyright 2019-2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { LogLevel, MatrixClient } from "matrix-bot-sdk" +import { ERROR_KIND_FATAL } from "../ErrorCache"; +import { logMessage } from "../LogProxy"; +import { RoomUpdateError } from "../models/RoomUpdateError"; +import { redactUserMessagesIn } from "../utils"; + +export interface QueuedRedaction { + /** The room which the redaction will take place in. */ + readonly roomId: string; + /** + * Carry out the redaction. + * Called by the EventRedactionQueue. + * @param client A MatrixClient to use to carry out the redaction. + */ + redact(client: MatrixClient): Promise + /** + * Used to test whether the redaction is the equivalent to another redaction. + * @param redaction Another QueuedRedaction to test if this redaction is an equivalent to. + */ + redactionEqual(redaction: QueuedRedaction): boolean +} + +/** + * Redacts all of the messages a user has sent to one room. + */ +export class RedactUserInRoom implements QueuedRedaction { + userId: string; + roomId: string; + + constructor(userId: string, roomId: string) { + this.userId = userId; + this.roomId = roomId; + } + + public async redact(client: MatrixClient) { + await logMessage(LogLevel.DEBUG, "Mjolnir", `Redacting events from ${this.userId} in room ${this.roomId}.`); + await redactUserMessagesIn(client, this.userId, [this.roomId]); + } + + public redactionEqual(redaction: QueuedRedaction): boolean { + if (redaction instanceof RedactUserInRoom) { + return redaction.userId === this.userId && redaction.roomId === this.roomId; + } else { + return false; + } + } +} +/** + * This is a queue for events so that other protections can happen first (e.g. applying room bans to every room). + */ +export class EventRedactionQueue { + /** + * This map is indexed by roomId and its values are a list of redactions waiting to be processed for that room. + */ + private toRedact: Map = new Map(); + + /** + * Test whether the redaction is already present in the queue. + * @param redaction a QueuedRedaction. + * @returns True if the queue already has the redaction, false otherwise. + */ + public has(redaction: QueuedRedaction): boolean { + return !!this.toRedact.get(redaction.roomId)?.find(r => r.redactionEqual(redaction)); + } + + /** + * Adds a `QueuedRedaction` to the queue. It will be processed when `process` is called. + * @param redaction A `QueuedRedaction` to await processing + * @returns `true` if the redaction was added to the queue, `false` if it is a duplicate of a redaction already present in the queue. + */ + public add(redaction: QueuedRedaction): boolean { + if (this.has(redaction)) { + return false; + } else { + let entry = this.toRedact.get(redaction.roomId); + if (entry) { + entry.push(redaction); + } else { + this.toRedact.set(redaction.roomId, [redaction]); + } + return true; + } + } + + /** + * Process the redaction queue, carrying out the action of each `QueuedRedaction` in sequence. + * If a redaction cannot be processed, the redaction is skipped and removed from the queue. + * We then carry on processing the next redactions. + * The reason we skip is at the moment is that we would have to think about all of the situations + * where we would not want failures to try again (e.g. messages were already redacted) and handle them explicitly. + * @param client The matrix client to use for processing redactions. + * @param limitToRoomId If the roomId is provided, only redactions for that room will be processed. + * @returns A description of any errors encountered by each QueuedRedaction that was processed. + */ + public async process(client: MatrixClient, limitToRoomId?: string): Promise { + const errors: RoomUpdateError[] = []; + const redact = async (currentBatch: QueuedRedaction[]) => { + for (const redaction of currentBatch) { + try { + await redaction.redact(client); + } catch (e) { + let roomError: RoomUpdateError; + if (e.roomId && e.errorMessage && e.errorKind) { + roomError = e; + } else { + const message = e.message || (e.body ? e.body.error : ''); + roomError = { + roomId: redaction.roomId, + errorMessage: message, + errorKind: ERROR_KIND_FATAL, + }; + } + errors.push(roomError); + } + } + } + if (limitToRoomId) { + // There might not actually be any queued redactions for this room. + let queuedRedactions = this.toRedact.get(limitToRoomId); + if (queuedRedactions) { + this.toRedact.delete(limitToRoomId); + await redact(queuedRedactions); + } + } else { + for (const [roomId, redactions] of this.toRedact) { + this.toRedact.delete(roomId); + await redact(redactions); + } + } + return errors; + } +} diff --git a/src/queues/AutomaticRedactionQueue.ts b/src/queues/UnlistedUserRedactionQueue.ts similarity index 81% rename from src/queues/AutomaticRedactionQueue.ts rename to src/queues/UnlistedUserRedactionQueue.ts index a2a4efb..1d3cbd6 100644 --- a/src/queues/AutomaticRedactionQueue.ts +++ b/src/queues/UnlistedUserRedactionQueue.ts @@ -13,12 +13,18 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - import { extractRequestError, LogLevel, LogService, MatrixClient, Permalinks } from "matrix-bot-sdk"; import { logMessage } from "../LogProxy"; import config from "../config"; -export class AutomaticRedactionQueue { +/** + * A queue of users who have been flagged for redaction typically by the flooding or image protection. + * Specifically any new events sent by a queued user will be redacted. + * This does not handle previously sent events, for that see the `EventRedactionQueue`. + * These users are not listed as banned in any watch list and so may continue + * to view a room until a moderator can investigate. + */ +export class UnlistedUserRedactionQueue { private usersToRedact: Set = new Set(); constructor() {