From 9b923d337d3bfea6e817b30055c9e07582615906 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 20 May 2021 10:01:30 +0200 Subject: [PATCH] write redactions during sync --- src/matrix/room/BaseRoom.js | 2 + src/matrix/room/Room.js | 11 ++- src/matrix/room/common.js | 2 + src/matrix/room/sending/PendingEvent.js | 2 +- src/matrix/room/sending/SendQueue.js | 3 +- .../room/timeline/entries/EventEntry.js | 6 +- .../room/timeline/persistence/GapWriter.js | 12 ++- .../timeline/persistence/RelationWriter.js | 99 +++++++++++++++++++ .../room/timeline/persistence/SyncWriter.js | 30 +++--- 9 files changed, 144 insertions(+), 23 deletions(-) create mode 100644 src/matrix/room/timeline/persistence/RelationWriter.js diff --git a/src/matrix/room/BaseRoom.js b/src/matrix/room/BaseRoom.js index 924e5316..e6e7ae33 100644 --- a/src/matrix/room/BaseRoom.js +++ b/src/matrix/room/BaseRoom.js @@ -288,6 +288,8 @@ export class BaseRoom extends EventEmitter { this._applyGapFill(extraGapFillChanges); } if (this._timeline) { + // these should not be added if not already there + this._timeline.replaceEntries(gapResult.updatedEntries); this._timeline.addOrReplaceEntries(gapResult.entries); } }); diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index 57382831..c743dcb8 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -106,9 +106,8 @@ export class Room extends BaseRoom { txn.roomState.removeAllForRoom(this.id); txn.roomMembers.removeAllForRoom(this.id); } - const {entries: newEntries, newLiveKey, memberChanges} = + const {entries: newEntries, updatedEntries, newLiveKey, memberChanges} = await log.wrap("syncWriter", log => this._syncWriter.writeSync(roomResponse, isRejoin, txn, log), log.level.Detail); - let allEntries = newEntries; if (decryptChanges) { const decryption = await log.wrap("decryptChanges", log => decryptChanges.write(txn, log)); log.set("decryptionResults", decryption.results.size); @@ -119,16 +118,18 @@ export class Room extends BaseRoom { decryption.applyToEntries(newEntries); if (retryEntries?.length) { decryption.applyToEntries(retryEntries); - allEntries = retryEntries.concat(allEntries); + updatedEntries.push(...retryEntries); } } - log.set("allEntries", allEntries.length); + log.set("newEntries", newEntries.length); + log.set("updatedEntries", updatedEntries.length); let shouldFlushKeyShares = false; // pass member changes to device tracker if (roomEncryption && this.isTrackingMembers && memberChanges?.size) { shouldFlushKeyShares = await roomEncryption.writeMemberChanges(memberChanges, txn, log); log.set("shouldFlushKeyShares", shouldFlushKeyShares); } + const allEntries = newEntries.concat(updatedEntries); // also apply (decrypted) timeline entries to the summary changes summaryChanges = summaryChanges.applyTimelineEntries( allEntries, isInitialSync, !this._isTimelineOpen, this._user.id); @@ -164,7 +165,7 @@ export class Room extends BaseRoom { summaryChanges, roomEncryption, newEntries, - updatedEntries: retryEntries || [], + updatedEntries, newLiveKey, removedPendingEvents, memberChanges, diff --git a/src/matrix/room/common.js b/src/matrix/room/common.js index 922ca115..721160e6 100644 --- a/src/matrix/room/common.js +++ b/src/matrix/room/common.js @@ -19,3 +19,5 @@ export function getPrevContentFromStateEvent(event) { // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz return event.unsigned?.prev_content || event.prev_content; } + +export const REDACTION_TYPE = "m.room.redaction"; diff --git a/src/matrix/room/sending/PendingEvent.js b/src/matrix/room/sending/PendingEvent.js index f62cbc64..3d0c3fe1 100644 --- a/src/matrix/room/sending/PendingEvent.js +++ b/src/matrix/room/sending/PendingEvent.js @@ -15,8 +15,8 @@ limitations under the License. */ import {createEnum} from "../../../utils/enum.js"; import {AbortError} from "../../../utils/error.js"; +import {REDACTION_TYPE} from "../common.js"; import {isTxnId} from "../../common.js"; -import {REDACTION_TYPE} from "./SendQueue.js"; export const SendStatus = createEnum( "Waiting", diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js index 1420b56e..533a29b3 100644 --- a/src/matrix/room/sending/SendQueue.js +++ b/src/matrix/room/sending/SendQueue.js @@ -18,8 +18,7 @@ import {SortedArray} from "../../../observable/list/SortedArray.js"; import {ConnectionError} from "../../error.js"; import {PendingEvent} from "./PendingEvent.js"; import {makeTxnId, isTxnId} from "../../common.js"; - -export const REDACTION_TYPE = "m.room.redaction"; +import {REDACTION_TYPE} from "../common.js"; export class SendQueue { constructor({roomId, storage, hsApi, pendingEvents}) { diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js index 88a0aa5e..5f2496d8 100644 --- a/src/matrix/room/timeline/entries/EventEntry.js +++ b/src/matrix/room/timeline/entries/EventEntry.js @@ -108,4 +108,8 @@ export class EventEntry extends BaseEntry { get decryptionError() { return this._decryptionError; } -} + + get relatedEventId() { + return this._eventEntry.event.redacts; + } +} \ No newline at end of file diff --git a/src/matrix/room/timeline/persistence/GapWriter.js b/src/matrix/room/timeline/persistence/GapWriter.js index ebd2bedf..860265f5 100644 --- a/src/matrix/room/timeline/persistence/GapWriter.js +++ b/src/matrix/room/timeline/persistence/GapWriter.js @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {RelationWriter} from "./RelationWriter.js"; import {EventKey} from "../EventKey.js"; import {EventEntry} from "../entries/EventEntry.js"; import {createEventEntry, directionalAppend} from "./common.js"; @@ -24,6 +25,7 @@ export class GapWriter { this._roomId = roomId; this._storage = storage; this._fragmentIdComparer = fragmentIdComparer; + this._relationWriter = new RelationWriter(roomId, fragmentIdComparer); } // events is in reverse-chronological order (last event comes at index 0) if backwards async _findOverlappingEvents(fragmentEntry, events, txn, log) { @@ -105,6 +107,7 @@ export class GapWriter { _storeEvents(events, startKey, direction, state, txn) { const entries = []; + const updatedEntries = []; // events is in reverse chronological order for backwards pagination, // e.g. order is moving away from the `from` point. let key = startKey; @@ -120,6 +123,10 @@ export class GapWriter { txn.timelineEvents.insert(eventStorageEntry); const eventEntry = new EventEntry(eventStorageEntry, this._fragmentIdComparer); directionalAppend(entries, eventEntry, direction); + const updatedRelationTargetEntry = this._relationWriter.writeRelation(eventEntry); + if (updatedRelationTargetEntry) { + updatedEntries.push(updatedRelationTargetEntry); + } } return entries; } @@ -201,7 +208,6 @@ export class GapWriter { // chunk is in reverse-chronological order when backwards const {chunk, start, state} = response; let {end} = response; - let entries; if (!Array.isArray(chunk)) { throw new Error("Invalid chunk in response"); @@ -240,9 +246,9 @@ export class GapWriter { end = null; } // create entries for all events in chunk, add them to entries - entries = this._storeEvents(nonOverlappingEvents, lastKey, direction, state, txn); + const {entries, updatedEntries} = this._storeEvents(nonOverlappingEvents, lastKey, direction, state, txn); const fragments = await this._updateFragments(fragmentEntry, neighbourFragmentEntry, end, entries, txn); - return {entries, fragments}; + return {entries, updatedEntries, fragments}; } } diff --git a/src/matrix/room/timeline/persistence/RelationWriter.js b/src/matrix/room/timeline/persistence/RelationWriter.js new file mode 100644 index 00000000..67a78bb3 --- /dev/null +++ b/src/matrix/room/timeline/persistence/RelationWriter.js @@ -0,0 +1,99 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {EventEntry} from "../entries/EventEntry.js"; +import {REDACTION_TYPE} from "../../common.js"; + +export class RelationWriter { + constructor(roomId, fragmentIdComparer) { + this._roomId = roomId; + this._fragmentIdComparer = fragmentIdComparer; + } + + // this needs to happen again after decryption too for edits + async writeRelation(sourceEntry, txn) { + if (sourceEntry.relatedEventId) { + const target = await txn.timelineEvents.getByEventId(this._roomId, sourceEntry.relatedEventId); + if (target) { + if (this._applyRelation(sourceEntry, target)) { + txn.timelineEvents.update(target); + return new EventEntry(target, this._fragmentIdComparer); + } + } + } + return; + } + + _applyRelation(sourceEntry, target) { + if (sourceEntry.eventType === REDACTION_TYPE) { + return this._applyRedaction(sourceEntry.event, target.event); + } else { + return false; + } + } + + _applyRedaction(redactionEvent, targetEvent) { + // TODO: should we make efforts to preserve the decrypted event type? + // probably ok not to, as we'll show whatever is deleted as "deleted message" + // reactions are the only thing that comes to mind, but we don't encrypt those (for now) + for (const key of Object.keys(targetEvent)) { + if (!_REDACT_KEEP_KEY_MAP[key]) { + delete targetEvent[key]; + } + } + const {content} = targetEvent; + const keepMap = _REDACT_KEEP_CONTENT_MAP[targetEvent.type]; + for (const key of Object.keys(content)) { + if (!keepMap?.[key]) { + delete content[key]; + } + } + targetEvent.unsigned = targetEvent.unsigned || {}; + targetEvent.unsigned.redacted_because = redactionEvent; + + return true; + } +} + +// copied over from matrix-js-sdk, copyright 2016 OpenMarket Ltd +/* _REDACT_KEEP_KEY_MAP gives the keys we keep when an event is redacted + * + * This is specified here: + * http://matrix.org/speculator/spec/HEAD/client_server/latest.html#redactions + * + * Also: + * - We keep 'unsigned' since that is created by the local server + * - We keep user_id for backwards-compat with v1 + */ +const _REDACT_KEEP_KEY_MAP = [ + 'event_id', 'type', 'room_id', 'user_id', 'sender', 'state_key', 'prev_state', + 'content', 'unsigned', 'origin_server_ts', +].reduce(function(ret, val) { + ret[val] = 1; return ret; +}, {}); + +// a map from event type to the .content keys we keep when an event is redacted +const _REDACT_KEEP_CONTENT_MAP = { + 'm.room.member': {'membership': 1}, + 'm.room.create': {'creator': 1}, + 'm.room.join_rules': {'join_rule': 1}, + 'm.room.power_levels': {'ban': 1, 'events': 1, 'events_default': 1, + 'kick': 1, 'redact': 1, 'state_default': 1, + 'users': 1, 'users_default': 1, + }, + 'm.room.aliases': {'aliases': 1}, +}; +// end of matrix-js-sdk code \ No newline at end of file diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js index 0aeec413..8bf31ed2 100644 --- a/src/matrix/room/timeline/persistence/SyncWriter.js +++ b/src/matrix/room/timeline/persistence/SyncWriter.js @@ -21,6 +21,7 @@ import {FragmentBoundaryEntry} from "../entries/FragmentBoundaryEntry.js"; import {createEventEntry} from "./common.js"; import {EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../members/RoomMember.js"; import {MemberWriter} from "./MemberWriter.js"; +import {RelationWriter} from "./RelationWriter.js"; // Synapse bug? where the m.room.create event appears twice in sync response // when first syncing the room @@ -40,6 +41,7 @@ export class SyncWriter { constructor({roomId, fragmentIdComparer}) { this._roomId = roomId; this._memberWriter = new MemberWriter(roomId); + this._relationWriter = new RelationWriter(roomId, fragmentIdComparer); this._fragmentIdComparer = fragmentIdComparer; this._lastLiveKey = null; } @@ -151,7 +153,9 @@ export class SyncWriter { } } - async _writeTimeline(entries, timeline, currentKey, memberChanges, txn, log) { + async _writeTimeline(timeline, currentKey, memberChanges, txn, log) { + const entries = []; + const updatedEntries = []; if (Array.isArray(timeline?.events) && timeline.events.length) { // only create a fragment when we will really write an event currentKey = await this._ensureLiveFragment(currentKey, entries, timeline, txn, log); @@ -161,15 +165,19 @@ export class SyncWriter { for(const event of events) { // store event in timeline currentKey = currentKey.nextKey(); - const entry = createEventEntry(currentKey, this._roomId, event); + const storageEntry = createEventEntry(currentKey, this._roomId, event); let member = await this._memberWriter.lookupMember(event.sender, event, events, txn); if (member) { - entry.displayName = member.displayName; - entry.avatarUrl = member.avatarUrl; + storageEntry.displayName = member.displayName; + storageEntry.avatarUrl = member.avatarUrl; + } + txn.timelineEvents.insert(storageEntry); + const entry = new EventEntry(storageEntry, this._fragmentIdComparer); + entries.push(entry); + const updatedRelationTargetEntry = await this._relationWriter.writeRelation(entry); + if (updatedRelationTargetEntry) { + updatedEntries.push(updatedRelationTargetEntry); } - txn.timelineEvents.insert(entry); - entries.push(new EventEntry(entry, this._fragmentIdComparer)); - // update state events after writing event, so for a member event, // we only update the member info after having written the member event // to the timeline, as we want that event to have the old profile info @@ -187,7 +195,7 @@ export class SyncWriter { } log.set("timelineStateEventCount", timelineStateEventCount); } - return currentKey; + return {currentKey, entries, updatedEntries}; } async _handleRejoinOverlap(timeline, txn, log) { @@ -226,7 +234,6 @@ export class SyncWriter { * @return {SyncWriterResult} */ async writeSync(roomResponse, isRejoin, txn, log) { - const entries = []; let {timeline} = roomResponse; // we have rejoined the room after having synced it before, // check for overlap with the last synced event @@ -238,9 +245,10 @@ export class SyncWriter { // important this happens before _writeTimeline so // members are available in the transaction await this._writeStateEvents(roomResponse, memberChanges, timeline?.limited, txn, log); - const currentKey = await this._writeTimeline(entries, timeline, this._lastLiveKey, memberChanges, txn, log); + const {currentKey, entries, updatedEntries} = + await this._writeTimeline(entries, updatedEntries, timeline, this._lastLiveKey, memberChanges, txn, log); log.set("memberChanges", memberChanges.size); - return {entries, newLiveKey: currentKey, memberChanges}; + return {entries, updatedEntries, newLiveKey: currentKey, memberChanges}; } afterSync(newLiveKey) {