From d7407ecf66404f48e532b35c0ad6f54eb24b592f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 11:39:01 +0200 Subject: [PATCH 01/26] WIP --- scripts/babel-test.js | 32 ++ src/matrix/e2ee/megolm/Decryption.js | 1 + .../e2ee/megolm/decryption/ISessionSource.ts | 26 ++ src/matrix/e2ee/megolm/decryption/RoomKey.js | 166 ---------- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 300 ++++++++++++++++++ .../e2ee/megolm/decryption/SessionCache.js | 4 +- .../idb/stores/InboundGroupSessionStore.ts | 2 +- src/utils/LRUCache.js | 10 + 8 files changed, 373 insertions(+), 168 deletions(-) create mode 100644 scripts/babel-test.js create mode 100644 src/matrix/e2ee/megolm/decryption/ISessionSource.ts delete mode 100644 src/matrix/e2ee/megolm/decryption/RoomKey.js create mode 100644 src/matrix/e2ee/megolm/decryption/RoomKey.ts diff --git a/scripts/babel-test.js b/scripts/babel-test.js new file mode 100644 index 00000000..24283dc3 --- /dev/null +++ b/scripts/babel-test.js @@ -0,0 +1,32 @@ +babel = require('@babel/standalone'); + +const code = ` +async function doit() { + const foo = {bar: 5}; + const mapped = Object.values(foo).map(n => n*n); + console.log(mapped); + await Promise.resolve(); +} +doit(); +`; + +const {code: babelCode} = babel.transform(code, { + babelrc: false, + configFile: false, + presets: [ + [ + "env", + { + useBuiltIns: "entry", + modules: false, + corejs: "3.4", + targets: "IE 11", + // we provide our own promise polyfill (es6-promise) + // with support for synchronous flushing of + // the queue for idb where needed + // exclude: ["es.promise", "es.promise.all-settled", "es.promise.finally"] + } + ] + ] +}); +console.log(babelCode); diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js index 8f4714ea..30e1480f 100644 --- a/src/matrix/e2ee/megolm/Decryption.js +++ b/src/matrix/e2ee/megolm/Decryption.js @@ -118,6 +118,7 @@ export class Decryption { // look only in the cache after looking into newKeys as it may contains that are better if (!sessionInfo) { sessionInfo = sessionCache.get(roomId, senderKey, sessionId); + // TODO: shouldn't we retain here? } if (!sessionInfo) { const sessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId); diff --git a/src/matrix/e2ee/megolm/decryption/ISessionSource.ts b/src/matrix/e2ee/megolm/decryption/ISessionSource.ts new file mode 100644 index 00000000..8a96ba12 --- /dev/null +++ b/src/matrix/e2ee/megolm/decryption/ISessionSource.ts @@ -0,0 +1,26 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//import type {InboundGroupSession} from "@matrix-org/olm"; +interface InboundGroupSession {} + +export interface ISessionSource { + deserializeIntoSession(session: InboundGroupSession): void; + get roomId(): string; + get senderKey(): string; + get sessionId(): string; + get claimedEd25519Key(): string; +} diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.js b/src/matrix/e2ee/megolm/decryption/RoomKey.js deleted file mode 100644 index 6fcd738b..00000000 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.js +++ /dev/null @@ -1,166 +0,0 @@ -import {SessionInfo} from "./SessionInfo.js"; - -export class BaseRoomKey { - constructor() { - this._sessionInfo = null; - this._isBetter = null; - this._eventIds = null; - } - - async createSessionInfo(olm, pickleKey, txn) { - if (this._isBetter === false) { - return; - } - const session = new olm.InboundGroupSession(); - try { - this._loadSessionKey(session); - this._isBetter = await this._isBetterThanKnown(session, olm, pickleKey, txn); - if (this._isBetter) { - const claimedKeys = {ed25519: this.claimedEd25519Key}; - this._sessionInfo = new SessionInfo(this.roomId, this.senderKey, session, claimedKeys); - // retain the session so we don't have to create a new session during write. - this._sessionInfo.retain(); - return this._sessionInfo; - } else { - session.free(); - return; - } - } catch (err) { - this._sessionInfo = null; - session.free(); - throw err; - } - } - - async _isBetterThanKnown(session, olm, pickleKey, txn) { - let isBetter = true; - // TODO: we could potentially have a small speedup here if we looked first in the SessionCache here... - const existingSessionEntry = await txn.inboundGroupSessions.get(this.roomId, this.senderKey, this.sessionId); - if (existingSessionEntry?.session) { - const existingSession = new olm.InboundGroupSession(); - try { - existingSession.unpickle(pickleKey, existingSessionEntry.session); - isBetter = session.first_known_index() < existingSession.first_known_index(); - } finally { - existingSession.free(); - } - } - // store the event ids that can be decrypted with this key - // before we overwrite them if called from `write`. - if (existingSessionEntry?.eventIds) { - this._eventIds = existingSessionEntry.eventIds; - } - return isBetter; - } - - async write(olm, pickleKey, txn) { - // we checked already and we had a better session in storage, so don't write - if (this._isBetter === false) { - return false; - } - if (!this._sessionInfo) { - await this.createSessionInfo(olm, pickleKey, txn); - } - if (this._sessionInfo) { - const session = this._sessionInfo.session; - const sessionEntry = { - roomId: this.roomId, - senderKey: this.senderKey, - sessionId: this.sessionId, - session: session.pickle(pickleKey), - claimedKeys: this._sessionInfo.claimedKeys, - }; - txn.inboundGroupSessions.set(sessionEntry); - this.dispose(); - return true; - } - return false; - } - - get eventIds() { - return this._eventIds; - } - - dispose() { - if (this._sessionInfo) { - this._sessionInfo.release(); - this._sessionInfo = null; - } - } -} - -class DeviceMessageRoomKey extends BaseRoomKey { - constructor(decryptionResult) { - super(); - this._decryptionResult = decryptionResult; - } - - get roomId() { return this._decryptionResult.event.content?.["room_id"]; } - get senderKey() { return this._decryptionResult.senderCurve25519Key; } - get sessionId() { return this._decryptionResult.event.content?.["session_id"]; } - get claimedEd25519Key() { return this._decryptionResult.claimedEd25519Key; } - - _loadSessionKey(session) { - const sessionKey = this._decryptionResult.event.content?.["session_key"]; - session.create(sessionKey); - } -} - -class BackupRoomKey extends BaseRoomKey { - constructor(roomId, sessionId, backupInfo) { - super(); - this._roomId = roomId; - this._sessionId = sessionId; - this._backupInfo = backupInfo; - } - - get roomId() { return this._roomId; } - get senderKey() { return this._backupInfo["sender_key"]; } - get sessionId() { return this._sessionId; } - get claimedEd25519Key() { return this._backupInfo["sender_claimed_keys"]?.["ed25519"]; } - - _loadSessionKey(session) { - const sessionKey = this._backupInfo["session_key"]; - session.import_session(sessionKey); - } -} - -export function fromDeviceMessage(dr) { - const roomId = dr.event.content?.["room_id"]; - const sessionId = dr.event.content?.["session_id"]; - const sessionKey = dr.event.content?.["session_key"]; - if ( - typeof roomId === "string" || - typeof sessionId === "string" || - typeof senderKey === "string" || - typeof sessionKey === "string" - ) { - return new DeviceMessageRoomKey(dr); - } -} - -/* -sessionInfo is a response from key backup and has the following keys: - algorithm - forwarding_curve25519_key_chain - sender_claimed_keys - sender_key - session_key - */ -export function fromBackup(roomId, sessionId, sessionInfo) { - const sessionKey = sessionInfo["session_key"]; - const senderKey = sessionInfo["sender_key"]; - // TODO: can we just trust this? - const claimedEd25519Key = sessionInfo["sender_claimed_keys"]?.["ed25519"]; - - if ( - typeof roomId === "string" && - typeof sessionId === "string" && - typeof senderKey === "string" && - typeof sessionKey === "string" && - typeof claimedEd25519Key === "string" - ) { - return new BackupRoomKey(roomId, sessionId, sessionInfo); - } -} - diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts new file mode 100644 index 00000000..8815a4e6 --- /dev/null +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -0,0 +1,300 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {InboundGroupSession} from "../../../storage/idb/stores/InboundGroupSessionStore"; +import type {Transaction} from "../../../storage/idb/Transaction"; +import type {DecryptionResult} from "../../DecryptionResult"; + +declare class OlmInboundGroupSession { + constructor(); + free(): void; + pickle(key: string | Uint8Array): string; + unpickle(key: string | Uint8Array, pickle: string); + create(session_key: string): string; + import_session(session_key: string): string; + decrypt(message: string): object; + session_id(): string; + first_known_index(): number; + export_session(message_index: number): string; +} + +export interface IRoomKey { + get roomId(): string; + get senderKey(): string; + get sessionId(): string; + get claimedEd25519Key(): string; + get eventIds(): string[] | undefined; + deserializeInto(session: OlmInboundGroupSession, pickleKey: string): void; +} + +export interface IIncomingRoomKey extends IRoomKey { + copyEventIds(value: string[]): void; +} + +export async function checkBetterKeyInStorage(key: IIncomingRoomKey, keyDeserialization: KeyDeserialization, txn: Transaction) { + let existingKey = keyDeserialization.cache.get(key.roomId, key.senderKey, key.sessionId); + if (!existingKey) { + const storageKey = await fromStorage(key.roomId, key.senderKey, key.sessionId, txn); + // store the event ids that can be decrypted with this key + // before we overwrite them if called from `write`. + if (storageKey) { + if (storageKey.eventIds) { + key.copyEventIds(storageKey.eventIds); + } + if (storageKey.hasSession) { + existingKey = storageKey; + } + } + } + if (existingKey) { + const isBetter = await keyDeserialization.useKey(key, newSession => { + return keyDeserialization.useKey(existingKey, existingSession => { + return newSession.first_known_index() < existingSession.first_known_index(); + }); + }); + return isBetter ? key : existingKey; + } else { + return key; + } +} + +async function write(olm, pickleKey, keyDeserialization, txn) { + // we checked already and we had a better session in storage, so don't write + if (this._isBetter === false) { + return false; + } + if (!this._sessionInfo) { + await this.createSessionInfo(olm, pickleKey, txn); + } + if (this._sessionInfo) { + // before calling write in parallel, we need to check keyDeserialization.running is false so we are sure our transaction will not be closed + const pickledSession = await keyDeserialization.useKey(this, session => session.pickle(pickleKey)); + const sessionEntry = { + roomId: this.roomId, + senderKey: this.senderKey, + sessionId: this.sessionId, + session: pickledSession, + claimedKeys: this._sessionInfo.claimedKeys, + }; + txn.inboundGroupSessions.set(sessionEntry); + this.dispose(); + return true; + } + return false; +} + +class BaseIncomingRoomKey { + private _eventIds?: string[]; + + get eventIds() { return this._eventIds; } + + copyEventIds(eventIds: string[]): void { + this._eventIds = eventIds; + } +} + +class DeviceMessageRoomKey extends BaseIncomingRoomKey implements IIncomingRoomKey { + private _decryptionResult: DecryptionResult; + + constructor(decryptionResult: DecryptionResult) { + super(); + this._decryptionResult = decryptionResult; + } + + get roomId() { return this._decryptionResult.event.content?.["room_id"]; } + get senderKey() { return this._decryptionResult.senderCurve25519Key; } + get sessionId() { return this._decryptionResult.event.content?.["session_id"]; } + get claimedEd25519Key() { return this._decryptionResult.claimedEd25519Key; } + + deserializeInto(session) { + const sessionKey = this._decryptionResult.event.content?.["session_key"]; + session.create(sessionKey); + } +} + +class BackupRoomKey extends BaseIncomingRoomKey implements IIncomingRoomKey { + private _roomId: string; + private _sessionId: string; + private _backupInfo: string; + + constructor(roomId, sessionId, backupInfo) { + super(); + this._roomId = roomId; + this._sessionId = sessionId; + this._backupInfo = backupInfo; + } + + get roomId() { return this._roomId; } + get senderKey() { return this._backupInfo["sender_key"]; } + get sessionId() { return this._sessionId; } + get claimedEd25519Key() { return this._backupInfo["sender_claimed_keys"]?.["ed25519"]; } + + deserializeInto(session) { + const sessionKey = this._backupInfo["session_key"]; + session.import_session(sessionKey); + } +} + +class StoredRoomKey implements IRoomKey { + private storageEntry: InboundGroupSession; + + constructor(storageEntry: InboundGroupSession) { + this.storageEntry = storageEntry; + } + + get roomId() { return this.storageEntry.roomId; } + get senderKey() { return this.storageEntry.senderKey; } + get sessionId() { return this.storageEntry.sessionId; } + get claimedEd25519Key() { return this.storageEntry.claimedKeys!["ed25519"]; } + get eventIds() { return this.storageEntry.eventIds; } + + deserializeInto(session, pickleKey) { + session.unpickle(pickleKey, this.storageEntry.session); + } + + get hasSession() { + // sessions are stored before they are received + // to keep track of events that need it to be decrypted. + // This is used to retry decryption of those events once the session is received. + return !!this.storageEntry.session; + } +} + +export function fromDeviceMessage(dr) { + const roomId = dr.event.content?.["room_id"]; + const sessionId = dr.event.content?.["session_id"]; + const sessionKey = dr.event.content?.["session_key"]; + if ( + typeof roomId === "string" || + typeof sessionId === "string" || + typeof senderKey === "string" || + typeof sessionKey === "string" + ) { + return new DeviceMessageRoomKey(dr); + } +} + +/* +sessionInfo is a response from key backup and has the following keys: + algorithm + forwarding_curve25519_key_chain + sender_claimed_keys + sender_key + session_key + */ +export function fromBackup(roomId, sessionId, sessionInfo) { + const sessionKey = sessionInfo["session_key"]; + const senderKey = sessionInfo["sender_key"]; + // TODO: can we just trust this? + const claimedEd25519Key = sessionInfo["sender_claimed_keys"]?.["ed25519"]; + + if ( + typeof roomId === "string" && + typeof sessionId === "string" && + typeof senderKey === "string" && + typeof sessionKey === "string" && + typeof claimedEd25519Key === "string" + ) { + return new BackupRoomKey(roomId, sessionId, sessionInfo); + } +} + +export async function fromStorage(roomId: string, senderKey: string, sessionId: string, txn: Transaction): Promise { + const existingSessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId); + if (existingSessionEntry) { + return new StoredRoomKey(existingSessionEntry); + } + return; +} +/* +Because Olm only has very limited memory available when compiled to wasm, +we limit the amount of sessions held in memory. +*/ +class KeyDeserialization { + + public readonly cache: SessionCache; + private pickleKey: string; + private olm: any; + private resolveUnusedEntry?: () => void; + private entryBecomesUnusedPromise?: Promise; + + constructor({olm, pickleKey, limit}) { + this.cache = new SessionCache(limit); + this.pickleKey = pickleKey; + this.olm = olm; + } + + async useKey(key: IRoomKey, callback: (session: OlmInboundGroupSession) => Promise | T): Promise { + const cacheEntry = await this.allocateEntry(key); + try { + const {session} = cacheEntry; + key.deserializeInto(session, this.pickleKey); + return await callback(session); + } finally { + this.freeEntry(cacheEntry); + } + } + + get running() { + return !!this.cache.find(entry => entry.inUse); + } + + private async allocateEntry(key): CacheEntry { + let entry; + if (this.cache.size >= MAX) { + while(!(entry = this.cache.find(entry => !entry.inUse))) { + await this.entryBecomesUnused(); + } + entry.inUse = true; + entry.key = key; + } else { + const session: OlmInboundGroupSession = new this.olm.InboundGroupSession(); + const entry = new CacheEntry(key, session); + this.cache.add(entry); + } + return entry; + } + + private freeEntry(entry) { + entry.inUse = false; + if (this.resolveUnusedEntry) { + this.resolveUnusedEntry(); + // promise is resolved now, we'll need a new one for next await so clear + this.entryBecomesUnusedPromise = this.resolveUnusedEntry = undefined; + } + } + + private entryBecomesUnused(): Promise { + if (!this.entryBecomesUnusedPromise) { + this.entryBecomesUnusedPromise = new Promise(resolve => { + this.resolveUnusedEntry = resolve; + }); + } + return this.entryBecomesUnusedPromise; + } +} + +class CacheEntry { + inUse: boolean; + session: OlmInboundGroupSession; + key: IRoomKey; + + constructor(key, session) { + this.key = key; + this.session = session; + this.inUse = true; + } +} diff --git a/src/matrix/e2ee/megolm/decryption/SessionCache.js b/src/matrix/e2ee/megolm/decryption/SessionCache.js index c5b2c0fb..712e2e50 100644 --- a/src/matrix/e2ee/megolm/decryption/SessionCache.js +++ b/src/matrix/e2ee/megolm/decryption/SessionCache.js @@ -33,11 +33,13 @@ export class SessionCache extends BaseLRUCache { * @return {SessionInfo?} */ get(roomId, senderKey, sessionId) { - return this._get(s => { + const sessionInfo = this._get(s => { return s.roomId === roomId && s.senderKey === senderKey && sessionId === s.sessionId; }); + sessionInfo?.retain(); + return sessionInfo; } add(sessionInfo) { diff --git a/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts b/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts index 5dc0205f..65ce99ce 100644 --- a/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts +++ b/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts @@ -17,7 +17,7 @@ limitations under the License. import {MIN_UNICODE, MAX_UNICODE} from "./common"; import {Store} from "../Store"; -interface InboundGroupSession { +export interface InboundGroupSession { roomId: string; senderKey: string; sessionId: string; diff --git a/src/utils/LRUCache.js b/src/utils/LRUCache.js index 185e5aeb..183fc995 100644 --- a/src/utils/LRUCache.js +++ b/src/utils/LRUCache.js @@ -54,6 +54,16 @@ export class BaseLRUCache { } } + find(callback) { + // iterate backwards so least recently used items are found first + for (let i = this._entries.length - 1; i >= 0; i -= 1) { + const entry = this._entries[i]; + if (callback(entry)) { + return entry; + } + } + } + _onEvictEntry() {} } From 5dc0c8c0b3f095f7491b1586bb26e00471f87517 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 13:38:54 +0200 Subject: [PATCH 02/26] make 'better' better --- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 106 +++++++++++-------- 1 file changed, 61 insertions(+), 45 deletions(-) diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index 8815a4e6..ed799759 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -41,47 +41,38 @@ export interface IRoomKey { } export interface IIncomingRoomKey extends IRoomKey { - copyEventIds(value: string[]): void; + get isBetter(): boolean | undefined; + checkIsBetterThanStorage(keyDeserialization: KeyDeserialization, txn: Transaction): Promise; + write(keyDeserialization: KeyDeserialization, txn: Transaction): Promise; } -export async function checkBetterKeyInStorage(key: IIncomingRoomKey, keyDeserialization: KeyDeserialization, txn: Transaction) { - let existingKey = keyDeserialization.cache.get(key.roomId, key.senderKey, key.sessionId); - if (!existingKey) { - const storageKey = await fromStorage(key.roomId, key.senderKey, key.sessionId, txn); - // store the event ids that can be decrypted with this key - // before we overwrite them if called from `write`. - if (storageKey) { - if (storageKey.eventIds) { - key.copyEventIds(storageKey.eventIds); - } - if (storageKey.hasSession) { - existingKey = storageKey; - } +abstract class BaseIncomingRoomKey implements IIncomingRoomKey { + private _eventIds?: string[]; + private _isBetter?: boolean; + + checkBetterKeyInStorage(keyDeserialization: KeyDeserialization, txn: Transaction): Promise { + return this._checkBetterKeyInStorage(keyDeserialization, undefined, txn); + } + + async write(keyDeserialization: KeyDeserialization, pickleKey: string, txn: Transaction): Promise { + // we checked already and we had a better session in storage, so don't write + let pickledSession; + if (this._isBetter === undefined) { + // if this key wasn't used to decrypt any messages in the same sync, + // we haven't checked if this is the best key yet, + // so do that now to not overwrite a better key. + // while we have the key deserialized, also pickle it to store it later on here. + await this._checkBetterKeyInStorage(keyDeserialization, session => { + pickledSession = session.pickle(pickleKey); + }, txn); + } + if (this._isBetter === false) { + return false; } - } - if (existingKey) { - const isBetter = await keyDeserialization.useKey(key, newSession => { - return keyDeserialization.useKey(existingKey, existingSession => { - return newSession.first_known_index() < existingSession.first_known_index(); - }); - }); - return isBetter ? key : existingKey; - } else { - return key; - } -} - -async function write(olm, pickleKey, keyDeserialization, txn) { - // we checked already and we had a better session in storage, so don't write - if (this._isBetter === false) { - return false; - } - if (!this._sessionInfo) { - await this.createSessionInfo(olm, pickleKey, txn); - } - if (this._sessionInfo) { // before calling write in parallel, we need to check keyDeserialization.running is false so we are sure our transaction will not be closed - const pickledSession = await keyDeserialization.useKey(this, session => session.pickle(pickleKey)); + if (!pickledSession) { + pickledSession = await keyDeserialization.useKey(this, session => session.pickle(pickleKey)); + } const sessionEntry = { roomId: this.roomId, senderKey: this.senderKey, @@ -90,19 +81,44 @@ async function write(olm, pickleKey, keyDeserialization, txn) { claimedKeys: this._sessionInfo.claimedKeys, }; txn.inboundGroupSessions.set(sessionEntry); - this.dispose(); return true; } - return false; -} - -class BaseIncomingRoomKey { - private _eventIds?: string[]; get eventIds() { return this._eventIds; } + get isBetter() { return this._isBetter; } - copyEventIds(eventIds: string[]): void { - this._eventIds = eventIds; + private async _checkBetterKeyInStorage(keyDeserialization: KeyDeserialization, callback?: (session: OlmInboundGroupSession) => void, txn: Transaction): Promise { + if (this._isBetter !== undefined) { + return this._isBetter; + } + let existingKey = keyDeserialization.cache.get(this.roomId, this.senderKey, this.sessionId); + if (!existingKey) { + const storageKey = await fromStorage(this.roomId, this.senderKey, this.sessionId, txn); + // store the event ids that can be decrypted with this key + // before we overwrite them if called from `write`. + if (storageKey) { + if (storageKey.hasSession) { + existingKey = storageKey; + } else if (storageKey.eventIds) { + this._eventIds = storageKey.eventIds; + } + } + } + if (existingKey) { + this._isBetter = await keyDeserialization.useKey(key, newSession => { + return keyDeserialization.useKey(existingKey, existingSession => { + const isBetter = newSession.first_known_index() < existingSession.first_known_index(); + if (isBetter && callback) { + callback(newSession); + } + return isBetter; + }); + }); + } else { + // no previous key, so we're the best \o/ + this._isBetter = true; + } + return this._isBetter; } } From cbf82fcd298b21e8fcf46b65addd9b5fad004e1f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 15:14:17 +0200 Subject: [PATCH 03/26] cleanup code so far --- .../e2ee/megolm/decryption/KeyLoader.ts | 111 ++++++++++++ src/matrix/e2ee/megolm/decryption/RoomKey.ts | 169 +++++------------- .../idb/stores/InboundGroupSessionStore.ts | 19 +- src/utils/LRUCache.js | 3 + 4 files changed, 167 insertions(+), 135 deletions(-) create mode 100644 src/matrix/e2ee/megolm/decryption/KeyLoader.ts diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts new file mode 100644 index 00000000..d72ec546 --- /dev/null +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -0,0 +1,111 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {SessionCache} from "./SessionCache"; +import {IRoomKey} from "./RoomKey"; + +export declare class OlmInboundGroupSession { + constructor(); + free(): void; + pickle(key: string | Uint8Array): string; + unpickle(key: string | Uint8Array, pickle: string); + create(session_key: string): string; + import_session(session_key: string): string; + decrypt(message: string): object; + session_id(): string; + first_known_index(): number; + export_session(message_index: number): string; +} + +/* +Because Olm only has very limited memory available when compiled to wasm, +we limit the amount of sessions held in memory. +*/ +export class KeyLoader { + + public readonly cache: SessionCache; + private pickleKey: string; + private olm: any; + private resolveUnusedEntry?: () => void; + private entryBecomesUnusedPromise?: Promise; + + constructor(olm: any, pickleKey: string, limit: number) { + this.cache = new SessionCache(limit); + this.pickleKey = pickleKey; + this.olm = olm; + } + + async useKey(key: IRoomKey, callback: (session: OlmInboundGroupSession, pickleKey: string) => Promise | T): Promise { + const cacheEntry = await this.allocateEntry(key); + try { + const {session} = cacheEntry; + key.loadInto(session, this.pickleKey); + return await callback(session, this.pickleKey); + } finally { + this.freeEntry(cacheEntry); + } + } + + get running() { + return !!this.cache.find(entry => entry.inUse); + } + + private async allocateEntry(key: IRoomKey): Promise { + let entry; + if (this.cache.size >= this.cache.limit) { + while(!(entry = this.cache.find(entry => !entry.inUse))) { + await this.entryBecomesUnused(); + } + entry.inUse = true; + entry.key = key; + } else { + const session: OlmInboundGroupSession = new this.olm.InboundGroupSession(); + const entry = new CacheEntry(key, session); + this.cache.add(entry); + } + return entry; + } + + private freeEntry(entry: CacheEntry) { + entry.inUse = false; + if (this.resolveUnusedEntry) { + this.resolveUnusedEntry(); + // promise is resolved now, we'll need a new one for next await so clear + this.entryBecomesUnusedPromise = this.resolveUnusedEntry = undefined; + } + } + + private entryBecomesUnused(): Promise { + if (!this.entryBecomesUnusedPromise) { + this.entryBecomesUnusedPromise = new Promise(resolve => { + this.resolveUnusedEntry = resolve; + }); + } + return this.entryBecomesUnusedPromise; + } +} + +class CacheEntry { + inUse: boolean; + session: OlmInboundGroupSession; + key: IRoomKey; + + constructor(key, session) { + this.key = key; + this.session = session; + this.inUse = true; + } +} diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index ed799759..c4c00d99 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -14,22 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import type {InboundGroupSession} from "../../../storage/idb/stores/InboundGroupSessionStore"; +import type {InboundGroupSessionEntry} from "../../../storage/idb/stores/InboundGroupSessionStore"; import type {Transaction} from "../../../storage/idb/Transaction"; import type {DecryptionResult} from "../../DecryptionResult"; - -declare class OlmInboundGroupSession { - constructor(); - free(): void; - pickle(key: string | Uint8Array): string; - unpickle(key: string | Uint8Array, pickle: string); - create(session_key: string): string; - import_session(session_key: string): string; - decrypt(message: string): object; - session_id(): string; - first_known_index(): number; - export_session(message_index: number): string; -} +import type {KeyLoader, OlmInboundGroupSession} from "./KeyLoader"; +import {SessionCache} from "./SessionCache"; export interface IRoomKey { get roomId(): string; @@ -37,24 +26,24 @@ export interface IRoomKey { get sessionId(): string; get claimedEd25519Key(): string; get eventIds(): string[] | undefined; - deserializeInto(session: OlmInboundGroupSession, pickleKey: string): void; + loadInto(session: OlmInboundGroupSession, pickleKey: string): void; } export interface IIncomingRoomKey extends IRoomKey { get isBetter(): boolean | undefined; - checkIsBetterThanStorage(keyDeserialization: KeyDeserialization, txn: Transaction): Promise; - write(keyDeserialization: KeyDeserialization, txn: Transaction): Promise; + checkBetterKeyInStorage(loader: KeyLoader, txn: Transaction): Promise; + write(loader: KeyLoader, txn: Transaction): Promise; } abstract class BaseIncomingRoomKey implements IIncomingRoomKey { private _eventIds?: string[]; private _isBetter?: boolean; - checkBetterKeyInStorage(keyDeserialization: KeyDeserialization, txn: Transaction): Promise { - return this._checkBetterKeyInStorage(keyDeserialization, undefined, txn); + checkBetterKeyInStorage(loader: KeyLoader, txn: Transaction): Promise { + return this._checkBetterKeyInStorage(loader, undefined, txn); } - async write(keyDeserialization: KeyDeserialization, pickleKey: string, txn: Transaction): Promise { + async write(loader: KeyLoader, txn: Transaction): Promise { // we checked already and we had a better session in storage, so don't write let pickledSession; if (this._isBetter === undefined) { @@ -62,23 +51,23 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { // we haven't checked if this is the best key yet, // so do that now to not overwrite a better key. // while we have the key deserialized, also pickle it to store it later on here. - await this._checkBetterKeyInStorage(keyDeserialization, session => { + await this._checkBetterKeyInStorage(loader, (session, pickleKey) => { pickledSession = session.pickle(pickleKey); }, txn); } if (this._isBetter === false) { return false; } - // before calling write in parallel, we need to check keyDeserialization.running is false so we are sure our transaction will not be closed + // before calling write in parallel, we need to check loader.running is false so we are sure our transaction will not be closed if (!pickledSession) { - pickledSession = await keyDeserialization.useKey(this, session => session.pickle(pickleKey)); + pickledSession = await loader.useKey(this, (session, pickleKey) => session.pickle(pickleKey)); } const sessionEntry = { roomId: this.roomId, senderKey: this.senderKey, sessionId: this.sessionId, session: pickledSession, - claimedKeys: this._sessionInfo.claimedKeys, + claimedKeys: {"ed25519": this.claimedEd25519Key}, }; txn.inboundGroupSessions.set(sessionEntry); return true; @@ -87,11 +76,11 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { get eventIds() { return this._eventIds; } get isBetter() { return this._isBetter; } - private async _checkBetterKeyInStorage(keyDeserialization: KeyDeserialization, callback?: (session: OlmInboundGroupSession) => void, txn: Transaction): Promise { + private async _checkBetterKeyInStorage(loader: KeyLoader, callback: (((session: OlmInboundGroupSession, pickleKey: string) => void) | undefined), txn: Transaction): Promise { if (this._isBetter !== undefined) { return this._isBetter; } - let existingKey = keyDeserialization.cache.get(this.roomId, this.senderKey, this.sessionId); + let existingKey = loader.cache.get(this.roomId, this.senderKey, this.sessionId); if (!existingKey) { const storageKey = await fromStorage(this.roomId, this.senderKey, this.sessionId, txn); // store the event ids that can be decrypted with this key @@ -105,11 +94,11 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { } } if (existingKey) { - this._isBetter = await keyDeserialization.useKey(key, newSession => { - return keyDeserialization.useKey(existingKey, existingSession => { + this._isBetter = await loader.useKey(this, newSession => { + return loader.useKey(existingKey, (existingSession, pickleKey) => { const isBetter = newSession.first_known_index() < existingSession.first_known_index(); if (isBetter && callback) { - callback(newSession); + callback(newSession, pickleKey); } return isBetter; }); @@ -120,9 +109,15 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { } return this._isBetter; } + + abstract get roomId(): string; + abstract get senderKey(): string; + abstract get sessionId(): string; + abstract get claimedEd25519Key(): string; + abstract loadInto(session: OlmInboundGroupSession, pickleKey: string): void; } -class DeviceMessageRoomKey extends BaseIncomingRoomKey implements IIncomingRoomKey { +class DeviceMessageRoomKey extends BaseIncomingRoomKey { private _decryptionResult: DecryptionResult; constructor(decryptionResult: DecryptionResult) { @@ -135,13 +130,13 @@ class DeviceMessageRoomKey extends BaseIncomingRoomKey implements IIncomingRoomK get sessionId() { return this._decryptionResult.event.content?.["session_id"]; } get claimedEd25519Key() { return this._decryptionResult.claimedEd25519Key; } - deserializeInto(session) { + loadInto(session) { const sessionKey = this._decryptionResult.event.content?.["session_key"]; session.create(sessionKey); } } -class BackupRoomKey extends BaseIncomingRoomKey implements IIncomingRoomKey { +class BackupRoomKey extends BaseIncomingRoomKey { private _roomId: string; private _sessionId: string; private _backupInfo: string; @@ -158,16 +153,16 @@ class BackupRoomKey extends BaseIncomingRoomKey implements IIncomingRoomKey { get sessionId() { return this._sessionId; } get claimedEd25519Key() { return this._backupInfo["sender_claimed_keys"]?.["ed25519"]; } - deserializeInto(session) { + loadInto(session) { const sessionKey = this._backupInfo["session_key"]; session.import_session(sessionKey); } } class StoredRoomKey implements IRoomKey { - private storageEntry: InboundGroupSession; + private storageEntry: InboundGroupSessionEntry; - constructor(storageEntry: InboundGroupSession) { + constructor(storageEntry: InboundGroupSessionEntry) { this.storageEntry = storageEntry; } @@ -177,7 +172,7 @@ class StoredRoomKey implements IRoomKey { get claimedEd25519Key() { return this.storageEntry.claimedKeys!["ed25519"]; } get eventIds() { return this.storageEntry.eventIds; } - deserializeInto(session, pickleKey) { + loadInto(session, pickleKey) { session.unpickle(pickleKey, this.storageEntry.session); } @@ -189,17 +184,16 @@ class StoredRoomKey implements IRoomKey { } } -export function fromDeviceMessage(dr) { - const roomId = dr.event.content?.["room_id"]; - const sessionId = dr.event.content?.["session_id"]; +export function fromDeviceMessage(dr: DecryptionResult): DeviceMessageRoomKey | undefined { const sessionKey = dr.event.content?.["session_key"]; + const key = new DeviceMessageRoomKey(dr); if ( - typeof roomId === "string" || - typeof sessionId === "string" || - typeof senderKey === "string" || + typeof key.roomId === "string" && + typeof key.sessionId === "string" && + typeof key.senderKey === "string" && typeof sessionKey === "string" ) { - return new DeviceMessageRoomKey(dr); + return key; } } @@ -211,11 +205,11 @@ sessionInfo is a response from key backup and has the following keys: sender_key session_key */ -export function fromBackup(roomId, sessionId, sessionInfo) { - const sessionKey = sessionInfo["session_key"]; - const senderKey = sessionInfo["sender_key"]; +export function fromBackup(roomId, sessionId, backupInfo): BackupRoomKey | undefined { + const sessionKey = backupInfo["session_key"]; + const senderKey = backupInfo["sender_key"]; // TODO: can we just trust this? - const claimedEd25519Key = sessionInfo["sender_claimed_keys"]?.["ed25519"]; + const claimedEd25519Key = backupInfo["sender_claimed_keys"]?.["ed25519"]; if ( typeof roomId === "string" && @@ -224,7 +218,7 @@ export function fromBackup(roomId, sessionId, sessionInfo) { typeof sessionKey === "string" && typeof claimedEd25519Key === "string" ) { - return new BackupRoomKey(roomId, sessionId, sessionInfo); + return new BackupRoomKey(roomId, sessionId, backupInfo); } } @@ -235,82 +229,3 @@ export async function fromStorage(roomId: string, senderKey: string, sessionId: } return; } -/* -Because Olm only has very limited memory available when compiled to wasm, -we limit the amount of sessions held in memory. -*/ -class KeyDeserialization { - - public readonly cache: SessionCache; - private pickleKey: string; - private olm: any; - private resolveUnusedEntry?: () => void; - private entryBecomesUnusedPromise?: Promise; - - constructor({olm, pickleKey, limit}) { - this.cache = new SessionCache(limit); - this.pickleKey = pickleKey; - this.olm = olm; - } - - async useKey(key: IRoomKey, callback: (session: OlmInboundGroupSession) => Promise | T): Promise { - const cacheEntry = await this.allocateEntry(key); - try { - const {session} = cacheEntry; - key.deserializeInto(session, this.pickleKey); - return await callback(session); - } finally { - this.freeEntry(cacheEntry); - } - } - - get running() { - return !!this.cache.find(entry => entry.inUse); - } - - private async allocateEntry(key): CacheEntry { - let entry; - if (this.cache.size >= MAX) { - while(!(entry = this.cache.find(entry => !entry.inUse))) { - await this.entryBecomesUnused(); - } - entry.inUse = true; - entry.key = key; - } else { - const session: OlmInboundGroupSession = new this.olm.InboundGroupSession(); - const entry = new CacheEntry(key, session); - this.cache.add(entry); - } - return entry; - } - - private freeEntry(entry) { - entry.inUse = false; - if (this.resolveUnusedEntry) { - this.resolveUnusedEntry(); - // promise is resolved now, we'll need a new one for next await so clear - this.entryBecomesUnusedPromise = this.resolveUnusedEntry = undefined; - } - } - - private entryBecomesUnused(): Promise { - if (!this.entryBecomesUnusedPromise) { - this.entryBecomesUnusedPromise = new Promise(resolve => { - this.resolveUnusedEntry = resolve; - }); - } - return this.entryBecomesUnusedPromise; - } -} - -class CacheEntry { - inUse: boolean; - session: OlmInboundGroupSession; - key: IRoomKey; - - constructor(key, session) { - this.key = key; - this.session = session; - this.inUse = true; - } -} diff --git a/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts b/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts index 65ce99ce..22093884 100644 --- a/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts +++ b/src/matrix/storage/idb/stores/InboundGroupSessionStore.ts @@ -17,24 +17,26 @@ limitations under the License. import {MIN_UNICODE, MAX_UNICODE} from "./common"; import {Store} from "../Store"; -export interface InboundGroupSession { +export interface InboundGroupSessionEntry { roomId: string; senderKey: string; sessionId: string; session?: string; claimedKeys?: { [algorithm : string] : string }; eventIds?: string[]; - key: string; } +type InboundGroupSessionStorageEntry = InboundGroupSessionEntry & { key: string }; + + function encodeKey(roomId: string, senderKey: string, sessionId: string): string { return `${roomId}|${senderKey}|${sessionId}`; } export class InboundGroupSessionStore { - private _store: Store; + private _store: Store; - constructor(store: Store) { + constructor(store: Store) { this._store = store; } @@ -44,13 +46,14 @@ export class InboundGroupSessionStore { return key === fetchedKey; } - get(roomId: string, senderKey: string, sessionId: string): Promise { + get(roomId: string, senderKey: string, sessionId: string): Promise { return this._store.get(encodeKey(roomId, senderKey, sessionId)); } - set(session: InboundGroupSession): void { - session.key = encodeKey(session.roomId, session.senderKey, session.sessionId); - this._store.put(session); + set(session: InboundGroupSessionEntry): void { + const storageEntry = session as InboundGroupSessionStorageEntry; + storageEntry.key = encodeKey(session.roomId, session.senderKey, session.sessionId); + this._store.put(storageEntry); } removeAllForRoom(roomId: string) { diff --git a/src/utils/LRUCache.js b/src/utils/LRUCache.js index 183fc995..98ded41b 100644 --- a/src/utils/LRUCache.js +++ b/src/utils/LRUCache.js @@ -24,6 +24,9 @@ export class BaseLRUCache { this._entries = []; } + get size() { return this._entries.length; } + get limit() { return this._limit; } + _get(findEntryFn) { const idx = this._entries.findIndex(findEntryFn); if (idx !== -1) { From 041cedbc58542951f9ba30f3e7a6483c60cc5b41 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 15:24:39 +0200 Subject: [PATCH 04/26] fix typescript extension change --- src/matrix/e2ee/megolm/Decryption.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.js index 30e1480f..a4771cc8 100644 --- a/src/matrix/e2ee/megolm/Decryption.js +++ b/src/matrix/e2ee/megolm/Decryption.js @@ -15,7 +15,7 @@ limitations under the License. */ import {DecryptionError} from "../common.js"; -import * as RoomKey from "./decryption/RoomKey.js"; +import * as RoomKey from "./decryption/RoomKey"; import {SessionInfo} from "./decryption/SessionInfo.js"; import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js"; import {SessionDecryption} from "./decryption/SessionDecryption.js"; From 4fa285e85a5e1bc4e2bd2bf06f9501428ddd046b Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 15:24:58 +0200 Subject: [PATCH 05/26] convert LRUCache to ts --- .../e2ee/megolm/decryption/SessionCache.js | 2 +- .../room/timeline/persistence/MemberWriter.js | 2 +- src/utils/{LRUCache.js => LRUCache.ts} | 65 +++++++++++-------- 3 files changed, 40 insertions(+), 29 deletions(-) rename src/utils/{LRUCache.js => LRUCache.ts} (73%) diff --git a/src/matrix/e2ee/megolm/decryption/SessionCache.js b/src/matrix/e2ee/megolm/decryption/SessionCache.js index 712e2e50..cbb868ea 100644 --- a/src/matrix/e2ee/megolm/decryption/SessionCache.js +++ b/src/matrix/e2ee/megolm/decryption/SessionCache.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {BaseLRUCache} from "../../../../utils/LRUCache.js"; +import {BaseLRUCache} from "../../../../utils/LRUCache"; const DEFAULT_CACHE_SIZE = 10; /** diff --git a/src/matrix/room/timeline/persistence/MemberWriter.js b/src/matrix/room/timeline/persistence/MemberWriter.js index b21a4461..1cdcb7d5 100644 --- a/src/matrix/room/timeline/persistence/MemberWriter.js +++ b/src/matrix/room/timeline/persistence/MemberWriter.js @@ -15,7 +15,7 @@ limitations under the License. */ import {MemberChange, RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../members/RoomMember.js"; -import {LRUCache} from "../../../../utils/LRUCache.js"; +import {LRUCache} from "../../../../utils/LRUCache"; export class MemberWriter { constructor(roomId) { diff --git a/src/utils/LRUCache.js b/src/utils/LRUCache.ts similarity index 73% rename from src/utils/LRUCache.js rename to src/utils/LRUCache.ts index 98ded41b..d275b9f1 100644 --- a/src/utils/LRUCache.js +++ b/src/utils/LRUCache.ts @@ -18,8 +18,12 @@ limitations under the License. * Very simple least-recently-used cache implementation * that should be fast enough for very small cache sizes */ -export class BaseLRUCache { - constructor(limit) { +export class BaseLRUCache { + + private _limit: number; + private _entries: T[]; + + constructor(limit: number) { this._limit = limit; this._entries = []; } @@ -27,7 +31,7 @@ export class BaseLRUCache { get size() { return this._entries.length; } get limit() { return this._limit; } - _get(findEntryFn) { + _get(findEntryFn: (T) => boolean) { const idx = this._entries.findIndex(findEntryFn); if (idx !== -1) { const entry = this._entries[idx]; @@ -40,7 +44,7 @@ export class BaseLRUCache { } } - _set(value, findEntryFn) { + _set(value: T, findEntryFn: (T) => boolean) { let indexToRemove = this._entries.findIndex(findEntryFn); this._entries.unshift(value); if (indexToRemove === -1) { @@ -57,7 +61,7 @@ export class BaseLRUCache { } } - find(callback) { + find(callback: (T) => boolean) { // iterate backwards so least recently used items are found first for (let i = this._entries.length - 1; i >= 0; i -= 1) { const entry = this._entries[i]; @@ -67,69 +71,76 @@ export class BaseLRUCache { } } - _onEvictEntry() {} + _onEvictEntry(entry: T) {} } -export class LRUCache extends BaseLRUCache { - constructor(limit, keyFn) { +export class LRUCache extends BaseLRUCache { + private _keyFn: (T) => K; + + constructor(limit, keyFn: (T) => K) { super(limit); this._keyFn = keyFn; } - get(key) { + get(key: K): T | undefined { return this._get(e => this._keyFn(e) === key); } - set(value) { + set(value: T) { const key = this._keyFn(value); this._set(value, e => this._keyFn(e) === key); } } export function tests() { + interface NameTuple { + id: number; + name: string; + } + return { "can retrieve added entries": assert => { - const cache = new LRUCache(2, e => e.id); + const cache = new LRUCache(2, e => e.id); cache.set({id: 1, name: "Alice"}); cache.set({id: 2, name: "Bob"}); - assert.equal(cache.get(1).name, "Alice"); - assert.equal(cache.get(2).name, "Bob"); + assert.equal(cache.get(1)!.name, "Alice"); + assert.equal(cache.get(2)!.name, "Bob"); }, "first entry is evicted first": assert => { - const cache = new LRUCache(2, e => e.id); + const cache = new LRUCache(2, e => e.id); cache.set({id: 1, name: "Alice"}); cache.set({id: 2, name: "Bob"}); cache.set({id: 3, name: "Charly"}); assert.equal(cache.get(1), undefined); - assert.equal(cache.get(2).name, "Bob"); - assert.equal(cache.get(3).name, "Charly"); - assert.equal(cache._entries.length, 2); + assert.equal(cache.get(2)!.name, "Bob"); + assert.equal(cache.get(3)!.name, "Charly"); + assert.equal(cache.size, 2); }, "second entry is evicted if first is requested": assert => { - const cache = new LRUCache(2, e => e.id); + const cache = new LRUCache(2, e => e.id); cache.set({id: 1, name: "Alice"}); cache.set({id: 2, name: "Bob"}); cache.get(1); cache.set({id: 3, name: "Charly"}); - assert.equal(cache.get(1).name, "Alice"); + assert.equal(cache.get(1)!.name, "Alice"); assert.equal(cache.get(2), undefined); - assert.equal(cache.get(3).name, "Charly"); - assert.equal(cache._entries.length, 2); + assert.equal(cache.get(3)!.name, "Charly"); + assert.equal(cache.size, 2); }, "setting an entry twice removes the first": assert => { - const cache = new LRUCache(2, e => e.id); + const cache = new LRUCache(2, e => e.id); cache.set({id: 1, name: "Alice"}); cache.set({id: 2, name: "Bob"}); cache.set({id: 1, name: "Al Ice"}); cache.set({id: 3, name: "Charly"}); - assert.equal(cache.get(1).name, "Al Ice"); + assert.equal(cache.get(1)!.name, "Al Ice"); assert.equal(cache.get(2), undefined); - assert.equal(cache.get(3).name, "Charly"); - assert.equal(cache._entries.length, 2); + assert.equal(cache.get(3)!.name, "Charly"); + assert.equal(cache.size, 2); }, "evict callback is called": assert => { let evictions = 0; - class CustomCache extends LRUCache { + class CustomCache extends LRUCache { _onEvictEntry(entry) { assert.equal(entry.name, "Alice"); evictions += 1; @@ -143,7 +154,7 @@ export function tests() { }, "evict callback is called when replacing entry with same identity": assert => { let evictions = 0; - class CustomCache extends LRUCache { + class CustomCache extends LRUCache { _onEvictEntry(entry) { assert.equal(entry.name, "Alice"); evictions += 1; From 3bafc89855245f9a9fa905a6a3998329a19d17f1 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 20 Oct 2021 15:25:11 +0200 Subject: [PATCH 06/26] remove unused draft code --- .../e2ee/megolm/decryption/ISessionSource.ts | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 src/matrix/e2ee/megolm/decryption/ISessionSource.ts diff --git a/src/matrix/e2ee/megolm/decryption/ISessionSource.ts b/src/matrix/e2ee/megolm/decryption/ISessionSource.ts deleted file mode 100644 index 8a96ba12..00000000 --- a/src/matrix/e2ee/megolm/decryption/ISessionSource.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright 2021 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -//import type {InboundGroupSession} from "@matrix-org/olm"; -interface InboundGroupSession {} - -export interface ISessionSource { - deserializeIntoSession(session: InboundGroupSession): void; - get roomId(): string; - get senderKey(): string; - get sessionId(): string; - get claimedEd25519Key(): string; -} From 66a77519d7a5b6e0605e34070a2ffcb24974c925 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 21 Oct 2021 11:12:54 +0200 Subject: [PATCH 07/26] implement key caching in KeyLoader merging session cache into it so we can better manage and recycle keys without exposing too low-level public methods on BaseLRUCache. Using refCount instead of inUse flag as a key can of course be used by multiple useKey calls at the same time. --- .../e2ee/megolm/decryption/KeyLoader.ts | 186 ++++++++++++++---- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 39 ++-- .../e2ee/megolm/decryption/SessionCache.js | 63 ------ src/utils/LRUCache.ts | 36 ++-- 4 files changed, 189 insertions(+), 135 deletions(-) delete mode 100644 src/matrix/e2ee/megolm/decryption/SessionCache.js diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index d72ec546..bde3065f 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -15,7 +15,8 @@ limitations under the License. */ import {SessionCache} from "./SessionCache"; -import {IRoomKey} from "./RoomKey"; +import {IRoomKey, isBetterThan} from "./RoomKey"; +import {BaseLRUCache} from "../../../../utils/LRUCache"; export declare class OlmInboundGroupSession { constructor(); @@ -30,82 +31,193 @@ export declare class OlmInboundGroupSession { export_session(message_index: number): string; } +// this is what cache.get(...) should return +function findIndexBestForSession(ops: KeyOperation[], roomId: string, senderKey: string, sessionId: string): number { + return ops.reduce((bestIdx, op, i, arr) => { + const bestOp = bestIdx === -1 ? undefined : arr[bestIdx]; + if (op.isForSameSession(roomId, senderKey, sessionId)) { + if (!bestOp || op.isBetter(bestOp)) { + return i; + } + } + return bestIdx; + }, -1); +} + + /* Because Olm only has very limited memory available when compiled to wasm, we limit the amount of sessions held in memory. */ -export class KeyLoader { +export class KeyLoader extends BaseLRUCache { - public readonly cache: SessionCache; + private runningOps: Set; + private unusedOps: Set; private pickleKey: string; private olm: any; - private resolveUnusedEntry?: () => void; - private entryBecomesUnusedPromise?: Promise; + private resolveUnusedOperation?: () => void; + private operationBecomesUnusedPromise?: Promise; constructor(olm: any, pickleKey: string, limit: number) { - this.cache = new SessionCache(limit); + super(limit); this.pickleKey = pickleKey; this.olm = olm; } + getCachedKey(roomId: string, senderKey: string, sessionId: string): IRoomKey | undefined { + const idx = this.findIndexBestForSession(roomId, senderKey, sessionId); + if (idx !== -1) { + return this._getByIndexAndMoveUp(idx)!.key; + } + } + async useKey(key: IRoomKey, callback: (session: OlmInboundGroupSession, pickleKey: string) => Promise | T): Promise { - const cacheEntry = await this.allocateEntry(key); + const keyOp = await this.allocateOperation(key); try { - const {session} = cacheEntry; - key.loadInto(session, this.pickleKey); - return await callback(session, this.pickleKey); + return await callback(keyOp.session, this.pickleKey); } finally { - this.freeEntry(cacheEntry); + this.releaseOperation(keyOp); } } get running() { - return !!this.cache.find(entry => entry.inUse); + return this._entries.some(op => op.refCount !== 0); } - private async allocateEntry(key: IRoomKey): Promise { - let entry; - if (this.cache.size >= this.cache.limit) { - while(!(entry = this.cache.find(entry => !entry.inUse))) { - await this.entryBecomesUnused(); + dispose() { + for (let i = 0; i < this._entries.length; i += 1) { + this._entries[i].dispose(); + } + // remove all entries + this._entries.splice(0, this._entries.length); + } + + private async allocateOperation(key: IRoomKey): Promise { + let idx; + while((idx = this.findIndexForAllocation(key)) === -1) { + await this.operationBecomesUnused(); + } + if (idx < this.size) { + const op = this._getByIndexAndMoveUp(idx)!; + // cache hit + if (op.isForKey(key)) { + op.refCount += 1; + return op; + } else { + // refCount should be 0 here + op.refCount = 1; + op.key = key; + key.loadInto(op.session, this.pickleKey); } - entry.inUse = true; - entry.key = key; + return op; } else { - const session: OlmInboundGroupSession = new this.olm.InboundGroupSession(); - const entry = new CacheEntry(key, session); - this.cache.add(entry); + // create new operation + const session = new this.olm.InboundGroupSession(); + key.loadInto(session, this.pickleKey); + const op = new KeyOperation(key, session); + this._set(op); + return op; } - return entry; } - private freeEntry(entry: CacheEntry) { - entry.inUse = false; - if (this.resolveUnusedEntry) { - this.resolveUnusedEntry(); + private releaseOperation(op: KeyOperation) { + op.refCount -= 1; + if (op.refCount <= 0 && this.resolveUnusedOperation) { + this.resolveUnusedOperation(); // promise is resolved now, we'll need a new one for next await so clear - this.entryBecomesUnusedPromise = this.resolveUnusedEntry = undefined; + this.operationBecomesUnusedPromise = this.resolveUnusedOperation = undefined; } } - private entryBecomesUnused(): Promise { - if (!this.entryBecomesUnusedPromise) { - this.entryBecomesUnusedPromise = new Promise(resolve => { - this.resolveUnusedEntry = resolve; + private operationBecomesUnused(): Promise { + if (!this.operationBecomesUnusedPromise) { + this.operationBecomesUnusedPromise = new Promise(resolve => { + this.resolveUnusedOperation = resolve; }); } - return this.entryBecomesUnusedPromise; + return this.operationBecomesUnusedPromise; + } + + private findIndexForAllocation(key: IRoomKey) { + let idx = this.findIndexSameKey(key); // cache hit + if (idx === -1) { + idx = this.findIndexSameSessionUnused(key); + if (idx === -1) { + if (this.size < this.limit) { + idx = this.size; + } else { + idx = this.findIndexOldestUnused(); + } + } + } + return idx; + } + + private findIndexBestForSession(roomId: string, senderKey: string, sessionId: string): number { + return this._entries.reduce((bestIdx, op, i, arr) => { + const bestOp = bestIdx === -1 ? undefined : arr[bestIdx]; + if (op.isForSameSession(roomId, senderKey, sessionId)) { + if (!bestOp || op.isBetter(bestOp)) { + return i; + } + } + return bestIdx; + }, -1); + } + + private findIndexSameKey(key: IRoomKey): number { + return this._entries.findIndex(op => { + return op.isForKey(key); + }); + } + + private findIndexSameSessionUnused(key: IRoomKey): number { + for (let i = this._entries.length - 1; i >= 0; i -= 1) { + const op = this._entries[i]; + if (op.refCount === 0 && op.isForSameSession(key.roomId, key.senderKey, key.sessionId)) { + return i; + } + } + return -1; + } + + private findIndexOldestUnused(): number { + for (let i = this._entries.length - 1; i >= 0; i -= 1) { + const op = this._entries[i]; + if (op.refCount === 0) { + return i; + } + } + return -1; } } -class CacheEntry { - inUse: boolean; +class KeyOperation { session: OlmInboundGroupSession; key: IRoomKey; + refCount: number; - constructor(key, session) { + constructor(key: IRoomKey, session: OlmInboundGroupSession) { this.key = key; this.session = session; - this.inUse = true; + this.refCount = 1; + } + + isForSameSession(roomId: string, senderKey: string, sessionId: string): boolean { + return this.key.roomId === roomId && this.key.senderKey === senderKey && this.key.sessionId === sessionId; + } + + // assumes isForSameSession is true + isBetter(other: KeyOperation) { + return isBetterThan(this.session, other.session); + } + + isForKey(key: IRoomKey) { + return this.key.serializationKey === key.serializationKey && + this.key.serializationType === key.serializationType; + } + + dispose() { + this.session.free(); } } diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index c4c00d99..14f94613 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -18,20 +18,25 @@ import type {InboundGroupSessionEntry} from "../../../storage/idb/stores/Inbound import type {Transaction} from "../../../storage/idb/Transaction"; import type {DecryptionResult} from "../../DecryptionResult"; import type {KeyLoader, OlmInboundGroupSession} from "./KeyLoader"; -import {SessionCache} from "./SessionCache"; export interface IRoomKey { get roomId(): string; get senderKey(): string; get sessionId(): string; get claimedEd25519Key(): string; + get serializationKey(): string; + get serializationType(): string; get eventIds(): string[] | undefined; loadInto(session: OlmInboundGroupSession, pickleKey: string): void; } +export function isBetterThan(newSession: OlmInboundGroupSession, existingSession: OlmInboundGroupSession) { + return newSession.first_known_index() < existingSession.first_known_index(); +} + export interface IIncomingRoomKey extends IRoomKey { get isBetter(): boolean | undefined; - checkBetterKeyInStorage(loader: KeyLoader, txn: Transaction): Promise; + checkBetterThanKeyInStorage(loader: KeyLoader, txn: Transaction): Promise; write(loader: KeyLoader, txn: Transaction): Promise; } @@ -39,8 +44,8 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { private _eventIds?: string[]; private _isBetter?: boolean; - checkBetterKeyInStorage(loader: KeyLoader, txn: Transaction): Promise { - return this._checkBetterKeyInStorage(loader, undefined, txn); + checkBetterThanKeyInStorage(loader: KeyLoader, txn: Transaction): Promise { + return this._checkBetterThanKeyInStorage(loader, undefined, txn); } async write(loader: KeyLoader, txn: Transaction): Promise { @@ -51,7 +56,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { // we haven't checked if this is the best key yet, // so do that now to not overwrite a better key. // while we have the key deserialized, also pickle it to store it later on here. - await this._checkBetterKeyInStorage(loader, (session, pickleKey) => { + await this._checkBetterThanKeyInStorage(loader, (session, pickleKey) => { pickledSession = session.pickle(pickleKey); }, txn); } @@ -76,7 +81,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { get eventIds() { return this._eventIds; } get isBetter() { return this._isBetter; } - private async _checkBetterKeyInStorage(loader: KeyLoader, callback: (((session: OlmInboundGroupSession, pickleKey: string) => void) | undefined), txn: Transaction): Promise { + private async _checkBetterThanKeyInStorage(loader: KeyLoader, callback: (((session: OlmInboundGroupSession, pickleKey: string) => void) | undefined), txn: Transaction): Promise { if (this._isBetter !== undefined) { return this._isBetter; } @@ -96,7 +101,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { if (existingKey) { this._isBetter = await loader.useKey(this, newSession => { return loader.useKey(existingKey, (existingSession, pickleKey) => { - const isBetter = newSession.first_known_index() < existingSession.first_known_index(); + const isBetter = isBetterThan(newSession, existingSession); if (isBetter && callback) { callback(newSession, pickleKey); } @@ -114,6 +119,8 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { abstract get senderKey(): string; abstract get sessionId(): string; abstract get claimedEd25519Key(): string; + abstract get serializationKey(): string; + abstract get serializationType(): string; abstract loadInto(session: OlmInboundGroupSession, pickleKey: string): void; } @@ -129,10 +136,11 @@ class DeviceMessageRoomKey extends BaseIncomingRoomKey { get senderKey() { return this._decryptionResult.senderCurve25519Key; } get sessionId() { return this._decryptionResult.event.content?.["session_id"]; } get claimedEd25519Key() { return this._decryptionResult.claimedEd25519Key; } + get serializationKey(): string { return this._decryptionResult.event.content?.["session_key"]; } + get serializationType(): string { return "create"; } loadInto(session) { - const sessionKey = this._decryptionResult.event.content?.["session_key"]; - session.create(sessionKey); + session.create(this.serializationKey); } } @@ -152,10 +160,11 @@ class BackupRoomKey extends BaseIncomingRoomKey { get senderKey() { return this._backupInfo["sender_key"]; } get sessionId() { return this._sessionId; } get claimedEd25519Key() { return this._backupInfo["sender_claimed_keys"]?.["ed25519"]; } - + get serializationKey(): string { return this._backupInfo["session_key"]; } + get serializationType(): string { return "import_session"; } + loadInto(session) { - const sessionKey = this._backupInfo["session_key"]; - session.import_session(sessionKey); + session.import_session(this.serializationKey); } } @@ -171,9 +180,11 @@ class StoredRoomKey implements IRoomKey { get sessionId() { return this.storageEntry.sessionId; } get claimedEd25519Key() { return this.storageEntry.claimedKeys!["ed25519"]; } get eventIds() { return this.storageEntry.eventIds; } - + get serializationKey(): string { return this.storageEntry.session || ""; } + get serializationType(): string { return "unpickle"; } + loadInto(session, pickleKey) { - session.unpickle(pickleKey, this.storageEntry.session); + session.unpickle(pickleKey, this.serializationKey); } get hasSession() { diff --git a/src/matrix/e2ee/megolm/decryption/SessionCache.js b/src/matrix/e2ee/megolm/decryption/SessionCache.js deleted file mode 100644 index cbb868ea..00000000 --- a/src/matrix/e2ee/megolm/decryption/SessionCache.js +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {BaseLRUCache} from "../../../../utils/LRUCache"; -const DEFAULT_CACHE_SIZE = 10; - -/** - * Cache of unpickled inbound megolm session. - */ -export class SessionCache extends BaseLRUCache { - constructor(limit) { - limit = typeof limit === "number" ? limit : DEFAULT_CACHE_SIZE; - super(limit); - } - - /** - * @param {string} roomId - * @param {string} senderKey - * @param {string} sessionId - * @return {SessionInfo?} - */ - get(roomId, senderKey, sessionId) { - const sessionInfo = this._get(s => { - return s.roomId === roomId && - s.senderKey === senderKey && - sessionId === s.sessionId; - }); - sessionInfo?.retain(); - return sessionInfo; - } - - add(sessionInfo) { - sessionInfo.retain(); - this._set(sessionInfo, s => { - return s.roomId === sessionInfo.roomId && - s.senderKey === sessionInfo.senderKey && - s.sessionId === sessionInfo.sessionId; - }); - } - - _onEvictEntry(sessionInfo) { - sessionInfo.release(); - } - - dispose() { - for (const sessionInfo of this._entries) { - sessionInfo.release(); - } - } -} diff --git a/src/utils/LRUCache.ts b/src/utils/LRUCache.ts index d275b9f1..05d8fef5 100644 --- a/src/utils/LRUCache.ts +++ b/src/utils/LRUCache.ts @@ -14,25 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ + +type FindCallback = (value: T) => boolean; /** * Very simple least-recently-used cache implementation * that should be fast enough for very small cache sizes */ export class BaseLRUCache { - private _limit: number; - private _entries: T[]; + public readonly limit: number; + protected _entries: T[]; constructor(limit: number) { - this._limit = limit; + this.limit = limit; this._entries = []; } get size() { return this._entries.length; } - get limit() { return this._limit; } - _get(findEntryFn: (T) => boolean) { - const idx = this._entries.findIndex(findEntryFn); + protected _get(findEntryFn: FindCallback) { + return this._getByIndexAndMoveUp(this._entries.findIndex(findEntryFn)); + } + + protected _getByIndexAndMoveUp(idx: number) { if (idx !== -1) { const entry = this._entries[idx]; // move to top @@ -44,11 +48,11 @@ export class BaseLRUCache { } } - _set(value: T, findEntryFn: (T) => boolean) { - let indexToRemove = this._entries.findIndex(findEntryFn); + protected _set(value: T, findEntryFn?: FindCallback) { + let indexToRemove = findEntryFn ? this._entries.findIndex(findEntryFn) : -1; this._entries.unshift(value); if (indexToRemove === -1) { - if (this._entries.length > this._limit) { + if (this._entries.length > this.limit) { indexToRemove = this._entries.length - 1; } } else { @@ -56,22 +60,12 @@ export class BaseLRUCache { indexToRemove += 1; } if (indexToRemove !== -1) { - this._onEvictEntry(this._entries[indexToRemove]); + this.onEvictEntry(this._entries[indexToRemove]); this._entries.splice(indexToRemove, 1); } } - find(callback: (T) => boolean) { - // iterate backwards so least recently used items are found first - for (let i = this._entries.length - 1; i >= 0; i -= 1) { - const entry = this._entries[i]; - if (callback(entry)) { - return entry; - } - } - } - - _onEvictEntry(entry: T) {} + protected onEvictEntry(entry: T) {} } export class LRUCache extends BaseLRUCache { From 77d10c93d67571092ed709208c7076aab199240f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 21 Oct 2021 14:40:51 +0200 Subject: [PATCH 08/26] convert groupby and megolm decryption utils to typescript --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 2 +- src/matrix/e2ee/RoomEncryption.js | 4 +-- .../megolm/decryption/{utils.js => utils.ts} | 30 ++++++++++--------- src/matrix/e2ee/olm/Decryption.js | 2 +- src/matrix/e2ee/olm/Encryption.js | 2 +- src/utils/{groupBy.js => groupBy.ts} | 10 +++---- 7 files changed, 27 insertions(+), 25 deletions(-) rename src/matrix/e2ee/megolm/decryption/{utils.js => utils.ts} (50%) rename src/utils/{groupBy.js => groupBy.ts} (79%) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 27f1d386..1f79d517 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -15,7 +15,7 @@ limitations under the License. */ import {OLM_ALGORITHM} from "./e2ee/common.js"; -import {countBy, groupBy} from "../utils/groupBy.js"; +import {countBy, groupBy} from "../utils/groupBy"; export class DeviceMessageHandler { constructor({storage}) { diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 36c8e084..af1bf9a8 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -33,7 +33,7 @@ import {MEGOLM_ALGORITHM} from "./e2ee/common.js"; import {RoomEncryption} from "./e2ee/RoomEncryption.js"; import {DeviceTracker} from "./e2ee/DeviceTracker.js"; import {LockMap} from "../utils/LockMap.js"; -import {groupBy} from "../utils/groupBy.js"; +import {groupBy} from "../utils/groupBy"; import { keyFromCredential as ssssKeyFromCredential, readKey as ssssReadKey, diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index aba7d07d..cd2b5d59 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -15,9 +15,9 @@ limitations under the License. */ import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js"; -import {groupEventsBySession} from "./megolm/decryption/utils.js"; +import {groupEventsBySession} from "./megolm/decryption/utils"; import {mergeMap} from "../../utils/mergeMap.js"; -import {groupBy} from "../../utils/groupBy.js"; +import {groupBy} from "../../utils/groupBy"; import {makeTxnId} from "../common.js"; const ENCRYPTED_TYPE = "m.room.encrypted"; diff --git a/src/matrix/e2ee/megolm/decryption/utils.js b/src/matrix/e2ee/megolm/decryption/utils.ts similarity index 50% rename from src/matrix/e2ee/megolm/decryption/utils.js rename to src/matrix/e2ee/megolm/decryption/utils.ts index c38b1416..4207006b 100644 --- a/src/matrix/e2ee/megolm/decryption/utils.js +++ b/src/matrix/e2ee/megolm/decryption/utils.ts @@ -14,44 +14,46 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {groupByWithCreator} from "../../../../utils/groupBy.js"; +import {groupByWithCreator} from "../../../../utils/groupBy"; +import type {TimelineEvent} from "../../../storage/types"; -function getSenderKey(event) { +function getSenderKey(event: TimelineEvent): string | undefined { return event.content?.["sender_key"]; } -function getSessionId(event) { +function getSessionId(event: TimelineEvent): string | undefined { return event.content?.["session_id"]; } -function getCiphertext(event) { +function getCiphertext(event: TimelineEvent): string | undefined { return event.content?.ciphertext; } -export function validateEvent(event) { +export function validateEvent(event: TimelineEvent) { return typeof getSenderKey(event) === "string" && typeof getSessionId(event) === "string" && typeof getCiphertext(event) === "string"; } -class SessionKeyGroup { +export class SessionKeyGroup { + public readonly events: TimelineEvent[]; constructor() { this.events = []; } - get senderKey() { - return getSenderKey(this.events[0]); + get senderKey(): string | undefined { + return getSenderKey(this.events[0]!); } - get sessionId() { - return getSessionId(this.events[0]); + get sessionId(): string | undefined { + return getSessionId(this.events[0]!); } } -export function groupEventsBySession(events) { - return groupByWithCreator(events, - event => `${getSenderKey(event)}|${getSessionId(event)}`, +export function groupEventsBySession(events: TimelineEvent[]): Map { + return groupByWithCreator(events, + (event: TimelineEvent) => `${getSenderKey(event)}|${getSessionId(event)}`, () => new SessionKeyGroup(), - (group, event) => group.events.push(event) + (group: SessionKeyGroup, event: TimelineEvent) => group.events.push(event) ); } diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js index 7556c367..c3777bd3 100644 --- a/src/matrix/e2ee/olm/Decryption.js +++ b/src/matrix/e2ee/olm/Decryption.js @@ -15,7 +15,7 @@ limitations under the License. */ import {DecryptionError} from "../common.js"; -import {groupBy} from "../../../utils/groupBy.js"; +import {groupBy} from "../../../utils/groupBy"; import {MultiLock} from "../../../utils/Lock.js"; import {Session} from "./Session.js"; import {DecryptionResult} from "../DecryptionResult.js"; diff --git a/src/matrix/e2ee/olm/Encryption.js b/src/matrix/e2ee/olm/Encryption.js index 3bc66ec3..1b720ae7 100644 --- a/src/matrix/e2ee/olm/Encryption.js +++ b/src/matrix/e2ee/olm/Encryption.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {groupByWithCreator} from "../../../utils/groupBy.js"; +import {groupByWithCreator} from "../../../utils/groupBy"; import {verifyEd25519Signature, OLM_ALGORITHM} from "../common.js"; import {createSessionEntry} from "./Session.js"; diff --git a/src/utils/groupBy.js b/src/utils/groupBy.ts similarity index 79% rename from src/utils/groupBy.js rename to src/utils/groupBy.ts index 9bed5298..2d91b209 100644 --- a/src/utils/groupBy.js +++ b/src/utils/groupBy.ts @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -export function groupBy(array, groupFn) { - return groupByWithCreator(array, groupFn, +export function groupBy(array: V[], groupFn: (V) => K): Map { + return groupByWithCreator(array, groupFn, () => {return [];}, (array, value) => array.push(value) ); } -export function groupByWithCreator(array, groupFn, createCollectionFn, addCollectionFn) { +export function groupByWithCreator(array: V[], groupFn: (V) => K, createCollectionFn: () => C, addCollectionFn: (C, V) => void): Map { return array.reduce((map, value) => { const key = groupFn(value); let collection = map.get(key); @@ -31,10 +31,10 @@ export function groupByWithCreator(array, groupFn, createCollectionFn, addCollec } addCollectionFn(collection, value); return map; - }, new Map()); + }, new Map()); } -export function countBy(events, mapper) { +export function countBy(events: V[], mapper: (V) => string | number): { [key: string]: number } { return events.reduce((counts, event) => { const mappedValue = mapper(event); if (!counts[mappedValue]) { From 45dc2162dc21702f7e28698f7e1e9f691758f009 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:30:20 +0200 Subject: [PATCH 09/26] fix unit tests --- src/utils/LRUCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/LRUCache.ts b/src/utils/LRUCache.ts index 05d8fef5..c5a7cd06 100644 --- a/src/utils/LRUCache.ts +++ b/src/utils/LRUCache.ts @@ -135,7 +135,7 @@ export function tests() { "evict callback is called": assert => { let evictions = 0; class CustomCache extends LRUCache { - _onEvictEntry(entry) { + onEvictEntry(entry) { assert.equal(entry.name, "Alice"); evictions += 1; } @@ -149,7 +149,7 @@ export function tests() { "evict callback is called when replacing entry with same identity": assert => { let evictions = 0; class CustomCache extends LRUCache { - _onEvictEntry(entry) { + onEvictEntry(entry) { assert.equal(entry.name, "Alice"); evictions += 1; } From 2ddb3fbf72a09dc61bedc6b133621e7b9a7f1179 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:45:55 +0200 Subject: [PATCH 10/26] cleanup --- .../e2ee/megolm/decryption/KeyLoader.ts | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index bde3065f..d1741f30 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -14,10 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SessionCache} from "./SessionCache"; import {IRoomKey, isBetterThan} from "./RoomKey"; import {BaseLRUCache} from "../../../../utils/LRUCache"; + +export declare class OlmDecryptionResult { + readonly plaintext: string; + readonly message_index: number; +} + export declare class OlmInboundGroupSession { constructor(); free(): void; @@ -25,34 +30,18 @@ export declare class OlmInboundGroupSession { unpickle(key: string | Uint8Array, pickle: string); create(session_key: string): string; import_session(session_key: string): string; - decrypt(message: string): object; + decrypt(message: string): OlmDecryptionResult; session_id(): string; first_known_index(): number; export_session(message_index: number): string; } -// this is what cache.get(...) should return -function findIndexBestForSession(ops: KeyOperation[], roomId: string, senderKey: string, sessionId: string): number { - return ops.reduce((bestIdx, op, i, arr) => { - const bestOp = bestIdx === -1 ? undefined : arr[bestIdx]; - if (op.isForSameSession(roomId, senderKey, sessionId)) { - if (!bestOp || op.isBetter(bestOp)) { - return i; - } - } - return bestIdx; - }, -1); -} - - /* Because Olm only has very limited memory available when compiled to wasm, we limit the amount of sessions held in memory. */ export class KeyLoader extends BaseLRUCache { - private runningOps: Set; - private unusedOps: Set; private pickleKey: string; private olm: any; private resolveUnusedOperation?: () => void; From d6e243321b67d73fe3ccca25015c694f9424f873 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:46:39 +0200 Subject: [PATCH 11/26] convert megolm/Decryption to typescript and adapt to KeyLoader --- .../megolm/{Decryption.js => Decryption.ts} | 97 ++++++++----------- 1 file changed, 41 insertions(+), 56 deletions(-) rename src/matrix/e2ee/megolm/{Decryption.js => Decryption.ts} (61%) diff --git a/src/matrix/e2ee/megolm/Decryption.js b/src/matrix/e2ee/megolm/Decryption.ts similarity index 61% rename from src/matrix/e2ee/megolm/Decryption.js rename to src/matrix/e2ee/megolm/Decryption.ts index a4771cc8..fdca3232 100644 --- a/src/matrix/e2ee/megolm/Decryption.js +++ b/src/matrix/e2ee/megolm/Decryption.ts @@ -15,23 +15,26 @@ limitations under the License. */ import {DecryptionError} from "../common.js"; -import * as RoomKey from "./decryption/RoomKey"; -import {SessionInfo} from "./decryption/SessionInfo.js"; import {DecryptionPreparation} from "./decryption/DecryptionPreparation.js"; -import {SessionDecryption} from "./decryption/SessionDecryption.js"; -import {SessionCache} from "./decryption/SessionCache.js"; +import {SessionDecryption} from "./decryption/SessionDecryption"; import {MEGOLM_ALGORITHM} from "../common.js"; -import {validateEvent, groupEventsBySession} from "./decryption/utils.js"; +import {validateEvent, groupEventsBySession} from "./decryption/utils"; +import {keyFromStorage, keyFromDeviceMessage, keyFromBackup} from "./decryption/RoomKey"; +import type {IRoomKey, IIncomingRoomKey} from "./decryption/RoomKey"; +import type {KeyLoader} from "./decryption/KeyLoader"; +import type {OlmWorker} from "../OlmWorker"; +import type {Transaction} from "../../storage/idb/Transaction"; +import type {TimelineEvent} from "../../storage/types"; +import type {DecryptionResult} from "../DecryptionResult"; +import type {LogItem} from "../../../logging/LogItem"; export class Decryption { - constructor({pickleKey, olm, olmWorker}) { - this._pickleKey = pickleKey; - this._olm = olm; - this._olmWorker = olmWorker; - } + private keyLoader: KeyLoader; + private olmWorker?: OlmWorker; - createSessionCache(size) { - return new SessionCache(size); + constructor(keyLoader: KeyLoader, olmWorker: OlmWorker | undefined) { + this.keyLoader = keyLoader; + this.olmWorker = olmWorker; } async addMissingKeyEventIds(roomId, senderKey, sessionId, eventIds, txn) { @@ -75,9 +78,9 @@ export class Decryption { * @param {[type]} txn [description] * @return {DecryptionPreparation} */ - async prepareDecryptAll(roomId, events, newKeys, sessionCache, txn) { + async prepareDecryptAll(roomId: string, events: TimelineEvent[], newKeys: IIncomingRoomKey[] | undefined, txn: Transaction) { const errors = new Map(); - const validEvents = []; + const validEvents: TimelineEvent[] = []; for (const event of events) { if (validateEvent(event)) { @@ -89,11 +92,11 @@ export class Decryption { const eventsBySession = groupEventsBySession(validEvents); - const sessionDecryptions = []; + const sessionDecryptions: SessionDecryption[] = []; await Promise.all(Array.from(eventsBySession.values()).map(async group => { - const sessionInfo = await this._getSessionInfo(roomId, group.senderKey, group.sessionId, newKeys, sessionCache, txn); - if (sessionInfo) { - sessionDecryptions.push(new SessionDecryption(sessionInfo, group.events, this._olmWorker)); + const key = await this.getRoomKey(roomId, group.senderKey!, group.sessionId!, newKeys, txn); + if (key) { + sessionDecryptions.push(new SessionDecryption(key, group.events, this.olmWorker, this.keyLoader)); } else { for (const event of group.events) { errors.set(event.event_id, new DecryptionError("MEGOLM_NO_SESSION", event)); @@ -104,64 +107,43 @@ export class Decryption { return new DecryptionPreparation(roomId, sessionDecryptions, errors); } - async _getSessionInfo(roomId, senderKey, sessionId, newKeys, sessionCache, txn) { - let sessionInfo; + private async getRoomKey(roomId: string, senderKey: string, sessionId: string, newKeys: IIncomingRoomKey[] | undefined, txn: Transaction): Promise { if (newKeys) { const key = newKeys.find(k => k.roomId === roomId && k.senderKey === senderKey && k.sessionId === sessionId); - if (key) { - sessionInfo = await key.createSessionInfo(this._olm, this._pickleKey, txn); - if (sessionInfo) { - sessionCache.add(sessionInfo); - } + if (key && await key.checkBetterThanKeyInStorage(this.keyLoader, txn)) { + return key; } } // look only in the cache after looking into newKeys as it may contains that are better - if (!sessionInfo) { - sessionInfo = sessionCache.get(roomId, senderKey, sessionId); - // TODO: shouldn't we retain here? + const cachedKey = this.keyLoader.getCachedKey(roomId, senderKey, sessionId); + if (cachedKey) { + return cachedKey; } - if (!sessionInfo) { - const sessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId); - if (sessionEntry && sessionEntry.session) { - let session = new this._olm.InboundGroupSession(); - try { - session.unpickle(this._pickleKey, sessionEntry.session); - sessionInfo = new SessionInfo(roomId, senderKey, session, sessionEntry.claimedKeys); - } catch (err) { - session.free(); - throw err; - } - sessionCache.add(sessionInfo); - } + const storageKey = await keyFromStorage(roomId, senderKey, sessionId, txn); + if (storageKey && storageKey.serializationKey) { + return storageKey; } - return sessionInfo; } /** * Writes the key as an inbound group session if there is not already a better key in the store - * @param {RoomKey} key - * @param {Transaction} txn a storage transaction with read/write on inboundGroupSessions - * @return {Promise} whether the key was the best for the sessio id and was written */ - writeRoomKey(key, txn) { - return key.write(this._olm, this._pickleKey, txn); + writeRoomKey(key: IIncomingRoomKey, txn: Transaction): Promise { + return key.write(this.keyLoader, txn); } /** * Extracts room keys from decrypted device messages. * The key won't be persisted yet, you need to call RoomKey.write for that. - * - * @param {Array} decryptionResults, any non megolm m.room_key messages will be ignored. - * @return {Array} an array with validated RoomKey's. Note that it is possible we already have a better version of this key in storage though; writing the key will tell you so. */ - roomKeysFromDeviceMessages(decryptionResults, log) { - let keys = []; + roomKeysFromDeviceMessages(decryptionResults: DecryptionResult[], log: LogItem): IIncomingRoomKey[] { + let keys: IIncomingRoomKey[] = []; for (const dr of decryptionResults) { if (dr.event?.type !== "m.room_key" || dr.event.content?.algorithm !== MEGOLM_ALGORITHM) { continue; } log.wrap("room_key", log => { - const key = RoomKey.fromDeviceMessage(dr); + const key = keyFromDeviceMessage(dr); if (key) { log.set("roomId", key.roomId); log.set("id", key.sessionId); @@ -175,8 +157,11 @@ export class Decryption { return keys; } - roomKeyFromBackup(roomId, sessionId, sessionInfo) { - return RoomKey.fromBackup(roomId, sessionId, sessionInfo); + roomKeyFromBackup(roomId: string, sessionId: string, sessionInfo: string): IIncomingRoomKey | undefined { + return keyFromBackup(roomId, sessionId, sessionInfo); + } + + dispose() { + this.keyLoader.dispose(); } } - From b55930f084bbeb4a80c027e0a2e178c43b8773b7 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:47:29 +0200 Subject: [PATCH 12/26] convert ReplayDetectionEntry to typescript --- ...ectionEntry.js => ReplayDetectionEntry.ts} | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) rename src/matrix/e2ee/megolm/decryption/{ReplayDetectionEntry.js => ReplayDetectionEntry.ts} (61%) diff --git a/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js b/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.ts similarity index 61% rename from src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js rename to src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.ts index e5ce2845..f3807c39 100644 --- a/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.js +++ b/src/matrix/e2ee/megolm/decryption/ReplayDetectionEntry.ts @@ -14,11 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ +import type {TimelineEvent} from "../../../storage/types"; + export class ReplayDetectionEntry { - constructor(sessionId, messageIndex, event) { + public readonly sessionId: string; + public readonly messageIndex: number; + public readonly event: TimelineEvent; + + constructor(sessionId: string, messageIndex: number, event: TimelineEvent) { this.sessionId = sessionId; this.messageIndex = messageIndex; - this.eventId = event.event_id; - this.timestamp = event.origin_server_ts; + this.event = event; + } + + get eventId(): string { + return this.event.event_id; + } + + get timestamp(): number { + return this.event.origin_server_ts; } } From ac23119838b93d29f0a51058eca6a25797be9791 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:48:04 +0200 Subject: [PATCH 13/26] convert SessionDecryption to TS and adapt to use KeyLoader --- src/matrix/e2ee/DecryptionResult.js | 4 +- .../megolm/decryption/SessionDecryption.js | 90 --------------- .../megolm/decryption/SessionDecryption.ts | 103 ++++++++++++++++++ src/matrix/e2ee/olm/Decryption.js | 2 +- 4 files changed, 106 insertions(+), 93 deletions(-) delete mode 100644 src/matrix/e2ee/megolm/decryption/SessionDecryption.js create mode 100644 src/matrix/e2ee/megolm/decryption/SessionDecryption.ts diff --git a/src/matrix/e2ee/DecryptionResult.js b/src/matrix/e2ee/DecryptionResult.js index c109e689..e1c2bcc4 100644 --- a/src/matrix/e2ee/DecryptionResult.js +++ b/src/matrix/e2ee/DecryptionResult.js @@ -29,10 +29,10 @@ limitations under the License. export class DecryptionResult { - constructor(event, senderCurve25519Key, claimedKeys) { + constructor(event, senderCurve25519Key, claimedEd25519Key) { this.event = event; this.senderCurve25519Key = senderCurve25519Key; - this.claimedEd25519Key = claimedKeys.ed25519; + this.claimedEd25519Key = claimedEd25519Key; this._device = null; this._roomTracked = true; } diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js b/src/matrix/e2ee/megolm/decryption/SessionDecryption.js deleted file mode 100644 index 137ae9f8..00000000 --- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.js +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {DecryptionResult} from "../../DecryptionResult.js"; -import {DecryptionError} from "../../common.js"; -import {ReplayDetectionEntry} from "./ReplayDetectionEntry.js"; - -/** - * Does the actual decryption of all events for a given megolm session in a batch - */ -export class SessionDecryption { - constructor(sessionInfo, events, olmWorker) { - sessionInfo.retain(); - this._sessionInfo = sessionInfo; - this._events = events; - this._olmWorker = olmWorker; - this._decryptionRequests = olmWorker ? [] : null; - } - - async decryptAll() { - const replayEntries = []; - const results = new Map(); - let errors; - const roomId = this._sessionInfo.roomId; - - await Promise.all(this._events.map(async event => { - try { - const {session} = this._sessionInfo; - const ciphertext = event.content.ciphertext; - let decryptionResult; - if (this._olmWorker) { - const request = this._olmWorker.megolmDecrypt(session, ciphertext); - this._decryptionRequests.push(request); - decryptionResult = await request.response(); - } else { - decryptionResult = session.decrypt(ciphertext); - } - const plaintext = decryptionResult.plaintext; - const messageIndex = decryptionResult.message_index; - let payload; - try { - payload = JSON.parse(plaintext); - } catch (err) { - throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err}); - } - if (payload.room_id !== roomId) { - throw new DecryptionError("MEGOLM_WRONG_ROOM", event, - {encryptedRoomId: payload.room_id, eventRoomId: roomId}); - } - replayEntries.push(new ReplayDetectionEntry(session.session_id(), messageIndex, event)); - const result = new DecryptionResult(payload, this._sessionInfo.senderKey, this._sessionInfo.claimedKeys); - results.set(event.event_id, result); - } catch (err) { - // ignore AbortError from cancelling decryption requests in dispose method - if (err.name === "AbortError") { - return; - } - if (!errors) { - errors = new Map(); - } - errors.set(event.event_id, err); - } - })); - - return {results, errors, replayEntries}; - } - - dispose() { - if (this._decryptionRequests) { - for (const r of this._decryptionRequests) { - r.abort(); - } - } - // TODO: cancel decryptions here - this._sessionInfo.release(); - } -} diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts new file mode 100644 index 00000000..3adf5bdb --- /dev/null +++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts @@ -0,0 +1,103 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {DecryptionResult} from "../../DecryptionResult.js"; +import {DecryptionError} from "../../common.js"; +import {ReplayDetectionEntry} from "./ReplayDetectionEntry"; +import type {IRoomKey} from "./RoomKey.js"; +import type {KeyLoader, OlmDecryptionResult} from "./KeyLoader"; +import type {OlmWorker} from "../../OlmWorker"; +import type {TimelineEvent} from "../../../storage/types"; + +interface DecryptAllResult { + readonly results: Map; + readonly errors?: Map; + readonly replayEntries: ReplayDetectionEntry[]; +} +/** + * Does the actual decryption of all events for a given megolm session in a batch + */ +export class SessionDecryption { + private key: IRoomKey; + private events: TimelineEvent[]; + private keyLoader: KeyLoader; + private olmWorker?: OlmWorker; + private decryptionRequests?: any[]; + + constructor(key: IRoomKey, events: TimelineEvent[], olmWorker: OlmWorker | undefined, keyLoader: KeyLoader) { + this.key = key; + this.events = events; + this.olmWorker = olmWorker; + this.keyLoader = keyLoader; + this.decryptionRequests = olmWorker ? [] : undefined; + } + + async decryptAll(): Promise { + const replayEntries: ReplayDetectionEntry[] = []; + const results: Map = new Map(); + let errors: Map | undefined; + + await this.keyLoader.useKey(this.key, async session => { + for (const event of this.events) { + try { + const ciphertext = event.content.ciphertext as string; + let decryptionResult: OlmDecryptionResult | undefined; + // TODO: pass all cipthertexts in one go to the megolm worker and don't deserialize the key until in the worker? + if (this.olmWorker) { + const request = this.olmWorker.megolmDecrypt(session, ciphertext); + this.decryptionRequests!.push(request); + decryptionResult = await request.response(); + } else { + decryptionResult = session.decrypt(ciphertext); + } + const {plaintext} = decryptionResult!; + let payload; + try { + payload = JSON.parse(plaintext); + } catch (err) { + throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, err}); + } + if (payload.room_id !== this.key.roomId) { + throw new DecryptionError("MEGOLM_WRONG_ROOM", event, + {encryptedRoomId: payload.room_id, eventRoomId: this.key.roomId}); + } + replayEntries.push(new ReplayDetectionEntry(this.key.sessionId, decryptionResult!.message_index, event)); + const result = new DecryptionResult(payload, this.key.senderKey, this.key.claimedEd25519Key); + results.set(event.event_id, result); + } catch (err) { + // ignore AbortError from cancelling decryption requests in dispose method + if (err.name === "AbortError") { + return; + } + if (!errors) { + errors = new Map(); + } + errors.set(event.event_id, err); + } + } + }); + + return {results, errors, replayEntries}; + } + + dispose() { + if (this.decryptionRequests) { + for (const r of this.decryptionRequests) { + r.abort(); + } + } + } +} diff --git a/src/matrix/e2ee/olm/Decryption.js b/src/matrix/e2ee/olm/Decryption.js index c3777bd3..0af3bd23 100644 --- a/src/matrix/e2ee/olm/Decryption.js +++ b/src/matrix/e2ee/olm/Decryption.js @@ -150,7 +150,7 @@ export class Decryption { throw new DecryptionError("PLAINTEXT_NOT_JSON", event, {plaintext, error}); } this._validatePayload(payload, event); - return new DecryptionResult(payload, senderKey, payload.keys); + return new DecryptionResult(payload, senderKey, payload.keys.ed25519); } else { throw new DecryptionError("OLM_NO_MATCHING_SESSION", event, {knownSessionIds: senderKeyDecryption.sessions.map(s => s.id)}); From 66a93ee10843916f71bdc7108af4043ed17bd744 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:48:53 +0200 Subject: [PATCH 14/26] adapt Session and RoomEncryption to megolm/Decryption API changes --- src/matrix/Session.js | 10 ++--- src/matrix/e2ee/RoomEncryption.js | 64 ++++++++----------------------- 2 files changed, 20 insertions(+), 54 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index af1bf9a8..0c257728 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -26,7 +26,7 @@ import {DeviceMessageHandler} from "./DeviceMessageHandler.js"; import {Account as E2EEAccount} from "./e2ee/Account.js"; import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js"; import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js"; -import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js"; +import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption"; import {SessionBackup} from "./e2ee/megolm/SessionBackup.js"; import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js"; import {MEGOLM_ALGORITHM} from "./e2ee/common.js"; @@ -137,11 +137,8 @@ export class Session { now: this._platform.clock.now, ownDeviceId: this._sessionInfo.deviceId, }); - this._megolmDecryption = new MegOlmDecryption({ - pickleKey: PICKLE_KEY, - olm: this._olm, - olmWorker: this._olmWorker, - }); + const keyLoader = new KeyLoader(this._olm, PICKLE_KEY, 20); + this._megolmDecryption = new MegOlmDecryption(keyLoader, this._olmWorker); this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption}); } @@ -319,6 +316,7 @@ export class Session { dispose() { this._olmWorker?.dispose(); this._sessionBackup?.dispose(); + this._megolmDecryption.dispose(); for (const room of this._rooms.values()) { room.dispose(); } diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index cd2b5d59..66b3366f 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -36,8 +36,6 @@ export class RoomEncryption { this._megolmDecryption = megolmDecryption; // content of the m.room.encryption event this._encryptionParams = encryptionParams; - this._megolmBackfillCache = this._megolmDecryption.createSessionCache(); - this._megolmSyncCache = this._megolmDecryption.createSessionCache(1); // caches devices to verify events this._senderDeviceCache = new Map(); this._storage = storage; @@ -76,9 +74,6 @@ export class RoomEncryption { } notifyTimelineClosed() { - // empty the backfill cache when closing the timeline - this._megolmBackfillCache.dispose(); - this._megolmBackfillCache = this._megolmDecryption.createSessionCache(); this._senderDeviceCache = new Map(); // purge the sender device cache } @@ -112,27 +107,8 @@ export class RoomEncryption { } validEvents.push(event); } - let customCache; - let sessionCache; - // we have different caches so we can keep them small but still - // have backfill and sync not invalidate each other - if (source === DecryptionSource.Sync) { - sessionCache = this._megolmSyncCache; - } else if (source === DecryptionSource.Timeline) { - sessionCache = this._megolmBackfillCache; - } else if (source === DecryptionSource.Retry) { - // when retrying, we could have mixed events from at the bottom of the timeline (sync) - // and somewhere else, so create a custom cache we use just for this operation. - customCache = this._megolmDecryption.createSessionCache(); - sessionCache = customCache; - } else { - throw new Error("Unknown source: " + source); - } const preparation = await this._megolmDecryption.prepareDecryptAll( - this._room.id, validEvents, newKeys, sessionCache, txn); - if (customCache) { - customCache.dispose(); - } + this._room.id, validEvents, newKeys, txn); return new DecryptionPreparation(preparation, errors, source, this, events); } @@ -204,37 +180,31 @@ export class RoomEncryption { return; } log.set("id", sessionId); - log.set("senderKey", senderKey); + log.set("senderKey", senderKey); try { const session = await this._sessionBackup.getSession(this._room.id, sessionId, log); if (session?.algorithm === MEGOLM_ALGORITHM) { - if (session["sender_key"] !== senderKey) { - log.set("wrong_sender_key", session["sender_key"]); - log.logLevel = log.level.Warn; - return; - } let roomKey = this._megolmDecryption.roomKeyFromBackup(this._room.id, sessionId, session); if (roomKey) { + if (roomKey.senderKey !== senderKey) { + log.set("wrong_sender_key", roomKey.senderKey); + log.logLevel = log.level.Warn; + return; + } let keyIsBestOne = false; let retryEventIds; + const txn = await this._storage.readWriteTxn([this._storage.storeNames.inboundGroupSessions]); try { - const txn = await this._storage.readWriteTxn([this._storage.storeNames.inboundGroupSessions]); - try { - keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn); - log.set("isBetter", keyIsBestOne); - if (keyIsBestOne) { - retryEventIds = roomKey.eventIds; - } - } catch (err) { - txn.abort(); - throw err; + keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn); + log.set("isBetter", keyIsBestOne); + if (keyIsBestOne) { + retryEventIds = roomKey.eventIds; } - await txn.complete(); - } finally { - // can still access properties on it afterwards - // this is just clearing the internal sessionInfo - roomKey.dispose(); + } catch (err) { + txn.abort(); + throw err; } + await txn.complete(); if (keyIsBestOne) { await log.wrap("retryDecryption", log => this._room.notifyRoomKey(roomKey, retryEventIds || [], log)); } @@ -466,8 +436,6 @@ export class RoomEncryption { dispose() { this._disposed = true; - this._megolmBackfillCache.dispose(); - this._megolmSyncCache.dispose(); } } From 1278288a429fcbb91a0f728f98fbfb19f4351a0f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:50:30 +0200 Subject: [PATCH 15/26] cleanup RoomKey to changes and better naming --- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index 14f94613..48af45a7 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -85,9 +85,9 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { if (this._isBetter !== undefined) { return this._isBetter; } - let existingKey = loader.cache.get(this.roomId, this.senderKey, this.sessionId); + let existingKey = loader.getCachedKey(this.roomId, this.senderKey, this.sessionId); if (!existingKey) { - const storageKey = await fromStorage(this.roomId, this.senderKey, this.sessionId, txn); + const storageKey = await keyFromStorage(this.roomId, this.senderKey, this.sessionId, txn); // store the event ids that can be decrypted with this key // before we overwrite them if called from `write`. if (storageKey) { @@ -99,8 +99,9 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { } } if (existingKey) { + const key = existingKey; this._isBetter = await loader.useKey(this, newSession => { - return loader.useKey(existingKey, (existingSession, pickleKey) => { + return loader.useKey(key, (existingSession, pickleKey) => { const isBetter = isBetterThan(newSession, existingSession); if (isBetter && callback) { callback(newSession, pickleKey); @@ -112,7 +113,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { // no previous key, so we're the best \o/ this._isBetter = true; } - return this._isBetter; + return this._isBetter!; } abstract get roomId(): string; @@ -195,7 +196,7 @@ class StoredRoomKey implements IRoomKey { } } -export function fromDeviceMessage(dr: DecryptionResult): DeviceMessageRoomKey | undefined { +export function keyFromDeviceMessage(dr: DecryptionResult): DeviceMessageRoomKey | undefined { const sessionKey = dr.event.content?.["session_key"]; const key = new DeviceMessageRoomKey(dr); if ( @@ -216,7 +217,7 @@ sessionInfo is a response from key backup and has the following keys: sender_key session_key */ -export function fromBackup(roomId, sessionId, backupInfo): BackupRoomKey | undefined { +export function keyFromBackup(roomId, sessionId, backupInfo): BackupRoomKey | undefined { const sessionKey = backupInfo["session_key"]; const senderKey = backupInfo["sender_key"]; // TODO: can we just trust this? @@ -233,7 +234,7 @@ export function fromBackup(roomId, sessionId, backupInfo): BackupRoomKey | undef } } -export async function fromStorage(roomId: string, senderKey: string, sessionId: string, txn: Transaction): Promise { +export async function keyFromStorage(roomId: string, senderKey: string, sessionId: string, txn: Transaction): Promise { const existingSessionEntry = await txn.inboundGroupSessions.get(roomId, senderKey, sessionId); if (existingSessionEntry) { return new StoredRoomKey(existingSessionEntry); From 2943cb525f6ba83334b0ab4c6c89cefca0808141 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:50:45 +0200 Subject: [PATCH 16/26] add comment about possible future optimization --- src/matrix/room/Room.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index cab0e13b..a0d7c693 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -76,6 +76,10 @@ export class Room extends BaseRoom { let eventsToDecrypt = roomResponse?.timeline?.events || []; // when new keys arrive, also see if any older events can now be retried to decrypt if (newKeys) { + // TODO: if a key is considered by roomEncryption.prepareDecryptAll to use for decryption, + // key.eventIds will be set. We could somehow try to reuse that work, but retrying also needs + // to happen if a key is not needed to decrypt this sync or there are indeed no encrypted messages + // in this sync at all. retryEntries = await this._getSyncRetryDecryptEntries(newKeys, roomEncryption, txn); if (retryEntries.length) { log.set("retry", retryEntries.length); From b7e3a54e15058dd9fbb753a5fd2f58639e3bfdc9 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 17:51:00 +0200 Subject: [PATCH 17/26] remove now usused code --- .../e2ee/megolm/decryption/SessionInfo.js | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 src/matrix/e2ee/megolm/decryption/SessionInfo.js diff --git a/src/matrix/e2ee/megolm/decryption/SessionInfo.js b/src/matrix/e2ee/megolm/decryption/SessionInfo.js deleted file mode 100644 index 098bc3de..00000000 --- a/src/matrix/e2ee/megolm/decryption/SessionInfo.js +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/** - * session loaded in memory with everything needed to create DecryptionResults - * and to store/retrieve it in the SessionCache - */ -export class SessionInfo { - constructor(roomId, senderKey, session, claimedKeys) { - this.roomId = roomId; - this.senderKey = senderKey; - this.session = session; - this.claimedKeys = claimedKeys; - this._refCounter = 0; - } - - get sessionId() { - return this.session?.session_id(); - } - - retain() { - this._refCounter += 1; - } - - release() { - this._refCounter -= 1; - if (this._refCounter <= 0) { - this.dispose(); - } - } - - dispose() { - this.session.free(); - this.session = null; - } -} From 6d8ec69a4d0218b61f85f4ac118d0b8aac86dd70 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 18:01:17 +0200 Subject: [PATCH 18/26] fix lint --- src/matrix/Session.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 0c257728..7047171e 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -27,6 +27,7 @@ import {Account as E2EEAccount} from "./e2ee/Account.js"; import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js"; import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js"; import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption"; +import {KeyLoader as MegOlmKeyLoader} from "./e2ee/megolm/decryption/KeyLoader"; import {SessionBackup} from "./e2ee/megolm/SessionBackup.js"; import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js"; import {MEGOLM_ALGORITHM} from "./e2ee/common.js"; @@ -137,7 +138,7 @@ export class Session { now: this._platform.clock.now, ownDeviceId: this._sessionInfo.deviceId, }); - const keyLoader = new KeyLoader(this._olm, PICKLE_KEY, 20); + const keyLoader = new MegOlmKeyLoader(this._olm, PICKLE_KEY, 20); this._megolmDecryption = new MegOlmDecryption(keyLoader, this._olmWorker); this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption}); } From 076f450ec7c498abd48196d27e6a3c94a2dabb5f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 18:01:26 +0200 Subject: [PATCH 19/26] this can be const --- src/matrix/e2ee/megolm/Decryption.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/e2ee/megolm/Decryption.ts b/src/matrix/e2ee/megolm/Decryption.ts index fdca3232..7ab0e15a 100644 --- a/src/matrix/e2ee/megolm/Decryption.ts +++ b/src/matrix/e2ee/megolm/Decryption.ts @@ -137,7 +137,7 @@ export class Decryption { * The key won't be persisted yet, you need to call RoomKey.write for that. */ roomKeysFromDeviceMessages(decryptionResults: DecryptionResult[], log: LogItem): IIncomingRoomKey[] { - let keys: IIncomingRoomKey[] = []; + const keys: IIncomingRoomKey[] = []; for (const dr of decryptionResults) { if (dr.event?.type !== "m.room_key" || dr.event.content?.algorithm !== MEGOLM_ALGORITHM) { continue; From 22361bdf42998a6d362f586de108ae3ab22aadf7 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 18:08:09 +0200 Subject: [PATCH 20/26] don't need to dispose room keys anymore, they are owned by the loader --- src/matrix/DeviceMessageHandler.js | 8 -------- src/matrix/Sync.js | 1 - 2 files changed, 9 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 1f79d517..0a606841 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -67,12 +67,4 @@ class SyncPreparation { this.newRoomKeys = newRoomKeys; this.newKeysByRoom = groupBy(newRoomKeys, r => r.roomId); } - - dispose() { - if (this.newRoomKeys) { - for (const k of this.newRoomKeys) { - k.dispose(); - } - } - } } diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index e010f90d..0be48007 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -464,7 +464,6 @@ class SessionSyncProcessState { dispose() { this.lock?.release(); - this.preparation?.dispose(); } } From 6bbce06d936779db9c95b1ca92ca36acd47f8927 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 22 Oct 2021 19:01:20 +0200 Subject: [PATCH 21/26] start writing tests for key loader --- .../e2ee/megolm/decryption/KeyLoader.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index d1741f30..1618a4a6 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -210,3 +210,70 @@ class KeyOperation { this.session.free(); } } + +export function tests() { + let instances = 0; + let idCounter = 0; + + class MockRoomKey implements IRoomKey { + private _roomId: string; + private _senderKey: string; + private _sessionId: string; + private _firstKnownIndex: number; + + constructor(roomId: string, senderKey: string, sessionId: string, firstKnownIndex: number) { + this._roomId = roomId; + this._senderKey = senderKey; + this._sessionId = sessionId; + this._firstKnownIndex = firstKnownIndex; + } + + get roomId(): string { return this._roomId; } + get senderKey(): string { return this._senderKey; } + get sessionId(): string { return this._sessionId; } + get claimedEd25519Key(): string { return "claimedEd25519Key"; } + get serializationKey(): string { return "key"; } + get serializationType(): string { return "type"; } + get eventIds(): string[] | undefined { return undefined; } + loadInto(session: OlmInboundGroupSession) { + const mockSession = session as MockInboundSession; + mockSession.sessionId = this.sessionId; + mockSession.firstKnownIndex = this._firstKnownIndex; + } + } + + class MockInboundSession { + public sessionId: string = ""; + public firstKnownIndex: number = 0; + + free(): void { instances -= 1; } + pickle(key: string | Uint8Array): string { return `${this.sessionId}-pickled-session`; } + unpickle(key: string | Uint8Array, pickle: string) {} + create(session_key: string): string { return `${this.sessionId}-created-session`; } + import_session(session_key: string): string { return ""; } + decrypt(message: string): OlmDecryptionResult { return {} as OlmDecryptionResult; } + session_id(): string { return this.sessionId; } + first_known_index(): number { return this.firstKnownIndex; } + export_session(message_index: number): string { return `${this.sessionId}-exported-session`; } + } + + const PICKLE_KEY = "🥒🔑"; + const olm = {InboundGroupSession: MockInboundSession}; + const roomId = "!abc:hs.tld"; + const aliceSenderKey = "abc"; + const bobSenderKey = "def"; + const sessionId = "s123"; + + return { + "load key gives correct session": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 5); + let callbackCalled = false; + await loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId, 1), session => { + callbackCalled = true; + assert.equal(session.session_id(), sessionId); + assert.equal(session.first_known_index(), 1); + }); + assert(callbackCalled); + }, + } +} From 74e8bc3bdae3465038f386bb40aad237ad93ed60 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 25 Oct 2021 17:19:48 +0200 Subject: [PATCH 22/26] write unit tests --- .../e2ee/megolm/decryption/KeyLoader.ts | 110 ++++++++++++++++-- 1 file changed, 102 insertions(+), 8 deletions(-) diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index 1618a4a6..375fb2ac 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -156,7 +156,7 @@ export class KeyLoader extends BaseLRUCache { private findIndexSameKey(key: IRoomKey): number { return this._entries.findIndex(op => { - return op.isForKey(key); + return op.isForSameSession(key.roomId, key.senderKey, key.sessionId) && op.isForKey(key); }); } @@ -213,7 +213,6 @@ class KeyOperation { export function tests() { let instances = 0; - let idCounter = 0; class MockRoomKey implements IRoomKey { private _roomId: string; @@ -232,7 +231,7 @@ export function tests() { get senderKey(): string { return this._senderKey; } get sessionId(): string { return this._sessionId; } get claimedEd25519Key(): string { return "claimedEd25519Key"; } - get serializationKey(): string { return "key"; } + get serializationKey(): string { return `key-${this.sessionId}-${this._firstKnownIndex}`; } get serializationType(): string { return "type"; } get eventIds(): string[] | undefined { return undefined; } loadInto(session: OlmInboundGroupSession) { @@ -246,6 +245,10 @@ export function tests() { public sessionId: string = ""; public firstKnownIndex: number = 0; + constructor() { + instances += 1; + } + free(): void { instances -= 1; } pickle(key: string | Uint8Array): string { return `${this.sessionId}-pickled-session`; } unpickle(key: string | Uint8Array, pickle: string) {} @@ -262,18 +265,109 @@ export function tests() { const roomId = "!abc:hs.tld"; const aliceSenderKey = "abc"; const bobSenderKey = "def"; - const sessionId = "s123"; + const sessionId1 = "s123"; + const sessionId2 = "s456"; + const sessionId3 = "s789"; return { "load key gives correct session": async assert => { - const loader = new KeyLoader(olm, PICKLE_KEY, 5); + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + let callback1Called = false; + let callback2Called = false; + const p1 = loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1), async session => { + callback1Called = true; + assert.equal(session.session_id(), sessionId1); + assert.equal(session.first_known_index(), 1); + await Promise.resolve(); // make sure they are busy in parallel + }); + const p2 = loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId2, 2), async session => { + callback2Called = true; + assert.equal(session.session_id(), sessionId2); + assert.equal(session.first_known_index(), 2); + await Promise.resolve(); // make sure they are busy in parallel + }); + assert.equal(loader.size, 2); + await Promise.all([p1, p2]); + assert(callback1Called); + assert(callback2Called); + }, + "keys with different first index are kept separate": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + let callback1Called = false; + let callback2Called = false; + const p1 = loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1), async session => { + callback1Called = true; + assert.equal(session.session_id(), sessionId1); + assert.equal(session.first_known_index(), 1); + await Promise.resolve(); // make sure they are busy in parallel + }); + const p2 = loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId1, 2), async session => { + callback2Called = true; + assert.equal(session.session_id(), sessionId1); + assert.equal(session.first_known_index(), 2); + await Promise.resolve(); // make sure they are busy in parallel + }); + assert.equal(loader.size, 2); + await Promise.all([p1, p2]); + assert(callback1Called); + assert(callback2Called); + }, + "useKey blocks as long as no free sessions are available": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 1); + let resolve; let callbackCalled = false; - await loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId, 1), session => { + loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1), async session => { + await new Promise(r => resolve = r); + }); + await Promise.resolve(); + assert.equal(loader.size, 1); + const promise = loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId2, 1), session => { callbackCalled = true; - assert.equal(session.session_id(), sessionId); + }); + assert.equal(callbackCalled, false); + resolve(); + await promise; + assert.equal(callbackCalled, true); + }, + "cache hit while key in use, then replace (check refCount works properly)": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 1); + let resolve1, resolve2; + const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1); + const p1 = loader.useKey(key1, async session => { + await new Promise(r => resolve1 = r); + }); + const p2 = loader.useKey(key1, async session => { + await new Promise(r => resolve2 = r); + }); + await Promise.resolve(); + assert.equal(loader.size, 1); + assert.equal(loader.running, true); + resolve1(); + await p1; + assert.equal(loader.running, true); + resolve2(); + await p2; + assert.equal(loader.running, false); + let callbackCalled = false; + await loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId2, 1), async session => { + callbackCalled = true; + assert.equal(session.session_id(), sessionId2); assert.equal(session.first_known_index(), 1); }); - assert(callbackCalled); + assert.equal(loader.size, 1); + assert.equal(callbackCalled, true); }, + "cache hit while key not in use": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + let resolve1, resolve2, invocations = 0; + const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1); + await loader.useKey(key1, async session => { invocations += 1; }); + assert.equal(loader.size, 1); + const cachedKey = loader.getCachedKey(roomId, aliceSenderKey, sessionId1)!; + assert.equal(cachedKey, key1); + await loader.useKey(cachedKey, async session => { invocations += 1; }); + assert.equal(loader.size, 1); + assert.equal(invocations, 2); + } } } From 3c2604b384226c52f8e1c4c48f6a4cb3838455b2 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 25 Oct 2021 17:33:33 +0200 Subject: [PATCH 23/26] test that sessions get free'd correctly --- src/matrix/e2ee/megolm/decryption/KeyLoader.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index 375fb2ac..cf276608 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -368,6 +368,17 @@ export function tests() { await loader.useKey(cachedKey, async session => { invocations += 1; }); assert.equal(loader.size, 1); assert.equal(invocations, 2); + }, + "dispose calls free on all sessions": async assert => { + instances = 0; + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + await loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1), async session => {}); + await loader.useKey(new MockRoomKey(roomId, aliceSenderKey, sessionId2, 1), async session => {}); + assert.equal(instances, 2); + assert.equal(loader.size, 2); + loader.dispose(); + assert.strictEqual(instances, 0, "instances"); + assert.strictEqual(loader.size, 0, "loader.size"); } } } From ab2f15b5a2364ab110aabce1f52481d63c69f34f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 25 Oct 2021 19:17:13 +0200 Subject: [PATCH 24/26] prevent cache hiding better keys in storage (+ tests) --- src/matrix/e2ee/megolm/Decryption.ts | 14 +-- .../e2ee/megolm/decryption/KeyLoader.ts | 103 +++++++++++++----- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 80 +++++++------- .../megolm/decryption/SessionDecryption.ts | 6 +- 4 files changed, 125 insertions(+), 78 deletions(-) diff --git a/src/matrix/e2ee/megolm/Decryption.ts b/src/matrix/e2ee/megolm/Decryption.ts index 7ab0e15a..cc577f3d 100644 --- a/src/matrix/e2ee/megolm/Decryption.ts +++ b/src/matrix/e2ee/megolm/Decryption.ts @@ -20,7 +20,7 @@ import {SessionDecryption} from "./decryption/SessionDecryption"; import {MEGOLM_ALGORITHM} from "../common.js"; import {validateEvent, groupEventsBySession} from "./decryption/utils"; import {keyFromStorage, keyFromDeviceMessage, keyFromBackup} from "./decryption/RoomKey"; -import type {IRoomKey, IIncomingRoomKey} from "./decryption/RoomKey"; +import type {RoomKey, IncomingRoomKey} from "./decryption/RoomKey"; import type {KeyLoader} from "./decryption/KeyLoader"; import type {OlmWorker} from "../OlmWorker"; import type {Transaction} from "../../storage/idb/Transaction"; @@ -78,7 +78,7 @@ export class Decryption { * @param {[type]} txn [description] * @return {DecryptionPreparation} */ - async prepareDecryptAll(roomId: string, events: TimelineEvent[], newKeys: IIncomingRoomKey[] | undefined, txn: Transaction) { + async prepareDecryptAll(roomId: string, events: TimelineEvent[], newKeys: IncomingRoomKey[] | undefined, txn: Transaction) { const errors = new Map(); const validEvents: TimelineEvent[] = []; @@ -107,7 +107,7 @@ export class Decryption { return new DecryptionPreparation(roomId, sessionDecryptions, errors); } - private async getRoomKey(roomId: string, senderKey: string, sessionId: string, newKeys: IIncomingRoomKey[] | undefined, txn: Transaction): Promise { + private async getRoomKey(roomId: string, senderKey: string, sessionId: string, newKeys: IncomingRoomKey[] | undefined, txn: Transaction): Promise { if (newKeys) { const key = newKeys.find(k => k.roomId === roomId && k.senderKey === senderKey && k.sessionId === sessionId); if (key && await key.checkBetterThanKeyInStorage(this.keyLoader, txn)) { @@ -128,7 +128,7 @@ export class Decryption { /** * Writes the key as an inbound group session if there is not already a better key in the store */ - writeRoomKey(key: IIncomingRoomKey, txn: Transaction): Promise { + writeRoomKey(key: IncomingRoomKey, txn: Transaction): Promise { return key.write(this.keyLoader, txn); } @@ -136,8 +136,8 @@ export class Decryption { * Extracts room keys from decrypted device messages. * The key won't be persisted yet, you need to call RoomKey.write for that. */ - roomKeysFromDeviceMessages(decryptionResults: DecryptionResult[], log: LogItem): IIncomingRoomKey[] { - const keys: IIncomingRoomKey[] = []; + roomKeysFromDeviceMessages(decryptionResults: DecryptionResult[], log: LogItem): IncomingRoomKey[] { + const keys: IncomingRoomKey[] = []; for (const dr of decryptionResults) { if (dr.event?.type !== "m.room_key" || dr.event.content?.algorithm !== MEGOLM_ALGORITHM) { continue; @@ -157,7 +157,7 @@ export class Decryption { return keys; } - roomKeyFromBackup(roomId: string, sessionId: string, sessionInfo: string): IIncomingRoomKey | undefined { + roomKeyFromBackup(roomId: string, sessionId: string, sessionInfo: string): IncomingRoomKey | undefined { return keyFromBackup(roomId, sessionId, sessionInfo); } diff --git a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts index cf276608..58f968c8 100644 --- a/src/matrix/e2ee/megolm/decryption/KeyLoader.ts +++ b/src/matrix/e2ee/megolm/decryption/KeyLoader.ts @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {IRoomKey, isBetterThan} from "./RoomKey"; +import {isBetterThan, IncomingRoomKey} from "./RoomKey"; import {BaseLRUCache} from "../../../../utils/LRUCache"; - +import type {RoomKey} from "./RoomKey"; export declare class OlmDecryptionResult { readonly plaintext: string; @@ -53,14 +53,14 @@ export class KeyLoader extends BaseLRUCache { this.olm = olm; } - getCachedKey(roomId: string, senderKey: string, sessionId: string): IRoomKey | undefined { - const idx = this.findIndexBestForSession(roomId, senderKey, sessionId); + getCachedKey(roomId: string, senderKey: string, sessionId: string): RoomKey | undefined { + const idx = this.findCachedKeyIndex(roomId, senderKey, sessionId); if (idx !== -1) { return this._getByIndexAndMoveUp(idx)!.key; } } - async useKey(key: IRoomKey, callback: (session: OlmInboundGroupSession, pickleKey: string) => Promise | T): Promise { + async useKey(key: RoomKey, callback: (session: OlmInboundGroupSession, pickleKey: string) => Promise | T): Promise { const keyOp = await this.allocateOperation(key); try { return await callback(keyOp.session, this.pickleKey); @@ -81,7 +81,7 @@ export class KeyLoader extends BaseLRUCache { this._entries.splice(0, this._entries.length); } - private async allocateOperation(key: IRoomKey): Promise { + private async allocateOperation(key: RoomKey): Promise { let idx; while((idx = this.findIndexForAllocation(key)) === -1) { await this.operationBecomesUnused(); @@ -127,14 +127,14 @@ export class KeyLoader extends BaseLRUCache { return this.operationBecomesUnusedPromise; } - private findIndexForAllocation(key: IRoomKey) { + private findIndexForAllocation(key: RoomKey) { let idx = this.findIndexSameKey(key); // cache hit if (idx === -1) { - idx = this.findIndexSameSessionUnused(key); - if (idx === -1) { - if (this.size < this.limit) { - idx = this.size; - } else { + if (this.size < this.limit) { + idx = this.size; + } else { + idx = this.findIndexSameSessionUnused(key); + if (idx === -1) { idx = this.findIndexOldestUnused(); } } @@ -142,10 +142,11 @@ export class KeyLoader extends BaseLRUCache { return idx; } - private findIndexBestForSession(roomId: string, senderKey: string, sessionId: string): number { + private findCachedKeyIndex(roomId: string, senderKey: string, sessionId: string): number { return this._entries.reduce((bestIdx, op, i, arr) => { const bestOp = bestIdx === -1 ? undefined : arr[bestIdx]; - if (op.isForSameSession(roomId, senderKey, sessionId)) { + // only operations that are the "best" for their session can be used, see comment on isBest + if (op.isBest === true && op.isForSameSession(roomId, senderKey, sessionId)) { if (!bestOp || op.isBetter(bestOp)) { return i; } @@ -154,20 +155,23 @@ export class KeyLoader extends BaseLRUCache { }, -1); } - private findIndexSameKey(key: IRoomKey): number { + private findIndexSameKey(key: RoomKey): number { return this._entries.findIndex(op => { return op.isForSameSession(key.roomId, key.senderKey, key.sessionId) && op.isForKey(key); }); } - private findIndexSameSessionUnused(key: IRoomKey): number { - for (let i = this._entries.length - 1; i >= 0; i -= 1) { - const op = this._entries[i]; + private findIndexSameSessionUnused(key: RoomKey): number { + return this._entries.reduce((worstIdx, op, i, arr) => { + const worst = worstIdx === -1 ? undefined : arr[worstIdx]; + // we try to pick the worst operation to overwrite, so the best one stays in the cache if (op.refCount === 0 && op.isForSameSession(key.roomId, key.senderKey, key.sessionId)) { - return i; + if (!worst || !op.isBetter(worst)) { + return i; + } } - } - return -1; + return worstIdx; + }, -1); } private findIndexOldestUnused(): number { @@ -183,10 +187,10 @@ export class KeyLoader extends BaseLRUCache { class KeyOperation { session: OlmInboundGroupSession; - key: IRoomKey; + key: RoomKey; refCount: number; - constructor(key: IRoomKey, session: OlmInboundGroupSession) { + constructor(key: RoomKey, session: OlmInboundGroupSession) { this.key = key; this.session = session; this.refCount = 1; @@ -201,7 +205,7 @@ class KeyOperation { return isBetterThan(this.session, other.session); } - isForKey(key: IRoomKey) { + isForKey(key: RoomKey) { return this.key.serializationKey === key.serializationKey && this.key.serializationType === key.serializationType; } @@ -209,18 +213,27 @@ class KeyOperation { dispose() { this.session.free(); } + + /** returns whether the key for this operation has been checked at some point against storage + * and was determined to be the better key, undefined if it hasn't been checked yet. + * Only keys that are the best keys can be returned by getCachedKey as returning a cache hit + * will usually not check for a better session in storage. Also see RoomKey.isBetter. */ + get isBest(): boolean | undefined { + return this.key.isBetter; + } } export function tests() { let instances = 0; - class MockRoomKey implements IRoomKey { + class MockRoomKey extends IncomingRoomKey { private _roomId: string; private _senderKey: string; private _sessionId: string; private _firstKnownIndex: number; constructor(roomId: string, senderKey: string, sessionId: string, firstKnownIndex: number) { + super(); this._roomId = roomId; this._senderKey = senderKey; this._sessionId = sessionId; @@ -267,7 +280,6 @@ export function tests() { const bobSenderKey = "def"; const sessionId1 = "s123"; const sessionId2 = "s456"; - const sessionId3 = "s789"; return { "load key gives correct session": async assert => { @@ -362,6 +374,7 @@ export function tests() { let resolve1, resolve2, invocations = 0; const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1); await loader.useKey(key1, async session => { invocations += 1; }); + key1.isBetter = true; assert.equal(loader.size, 1); const cachedKey = loader.getCachedKey(roomId, aliceSenderKey, sessionId1)!; assert.equal(cachedKey, key1); @@ -379,6 +392,42 @@ export function tests() { loader.dispose(); assert.strictEqual(instances, 0, "instances"); assert.strictEqual(loader.size, 0, "loader.size"); - } + }, + "checkBetterThanKeyInStorage false with cache": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 2); + await loader.useKey(key1, async session => {}); + // fake we've checked with storage that this is the best key, + // and as long is it remains the best key with newly added keys, + // it will be returned from getCachedKey (as called from checkBetterThanKeyInStorage) + key1.isBetter = true; + const key2 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 3); + // this will hit cache of key 1 so we pass in null as txn + const isBetter = await key2.checkBetterThanKeyInStorage(loader, null as any); + assert.strictEqual(isBetter, false); + assert.strictEqual(key2.isBetter, false); + }, + "checkBetterThanKeyInStorage true with cache": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 2); + key1.isBetter = true; // fake we've check with storage so far (not including key2) this is the best key + await loader.useKey(key1, async session => {}); + const key2 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 1); + // this will hit cache of key 1 so we pass in null as txn + const isBetter = await key2.checkBetterThanKeyInStorage(loader, null as any); + assert.strictEqual(isBetter, true); + assert.strictEqual(key2.isBetter, true); + }, + "prefer to remove worst key for a session from cache": async assert => { + const loader = new KeyLoader(olm, PICKLE_KEY, 2); + const key1 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 2); + await loader.useKey(key1, async session => {}); + key1.isBetter = true; // set to true just so it gets returned from getCachedKey + const key2 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 4); + await loader.useKey(key2, async session => {}); + const key3 = new MockRoomKey(roomId, aliceSenderKey, sessionId1, 3); + await loader.useKey(key3, async session => {}); + assert.strictEqual(loader.getCachedKey(roomId, aliceSenderKey, sessionId1), key1); + }, } } diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index 48af45a7..ad1cd8a0 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -19,30 +19,33 @@ import type {Transaction} from "../../../storage/idb/Transaction"; import type {DecryptionResult} from "../../DecryptionResult"; import type {KeyLoader, OlmInboundGroupSession} from "./KeyLoader"; -export interface IRoomKey { - get roomId(): string; - get senderKey(): string; - get sessionId(): string; - get claimedEd25519Key(): string; - get serializationKey(): string; - get serializationType(): string; - get eventIds(): string[] | undefined; - loadInto(session: OlmInboundGroupSession, pickleKey: string): void; +export abstract class RoomKey { + private _isBetter: boolean | undefined; + + abstract get roomId(): string; + abstract get senderKey(): string; + abstract get sessionId(): string; + abstract get claimedEd25519Key(): string; + abstract get serializationKey(): string; + abstract get serializationType(): string; + abstract get eventIds(): string[] | undefined; + abstract loadInto(session: OlmInboundGroupSession, pickleKey: string): void; + /* Whether the key has been checked against storage (or is from storage) + * to be the better key for a given session. Given that all keys are checked to be better + * as part of writing, we can trust that when this returns true, it really is the best key + * available between storage and cached keys in memory. This is why keys with this field set to + * true are used by the key loader to return cached keys. Also see KeyOperation.isBest there. */ + get isBetter(): boolean | undefined { return this._isBetter; } + // should only be set in key.checkBetterThanKeyInStorage + set isBetter(value: boolean | undefined) { this._isBetter = value; } } export function isBetterThan(newSession: OlmInboundGroupSession, existingSession: OlmInboundGroupSession) { return newSession.first_known_index() < existingSession.first_known_index(); } -export interface IIncomingRoomKey extends IRoomKey { - get isBetter(): boolean | undefined; - checkBetterThanKeyInStorage(loader: KeyLoader, txn: Transaction): Promise; - write(loader: KeyLoader, txn: Transaction): Promise; -} - -abstract class BaseIncomingRoomKey implements IIncomingRoomKey { +export abstract class IncomingRoomKey extends RoomKey { private _eventIds?: string[]; - private _isBetter?: boolean; checkBetterThanKeyInStorage(loader: KeyLoader, txn: Transaction): Promise { return this._checkBetterThanKeyInStorage(loader, undefined, txn); @@ -51,7 +54,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { async write(loader: KeyLoader, txn: Transaction): Promise { // we checked already and we had a better session in storage, so don't write let pickledSession; - if (this._isBetter === undefined) { + if (this.isBetter === undefined) { // if this key wasn't used to decrypt any messages in the same sync, // we haven't checked if this is the best key yet, // so do that now to not overwrite a better key. @@ -60,7 +63,7 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { pickledSession = session.pickle(pickleKey); }, txn); } - if (this._isBetter === false) { + if (this.isBetter === false) { return false; } // before calling write in parallel, we need to check loader.running is false so we are sure our transaction will not be closed @@ -79,11 +82,10 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { } get eventIds() { return this._eventIds; } - get isBetter() { return this._isBetter; } private async _checkBetterThanKeyInStorage(loader: KeyLoader, callback: (((session: OlmInboundGroupSession, pickleKey: string) => void) | undefined), txn: Transaction): Promise { - if (this._isBetter !== undefined) { - return this._isBetter; + if (this.isBetter !== undefined) { + return this.isBetter; } let existingKey = loader.getCachedKey(this.roomId, this.senderKey, this.sessionId); if (!existingKey) { @@ -100,32 +102,26 @@ abstract class BaseIncomingRoomKey implements IIncomingRoomKey { } if (existingKey) { const key = existingKey; - this._isBetter = await loader.useKey(this, newSession => { - return loader.useKey(key, (existingSession, pickleKey) => { - const isBetter = isBetterThan(newSession, existingSession); - if (isBetter && callback) { + await loader.useKey(this, async newSession => { + await loader.useKey(key, (existingSession, pickleKey) => { + // set isBetter as soon as possible, on both keys compared, + // as it is is used to determine whether a key can be used for the cache + this.isBetter = isBetterThan(newSession, existingSession); + key.isBetter = !this.isBetter; + if (this.isBetter && callback) { callback(newSession, pickleKey); } - return isBetter; }); }); } else { // no previous key, so we're the best \o/ - this._isBetter = true; + this.isBetter = true; } - return this._isBetter!; + return this.isBetter!; } - - abstract get roomId(): string; - abstract get senderKey(): string; - abstract get sessionId(): string; - abstract get claimedEd25519Key(): string; - abstract get serializationKey(): string; - abstract get serializationType(): string; - abstract loadInto(session: OlmInboundGroupSession, pickleKey: string): void; } -class DeviceMessageRoomKey extends BaseIncomingRoomKey { +class DeviceMessageRoomKey extends IncomingRoomKey { private _decryptionResult: DecryptionResult; constructor(decryptionResult: DecryptionResult) { @@ -145,7 +141,7 @@ class DeviceMessageRoomKey extends BaseIncomingRoomKey { } } -class BackupRoomKey extends BaseIncomingRoomKey { +class BackupRoomKey extends IncomingRoomKey { private _roomId: string; private _sessionId: string; private _backupInfo: string; @@ -169,10 +165,12 @@ class BackupRoomKey extends BaseIncomingRoomKey { } } -class StoredRoomKey implements IRoomKey { +class StoredRoomKey extends RoomKey { private storageEntry: InboundGroupSessionEntry; constructor(storageEntry: InboundGroupSessionEntry) { + super(); + this.isBetter = true; // usually the key in storage is the best until checks prove otherwise this.storageEntry = storageEntry; } @@ -192,7 +190,7 @@ class StoredRoomKey implements IRoomKey { // sessions are stored before they are received // to keep track of events that need it to be decrypted. // This is used to retry decryption of those events once the session is received. - return !!this.storageEntry.session; + return !!this.serializationKey; } } diff --git a/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts index 3adf5bdb..7e466806 100644 --- a/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts +++ b/src/matrix/e2ee/megolm/decryption/SessionDecryption.ts @@ -17,7 +17,7 @@ limitations under the License. import {DecryptionResult} from "../../DecryptionResult.js"; import {DecryptionError} from "../../common.js"; import {ReplayDetectionEntry} from "./ReplayDetectionEntry"; -import type {IRoomKey} from "./RoomKey.js"; +import type {RoomKey} from "./RoomKey.js"; import type {KeyLoader, OlmDecryptionResult} from "./KeyLoader"; import type {OlmWorker} from "../../OlmWorker"; import type {TimelineEvent} from "../../../storage/types"; @@ -31,13 +31,13 @@ interface DecryptAllResult { * Does the actual decryption of all events for a given megolm session in a batch */ export class SessionDecryption { - private key: IRoomKey; + private key: RoomKey; private events: TimelineEvent[]; private keyLoader: KeyLoader; private olmWorker?: OlmWorker; private decryptionRequests?: any[]; - constructor(key: IRoomKey, events: TimelineEvent[], olmWorker: OlmWorker | undefined, keyLoader: KeyLoader) { + constructor(key: RoomKey, events: TimelineEvent[], olmWorker: OlmWorker | undefined, keyLoader: KeyLoader) { this.key = key; this.events = events; this.olmWorker = olmWorker; From 805c2657f2dbcca9595dd8950c3eb7591fb64e06 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 26 Oct 2021 11:07:17 +0200 Subject: [PATCH 25/26] remove unrelated file --- scripts/babel-test.js | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 scripts/babel-test.js diff --git a/scripts/babel-test.js b/scripts/babel-test.js deleted file mode 100644 index 24283dc3..00000000 --- a/scripts/babel-test.js +++ /dev/null @@ -1,32 +0,0 @@ -babel = require('@babel/standalone'); - -const code = ` -async function doit() { - const foo = {bar: 5}; - const mapped = Object.values(foo).map(n => n*n); - console.log(mapped); - await Promise.resolve(); -} -doit(); -`; - -const {code: babelCode} = babel.transform(code, { - babelrc: false, - configFile: false, - presets: [ - [ - "env", - { - useBuiltIns: "entry", - modules: false, - corejs: "3.4", - targets: "IE 11", - // we provide our own promise polyfill (es6-promise) - // with support for synchronous flushing of - // the queue for idb where needed - // exclude: ["es.promise", "es.promise.all-settled", "es.promise.finally"] - } - ] - ] -}); -console.log(babelCode); From 67dd929951ee4c60052d949b421f6fa54d060528 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 26 Oct 2021 11:14:46 +0200 Subject: [PATCH 26/26] put key session check in method --- src/matrix/e2ee/megolm/Decryption.ts | 2 +- src/matrix/e2ee/megolm/decryption/RoomKey.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/matrix/e2ee/megolm/Decryption.ts b/src/matrix/e2ee/megolm/Decryption.ts index cc577f3d..842d423d 100644 --- a/src/matrix/e2ee/megolm/Decryption.ts +++ b/src/matrix/e2ee/megolm/Decryption.ts @@ -109,7 +109,7 @@ export class Decryption { private async getRoomKey(roomId: string, senderKey: string, sessionId: string, newKeys: IncomingRoomKey[] | undefined, txn: Transaction): Promise { if (newKeys) { - const key = newKeys.find(k => k.roomId === roomId && k.senderKey === senderKey && k.sessionId === sessionId); + const key = newKeys.find(k => k.isForSession(roomId, senderKey, sessionId)); if (key && await key.checkBetterThanKeyInStorage(this.keyLoader, txn)) { return key; } diff --git a/src/matrix/e2ee/megolm/decryption/RoomKey.ts b/src/matrix/e2ee/megolm/decryption/RoomKey.ts index ad1cd8a0..81f1a9be 100644 --- a/src/matrix/e2ee/megolm/decryption/RoomKey.ts +++ b/src/matrix/e2ee/megolm/decryption/RoomKey.ts @@ -22,6 +22,10 @@ import type {KeyLoader, OlmInboundGroupSession} from "./KeyLoader"; export abstract class RoomKey { private _isBetter: boolean | undefined; + isForSession(roomId: string, senderKey: string, sessionId: string) { + return this.roomId === roomId && this.senderKey === senderKey && this.sessionId === sessionId; + } + abstract get roomId(): string; abstract get senderKey(): string; abstract get sessionId(): string;