From 77bd0d3f3c419cd62007b055aff9b6a6af854459 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 29 Sep 2021 11:49:58 +0200 Subject: [PATCH 1/7] store e2ee session values as well in localStorage --- src/matrix/SessionContainer.js | 1 + src/matrix/e2ee/Account.js | 8 +-- src/matrix/e2ee/common.js | 2 +- src/matrix/storage/idb/QueryTarget.ts | 6 +++ src/matrix/storage/idb/Storage.ts | 9 +++- src/matrix/storage/idb/StorageFactory.ts | 24 +++++---- src/matrix/storage/idb/Transaction.ts | 6 ++- src/matrix/storage/idb/schema.ts | 12 +++-- src/matrix/storage/idb/stores/SessionStore.ts | 51 ++++++++++++++++++- src/matrix/storage/idb/types.ts | 23 +++++++++ src/mocks/Storage.ts | 41 ++++++++++++++- 11 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 src/matrix/storage/idb/types.ts diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js index b58589d0..7248e11e 100644 --- a/src/matrix/SessionContainer.js +++ b/src/matrix/SessionContainer.js @@ -233,6 +233,7 @@ export class SessionContainer { platform: this._platform, }); await this._session.load(log); + // TODO: check instead storage doesn't have an identity if (isNewLogin) { this._status.set(LoadStatus.SessionSetup); await log.wrap("createIdentity", log => this._session.createIdentity(log)); diff --git a/src/matrix/e2ee/Account.js b/src/matrix/e2ee/Account.js index 9314d590..1de43ccd 100644 --- a/src/matrix/e2ee/Account.js +++ b/src/matrix/e2ee/Account.js @@ -15,12 +15,12 @@ limitations under the License. */ import anotherjson from "../../../lib/another-json/index.js"; -import {SESSION_KEY_PREFIX, OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./common.js"; +import {SESSION_E2EE_KEY_PREFIX, OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./common.js"; // use common prefix so it's easy to clear properties that are not e2ee related during session clear -const ACCOUNT_SESSION_KEY = SESSION_KEY_PREFIX + "olmAccount"; -const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_KEY_PREFIX + "areDeviceKeysUploaded"; -const SERVER_OTK_COUNT_SESSION_KEY = SESSION_KEY_PREFIX + "serverOTKCount"; +const ACCOUNT_SESSION_KEY = SESSION_E2EE_KEY_PREFIX + "olmAccount"; +const DEVICE_KEY_FLAG_SESSION_KEY = SESSION_E2EE_KEY_PREFIX + "areDeviceKeysUploaded"; +const SERVER_OTK_COUNT_SESSION_KEY = SESSION_E2EE_KEY_PREFIX + "serverOTKCount"; export class Account { static async load({olm, pickleKey, hsApi, userId, deviceId, olmWorker, txn}) { diff --git a/src/matrix/e2ee/common.js b/src/matrix/e2ee/common.js index 52995765..fa970236 100644 --- a/src/matrix/e2ee/common.js +++ b/src/matrix/e2ee/common.js @@ -20,7 +20,7 @@ import {createEnum} from "../../utils/enum.js"; export const DecryptionSource = createEnum("Sync", "Timeline", "Retry"); // use common prefix so it's easy to clear properties that are not e2ee related during session clear -export const SESSION_KEY_PREFIX = "e2ee:"; +export const SESSION_E2EE_KEY_PREFIX = "e2ee:"; export const OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2"; export const MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2"; diff --git a/src/matrix/storage/idb/QueryTarget.ts b/src/matrix/storage/idb/QueryTarget.ts index 1ef3aacd..e84c41e1 100644 --- a/src/matrix/storage/idb/QueryTarget.ts +++ b/src/matrix/storage/idb/QueryTarget.ts @@ -22,6 +22,7 @@ import {IDBKey} from "./Transaction"; export interface ITransaction { idbFactory: IDBFactory; IDBKeyRange: typeof IDBKeyRange; + databaseName: string; addWriteError(error: StorageError, refItem: LogItem | undefined, operationName: string, keys: IDBKey[] | undefined); } @@ -55,6 +56,10 @@ export class QueryTarget { return this._transaction.IDBKeyRange; } + get databaseName(): string { + return this._transaction.databaseName; + } + _openCursor(range?: IDBQuery, direction?: IDBCursorDirection): IDBRequest { if (range && direction) { return this._target.openCursor(range, direction); @@ -269,6 +274,7 @@ import {QueryTargetWrapper, Store} from "./Store"; export function tests() { class MockTransaction extends MockIDBImpl { + get databaseName(): string { return "mockdb"; } addWriteError(error: StorageError, refItem: LogItem | undefined, operationName: string, keys: IDBKey[] | undefined) {} } diff --git a/src/matrix/storage/idb/Storage.ts b/src/matrix/storage/idb/Storage.ts index 72be55ce..53ee7bc0 100644 --- a/src/matrix/storage/idb/Storage.ts +++ b/src/matrix/storage/idb/Storage.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {IDOMStorage} from "./types"; import {Transaction} from "./Transaction"; import { STORE_NAMES, StoreNames, StorageError } from "../common"; import { reqAsPromise } from "./utils"; @@ -29,13 +30,15 @@ export class Storage { readonly idbFactory: IDBFactory readonly IDBKeyRange: typeof IDBKeyRange; readonly storeNames: typeof StoreNames; + readonly localStorage: IDOMStorage; - constructor(idbDatabase: IDBDatabase, idbFactory: IDBFactory, _IDBKeyRange: typeof IDBKeyRange, hasWebkitEarlyCloseTxnBug: boolean, logger: BaseLogger) { + constructor(idbDatabase: IDBDatabase, idbFactory: IDBFactory, _IDBKeyRange: typeof IDBKeyRange, hasWebkitEarlyCloseTxnBug: boolean, localStorage: IDOMStorage, logger: BaseLogger) { this._db = idbDatabase; this.idbFactory = idbFactory; this.IDBKeyRange = _IDBKeyRange; this._hasWebkitEarlyCloseTxnBug = hasWebkitEarlyCloseTxnBug; this.storeNames = StoreNames; + this.localStorage = localStorage; this.logger = logger; } @@ -79,4 +82,8 @@ export class Storage { close(): void { this._db.close(); } + + get databaseName(): string { + return this._db.name; + } } diff --git a/src/matrix/storage/idb/StorageFactory.ts b/src/matrix/storage/idb/StorageFactory.ts index 59c988a0..71201842 100644 --- a/src/matrix/storage/idb/StorageFactory.ts +++ b/src/matrix/storage/idb/StorageFactory.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {IDOMStorage} from "./types"; import {Storage} from "./Storage"; import { openDatabase, reqAsPromise } from "./utils"; import { exportSession, importSession, Export } from "./export"; @@ -23,8 +24,8 @@ import { BaseLogger } from "../../../logging/BaseLogger.js"; import { LogItem } from "../../../logging/LogItem.js"; const sessionName = (sessionId: string) => `hydrogen_session_${sessionId}`; -const openDatabaseWithSessionId = function(sessionId: string, idbFactory: IDBFactory, log: LogItem) { - const create = (db, txn, oldVersion, version) => createStores(db, txn, oldVersion, version, log); +const openDatabaseWithSessionId = function(sessionId: string, idbFactory: IDBFactory, localStorage: IDOMStorage, log: LogItem) { + const create = (db, txn, oldVersion, version) => createStores(db, txn, oldVersion, version, localStorage, log); return openDatabase(sessionName(sessionId), create, schema.length, idbFactory); } @@ -52,12 +53,14 @@ async function requestPersistedStorage(): Promise { export class StorageFactory { private _serviceWorkerHandler: ServiceWorkerHandler; private _idbFactory: IDBFactory; - private _IDBKeyRange: typeof IDBKeyRange + private _IDBKeyRange: typeof IDBKeyRange; + private _localStorage: IDOMStorage; - constructor(serviceWorkerHandler: ServiceWorkerHandler, idbFactory: IDBFactory = window.indexedDB, _IDBKeyRange = window.IDBKeyRange) { + constructor(serviceWorkerHandler: ServiceWorkerHandler, idbFactory: IDBFactory = window.indexedDB, _IDBKeyRange = window.IDBKeyRange, localStorage: IDOMStorage = window.localStorage) { this._serviceWorkerHandler = serviceWorkerHandler; this._idbFactory = idbFactory; this._IDBKeyRange = _IDBKeyRange; + this._localStorage = localStorage; } async create(sessionId: string, log: LogItem): Promise { @@ -70,8 +73,8 @@ export class StorageFactory { }); const hasWebkitEarlyCloseTxnBug = await detectWebkitEarlyCloseTxnBug(this._idbFactory); - const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, log); - return new Storage(db, this._idbFactory, this._IDBKeyRange, hasWebkitEarlyCloseTxnBug, log.logger); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, this._localStorage, log); + return new Storage(db, this._idbFactory, this._IDBKeyRange, hasWebkitEarlyCloseTxnBug, this._localStorage, log.logger); } delete(sessionId: string): Promise { @@ -81,21 +84,22 @@ export class StorageFactory { } async export(sessionId: string, log: LogItem): Promise { - const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, log); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, this._localStorage, log); return await exportSession(db); } async import(sessionId: string, data: Export, log: LogItem): Promise { - const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, log); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, this._localStorage, log); return await importSession(db, data); } } -async function createStores(db: IDBDatabase, txn: IDBTransaction, oldVersion: number | null, version: number, log: LogItem): Promise { +async function createStores(db: IDBDatabase, txn: IDBTransaction, oldVersion: number | null, version: number, localStorage: IDOMStorage, log: LogItem): Promise { const startIdx = oldVersion || 0; return log.wrap({l: "storage migration", oldVersion, version}, async log => { for(let i = startIdx; i < version; ++i) { - await log.wrap(`v${i + 1}`, log => schema[i](db, txn, log)); + const migrationFunc = schema[i]; + await log.wrap(`v${i + 1}`, log => migrationFunc(db, txn, localStorage, log)); } }); } diff --git a/src/matrix/storage/idb/Transaction.ts b/src/matrix/storage/idb/Transaction.ts index 9de4caf2..3bc4aed2 100644 --- a/src/matrix/storage/idb/Transaction.ts +++ b/src/matrix/storage/idb/Transaction.ts @@ -73,6 +73,10 @@ export class Transaction { return this._storage.IDBKeyRange; } + get databaseName(): string { + return this._storage.databaseName; + } + get logger(): BaseLogger { return this._storage.logger; } @@ -94,7 +98,7 @@ export class Transaction { } get session(): SessionStore { - return this._store(StoreNames.session, idbStore => new SessionStore(idbStore)); + return this._store(StoreNames.session, idbStore => new SessionStore(idbStore, this._storage.localStorage)); } get roomSummary(): RoomSummaryStore { diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index a072e34f..f77d7753 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -1,3 +1,4 @@ +import {IDOMStorage} from "./types"; import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js"; import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js"; @@ -7,10 +8,13 @@ import {RoomStateEntry} from "./stores/RoomStateStore"; import {SessionStore} from "./stores/SessionStore"; import {encodeScopeTypeKey} from "./stores/OperationStore"; import {MAX_UNICODE} from "./stores/common"; +import {LogItem} from "../../../logging/LogItem.js"; + +export type MigrationFunc = (db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: LogItem) => Promise | void; // FUNCTIONS SHOULD ONLY BE APPENDED!! // the index in the array is the database version -export const schema = [ +export const schema: MigrationFunc[] = [ createInitialStores, createMemberStore, migrateSession, @@ -64,7 +68,7 @@ async function createMemberStore(db: IDBDatabase, txn: IDBTransaction): Promise< }); } //v3 -async function migrateSession(db: IDBDatabase, txn: IDBTransaction): Promise { +async function migrateSession(db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage): Promise { const session = txn.objectStore("session"); try { const PRE_MIGRATION_KEY = 1; @@ -73,7 +77,7 @@ async function migrateSession(db: IDBDatabase, txn: IDBTransaction): Promise(roomSummaryStore.openCursor(), roomSummary => { diff --git a/src/matrix/storage/idb/stores/SessionStore.ts b/src/matrix/storage/idb/stores/SessionStore.ts index 859d3319..b811c6d8 100644 --- a/src/matrix/storage/idb/stores/SessionStore.ts +++ b/src/matrix/storage/idb/stores/SessionStore.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ import {Store} from "../Store"; +import {IDOMStorage} from "../types"; +import {SESSION_E2EE_KEY_PREFIX} from "../../../e2ee/common.js"; export interface SessionEntry { key: string; @@ -22,9 +24,11 @@ export interface SessionEntry { export class SessionStore { private _sessionStore: Store + private _localStorage: IDOMStorage; - constructor(sessionStore: Store) { + constructor(sessionStore: Store, localStorage: IDOMStorage) { this._sessionStore = sessionStore; + this._localStorage = localStorage; } async get(key: string): Promise { @@ -34,15 +38,60 @@ export class SessionStore { } } + _writeKeyToLocalStorage(key: string, value: any) { + // we backup to localStorage so when idb gets cleared for some reason, we don't lose our e2ee identity + try { + const lsKey = `${this._sessionStore.databaseName}.session.${key}`; + const lsValue = JSON.stringify(value); + this._localStorage.setItem(lsKey, lsValue); + } catch (err) { + console.error("could not write to localStorage", err); + } + } + + writeToLocalStorage() { + this._sessionStore.iterateValues(undefined, (value: any, key: string) => { + if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { + this._writeKeyToLocalStorage(key, value); + } + return false; + }); + } + + tryRestoreFromLocalStorage(): boolean { + let success = false; + const lsPrefix = `${this._sessionStore.databaseName}.session.`; + const prefix = `${lsPrefix}${SESSION_E2EE_KEY_PREFIX}`; + for(let i = 0; i < this._localStorage.length; i += 1) { + const lsKey = this._localStorage.key(i)!; + if (lsKey.startsWith(prefix)) { + const value = JSON.parse(this._localStorage.getItem(lsKey)!); + const key = lsKey.substr(lsPrefix.length); + this._sessionStore.put({key, value}); + success = true; + } + } + return success; + } + set(key: string, value: any): void { + if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { + this._writeKeyToLocalStorage(key, value); + } this._sessionStore.put({key, value}); } add(key: string, value: any): void { + if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { + this._writeKeyToLocalStorage(key, value); + } this._sessionStore.add({key, value}); } remove(key: string): void { + if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { + this._localStorage.removeItem(this._sessionStore.databaseName + key); + } this._sessionStore.delete(key); } } diff --git a/src/matrix/storage/idb/types.ts b/src/matrix/storage/idb/types.ts new file mode 100644 index 00000000..15232623 --- /dev/null +++ b/src/matrix/storage/idb/types.ts @@ -0,0 +1,23 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface IDOMStorage { + getItem(key: string): string | null; + setItem(key: string, value: string): void; + removeItem(key: string): void; + key(n: number): string | null; + readonly length: number; +} diff --git a/src/mocks/Storage.ts b/src/mocks/Storage.ts index 2adf8001..5dba796a 100644 --- a/src/mocks/Storage.ts +++ b/src/mocks/Storage.ts @@ -16,12 +16,13 @@ limitations under the License. import {FDBFactory, FDBKeyRange} from "../../lib/fake-indexeddb/index.js"; import {StorageFactory} from "../matrix/storage/idb/StorageFactory"; +import {IDOMStorage} from "../matrix/storage/idb/types"; import {Storage} from "../matrix/storage/idb/Storage"; import {Instance as nullLogger} from "../logging/NullLogger.js"; import {openDatabase, CreateObjectStore} from "../matrix/storage/idb/utils"; export function createMockStorage(): Promise { - return new StorageFactory(null as any, new FDBFactory(), FDBKeyRange).create("1", nullLogger.item); + return new StorageFactory(null as any, new FDBFactory(), FDBKeyRange, new MockLocalStorage()).create("1", nullLogger.item); } export function createMockDatabase(name: string, createObjectStore: CreateObjectStore, impl: MockIDBImpl): Promise { @@ -39,3 +40,41 @@ export class MockIDBImpl { return FDBKeyRange; } } + +class MockLocalStorage implements IDOMStorage { + private _map: Map; + + constructor() { + this._map = new Map(); + } + + getItem(key: string): string | null { + return this._map.get(key) || null; + } + + setItem(key: string, value: string) { + this._map.set(key, value); + } + + removeItem(key: string): void { + this._map.delete(key); + } + + get length(): number { + return this._map.size; + } + + key(n: number): string | null { + const it = this._map.keys(); + let i = -1; + let result; + while (i < n) { + result = it.next(); + if (result.done) { + return null; + } + i += 1; + } + return result?.value || null; + } +} From 2ef72510797e93a1b56faec72cb5430577ce709c Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 29 Sep 2021 19:20:27 +0200 Subject: [PATCH 2/7] move ssssKey to e2ee prefix as well so it gets backed up too --- src/matrix/ssss/index.js | 7 +- src/matrix/storage/idb/schema.ts | 13 ++- src/matrix/storage/idb/stores/SessionStore.ts | 23 ++-- src/utils/typedJSON.ts | 107 ++++++++++++++++++ 4 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 src/utils/typedJSON.ts diff --git a/src/matrix/ssss/index.js b/src/matrix/ssss/index.js index 34ee63b8..e1baf9c9 100644 --- a/src/matrix/ssss/index.js +++ b/src/matrix/ssss/index.js @@ -17,6 +17,9 @@ limitations under the License. import {KeyDescription, Key} from "./common.js"; import {keyFromPassphrase} from "./passphrase.js"; import {keyFromRecoveryKey} from "./recoveryKey.js"; +import {SESSION_E2EE_KEY_PREFIX} from "../e2ee/common.js"; + +const SSSS_KEY = `${SESSION_E2EE_KEY_PREFIX}ssssKey`; async function readDefaultKeyDescription(storage) { const txn = await storage.readTxn([ @@ -35,11 +38,11 @@ async function readDefaultKeyDescription(storage) { } export async function writeKey(key, txn) { - txn.session.set("ssssKey", {id: key.id, binaryKey: key.binaryKey}); + txn.session.set(SSSS_KEY, {id: key.id, binaryKey: key.binaryKey}); } export async function readKey(txn) { - const keyData = await txn.session.get("ssssKey"); + const keyData = await txn.session.get(SSSS_KEY); if (!keyData) { return; } diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index f77d7753..14e0bc10 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -2,6 +2,7 @@ import {IDOMStorage} from "./types"; import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js"; import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js"; +import {SESSION_E2EE_KEY_PREFIX} from "../../e2ee/common.js"; import {SummaryData} from "../../room/RoomSummary"; import {RoomMemberStore, MemberData} from "./stores/RoomMemberStore"; import {RoomStateEntry} from "./stores/RoomStateStore"; @@ -25,7 +26,8 @@ export const schema: MigrationFunc[] = [ createArchivedRoomSummaryStore, migrateOperationScopeIndex, createTimelineRelationsStore, - fixMissingRoomsInUserIdentities + fixMissingRoomsInUserIdentities, + changeSSSSKeyPrefix, ]; // TODO: how to deal with git merge conflicts of this array? @@ -204,3 +206,12 @@ async function fixMissingRoomsInUserIdentities(db: IDBDatabase, txn: IDBTransact }); } } + +// v12 move ssssKey to e2ee:ssssKey so it will get backed up in the next step +async function changeSSSSKeyPrefix(db: IDBDatabase, txn: IDBTransaction) { + const session = txn.objectStore("session"); + const ssssKey = await reqAsPromise(session.get("ssssKey")); + if (ssssKey) { + session.put({key: `${SESSION_E2EE_KEY_PREFIX}ssssKey`, value: ssssKey}); + } +} diff --git a/src/matrix/storage/idb/stores/SessionStore.ts b/src/matrix/storage/idb/stores/SessionStore.ts index b811c6d8..2158dc23 100644 --- a/src/matrix/storage/idb/stores/SessionStore.ts +++ b/src/matrix/storage/idb/stores/SessionStore.ts @@ -16,6 +16,8 @@ limitations under the License. import {Store} from "../Store"; import {IDOMStorage} from "../types"; import {SESSION_E2EE_KEY_PREFIX} from "../../../e2ee/common.js"; +import {LogItem} from "../../../../logging/LogItem.js"; +import {parse, stringify} from "../../../../utils/typedJSON"; export interface SessionEntry { key: string; @@ -42,33 +44,38 @@ export class SessionStore { // we backup to localStorage so when idb gets cleared for some reason, we don't lose our e2ee identity try { const lsKey = `${this._sessionStore.databaseName}.session.${key}`; - const lsValue = JSON.stringify(value); + const lsValue = stringify(value); this._localStorage.setItem(lsKey, lsValue); } catch (err) { console.error("could not write to localStorage", err); } } - writeToLocalStorage() { - this._sessionStore.iterateValues(undefined, (value: any, key: string) => { + writeE2EEIdentityToLocalStorage() { + this._sessionStore.iterateValues(undefined, (entry: SessionEntry, key: string) => { if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { - this._writeKeyToLocalStorage(key, value); + this._writeKeyToLocalStorage(key, entry.value); } return false; }); } - tryRestoreFromLocalStorage(): boolean { + async tryRestoreE2EEIdentityFromLocalStorage(log: LogItem): Promise { let success = false; const lsPrefix = `${this._sessionStore.databaseName}.session.`; const prefix = `${lsPrefix}${SESSION_E2EE_KEY_PREFIX}`; for(let i = 0; i < this._localStorage.length; i += 1) { const lsKey = this._localStorage.key(i)!; if (lsKey.startsWith(prefix)) { - const value = JSON.parse(this._localStorage.getItem(lsKey)!); + const value = parse(this._localStorage.getItem(lsKey)!); const key = lsKey.substr(lsPrefix.length); - this._sessionStore.put({key, value}); - success = true; + // we check if we don't have this key already, as we don't want to override anything + const hasKey = (await this._sessionStore.getKey(key)) === key; + log.set(key, !hasKey); + if (!hasKey) { + this._sessionStore.put({key, value}); + success = true; + } } } return success; diff --git a/src/utils/typedJSON.ts b/src/utils/typedJSON.ts new file mode 100644 index 00000000..2a8a3c60 --- /dev/null +++ b/src/utils/typedJSON.ts @@ -0,0 +1,107 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export function stringify(value: any): string { + return JSON.stringify(encodeValue(value)); +} + +export function parse(value: string): any { + return decodeValue(JSON.parse(value)); +} + + +function encodeValue(value: any): any { + switch (typeof value) { + case "object": { + if (value === null || Array.isArray(value)) { + return value; + } + // TypedArray + if (value.byteLength) { + return {_type: value.constructor.name, value: Array.from(value)}; + } + let newObj = {}; + for (const prop in value) { + if (value.hasOwnProperty(prop)) { + newObj[prop] = encodeValue(value[prop]); + } + } + return newObj; + } + default: + return value; + } +} + +function decodeValue(value: any): any { + switch (typeof value) { + case "object": { + if (value === null || Array.isArray(value)) { + return value; + } + if (typeof value._type === "string") { + switch (value._type) { + case "Int8Array": return Int8Array.from(value.value); + case "Uint8Array": return Uint8Array.from(value.value); + case "Uint8ClampedArray": return Uint8ClampedArray.from(value.value); + case "Int16Array": return Int16Array.from(value.value); + case "Uint16Array": return Uint16Array.from(value.value); + case "Int32Array": return Int32Array.from(value.value); + case "Uint32Array": return Uint32Array.from(value.value); + case "Float32Array": return Float32Array.from(value.value); + case "Float64Array": return Float64Array.from(value.value); + case "BigInt64Array": return BigInt64Array.from(value.value); + case "BigUint64Array": return BigUint64Array.from(value.value); + default: + return value.value; + } + } + let newObj = {}; + for (const prop in value) { + if (value.hasOwnProperty(prop)) { + newObj[prop] = decodeValue(value[prop]); + } + } + return newObj; + } + default: + return value; + } +} + +export function tests() { + return { + "Uint8Array and primitives": assert => { + const value = { + foo: "bar", + bar: 5, + baz: false, + fuzz: new Uint8Array([3, 1, 2]) + }; + const serialized = stringify(value); + assert.strictEqual(typeof serialized, "string"); + const deserialized = parse(serialized); + assert.strictEqual(deserialized.foo, "bar"); + assert.strictEqual(deserialized.bar, 5); + assert.strictEqual(deserialized.baz, false); + assert(deserialized.fuzz instanceof Uint8Array); + assert.strictEqual(deserialized.fuzz.length, 3); + assert.strictEqual(deserialized.fuzz[0], 3); + assert.strictEqual(deserialized.fuzz[1], 1); + assert.strictEqual(deserialized.fuzz[2], 2); + } + } +} From 3a064d67967f1409edf18ae999db110a51d51884 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 29 Sep 2021 19:21:42 +0200 Subject: [PATCH 3/7] a IDBRequest when opening the database does not have a source --- src/matrix/storage/idb/error.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/storage/idb/error.ts b/src/matrix/storage/idb/error.ts index fb602168..abfbe415 100644 --- a/src/matrix/storage/idb/error.ts +++ b/src/matrix/storage/idb/error.ts @@ -33,10 +33,10 @@ export class IDBError extends StorageError { storeName: string; databaseName: string; - constructor(message: string, sourceOrCursor: IDBIndex | IDBCursor | IDBObjectStore, cause: DOMException | null = null) { - const source = "source" in sourceOrCursor ? sourceOrCursor.source : sourceOrCursor; - const storeName = _sourceName(source); - const databaseName = _sourceDatabase(source); + constructor(message: string, sourceOrCursor: IDBIndex | IDBCursor | IDBObjectStore | null, cause: DOMException | null = null) { + const source = (sourceOrCursor && "source" in sourceOrCursor) ? sourceOrCursor.source : sourceOrCursor; + const storeName = source ? _sourceName(source) : ""; + const databaseName = source ? _sourceDatabase(source) : ""; let fullMessage = `${message} on ${databaseName}.${storeName}`; if (cause) { fullMessage += ": "; From a1c66738db3d269bffc084dcee23ce2a2a6dfce2 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 29 Sep 2021 19:39:26 +0200 Subject: [PATCH 4/7] migration to initialize & restore e2ee identity backup --- src/matrix/storage/idb/QueryTarget.ts | 2 ++ src/matrix/storage/idb/schema.ts | 28 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/matrix/storage/idb/QueryTarget.ts b/src/matrix/storage/idb/QueryTarget.ts index e84c41e1..01f46e1a 100644 --- a/src/matrix/storage/idb/QueryTarget.ts +++ b/src/matrix/storage/idb/QueryTarget.ts @@ -19,6 +19,8 @@ import {StorageError} from "../common"; import {LogItem} from "../../../logging/LogItem.js"; import {IDBKey} from "./Transaction"; +// this is the part of the Transaction class API that is used here and in the Store subclass, +// to make it easier to replace it with alternative implementations in schema.ts and unit tests export interface ITransaction { idbFactory: IDBFactory; IDBKeyRange: typeof IDBKeyRange; diff --git a/src/matrix/storage/idb/schema.ts b/src/matrix/storage/idb/schema.ts index 14e0bc10..c7d41a6f 100644 --- a/src/matrix/storage/idb/schema.ts +++ b/src/matrix/storage/idb/schema.ts @@ -1,4 +1,5 @@ import {IDOMStorage} from "./types"; +import {ITransaction} from "./QueryTarget"; import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils"; import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js"; import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js"; @@ -7,6 +8,7 @@ import {SummaryData} from "../../room/RoomSummary"; import {RoomMemberStore, MemberData} from "./stores/RoomMemberStore"; import {RoomStateEntry} from "./stores/RoomStateStore"; import {SessionStore} from "./stores/SessionStore"; +import {Store} from "./Store"; import {encodeScopeTypeKey} from "./stores/OperationStore"; import {MAX_UNICODE} from "./stores/common"; import {LogItem} from "../../../logging/LogItem.js"; @@ -28,6 +30,7 @@ export const schema: MigrationFunc[] = [ createTimelineRelationsStore, fixMissingRoomsInUserIdentities, changeSSSSKeyPrefix, + backupAndRestoreE2EEAccountToLocalStorage ]; // TODO: how to deal with git merge conflicts of this array? @@ -215,3 +218,28 @@ async function changeSSSSKeyPrefix(db: IDBDatabase, txn: IDBTransaction) { session.put({key: `${SESSION_E2EE_KEY_PREFIX}ssssKey`, value: ssssKey}); } } +// v13 +async function backupAndRestoreE2EEAccountToLocalStorage(db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: LogItem) { + const session = txn.objectStore("session"); + // the Store object gets passed in several things through the Transaction class (a wrapper around IDBTransaction), + // the only thing we should need here is the databaseName though, so we mock it out. + // ideally we should have an easier way to go from the idb primitive layer to the specific store classes where + // we implement logic, but for now we need this. + const databaseNameHelper: ITransaction = { + databaseName: db.name, + get idbFactory(): IDBFactory { throw new Error("unused");}, + get IDBKeyRange(): typeof IDBKeyRange { throw new Error("unused");}, + addWriteError() {}, + }; + const sessionStore = new SessionStore(new Store(session, databaseNameHelper), localStorage); + // if we already have an e2ee identity, write a backup to local storage. + // further updates to e2ee keys in the session store will also write to local storage from 0.2.15 on, + // but here we make sure a backup is immediately created after installing the update and we don't wait until + // the olm account needs to change + sessionStore.writeE2EEIdentityToLocalStorage(); + // and if we already have a backup, restore it now for any missing key in idb. + // this will restore the backup every time the idb database is dropped as it will + // run through all the migration steps when recreating it. + const restored = await sessionStore.tryRestoreE2EEIdentityFromLocalStorage(log); + log.set("restored", restored); +} From e0b9a3fa505fdcdb1a3f37a73717d52ab6f13436 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 29 Sep 2021 20:07:42 +0200 Subject: [PATCH 5/7] create e2ee identity also when storage got cleared without backup --- src/matrix/Session.js | 4 ++++ src/matrix/SessionContainer.js | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 63aece42..87a3150c 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -237,6 +237,10 @@ export class Session { return this._sessionBackup; } + get hasIdentity() { + return !!this._e2eeAccount; + } + /** @internal */ async createIdentity(log) { if (this._olm) { diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js index 7248e11e..479bb7a8 100644 --- a/src/matrix/SessionContainer.js +++ b/src/matrix/SessionContainer.js @@ -233,8 +233,7 @@ export class SessionContainer { platform: this._platform, }); await this._session.load(log); - // TODO: check instead storage doesn't have an identity - if (isNewLogin) { + if (!this._session.hasIdentity) { this._status.set(LoadStatus.SessionSetup); await log.wrap("createIdentity", log => this._session.createIdentity(log)); } From ae68264db4a08e12ac54fac19726db92853c25f2 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 30 Sep 2021 08:42:00 +0200 Subject: [PATCH 6/7] don't use switch where single if/else works --- src/utils/typedJSON.ts | 87 ++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 49 deletions(-) diff --git a/src/utils/typedJSON.ts b/src/utils/typedJSON.ts index 2a8a3c60..928df32e 100644 --- a/src/utils/typedJSON.ts +++ b/src/utils/typedJSON.ts @@ -22,63 +22,52 @@ export function parse(value: string): any { return decodeValue(JSON.parse(value)); } - function encodeValue(value: any): any { - switch (typeof value) { - case "object": { - if (value === null || Array.isArray(value)) { - return value; - } - // TypedArray - if (value.byteLength) { - return {_type: value.constructor.name, value: Array.from(value)}; - } - let newObj = {}; - for (const prop in value) { - if (value.hasOwnProperty(prop)) { - newObj[prop] = encodeValue(value[prop]); - } - } - return newObj; + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + // TypedArray + if (value.byteLength) { + return {_type: value.constructor.name, value: Array.from(value)}; } - default: - return value; + let newObj = {}; + for (const prop in value) { + if (value.hasOwnProperty(prop)) { + newObj[prop] = encodeValue(value[prop]); + } + } + return newObj; + } else { + return value; } } function decodeValue(value: any): any { - switch (typeof value) { - case "object": { - if (value === null || Array.isArray(value)) { - return value; + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + if (typeof value._type === "string") { + switch (value._type) { + case "Int8Array": return Int8Array.from(value.value); + case "Uint8Array": return Uint8Array.from(value.value); + case "Uint8ClampedArray": return Uint8ClampedArray.from(value.value); + case "Int16Array": return Int16Array.from(value.value); + case "Uint16Array": return Uint16Array.from(value.value); + case "Int32Array": return Int32Array.from(value.value); + case "Uint32Array": return Uint32Array.from(value.value); + case "Float32Array": return Float32Array.from(value.value); + case "Float64Array": return Float64Array.from(value.value); + case "BigInt64Array": return BigInt64Array.from(value.value); + case "BigUint64Array": return BigUint64Array.from(value.value); + default: + return value.value; } - if (typeof value._type === "string") { - switch (value._type) { - case "Int8Array": return Int8Array.from(value.value); - case "Uint8Array": return Uint8Array.from(value.value); - case "Uint8ClampedArray": return Uint8ClampedArray.from(value.value); - case "Int16Array": return Int16Array.from(value.value); - case "Uint16Array": return Uint16Array.from(value.value); - case "Int32Array": return Int32Array.from(value.value); - case "Uint32Array": return Uint32Array.from(value.value); - case "Float32Array": return Float32Array.from(value.value); - case "Float64Array": return Float64Array.from(value.value); - case "BigInt64Array": return BigInt64Array.from(value.value); - case "BigUint64Array": return BigUint64Array.from(value.value); - default: - return value.value; - } - } - let newObj = {}; - for (const prop in value) { - if (value.hasOwnProperty(prop)) { - newObj[prop] = decodeValue(value[prop]); - } - } - return newObj; } - default: - return value; + let newObj = {}; + for (const prop in value) { + if (value.hasOwnProperty(prop)) { + newObj[prop] = decodeValue(value[prop]); + } + } + return newObj; + } else { + return value; } } From 42f1603d8191a9e6fc700a5e1bcd58b2a1b2e179 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 30 Sep 2021 09:25:35 +0200 Subject: [PATCH 7/7] use correct prefix to remove local storage value --- src/matrix/storage/idb/stores/SessionStore.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/matrix/storage/idb/stores/SessionStore.ts b/src/matrix/storage/idb/stores/SessionStore.ts index 2158dc23..785835b8 100644 --- a/src/matrix/storage/idb/stores/SessionStore.ts +++ b/src/matrix/storage/idb/stores/SessionStore.ts @@ -33,6 +33,10 @@ export class SessionStore { this._localStorage = localStorage; } + private get _localStorageKeyPrefix(): string { + return `${this._sessionStore.databaseName}.session.`; + } + async get(key: string): Promise { const entry = await this._sessionStore.get(key); if (entry) { @@ -43,7 +47,7 @@ export class SessionStore { _writeKeyToLocalStorage(key: string, value: any) { // we backup to localStorage so when idb gets cleared for some reason, we don't lose our e2ee identity try { - const lsKey = `${this._sessionStore.databaseName}.session.${key}`; + const lsKey = this._localStorageKeyPrefix + key; const lsValue = stringify(value); this._localStorage.setItem(lsKey, lsValue); } catch (err) { @@ -62,8 +66,8 @@ export class SessionStore { async tryRestoreE2EEIdentityFromLocalStorage(log: LogItem): Promise { let success = false; - const lsPrefix = `${this._sessionStore.databaseName}.session.`; - const prefix = `${lsPrefix}${SESSION_E2EE_KEY_PREFIX}`; + const lsPrefix = this._localStorageKeyPrefix; + const prefix = lsPrefix + SESSION_E2EE_KEY_PREFIX; for(let i = 0; i < this._localStorage.length; i += 1) { const lsKey = this._localStorage.key(i)!; if (lsKey.startsWith(prefix)) { @@ -97,7 +101,7 @@ export class SessionStore { remove(key: string): void { if (key.startsWith(SESSION_E2EE_KEY_PREFIX)) { - this._localStorage.removeItem(this._sessionStore.databaseName + key); + this._localStorage.removeItem(this._localStorageKeyPrefix + key); } this._sessionStore.delete(key); }