From edbac2561374a1b764d660af439d69282d2294e4 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:31:13 +0200 Subject: [PATCH 1/9] prepare storage to work with alternative idb impl --- src/matrix/storage/idb/Storage.js | 7 +++-- src/matrix/storage/idb/StorageFactory.js | 30 +++++++++++-------- src/matrix/storage/idb/Store.js | 4 +++ src/matrix/storage/idb/Transaction.js | 5 ++-- src/matrix/storage/idb/quirks.js | 4 +-- .../storage/idb/stores/DeviceIdentityStore.js | 6 ++-- .../idb/stores/GroupSessionDecryptionStore.js | 2 +- .../idb/stores/InboundGroupSessionStore.js | 2 +- .../storage/idb/stores/OlmSessionStore.js | 4 +-- .../storage/idb/stores/OperationStore.js | 2 +- .../storage/idb/stores/PendingEventStore.js | 8 ++--- .../storage/idb/stores/RoomMemberStore.js | 6 ++-- .../storage/idb/stores/RoomStateStore.js | 2 +- .../storage/idb/stores/TimelineEventStore.js | 21 ++++++------- .../idb/stores/TimelineFragmentStore.js | 2 +- src/matrix/storage/idb/utils.js | 4 +-- 16 files changed, 61 insertions(+), 48 deletions(-) diff --git a/src/matrix/storage/idb/Storage.js b/src/matrix/storage/idb/Storage.js index 93d55f34..1f96e347 100644 --- a/src/matrix/storage/idb/Storage.js +++ b/src/matrix/storage/idb/Storage.js @@ -21,8 +21,9 @@ import { reqAsPromise } from "./utils.js"; const WEBKITEARLYCLOSETXNBUG_BOGUS_KEY = "782rh281re38-boguskey"; export class Storage { - constructor(idbDatabase, hasWebkitEarlyCloseTxnBug) { + constructor(idbDatabase, IDBKeyRange, hasWebkitEarlyCloseTxnBug) { this._db = idbDatabase; + this._IDBKeyRange = IDBKeyRange; this._hasWebkitEarlyCloseTxnBug = hasWebkitEarlyCloseTxnBug; const nameMap = STORE_NAMES.reduce((nameMap, name) => { nameMap[name] = name; @@ -47,7 +48,7 @@ export class Storage { if (this._hasWebkitEarlyCloseTxnBug) { await reqAsPromise(txn.objectStore(storeNames[0]).get(WEBKITEARLYCLOSETXNBUG_BOGUS_KEY)); } - return new Transaction(txn, storeNames); + return new Transaction(txn, storeNames, this._IDBKeyRange); } catch(err) { throw new StorageError("readTxn failed", err); } @@ -62,7 +63,7 @@ export class Storage { if (this._hasWebkitEarlyCloseTxnBug) { await reqAsPromise(txn.objectStore(storeNames[0]).get(WEBKITEARLYCLOSETXNBUG_BOGUS_KEY)); } - return new Transaction(txn, storeNames); + return new Transaction(txn, storeNames, this._IDBKeyRange); } catch(err) { throw new StorageError("readWriteTxn failed", err); } diff --git a/src/matrix/storage/idb/StorageFactory.js b/src/matrix/storage/idb/StorageFactory.js index 13860f67..719d2672 100644 --- a/src/matrix/storage/idb/StorageFactory.js +++ b/src/matrix/storage/idb/StorageFactory.js @@ -21,14 +21,18 @@ import { schema } from "./schema.js"; import { detectWebkitEarlyCloseTxnBug } from "./quirks.js"; const sessionName = sessionId => `hydrogen_session_${sessionId}`; -const openDatabaseWithSessionId = sessionId => openDatabase(sessionName(sessionId), createStores, schema.length); +const openDatabaseWithSessionId = function(sessionId, idbFactory) { + return openDatabase(sessionName(sessionId), createStores, schema.length, idbFactory); +} async function requestPersistedStorage() { - if (navigator?.storage?.persist) { - return await navigator.storage.persist(); - } else if (document.requestStorageAccess) { + // don't assume browser so we can run in node with fake-idb + const glob = this; + if (glob?.navigator?.storage?.persist) { + return await glob.navigator.storage.persist(); + } else if (glob?.document.requestStorageAccess) { try { - await document.requestStorageAccess(); + await glob.document.requestStorageAccess(); return true; } catch (err) { return false; @@ -39,8 +43,10 @@ async function requestPersistedStorage() { } export class StorageFactory { - constructor(serviceWorkerHandler) { + constructor(serviceWorkerHandler, idbFactory = window.indexedDB, IDBKeyRange = window.IDBKeyRange) { this._serviceWorkerHandler = serviceWorkerHandler; + this._idbFactory = idbFactory; + this._IDBKeyRange = IDBKeyRange; } async create(sessionId) { @@ -52,24 +58,24 @@ export class StorageFactory { } }); - const hasWebkitEarlyCloseTxnBug = await detectWebkitEarlyCloseTxnBug(); - const db = await openDatabaseWithSessionId(sessionId); - return new Storage(db, hasWebkitEarlyCloseTxnBug); + const hasWebkitEarlyCloseTxnBug = await detectWebkitEarlyCloseTxnBug(this._idbFactory); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory); + return new Storage(db, this._IDBKeyRange, hasWebkitEarlyCloseTxnBug); } delete(sessionId) { const databaseName = sessionName(sessionId); - const req = indexedDB.deleteDatabase(databaseName); + const req = this._idbFactory.deleteDatabase(databaseName); return reqAsPromise(req); } async export(sessionId) { - const db = await openDatabaseWithSessionId(sessionId); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory); return await exportSession(db); } async import(sessionId, data) { - const db = await openDatabaseWithSessionId(sessionId); + const db = await openDatabaseWithSessionId(sessionId, this._idbFactory); return await importSession(db, data); } } diff --git a/src/matrix/storage/idb/Store.js b/src/matrix/storage/idb/Store.js index 22851af0..2cafe500 100644 --- a/src/matrix/storage/idb/Store.js +++ b/src/matrix/storage/idb/Store.js @@ -126,6 +126,10 @@ export class Store extends QueryTarget { this._transaction = transaction; } + get IDBKeyRange() { + return this._transaction.IDBKeyRange; + } + get _idbStore() { return this._target; } diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js index d497077b..a2041d31 100644 --- a/src/matrix/storage/idb/Transaction.js +++ b/src/matrix/storage/idb/Transaction.js @@ -35,10 +35,11 @@ import {OperationStore} from "./stores/OperationStore.js"; import {AccountDataStore} from "./stores/AccountDataStore.js"; export class Transaction { - constructor(txn, allowedStoreNames) { + constructor(txn, allowedStoreNames, IDBKeyRange) { this._txn = txn; this._allowedStoreNames = allowedStoreNames; this._stores = {}; + this.IDBKeyRange = IDBKeyRange; } _idbStore(name) { @@ -46,7 +47,7 @@ export class Transaction { // more specific error? this is a bug, so maybe not ... throw new StorageError(`Invalid store for transaction: ${name}, only ${this._allowedStoreNames.join(", ")} are allowed.`); } - return new Store(this._txn.objectStore(name)); + return new Store(this._txn.objectStore(name), this); } _store(name, mapStore) { diff --git a/src/matrix/storage/idb/quirks.js b/src/matrix/storage/idb/quirks.js index 8f4c1826..9739b7da 100644 --- a/src/matrix/storage/idb/quirks.js +++ b/src/matrix/storage/idb/quirks.js @@ -18,12 +18,12 @@ limitations under the License. import {openDatabase, txnAsPromise, reqAsPromise} from "./utils.js"; // filed as https://bugs.webkit.org/show_bug.cgi?id=222746 -export async function detectWebkitEarlyCloseTxnBug() { +export async function detectWebkitEarlyCloseTxnBug(idbFactory) { const dbName = "hydrogen_webkit_test_inactive_txn_bug"; try { const db = await openDatabase(dbName, db => { db.createObjectStore("test", {keyPath: "key"}); - }, 1); + }, 1, idbFactory); const readTxn = db.transaction(["test"], "readonly"); await reqAsPromise(readTxn.objectStore("test").get("somekey")); // schedule a macro task in between the two txns diff --git a/src/matrix/storage/idb/stores/DeviceIdentityStore.js b/src/matrix/storage/idb/stores/DeviceIdentityStore.js index fed8878b..bfc5b30b 100644 --- a/src/matrix/storage/idb/stores/DeviceIdentityStore.js +++ b/src/matrix/storage/idb/stores/DeviceIdentityStore.js @@ -31,7 +31,7 @@ export class DeviceIdentityStore { } getAllForUserId(userId) { - const range = IDBKeyRange.lowerBound(encodeKey(userId, "")); + const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, "")); return this._store.selectWhile(range, device => { return device.userId === userId; }); @@ -39,7 +39,7 @@ export class DeviceIdentityStore { async getAllDeviceIds(userId) { const deviceIds = []; - const range = IDBKeyRange.lowerBound(encodeKey(userId, "")); + const range = this._store.IDBKeyRange.lowerBound(encodeKey(userId, "")); await this._store.iterateKeys(range, key => { const decodedKey = decodeKey(key); // prevent running into the next room @@ -72,7 +72,7 @@ export class DeviceIdentityStore { removeAllForUser(userId) { // exclude both keys as they are theoretical min and max, // but we should't have a match for just the room id, or room id with max - const range = IDBKeyRange.bound(encodeKey(userId, MIN_UNICODE), encodeKey(userId, MAX_UNICODE), true, true); + const range = this._store.IDBKeyRange.bound(encodeKey(userId, MIN_UNICODE), encodeKey(userId, MAX_UNICODE), true, true); this._store.delete(range); } } diff --git a/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js index df3c6fbe..f3a721f1 100644 --- a/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js +++ b/src/matrix/storage/idb/stores/GroupSessionDecryptionStore.js @@ -35,7 +35,7 @@ export class GroupSessionDecryptionStore { } removeAllForRoom(roomId) { - const range = IDBKeyRange.bound( + const range = this._store.IDBKeyRange.bound( encodeKey(roomId, MIN_UNICODE, MIN_UNICODE), encodeKey(roomId, MAX_UNICODE, MAX_UNICODE) ); diff --git a/src/matrix/storage/idb/stores/InboundGroupSessionStore.js b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js index 928f7572..f5dcdc39 100644 --- a/src/matrix/storage/idb/stores/InboundGroupSessionStore.js +++ b/src/matrix/storage/idb/stores/InboundGroupSessionStore.js @@ -41,7 +41,7 @@ export class InboundGroupSessionStore { } removeAllForRoom(roomId) { - const range = IDBKeyRange.bound( + const range = this._store.IDBKeyRange.bound( encodeKey(roomId, MIN_UNICODE, MIN_UNICODE), encodeKey(roomId, MAX_UNICODE, MAX_UNICODE) ); diff --git a/src/matrix/storage/idb/stores/OlmSessionStore.js b/src/matrix/storage/idb/stores/OlmSessionStore.js index 4648f09c..d81cc048 100644 --- a/src/matrix/storage/idb/stores/OlmSessionStore.js +++ b/src/matrix/storage/idb/stores/OlmSessionStore.js @@ -30,7 +30,7 @@ export class OlmSessionStore { async getSessionIds(senderKey) { const sessionIds = []; - const range = IDBKeyRange.lowerBound(encodeKey(senderKey, "")); + const range = this._store.IDBKeyRange.lowerBound(encodeKey(senderKey, "")); await this._store.iterateKeys(range, key => { const decodedKey = decodeKey(key); // prevent running into the next room @@ -44,7 +44,7 @@ export class OlmSessionStore { } getAll(senderKey) { - const range = IDBKeyRange.lowerBound(encodeKey(senderKey, "")); + const range = this._store.IDBKeyRange.lowerBound(encodeKey(senderKey, "")); return this._store.selectWhile(range, session => { return session.senderKey === senderKey; }); diff --git a/src/matrix/storage/idb/stores/OperationStore.js b/src/matrix/storage/idb/stores/OperationStore.js index 09e4552f..e2040e30 100644 --- a/src/matrix/storage/idb/stores/OperationStore.js +++ b/src/matrix/storage/idb/stores/OperationStore.js @@ -55,7 +55,7 @@ export class OperationStore { } async removeAllForScope(scope) { - const range = IDBKeyRange.bound( + const range = this._store.IDBKeyRange.bound( encodeScopeTypeKey(scope, MIN_UNICODE), encodeScopeTypeKey(scope, MAX_UNICODE) ); diff --git a/src/matrix/storage/idb/stores/PendingEventStore.js b/src/matrix/storage/idb/stores/PendingEventStore.js index a6727ec5..455cad2c 100644 --- a/src/matrix/storage/idb/stores/PendingEventStore.js +++ b/src/matrix/storage/idb/stores/PendingEventStore.js @@ -33,7 +33,7 @@ export class PendingEventStore { } async getMaxQueueIndex(roomId) { - const range = IDBKeyRange.bound( + const range = this._eventStore.IDBKeyRange.bound( encodeKey(roomId, KeyLimits.minStorageKey), encodeKey(roomId, KeyLimits.maxStorageKey), false, @@ -46,12 +46,12 @@ export class PendingEventStore { } remove(roomId, queueIndex) { - const keyRange = IDBKeyRange.only(encodeKey(roomId, queueIndex)); + const keyRange = this._eventStore.IDBKeyRange.only(encodeKey(roomId, queueIndex)); this._eventStore.delete(keyRange); } async exists(roomId, queueIndex) { - const keyRange = IDBKeyRange.only(encodeKey(roomId, queueIndex)); + const keyRange = this._eventStore.IDBKeyRange.only(encodeKey(roomId, queueIndex)); const key = await this._eventStore.getKey(keyRange); return !!key; } @@ -72,7 +72,7 @@ export class PendingEventStore { removeAllForRoom(roomId) { const minKey = encodeKey(roomId, KeyLimits.minStorageKey); const maxKey = encodeKey(roomId, KeyLimits.maxStorageKey); - const range = IDBKeyRange.bound(minKey, maxKey); + const range = this._eventStore.IDBKeyRange.bound(minKey, maxKey); this._eventStore.delete(range); } } diff --git a/src/matrix/storage/idb/stores/RoomMemberStore.js b/src/matrix/storage/idb/stores/RoomMemberStore.js index 340e48a1..f900ff0b 100644 --- a/src/matrix/storage/idb/stores/RoomMemberStore.js +++ b/src/matrix/storage/idb/stores/RoomMemberStore.js @@ -42,7 +42,7 @@ export class RoomMemberStore { } getAll(roomId) { - const range = IDBKeyRange.lowerBound(encodeKey(roomId, "")); + const range = this._roomMembersStore.IDBKeyRange.lowerBound(encodeKey(roomId, "")); return this._roomMembersStore.selectWhile(range, member => { return member.roomId === roomId; }); @@ -50,7 +50,7 @@ export class RoomMemberStore { async getAllUserIds(roomId) { const userIds = []; - const range = IDBKeyRange.lowerBound(encodeKey(roomId, "")); + const range = this._roomMembersStore.IDBKeyRange.lowerBound(encodeKey(roomId, "")); await this._roomMembersStore.iterateKeys(range, key => { const decodedKey = decodeKey(key); // prevent running into the next room @@ -66,7 +66,7 @@ export class RoomMemberStore { removeAllForRoom(roomId) { // exclude both keys as they are theoretical min and max, // but we should't have a match for just the room id, or room id with max - const range = IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true); + const range = this._roomMembersStore.IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true); this._roomMembersStore.delete(range); } } diff --git a/src/matrix/storage/idb/stores/RoomStateStore.js b/src/matrix/storage/idb/stores/RoomStateStore.js index 84dd4ec8..73bde1ec 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.js +++ b/src/matrix/storage/idb/stores/RoomStateStore.js @@ -44,7 +44,7 @@ export class RoomStateStore { removeAllForRoom(roomId) { // exclude both keys as they are theoretical min and max, // but we should't have a match for just the room id, or room id with max - const range = IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true); + const range = this._roomStateStore.IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true); this._roomStateStore.delete(range); } } diff --git a/src/matrix/storage/idb/stores/TimelineEventStore.js b/src/matrix/storage/idb/stores/TimelineEventStore.js index 9c40aad2..b4b204a8 100644 --- a/src/matrix/storage/idb/stores/TimelineEventStore.js +++ b/src/matrix/storage/idb/stores/TimelineEventStore.js @@ -33,7 +33,8 @@ function decodeEventIdKey(eventIdKey) { } class Range { - constructor(only, lower, upper, lowerOpen, upperOpen) { + constructor(IDBKeyRange, only, lower, upper, lowerOpen, upperOpen) { + this._IDBKeyRange = IDBKeyRange; this._only = only; this._lower = lower; this._upper = upper; @@ -45,12 +46,12 @@ class Range { try { // only if (this._only) { - return IDBKeyRange.only(encodeKey(roomId, this._only.fragmentId, this._only.eventIndex)); + return this._IDBKeyRange.only(encodeKey(roomId, this._only.fragmentId, this._only.eventIndex)); } // lowerBound // also bound as we don't want to move into another roomId if (this._lower && !this._upper) { - return IDBKeyRange.bound( + return this._IDBKeyRange.bound( encodeKey(roomId, this._lower.fragmentId, this._lower.eventIndex), encodeKey(roomId, this._lower.fragmentId, KeyLimits.maxStorageKey), this._lowerOpen, @@ -60,7 +61,7 @@ class Range { // upperBound // also bound as we don't want to move into another roomId if (!this._lower && this._upper) { - return IDBKeyRange.bound( + return this._IDBKeyRange.bound( encodeKey(roomId, this._upper.fragmentId, KeyLimits.minStorageKey), encodeKey(roomId, this._upper.fragmentId, this._upper.eventIndex), false, @@ -69,7 +70,7 @@ class Range { } // bound if (this._lower && this._upper) { - return IDBKeyRange.bound( + return this._IDBKeyRange.bound( encodeKey(roomId, this._lower.fragmentId, this._lower.eventIndex), encodeKey(roomId, this._upper.fragmentId, this._upper.eventIndex), this._lowerOpen, @@ -107,7 +108,7 @@ export class TimelineEventStore { * @return {Range} the created range */ onlyRange(eventKey) { - return new Range(eventKey); + return new Range(this._timelineStore.IDBKeyRange, eventKey); } /** Creates a range that includes all keys before eventKey, and optionally also the key itself. @@ -116,7 +117,7 @@ export class TimelineEventStore { * @return {Range} the created range */ upperBoundRange(eventKey, open=false) { - return new Range(undefined, undefined, eventKey, undefined, open); + return new Range(this._timelineStore.IDBKeyRange, undefined, undefined, eventKey, undefined, open); } /** Creates a range that includes all keys after eventKey, and optionally also the key itself. @@ -125,7 +126,7 @@ export class TimelineEventStore { * @return {Range} the created range */ lowerBoundRange(eventKey, open=false) { - return new Range(undefined, eventKey, undefined, open); + return new Range(this._timelineStore.IDBKeyRange, undefined, eventKey, undefined, open); } /** Creates a range that includes all keys between `lower` and `upper`, and optionally the given keys as well. @@ -136,7 +137,7 @@ export class TimelineEventStore { * @return {Range} the created range */ boundRange(lower, upper, lowerOpen=false, upperOpen=false) { - return new Range(undefined, lower, upper, lowerOpen, upperOpen); + return new Range(this._timelineStore.IDBKeyRange, undefined, lower, upper, lowerOpen, upperOpen); } /** Looks up the last `amount` entries in the timeline for `roomId`. @@ -261,7 +262,7 @@ export class TimelineEventStore { removeAllForRoom(roomId) { const minKey = encodeKey(roomId, KeyLimits.minStorageKey, KeyLimits.minStorageKey); const maxKey = encodeKey(roomId, KeyLimits.maxStorageKey, KeyLimits.maxStorageKey); - const range = IDBKeyRange.bound(minKey, maxKey); + const range = this._timelineStore.IDBKeyRange.bound(minKey, maxKey); this._timelineStore.delete(range); } } diff --git a/src/matrix/storage/idb/stores/TimelineFragmentStore.js b/src/matrix/storage/idb/stores/TimelineFragmentStore.js index 10eaede1..96ff441e 100644 --- a/src/matrix/storage/idb/stores/TimelineFragmentStore.js +++ b/src/matrix/storage/idb/stores/TimelineFragmentStore.js @@ -29,7 +29,7 @@ export class TimelineFragmentStore { _allRange(roomId) { try { - return IDBKeyRange.bound( + return this._store.IDBKeyRange.bound( encodeKey(roomId, KeyLimits.minStorageKey), encodeKey(roomId, KeyLimits.maxStorageKey) ); diff --git a/src/matrix/storage/idb/utils.js b/src/matrix/storage/idb/utils.js index 2e6c2152..fbb82fc5 100644 --- a/src/matrix/storage/idb/utils.js +++ b/src/matrix/storage/idb/utils.js @@ -64,8 +64,8 @@ export function decodeUint32(str) { return parseInt(str, 16); } -export function openDatabase(name, createObjectStore, version) { - const req = indexedDB.open(name, version); +export function openDatabase(name, createObjectStore, version, idbFactory = window.indexedDB) { + const req = idbFactory.open(name, version); req.onupgradeneeded = (ev) => { const db = ev.target.result; const txn = ev.target.transaction; From 6a5679a33852424a812dfa99474cac42f7df6d1c Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:32:03 +0200 Subject: [PATCH 2/9] create mock storage using fake-indexeddb --- package.json | 1 + scripts/build.mjs | 4 + scripts/package-overrides/fake-indexeddb.js | 4 + scripts/post-install.js | 9 +++ src/mocks/Storage.js | 22 +++++ yarn.lock | 89 ++++++++++++++++++++- 6 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 scripts/package-overrides/fake-indexeddb.js create mode 100644 src/mocks/Storage.js diff --git a/package.json b/package.json index eb2dde61..349f8506 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "commander": "^6.0.0", "core-js": "^3.6.5", "eslint": "^7.25.0", + "fake-indexeddb": "^3.1.2", "finalhandler": "^1.1.1", "impunity": "^1.0.0", "mdn-polyfills": "^5.20.0", diff --git a/scripts/build.mjs b/scripts/build.mjs index b8467760..997d974f 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -191,6 +191,8 @@ async function buildJs(mainFile, extraFiles, importOverrides) { plugins.push(overridesAsRollupPlugin(importOverrides)); } const bundle = await rollup({ + // for fake-indexeddb, so usage for tests only doesn't put it in bundle + treeshake: {moduleSideEffects: false}, input: extraFiles.concat(mainFile), plugins }); @@ -230,6 +232,8 @@ async function buildJsLegacy(mainFile, extraFiles, importOverrides) { plugins.push(nodeResolve(), babelPlugin); // create js bundle const rollupConfig = { + // for fake-indexeddb, so usage for tests only doesn't put it in bundle + treeshake: {moduleSideEffects: false}, // important the extraFiles come first, // so polyfills are available in the global scope // if needed for the mainfile diff --git a/scripts/package-overrides/fake-indexeddb.js b/scripts/package-overrides/fake-indexeddb.js new file mode 100644 index 00000000..65159813 --- /dev/null +++ b/scripts/package-overrides/fake-indexeddb.js @@ -0,0 +1,4 @@ +// we have our own main file for this module as we need both these symbols to +// be exported, and we also don't want to auto behaviour that modifies global vars +exports.FDBFactory = require("fake-indexeddb/lib/FDBFactory.js"); +exports.FDBKeyRange = require("fake-indexeddb/lib/FDBKeyRange.js"); \ No newline at end of file diff --git a/scripts/post-install.js b/scripts/post-install.js index 3addd344..828474ef 100644 --- a/scripts/post-install.js +++ b/scripts/post-install.js @@ -51,6 +51,7 @@ function packageIterator(request, start, defaultIterator) { async function commonjsToESM(src, dst) { // create js bundle const bundle = await rollup({ + treeshake: {moduleSideEffects: false}, input: src, plugins: [commonjs(), nodeResolve({ browser: true, @@ -111,6 +112,14 @@ async function populateLib() { require.resolve('es6-promise/lib/es6-promise/promise.js'), path.join(libDir, "es6-promise/index.js") ); + // fake-indexeddb, used for tests (but unresolvable bare imports also makes the build complain) + // and might want to use it for in-memory storage too, although we probably do ts->es6 with esm + // directly rather than ts->es5->es6 as we do now. The bundle is 240K currently. + await fs.mkdir(path.join(libDir, "fake-indexeddb/")); + await commonjsToESM( + path.join(projectDir, "/scripts/package-overrides/fake-indexeddb.js"), + path.join(libDir, "fake-indexeddb/index.js") + ); } populateLib(); diff --git a/src/mocks/Storage.js b/src/mocks/Storage.js new file mode 100644 index 00000000..0cddc7bc --- /dev/null +++ b/src/mocks/Storage.js @@ -0,0 +1,22 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {FDBFactory, FDBKeyRange} from "../../lib/fake-indexeddb/index.js"; +import {StorageFactory} from "../matrix/storage/idb/StorageFactory.js"; + +export function createMockStorage() { + return new StorageFactory(null, new FDBFactory(), FDBKeyRange).create(1); +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index d29a6d84..0a6bc448 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1045,6 +1045,11 @@ base-x@^3.0.2: dependencies: safe-buffer "^5.0.1" +base64-arraybuffer-es6@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/base64-arraybuffer-es6/-/base64-arraybuffer-es6-0.7.0.tgz#dbe1e6c87b1bf1ca2875904461a7de40f21abc86" + integrity sha512-ESyU/U1CFZDJUdr+neHRhNozeCv72Y7Vm0m1DCbjX3KBjT6eYocvAJlSk6+8+HkVwXlT1FNxhGW6q3UKAlCvvw== + base64-arraybuffer@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.2.0.tgz#4b944fac0191aa5907afe2d8c999ccc57ce80f45" @@ -1209,6 +1214,11 @@ core-js-compat@^3.6.2: browserslist "^4.8.5" semver "7.0.0" +core-js@^2.5.3: + version "2.6.12" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" + integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== + core-js@^3.6.5: version "3.6.5" resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.6.5.tgz#7395dc273af37fb2e50e9bd3d9fe841285231d1a" @@ -1324,6 +1334,13 @@ domelementtype@^2.0.1: resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.0.1.tgz#1f8bdfe91f5a78063274e803b4bdcedf6e94f94d" integrity sha512-5HOHUDsYZWV8FGWN0Njbr/Rn7f/eWSQi1v7+HsUVwXgn8nWWlL64zKDkS0n8ZmQ3mlWOMuXOnR+7Nx/5tMO5AQ== +domexception@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90" + integrity sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug== + dependencies: + webidl-conversions "^4.0.2" + domhandler@^2.3.0: version "2.4.2" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803" @@ -1549,6 +1566,14 @@ extend@^3.0.1: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== +fake-indexeddb@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-3.1.2.tgz#8073a12ed3b254f7afc064f3cc2629f0110a5303" + integrity sha512-W60eRBrE8r9o/EePyyUc63sr2I/MI9p3zVwLlC1WI1xdmQVqqM6+wec9KDWDz2EZyvJKhrDvy3cGC6hK8L1pfg== + dependencies: + realistic-structured-clone "^2.0.1" + setimmediate "^1.0.5" + fast-deep-equal@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -1914,7 +1939,7 @@ lodash@^4.17.19: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== -lodash@^4.17.21: +lodash@^4.17.21, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -2218,7 +2243,7 @@ progress@^2.0.0: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -punycode@^2.1.0: +punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== @@ -2244,6 +2269,16 @@ readable-stream@^3.1.1: string_decoder "^1.1.1" util-deprecate "^1.0.1" +realistic-structured-clone@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/realistic-structured-clone/-/realistic-structured-clone-2.0.2.tgz#2f8ec225b1f9af20efc79ac96a09043704414959" + integrity sha512-5IEvyfuMJ4tjQOuKKTFNvd+H9GSbE87IcendSBannE28PTrbolgaVg5DdEApRKhtze794iXqVUFKV60GLCNKEg== + dependencies: + core-js "^2.5.3" + domexception "^1.0.1" + typeson "^5.8.2" + typeson-registry "^1.0.0-alpha.20" + regenerate-unicode-properties@^8.2.0: version "8.2.0" resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec" @@ -2411,6 +2446,11 @@ serve-static@^1.13.2: parseurl "~1.3.3" send "0.17.1" +setimmediate@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" + integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU= + setprototypeof@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" @@ -2548,6 +2588,13 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +tr46@^2.0.2: + version "2.1.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" + integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== + dependencies: + punycode "^2.1.1" + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -2565,6 +2612,25 @@ type-fest@^0.8.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +typeson-registry@^1.0.0-alpha.20: + version "1.0.0-alpha.39" + resolved "https://registry.yarnpkg.com/typeson-registry/-/typeson-registry-1.0.0-alpha.39.tgz#9e0f5aabd5eebfcffd65a796487541196f4b1211" + integrity sha512-NeGDEquhw+yfwNhguLPcZ9Oj0fzbADiX4R0WxvoY8nGhy98IbzQy1sezjoEFWOywOboj/DWehI+/aUlRVrJnnw== + dependencies: + base64-arraybuffer-es6 "^0.7.0" + typeson "^6.0.0" + whatwg-url "^8.4.0" + +typeson@^5.8.2: + version "5.18.2" + resolved "https://registry.yarnpkg.com/typeson/-/typeson-5.18.2.tgz#0d217fc0e11184a66aa7ca0076d9aa7707eb7bc2" + integrity sha512-Vetd+OGX05P4qHyHiSLdHZ5Z5GuQDrHHwSdjkqho9NSCYVSLSfRMjklD/unpHH8tXBR9Z/R05rwJSuMpMFrdsw== + +typeson@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/typeson/-/typeson-6.1.0.tgz#5b2a53705a5f58ff4d6f82f965917cabd0d7448b" + integrity sha512-6FTtyGr8ldU0pfbvW/eOZrEtEkczHRUtduBnA90Jh9kMPCiFNnXIon3vF41N0S4tV1HHQt4Hk1j4srpESziCaA== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -2610,6 +2676,25 @@ v8-compile-cache@^2.0.3: resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== +webidl-conversions@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" + integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg== + +webidl-conversions@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" + integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== + +whatwg-url@^8.4.0: + version "8.5.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.5.0.tgz#7752b8464fc0903fec89aa9846fc9efe07351fd3" + integrity sha512-fy+R77xWv0AiqfLl4nuGUlQ3/6b5uNfQ4WAbGQVMYshCTCCPK9psC1nWh3XHuxGVCtlcDDQPQW1csmmIQo+fwg== + dependencies: + lodash "^4.7.0" + tr46 "^2.0.2" + webidl-conversions "^6.1.0" + which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" From cf6e14ed8b5572eafe18fca396c5860f3433fb82 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:32:19 +0200 Subject: [PATCH 3/9] utility for homeserver api mocking --- src/mocks/HomeServer.js | 62 +++++++++++++++++++++++++++++++++++++++++ src/mocks/Request.js | 15 ++++++++-- 2 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 src/mocks/HomeServer.js diff --git a/src/mocks/HomeServer.js b/src/mocks/HomeServer.js new file mode 100644 index 00000000..072344ef --- /dev/null +++ b/src/mocks/HomeServer.js @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseRequest} from "./Request.js"; + +// a request as returned by the HomeServerApi +class HomeServerRequest extends BaseRequest { + constructor(args) { + super(); + this.arguments = args; + } + + respond(body) { + return this._respond(body); + } +} + +class Target { + constructor() { + this.requests = {}; + } +} + +function handleMethod(target, name, ...args) { + let requests = target.requests[name] + if (!requests) { + target.requests[name] = requests = []; + } + const request = new HomeServerRequest(args); + requests.push(request); + return request; +} + +class Handler { + get(target, prop) { + return handleMethod.bind(null, target, prop); + } +} + +export class HomeServer { + constructor() { + this._target = new Target(); + this.api = new Proxy(this._target, new Handler()); + } + + get requests() { + return this._target.requests; + } +} \ No newline at end of file diff --git a/src/mocks/Request.js b/src/mocks/Request.js index da8693b1..1984f06f 100644 --- a/src/mocks/Request.js +++ b/src/mocks/Request.js @@ -16,17 +16,19 @@ limitations under the License. import {AbortError} from "../utils/error.js"; -export class Request { +export class BaseRequest { constructor() { this._responsePromise = new Promise((resolve, reject) => { this.resolve = resolve; this.reject = reject; }); + this.responded = false; this.aborted = false; } - respond(status, body) { - this.resolve({status, body}); + _respond(value) { + this.responded = true; + this.resolve(value); return this; } @@ -39,3 +41,10 @@ export class Request { return this._responsePromise; } } + +// this is a NetworkRequest as used by HomeServerApi +export class Request extends BaseRequest { + respond(status, body) { + return this._respond({status, body}); + } +} From ffb6cb372aacf511cbf2a952689a7a080298d9c8 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:32:34 +0200 Subject: [PATCH 4/9] utility to create events in tests --- src/mocks/event.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/mocks/event.js diff --git a/src/mocks/event.js b/src/mocks/event.js new file mode 100644 index 00000000..b7d20c8a --- /dev/null +++ b/src/mocks/event.js @@ -0,0 +1,31 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export function event(type, id = null) { + return {type, event_id: id}; +} + +export function withContent(event, content) { + return Object.assign({}, event, {content}); +} + +export function withTextBody(event, body) { + return withContent(event, {body, msgtype: "m.text"}); +} + +export function withTxnId(event, txnId) { + return Object.assign({}, event, {unsigned: {transaction_id: txnId}}); +} From 8369dcef5fc16b4caaf83b713931cf48661654a2 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:32:50 +0200 Subject: [PATCH 5/9] utility to wait until a value becomes truthy in a test --- src/mocks/poll.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/mocks/poll.js diff --git a/src/mocks/poll.js b/src/mocks/poll.js new file mode 100644 index 00000000..36c8e256 --- /dev/null +++ b/src/mocks/poll.js @@ -0,0 +1,27 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export async function poll(fn) { + let result; + do { + const result = fn(); + if (result) { + return result; + } else { + await new Promise(setImmediate); + } + } while (1); +} \ No newline at end of file From 87ebdbb0b45ffb02459a10830b8096ecdd7c78f5 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:33:15 +0200 Subject: [PATCH 6/9] add failing test for unhandled race between sync and send see comments in code --- src/matrix/room/sending/SendQueue.js | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js index 7a648171..446e0ce8 100644 --- a/src/matrix/room/sending/SendQueue.js +++ b/src/matrix/room/sending/SendQueue.js @@ -303,3 +303,45 @@ export class SendQueue { } } } + +import {HomeServer as MockHomeServer} from "../../../mocks/HomeServer.js"; +import {createMockStorage} from "../../../mocks/Storage.js"; +import {NullLogger} from "../../../logging/NullLogger.js"; +import {event, withContent, withTextBody, withTxnId} from "../../../mocks/event.js"; +import {poll} from "../../../mocks/poll.js"; + +export function tests() { + const logger = new NullLogger(); + return { + "enqueue second message when remote echo of first arrives before /send returns": async assert => { + const storage = await createMockStorage(); + const hs = new MockHomeServer(); + // 1. enqueue and start send event 1 + const queue = new SendQueue({roomId: "!abc", storage, hsApi: hs.api}); + const event1 = withTextBody(event("m.room.message", "$123"), "message 1"); + await logger.run("event1", log => queue.enqueueEvent(event1.type, event1.content, null, log)); + assert.equal(queue.pendingEvents.length, 1); + const sendRequest1 = hs.requests.send[0]; + // 2. receive remote echo, before /send has returned + const remoteEcho = withTxnId(event1, sendRequest1.arguments[2]); + const txn = await storage.readWriteTxn([storage.storeNames.pendingEvents]); + const removal = await logger.run("remote echo", log => queue.removeRemoteEchos([remoteEcho], txn, log)); + await txn.complete(); + assert.equal(removal.length, 1); + queue.emitRemovals(removal); + assert.equal(queue.pendingEvents.length, 0); + // 3. now enqueue event 2 + const event2 = withTextBody(event("m.room.message", "$456"), "message 2"); + await logger.run("event2", log => queue.enqueueEvent(event2.type, event2.content, null, log)); + // even though the first pending event has been removed by the remote echo, + // the second should get the next index, as the send loop is still blocking on the first one + assert.equal(Array.from(queue.pendingEvents)[0].queueIndex, 2); + // 4. send for event 1 comes back + sendRequest1.respond({event_id: event1.event_id}); + // 5. now expect second send request for event 2 + const sendRequest2 = await poll(() => hs.requests.send[1]); + sendRequest2.respond({event_id: event2.event_id}); + await poll(() => !queue._isSending); + } + } +} \ No newline at end of file From c3754fdbd4ef8b1019b6ac17736ffab087f33844 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:34:02 +0200 Subject: [PATCH 7/9] fix the race, make test succeed --- src/matrix/room/sending/SendQueue.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js index 446e0ce8..908de836 100644 --- a/src/matrix/room/sending/SendQueue.js +++ b/src/matrix/room/sending/SendQueue.js @@ -31,6 +31,7 @@ export class SendQueue { this._isSending = false; this._offline = false; this._roomEncryption = null; + this._currentQueueIndex = 0; } _createPendingEvent(data, attachments = null) { @@ -55,6 +56,7 @@ export class SendQueue { await log.wrap("send event", async log => { log.set("queueIndex", pendingEvent.queueIndex); try { + this._currentQueueIndex = pendingEvent.queueIndex; await this._sendEvent(pendingEvent, log); } catch(err) { if (err instanceof ConnectionError) { @@ -75,6 +77,8 @@ export class SendQueue { pendingEvent.setError(err); } } + } finally { + this._currentQueueIndex = 0; } }); } @@ -274,7 +278,11 @@ export class SendQueue { let pendingEvent; try { const pendingEventsStore = txn.pendingEvents; - const maxQueueIndex = await pendingEventsStore.getMaxQueueIndex(this._roomId) || 0; + const maxStorageQueueIndex = await pendingEventsStore.getMaxQueueIndex(this._roomId) || 0; + // don't use the queueIndex of the pendingEvent currently waiting for /send to return + // if the remote echo already removed the pendingEvent in storage, as the send loop + // wouldn't be able to detect the remote echo already arrived and end up overwriting the new event + const maxQueueIndex = Math.max(maxStorageQueueIndex, this._currentQueueIndex); const queueIndex = maxQueueIndex + 1; const needsEncryption = eventType !== REDACTION_TYPE && !!this._roomEncryption; pendingEvent = this._createPendingEvent({ From bf951bd3220f7d9156545b366cf2b207e82d4658 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:34:14 +0200 Subject: [PATCH 8/9] fix logging wrong value --- src/matrix/room/sending/SendQueue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js index 908de836..6d7ce596 100644 --- a/src/matrix/room/sending/SendQueue.js +++ b/src/matrix/room/sending/SendQueue.js @@ -245,7 +245,7 @@ export class SendQueue { relatedTxnId = pe.txnId; } } - log.set("relatedTxnId", eventIdOrTxnId); + log.set("relatedTxnId", relatedTxnId); log.set("relatedEventId", relatedEventId); await this._enqueueEvent(REDACTION_TYPE, {reason}, null, relatedTxnId, relatedEventId, log); } From cef94fea45b18c06772e7c43d12092ff9b5ce651 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 2 Jun 2021 12:38:41 +0200 Subject: [PATCH 9/9] fix lint --- src/matrix/room/sending/SendQueue.js | 2 +- src/mocks/poll.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js index 6d7ce596..cdab31a8 100644 --- a/src/matrix/room/sending/SendQueue.js +++ b/src/matrix/room/sending/SendQueue.js @@ -315,7 +315,7 @@ export class SendQueue { import {HomeServer as MockHomeServer} from "../../../mocks/HomeServer.js"; import {createMockStorage} from "../../../mocks/Storage.js"; import {NullLogger} from "../../../logging/NullLogger.js"; -import {event, withContent, withTextBody, withTxnId} from "../../../mocks/event.js"; +import {event, withTextBody, withTxnId} from "../../../mocks/event.js"; import {poll} from "../../../mocks/poll.js"; export function tests() { diff --git a/src/mocks/poll.js b/src/mocks/poll.js index 36c8e256..40348bb3 100644 --- a/src/mocks/poll.js +++ b/src/mocks/poll.js @@ -21,7 +21,7 @@ export async function poll(fn) { if (result) { return result; } else { - await new Promise(setImmediate); + await new Promise(setImmediate); //eslint-disable-line no-undef } - } while (1); + } while (1); //eslint-disable-line no-constant-condition } \ No newline at end of file