From 53cdabb459d0da32059afb38396b10a8e349bba1 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Sat, 11 May 2019 13:10:31 +0200 Subject: [PATCH] store method to find events to connect with when filling gaps as fragments can be unaware of their chronological relationship, we need to check whether the events received from /messages or /context already exists, so we can later hook up the fragments. --- prototypes/idb-continue-key.html | 165 ++++++++++++++++++ src/matrix/storage/idb/query-target.js | 44 ++++- .../storage/idb/stores/RoomTimelineStore.js | 53 ++++-- src/matrix/storage/idb/utils.js | 10 +- 4 files changed, 248 insertions(+), 24 deletions(-) create mode 100644 prototypes/idb-continue-key.html diff --git a/prototypes/idb-continue-key.html b/prototypes/idb-continue-key.html new file mode 100644 index 00000000..dc488deb --- /dev/null +++ b/prototypes/idb-continue-key.html @@ -0,0 +1,165 @@ + + + + + + diff --git a/src/matrix/storage/idb/query-target.js b/src/matrix/storage/idb/query-target.js index 547c2fdd..ec5c4530 100644 --- a/src/matrix/storage/idb/query-target.js +++ b/src/matrix/storage/idb/query-target.js @@ -38,7 +38,7 @@ export default class QueryTarget { const results = []; await iterateCursor(cursor, (value) => { results.push(value); - return false; + return {done: false}; }); return results; } @@ -59,12 +59,48 @@ export default class QueryTarget { return this._find(range, predicate, "prev"); } + /** + * Checks if a given set of keys exist. + * Calls `callback(key, found)` for each key in `keys`, in key sorting order (or reversed if backwards=true). + * If the callback returns true, the search is halted and callback won't be called again. + * `callback` is called with the same instances of the key as given in `keys`, so direct comparison can be used. + */ + async findExistingKeys(keys, backwards, callback) { + const direction = backwards ? "prev" : "next"; + const compareKeys = (a, b) => backwards ? -indexedDB.cmp(a, b) : indexedDB.cmp(a, b); + const sortedKeys = keys.slice().sort(compareKeys); + const firstKey = backwards ? sortedKeys[sortedKeys.length - 1] : sortedKeys[0]; + const lastKey = backwards ? sortedKeys[0] : sortedKeys[sortedKeys.length - 1]; + const cursor = this._target.openKeyCursor(IDBKeyRange.bound(firstKey, lastKey), direction); + let i = 0; + let consumerDone = false; + await iterateCursor(cursor, (value, key) => { + // while key is larger than next key, advance and report false + while(i < sortedKeys.length && compareKeys(sortedKeys[i], key) < 0 && !consumerDone) { + consumerDone = callback(sortedKeys[i], false); + ++i; + } + if (i < sortedKeys.length && compareKeys(sortedKeys[i], key) === 0 && !consumerDone) { + consumerDone = callback(sortedKeys[i], true); + ++i; + } + const done = consumerDone || i >= sortedKeys.length; + const jumpTo = !done && sortedKeys[i]; + return {done, jumpTo}; + }); + // report null for keys we didn't to at the end + while (!consumerDone && i < sortedKeys.length) { + consumerDone = callback(sortedKeys[i], false); + ++i; + } + } + _reduce(range, reducer, initialValue, direction) { let reducedValue = initialValue; const cursor = this._target.openCursor(range, direction); return iterateCursor(cursor, (value) => { reducedValue = reducer(reducedValue, value); - return true; + return {done: false}; }); } @@ -79,7 +115,7 @@ export default class QueryTarget { const results = []; await iterateCursor(cursor, (value) => { results.push(value); - return predicate(results); + return {done: predicate(results)}; }); return results; } @@ -92,7 +128,7 @@ export default class QueryTarget { if (found) { result = value; } - return found; + return {done: found}; }); if (found) { return result; diff --git a/src/matrix/storage/idb/stores/RoomTimelineStore.js b/src/matrix/storage/idb/stores/RoomTimelineStore.js index 8b47a501..950685d7 100644 --- a/src/matrix/storage/idb/stores/RoomTimelineStore.js +++ b/src/matrix/storage/idb/stores/RoomTimelineStore.js @@ -147,24 +147,47 @@ export default class RoomTimelineStore { return events; } - /** Looks up the first, if any, event entry (so excluding gap entries) after `sortKey`. + /** Finds the first (or last if `findLast=true`) eventId that occurs in the store, if any. + * For optimal performance, `eventIds` should be in chronological order. + * + * The order in which results are returned might be different than `eventIds`. + * Call the return value to obtain the next {id, event} pair. * @param {string} roomId - * @param {SortKey} sortKey - * @return {Promise<(?Entry)>} a promise resolving to entry, if any. + * @param {string[]} eventIds + * @return {Function} */ - nextEvent(roomId, sortKey) { - const range = this.lowerBoundRange(sortKey, true).asIDBKeyRange(roomId); - return this._timelineStore.find(range, entry => !!entry.event); - } + // performance comment from above refers to the fact that their *might* + // be a correlation between event_id sorting order and chronology. + // In that case we could avoid running over all eventIds, as the reported order by findExistingKeys + // would match the order of eventIds. That's why findLast is also passed as backwards to keysExist. + // also passing them in chronological order makes sense as that's how we'll receive them almost always. + async findFirstOrLastOccurringEventId(roomId, eventIds, findLast = false) { + const byEventId = this._timelineStore.index("byEventId"); + const keys = eventIds.map(eventId => [roomId, eventId]); + const results = new Array(keys.length); + let firstFoundEventId; - /** Looks up the first, if any, event entry (so excluding gap entries) before `sortKey`. - * @param {string} roomId - * @param {SortKey} sortKey - * @return {Promise<(?Entry)>} a promise resolving to entry, if any. - */ - previousEvent(roomId, sortKey) { - const range = this.upperBoundRange(sortKey, true).asIDBKeyRange(roomId); - return this._timelineStore.findReverse(range, entry => !!entry.event); + // find first result that is found and has no undefined results before it + function firstFoundAndPrecedingResolved() { + let inc = findLast ? -1 : 1; + let start = findLast ? results.length - 1 : 0; + for(let i = start; i >= 0 && i < results.length; i += inc) { + if (results[i] === undefined) { + return; + } else if(results[i] === true) { + return keys[i]; + } + } + } + + await byEventId.findExistingKeys(keys, findLast, (key, found) => { + const index = keys.indexOf(key); + results[index] = found; + firstFoundEventId = firstFoundAndPrecedingResolved(); + return !!firstFoundEventId; + }); + + return firstFoundEventId; } /** Inserts a new entry into the store. The combination of roomId and sortKey should not exist yet, or an error is thrown. diff --git a/src/matrix/storage/idb/utils.js b/src/matrix/storage/idb/utils.js index 1d522f24..83ef3411 100644 --- a/src/matrix/storage/idb/utils.js +++ b/src/matrix/storage/idb/utils.js @@ -35,11 +35,11 @@ export function iterateCursor(cursor, processValue) { resolve(false); return; // end of results } - const isDone = processValue(cursor.value); - if (isDone) { + const {done, jumpTo} = processValue(cursor.value, cursor.key); + if (done) { resolve(true); } else { - cursor.continue(); + cursor.continue(jumpTo); } }; }); @@ -49,7 +49,7 @@ export async function fetchResults(cursor, isDone) { const results = []; await iterateCursor(cursor, (value) => { results.push(value); - return isDone(results); + return {done: isDone(results)}; }); return results; } @@ -100,4 +100,4 @@ export async function findStoreValue(db, storeName, toCursor, matchesValue) { throw new Error("Value not found"); } return match; -} \ No newline at end of file +}