mystiq
/
hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

store method to find events to connect with when filling gaps 

as fragments can be unaware of their chronological relationship,
we need to check whether the events received from /messages or /context
already exists, so we can later hook up the fragments.
This commit is contained in:
Bruno Windels 2019-05-11 13:10:31 +02:00
parent 35a5e3f21a
commit 53cdabb459
4 changed files with 248 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
prototypes/idb-continue-key.html Normal file
View File

@ -0,0 +1,165 @@
<html>
<head><meta charset="utf-8"></head>
<body>
<script type="text/javascript">
function reqAsPromise(req) {
return new Promise((resolve, reject) => {
req.onsuccess = () => resolve(req);
req.onerror = (err) => reject(err);
});
}
function txnAsPromise(txn) {
return new Promise((resolve, reject) => {
txn.addEventListener("complete", resolve);
txn.addEventListener("abort", reject);
});
}
function iterateCursor(cursor, processValue) {
// TODO: does cursor already have a value here??
return new Promise((resolve, reject) => {
cursor.onerror = (event) => {
reject(new Error("Query failed: " + event.target.errorCode));
};
// collect results
cursor.onsuccess = (event) => {
const cursor = event.target.result;
if (!cursor) {
resolve(false);
return; // end of results
}
const {done, jumpTo} = processValue(cursor.value, cursor.key);
if (done) {
resolve(true);
} else {
cursor.continue(jumpTo);
}
};
});
}
/**
* Checks if a given set of keys exist.
* Calls `callback(key, found)` for each key in `keys`, in an unspecified order.
* If the callback returns true, the search is halted and callback won't be called again.
*/
async function findKeys(target, keys, backwards, callback) {
const direction = backwards ? "prev" : "next";
const compareKeys = (a, b) => backwards ? -indexedDB.cmp(a, b) : indexedDB.cmp(a, b);
const sortedKeys = keys.slice().sort(compareKeys);
console.log(sortedKeys);
const firstKey = backwards ? sortedKeys[sortedKeys.length - 1] : sortedKeys[0];
const lastKey = backwards ? sortedKeys[0] : sortedKeys[sortedKeys.length - 1];
const cursor = target.openKeyCursor(IDBKeyRange.bound(firstKey, lastKey), direction);
let i = 0;
let consumerDone = false;
await iterateCursor(cursor, (value, key) => {
// while key is larger than next key, advance and report false
while(i < sortedKeys.length && compareKeys(sortedKeys[i], key) < 0 && !consumerDone) {
console.log("before match", sortedKeys[i]);
consumerDone = callback(sortedKeys[i], false);
++i;
}
if (i < sortedKeys.length && compareKeys(sortedKeys[i], key) === 0 && !consumerDone) {
console.log("match", sortedKeys[i]);
consumerDone = callback(sortedKeys[i], true);
++i;
}
const done = consumerDone || i >= sortedKeys.length;
const jumpTo = !done && sortedKeys[i];
return {done, jumpTo};
});
// report null for keys we didn't to at the end
while (!consumerDone && i < sortedKeys.length) {
console.log("afterwards", sortedKeys[i]);
consumerDone = callback(sortedKeys[i], false);
++i;
}
}
async function findFirstOrLastOccurringEventId(target, roomId, eventIds, findLast = false) {
const keys = eventIds.map(eventId => [roomId, eventId]);
const results = new Array(keys.length);
let firstFoundEventId;
// find first result that is found and has no undefined results before it
function firstFoundAndPrecedingResolved() {
let inc = findLast ? -1 : 1;
let start = findLast ? results.length - 1 : 0;
for(let i = start; i >= 0 && i < results.length; i += inc) {
if (results[i] === undefined) {
return;
} else if(results[i] === true) {
return keys[i];
}
}
}
await findKeys(target, keys, findLast, (key, found) => {
const index = keys.indexOf(key);
results[index] = found;
firstFoundEventId = firstFoundAndPrecedingResolved();
return !!firstFoundEventId;
});
return firstFoundEventId;
}
(async () => {
let db;
let isNew = false;
// open db
{
const req = window.indexedDB.open("prototype-idb-continue-key");
req.onupgradeneeded = (ev) => {
const db = ev.target.result;
db.createObjectStore("timeline", {keyPath: ["roomId", "eventId"]});
isNew = true;
};
db = (await reqAsPromise(req)).result;
}
const roomId = "!abcdef:localhost";
if (isNew) {
const txn = db.transaction(["timeline"], "readwrite");
const store = txn.objectStore("timeline");
for (var i = 1; i <= 100; ++i) {
store.add({roomId, eventId: `$${i * 10}`});
}
await txnAsPromise(txn);
}
console.log("show all in order we get them");
{
const txn = db.transaction(["timeline"], "readonly");
const store = txn.objectStore("timeline");
const cursor = store.openKeyCursor();
await iterateCursor(cursor, (value, key) => {
console.log(key);
return {done: false};
});
}
console.log("run findKeys");
{
const txn = db.transaction(["timeline"], "readonly");
const store = txn.objectStore("timeline");
const eventIds = ["$992", "$1000", "$1010", "$991", "$500", "$990"];
// const eventIds = ["$992", "$1010"];
const keys = eventIds.map(eventId => [roomId, eventId]);
await findKeys(store, keys, false, (key, found) => {
console.log(key, found);
});
}
console.log("run findFirstOrLastOccurringEventId");
{
const txn = db.transaction(["timeline"], "readonly");
const store = txn.objectStore("timeline");
const eventIds = ["$992", "$1000", "$1010", "$991", "$500", "$990", "$123"];
const firstMatch = await findFirstOrLastOccurringEventId(store, roomId, eventIds, false);
console.log("firstMatch", firstMatch);
const lastMatch = await findFirstOrLastOccurringEventId(store, roomId, eventIds, true);
console.log("lastMatch", lastMatch);
}
})();
</script>
</body>
</html>

44
src/matrix/storage/idb/query-target.js
View File

@ -38,7 +38,7 @@ export default class QueryTarget {
const results = [];
await iterateCursor(cursor, (value) => {
results.push(value);
return false;
return {done: false};
});
return results;
}
@ -59,12 +59,48 @@ export default class QueryTarget {
return this._find(range, predicate, "prev");
}
/**
* Checks if a given set of keys exist.
* Calls `callback(key, found)` for each key in `keys`, in key sorting order (or reversed if backwards=true).
* If the callback returns true, the search is halted and callback won't be called again.
* `callback` is called with the same instances of the key as given in `keys`, so direct comparison can be used.
*/
async findExistingKeys(keys, backwards, callback) {
const direction = backwards ? "prev" : "next";
const compareKeys = (a, b) => backwards ? -indexedDB.cmp(a, b) : indexedDB.cmp(a, b);
const sortedKeys = keys.slice().sort(compareKeys);
const firstKey = backwards ? sortedKeys[sortedKeys.length - 1] : sortedKeys[0];
const lastKey = backwards ? sortedKeys[0] : sortedKeys[sortedKeys.length - 1];
const cursor = this._target.openKeyCursor(IDBKeyRange.bound(firstKey, lastKey), direction);
let i = 0;
let consumerDone = false;
await iterateCursor(cursor, (value, key) => {
// while key is larger than next key, advance and report false
while(i < sortedKeys.length && compareKeys(sortedKeys[i], key) < 0 && !consumerDone) {
consumerDone = callback(sortedKeys[i], false);
++i;
}
if (i < sortedKeys.length && compareKeys(sortedKeys[i], key) === 0 && !consumerDone) {
consumerDone = callback(sortedKeys[i], true);
++i;
}
const done = consumerDone || i >= sortedKeys.length;
const jumpTo = !done && sortedKeys[i];
return {done, jumpTo};
});
// report null for keys we didn't to at the end
while (!consumerDone && i < sortedKeys.length) {
consumerDone = callback(sortedKeys[i], false);
++i;
}
}
_reduce(range, reducer, initialValue, direction) {
let reducedValue = initialValue;
const cursor = this._target.openCursor(range, direction);
return iterateCursor(cursor, (value) => {
reducedValue = reducer(reducedValue, value);
return true;
return {done: false};
});
}
@ -79,7 +115,7 @@ export default class QueryTarget {
const results = [];
await iterateCursor(cursor, (value) => {
results.push(value);
return predicate(results);
return {done: predicate(results)};
});
return results;
}
@ -92,7 +128,7 @@ export default class QueryTarget {
if (found) {
result = value;
}
return found;
return {done: found};
});
if (found) {
return result;

53
src/matrix/storage/idb/stores/RoomTimelineStore.js
View File

@ -147,24 +147,47 @@ export default class RoomTimelineStore {
return events;
}
/** Looks up the first, if any, event entry (so excluding gap entries) after `sortKey`.
/** Finds the first (or last if `findLast=true`) eventId that occurs in the store, if any.
* For optimal performance, `eventIds` should be in chronological order.
*
* The order in which results are returned might be different than `eventIds`.
* Call the return value to obtain the next {id, event} pair.
* @param {string} roomId
* @param {SortKey} sortKey
* @return {Promise<(?Entry)>} a promise resolving to entry, if any.
* @param {string[]} eventIds
* @return {Function<Promise>}
*/
nextEvent(roomId, sortKey) {
const range = this.lowerBoundRange(sortKey, true).asIDBKeyRange(roomId);
return this._timelineStore.find(range, entry => !!entry.event);
}
// performance comment from above refers to the fact that their *might*
// be a correlation between event_id sorting order and chronology.
// In that case we could avoid running over all eventIds, as the reported order by findExistingKeys
// would match the order of eventIds. That's why findLast is also passed as backwards to keysExist.
// also passing them in chronological order makes sense as that's how we'll receive them almost always.
async findFirstOrLastOccurringEventId(roomId, eventIds, findLast = false) {
const byEventId = this._timelineStore.index("byEventId");
const keys = eventIds.map(eventId => [roomId, eventId]);
const results = new Array(keys.length);
let firstFoundEventId;
/** Looks up the first, if any, event entry (so excluding gap entries) before `sortKey`.
* @param {string} roomId
* @param {SortKey} sortKey
* @return {Promise<(?Entry)>} a promise resolving to entry, if any.
*/
previousEvent(roomId, sortKey) {
const range = this.upperBoundRange(sortKey, true).asIDBKeyRange(roomId);
return this._timelineStore.findReverse(range, entry => !!entry.event);
// find first result that is found and has no undefined results before it
function firstFoundAndPrecedingResolved() {
let inc = findLast ? -1 : 1;
let start = findLast ? results.length - 1 : 0;
for(let i = start; i >= 0 && i < results.length; i += inc) {
if (results[i] === undefined) {
return;
} else if(results[i] === true) {
return keys[i];
}
}
}
await byEventId.findExistingKeys(keys, findLast, (key, found) => {
const index = keys.indexOf(key);
results[index] = found;
firstFoundEventId = firstFoundAndPrecedingResolved();
return !!firstFoundEventId;
});
return firstFoundEventId;
}
/** Inserts a new entry into the store. The combination of roomId and sortKey should not exist yet, or an error is thrown.

10
src/matrix/storage/idb/utils.js
View File

@ -35,11 +35,11 @@ export function iterateCursor(cursor, processValue) {
resolve(false);
return; // end of results
}
const isDone = processValue(cursor.value);
if (isDone) {
const {done, jumpTo} = processValue(cursor.value, cursor.key);
if (done) {
resolve(true);
} else {
cursor.continue();
cursor.continue(jumpTo);
}
};
});
@ -49,7 +49,7 @@ export async function fetchResults(cursor, isDone) {
const results = [];
await iterateCursor(cursor, (value) => {
results.push(value);
return isDone(results);
return {done: isDone(results)};
});
return results;
}
@ -100,4 +100,4 @@ export async function findStoreValue(db, storeName, toCursor, matchesValue) {
throw new Error("Value not found");
}
return match;
}
}