diff --git a/doc/FRAGMENTS.md b/doc/FRAGMENTS.md index e68b5051..a63a96ea 100644 --- a/doc/FRAGMENTS.md +++ b/doc/FRAGMENTS.md @@ -17,7 +17,7 @@ how will fragments be exposed in timeline store? - all read operations are passed a fragment id - adapt persister - - persist fragments in /sync + - DONE: persist fragments in /sync - load n items before and after key - fill gaps / fragment filling - add live fragment id optimization if we haven't done so already diff --git a/src/matrix/room/room.js b/src/matrix/room/room.js index 05560cb5..b2ad1312 100644 --- a/src/matrix/room/room.js +++ b/src/matrix/room/room.js @@ -1,7 +1,8 @@ import EventEmitter from "../../EventEmitter.js"; import RoomSummary from "./summary.js"; -import RoomPersister from "./persister.js"; import Timeline from "./timeline.js"; +import SyncPersister from "./timeline/persistence/SyncPersister.js"; +import FragmentIdComparer from "./timeline/FragmentIdComparer.js"; export default class Room extends EventEmitter { constructor({roomId, storage, hsApi, emitCollectionChange}) { @@ -10,14 +11,15 @@ export default class Room extends EventEmitter { this._storage = storage; this._hsApi = hsApi; this._summary = new RoomSummary(roomId); - this._persister = new RoomPersister({roomId, storage}); + this._fragmentIdComparer = new FragmentIdComparer([]); + this._syncPersister = new SyncPersister({roomId, storage, fragmentIdComparer: this._fragmentIdComparer}); this._emitCollectionChange = emitCollectionChange; this._timeline = null; } persistSync(roomResponse, membership, txn) { const summaryChanged = this._summary.applySync(roomResponse, membership, txn); - const newTimelineEntries = this._persister.persistSync(roomResponse, txn); + const newTimelineEntries = this._syncPersister.persistSync(roomResponse, txn); return {summaryChanged, newTimelineEntries}; } @@ -33,7 +35,7 @@ export default class Room extends EventEmitter { load(summary, txn) { this._summary.load(summary); - return this._persister.load(txn); + return this._syncPersister.load(txn); } get name() { @@ -51,7 +53,6 @@ export default class Room extends EventEmitter { this._timeline = new Timeline({ roomId: this.id, storage: this._storage, - persister: this._persister, hsApi: this._hsApi, closeCallback: () => this._timeline = null, }); diff --git a/src/matrix/room/timeline/FragmentIdIndex.js b/src/matrix/room/timeline/FragmentIdComparer.js similarity index 99% rename from src/matrix/room/timeline/FragmentIdIndex.js rename to src/matrix/room/timeline/FragmentIdComparer.js index e67be9c2..7ffb9422 100644 --- a/src/matrix/room/timeline/FragmentIdIndex.js +++ b/src/matrix/room/timeline/FragmentIdComparer.js @@ -106,7 +106,7 @@ class Island { /* index for fast lookup of how two fragments can be sorted */ -export default class FragmentIdIndex { +export default class FragmentIdComparer { constructor(fragments) { this.rebuild(fragments); } diff --git a/src/matrix/room/persister.js b/src/matrix/room/timeline/persistence/SyncPersister.js similarity index 63% rename from src/matrix/room/persister.js rename to src/matrix/room/timeline/persistence/SyncPersister.js index 94d811dc..eaa3bfdd 100644 --- a/src/matrix/room/persister.js +++ b/src/matrix/room/timeline/persistence/SyncPersister.js @@ -1,40 +1,13 @@ -import EventKey from "./timeline/EventKey.js"; -import FragmentIdIndex from "./timeline/FragmentIdIndex.js"; -import EventEntry from "./timeline/entries/EventEntry.js"; -import FragmentBoundaryEntry from "./timeline/entries/FragmentBoundaryEntry.js"; +import EventKey from "../EventKey.js"; +import EventEntry from "../entries/EventEntry.js"; +import FragmentBoundaryEntry from "../entries/FragmentBoundaryEntry.js"; +import {createEventEntry} from "./common.js"; -function gapEntriesAreEqual(a, b) { - if (!a || !b || !a.gap || !b.gap) { - return false; - } - const gapA = a.gap, gapB = b.gap; - return gapA.prev_batch === gapB.prev_batch && gapA.next_batch === gapB.next_batch; -} - -function replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards) { - let replacedRange; - if (neighbourEventKey) { - replacedRange = backwards ? - roomTimeline.boundRange(neighbourEventKey, gapKey, false, true) : - roomTimeline.boundRange(gapKey, neighbourEventKey, true, false); - } else { - replacedRange = roomTimeline.onlyRange(gapKey); - } - - const removedEntries = roomTimeline.getAndRemoveRange(this._roomId, replacedRange); - for (let entry of newEntries) { - roomTimeline.add(entry); - } - - return removedEntries; -} - -export default class RoomPersister { - constructor({roomId, storage}) { +export default class SyncPersister { + constructor({roomId, storage, fragmentIdComparer}) { this._roomId = roomId; this._storage = storage; - this._lastLiveKey = null; - this._fragmentIdIndex = new FragmentIdIndex([]); //only used when timeline is loaded ... e.g. "certain" methods on this class... split up? + this._fragmentIdComparer = fragmentIdComparer; } async load(txn) { @@ -52,77 +25,6 @@ export default class RoomPersister { console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString()); } - async persistGapFill(gapEntry, response) { - const backwards = !!gapEntry.prev_batch; - const {chunk, start, end} = response; - if (!Array.isArray(chunk)) { - throw new Error("Invalid chunk in response"); - } - if (typeof end !== "string") { - throw new Error("Invalid end token in response"); - } - if ((backwards && start !== gapEntry.prev_batch) || (!backwards && start !== gapEntry.next_batch)) { - throw new Error("start is not equal to prev_batch or next_batch"); - } - - const gapKey = gapEntry.sortKey; - const txn = await this._storage.readWriteTxn([this._storage.storeNames.roomTimeline]); - let result; - try { - const roomTimeline = txn.roomTimeline; - // make sure what we've been given is actually persisted - // in the timeline, otherwise we're replacing something - // that doesn't exist (maybe it has been replaced already, or ...) - const persistedEntry = await roomTimeline.get(this._roomId, gapKey); - if (!gapEntriesAreEqual(gapEntry, persistedEntry)) { - throw new Error("Gap is not present in the timeline"); - } - // find the previous event before the gap we could merge with - const neighbourEventEntry = await (backwards ? - roomTimeline.previousEvent(this._roomId, gapKey) : - roomTimeline.nextEvent(this._roomId, gapKey)); - const neighbourEventId = neighbourEventEntry ? neighbourEventEntry.event.event_id : undefined; - const {newEntries, eventFound} = this._createNewGapEntries(chunk, end, gapKey, neighbourEventId, backwards); - const neighbourEventKey = eventFound ? neighbourEventEntry.sortKey : undefined; - const replacedEntries = replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards); - result = {newEntries, replacedEntries}; - } catch (err) { - txn.abort(); - throw err; - } - - await txn.complete(); - - return result; - } - - _createNewGapEntries(chunk, nextPaginationToken, gapKey, neighbourEventId, backwards) { - if (backwards) { - // if backwards, the last events are the ones closest to the gap, - // and need to be assigned a key derived from the gap first, - // so swap order to only need one loop for both directions - chunk.reverse(); - } - let sortKey = gapKey; - const {newEntries, eventFound} = chunk.reduce((acc, event) => { - acc.eventFound = acc.eventFound || event.event_id === neighbourEventId; - if (!acc.eventFound) { - acc.newEntries.push(this._createEventEntry(sortKey, event)); - sortKey = backwards ? sortKey.previousKey() : sortKey.nextKey(); - } - }, {newEntries: [], eventFound: false}); - - if (!eventFound) { - // as we're replacing an existing gap, no need to increment the gap index - newEntries.push(this._createGapEntry(sortKey, nextPaginationToken, backwards)); - } - if (backwards) { - // swap resulting array order again if needed - newEntries.reverse(); - } - return {newEntries, eventFound}; - } - async _createLiveFragment(txn, previousToken) { const liveFragment = await txn.roomFragments.liveFragment(this._roomId); if (!liveFragment) { @@ -160,7 +62,7 @@ export default class RoomPersister { nextToken: null }; txn.roomFragments.add(newFragment); - return newFragment; + return {oldFragment, newFragment}; } async persistSync(roomResponse, txn) { @@ -172,23 +74,23 @@ export default class RoomPersister { // (but don't fail if it isn't, we won't be able to back-paginate though) let liveFragment = await this._createLiveFragment(txn, timeline.prev_batch); this._lastLiveKey = new EventKey(liveFragment.id, EventKey.defaultLiveKey.eventIndex); - entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdIndex)); + entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdComparer)); } else if (timeline.limited) { // replace live fragment for limited sync, *only* if we had a live fragment already const oldFragmentId = this._lastLiveKey.fragmentId; this._lastLiveKey = this._lastLiveKey.nextFragmentKey(); - const [oldFragment, newFragment] = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn); - entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdIndex)); - entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdIndex)); + const {oldFragment, newFragment} = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn); + entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdComparer)); + entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdComparer)); } let currentKey = this._lastLiveKey; const timeline = roomResponse.timeline; if (timeline.events) { for(const event of timeline.events) { currentKey = currentKey.nextKey(); - const entry = this._createEventEntry(currentKey, event); + const entry = createEventEntry(currentKey, event); txn.roomTimeline.insert(entry); - entries.push(new EventEntry(entry, this._fragmentIdIndex)); + entries.push(new EventEntry(entry, this._fragmentIdComparer)); } } // right thing to do? if the txn fails, not sure we'll continue anyways ... @@ -217,14 +119,6 @@ export default class RoomPersister { return entries; } - - _createEventEntry(key, event) { - return { - fragmentId: key.fragmentId, - eventIndex: key.eventIndex, - event: event, - }; - } } //#ifdef TESTS diff --git a/src/matrix/room/timeline/persistence/common.js b/src/matrix/room/timeline/persistence/common.js new file mode 100644 index 00000000..ac5a6c9d --- /dev/null +++ b/src/matrix/room/timeline/persistence/common.js @@ -0,0 +1,7 @@ +export function createEventEntry(key, event) { + return { + fragmentId: key.fragmentId, + eventIndex: key.eventIndex, + event: event, + }; +} \ No newline at end of file diff --git a/src/matrix/storage/idb/storage.js b/src/matrix/storage/idb/storage.js index b4a6659c..ca7e2256 100644 --- a/src/matrix/storage/idb/storage.js +++ b/src/matrix/storage/idb/storage.js @@ -1,6 +1,5 @@ import Transaction from "./transaction.js"; - -export const STORE_NAMES = ["session", "roomState", "roomSummary", "roomTimeline"]; +import { STORE_NAMES } from "../common.js"; export default class Storage { constructor(idbDatabase) { @@ -30,4 +29,4 @@ export default class Storage { const txn = this._db.transaction(storeNames, "readwrite"); return new Transaction(txn, storeNames); } -} \ No newline at end of file +} diff --git a/src/matrix/storage/idb/stores/RoomTimelineStore.js b/src/matrix/storage/idb/stores/TimelineEventStore.js similarity index 73% rename from src/matrix/storage/idb/stores/RoomTimelineStore.js rename to src/matrix/storage/idb/stores/TimelineEventStore.js index 0a540dd5..a6ce5617 100644 --- a/src/matrix/storage/idb/stores/RoomTimelineStore.js +++ b/src/matrix/storage/idb/stores/TimelineEventStore.js @@ -1,4 +1,4 @@ -import SortKey from "../../../room/timeline/SortKey.js"; +import EventKey from "../../../room/timeline/EventKey.js"; class Range { constructor(only, lower, upper, lowerOpen, upperOpen) { @@ -12,14 +12,14 @@ class Range { asIDBKeyRange(roomId) { // only if (this._only) { - return IDBKeyRange.only([roomId, this._only.buffer]); + return IDBKeyRange.only([roomId, this._only.fragmentId, this._only.eventIndex]); } // lowerBound // also bound as we don't want to move into another roomId if (this._lower && !this._upper) { return IDBKeyRange.bound( - [roomId, this._lower.buffer], - [roomId, SortKey.maxKey.buffer], + [roomId, this._lower.fragmentId, this._lower.eventIndex], + [roomId, EventKey.maxKey.fragmentId, EventKey.maxKey.eventIndex], this._lowerOpen, false ); @@ -28,8 +28,8 @@ class Range { // also bound as we don't want to move into another roomId if (!this._lower && this._upper) { return IDBKeyRange.bound( - [roomId, SortKey.minKey.buffer], - [roomId, this._upper.buffer], + [roomId, EventKey.minKey.fragmentId, EventKey.minKey.eventIndex], + [roomId, this._upper.fragmentId, this._upper.eventIndex], false, this._upperOpen ); @@ -37,8 +37,8 @@ class Range { // bound if (this._lower && this._upper) { return IDBKeyRange.bound( - [roomId, this._lower.buffer], - [roomId, this._upper.buffer], + [roomId, this._lower.fragmentId, this._lower.eventIndex], + [roomId, this._upper.fragmentId, this._upper.eventIndex], this._lowerOpen, this._upperOpen ); @@ -57,44 +57,44 @@ class Range { * * @typedef {Object} Entry * @property {string} roomId - * @property {SortKey} sortKey + * @property {EventKey} eventKey * @property {?Event} event if an event entry, the event * @property {?Gap} gap if a gap entry, the gap */ -export default class RoomTimelineStore { +export default class TimelineEventStore { constructor(timelineStore) { this._timelineStore = timelineStore; } /** Creates a range that only includes the given key - * @param {SortKey} sortKey the key + * @param {EventKey} eventKey the key * @return {Range} the created range */ - onlyRange(sortKey) { - return new Range(sortKey); + onlyRange(eventKey) { + return new Range(eventKey); } - /** Creates a range that includes all keys before sortKey, and optionally also the key itself. - * @param {SortKey} sortKey the key + /** Creates a range that includes all keys before eventKey, and optionally also the key itself. + * @param {EventKey} eventKey the key * @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the upper end. * @return {Range} the created range */ - upperBoundRange(sortKey, open=false) { - return new Range(undefined, undefined, sortKey, undefined, open); + upperBoundRange(eventKey, open=false) { + return new Range(undefined, undefined, eventKey, undefined, open); } - /** Creates a range that includes all keys after sortKey, and optionally also the key itself. - * @param {SortKey} sortKey the key + /** Creates a range that includes all keys after eventKey, and optionally also the key itself. + * @param {EventKey} eventKey the key * @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the lower end. * @return {Range} the created range */ - lowerBoundRange(sortKey, open=false) { - return new Range(undefined, sortKey, undefined, open); + lowerBoundRange(eventKey, open=false) { + return new Range(undefined, eventKey, undefined, open); } /** Creates a range that includes all keys between `lower` and `upper`, and optionally the given keys as well. - * @param {SortKey} lower the lower key - * @param {SortKey} upper the upper key + * @param {EventKey} lower the lower key + * @param {EventKey} upper the upper key * @param {boolean} [lowerOpen=false] whether the lower key is included (false) or excluded (true) from the range. * @param {boolean} [upperOpen=false] whether the upper key is included (false) or excluded (true) from the range. * @return {Range} the created range @@ -110,9 +110,9 @@ export default class RoomTimelineStore { * @return {Promise} a promise resolving to an array with 0 or more entries, in ascending order. */ async lastEvents(roomId, fragmentId, amount) { - const sortKey = SortKey.maxKey; - sortKey.fragmentId = fragmentId; - return this.eventsBefore(roomId, sortKey, amount); + const eventKey = EventKey.maxKey; + eventKey.fragmentId = fragmentId; + return this.eventsBefore(roomId, eventKey, amount); } /** Looks up the first `amount` entries in the timeline for `roomId`. @@ -122,32 +122,32 @@ export default class RoomTimelineStore { * @return {Promise} a promise resolving to an array with 0 or more entries, in ascending order. */ async firstEvents(roomId, fragmentId, amount) { - const sortKey = SortKey.minKey; - sortKey.fragmentId = fragmentId; - return this.eventsAfter(roomId, sortKey, amount); + const eventKey = EventKey.minKey; + eventKey.fragmentId = fragmentId; + return this.eventsAfter(roomId, eventKey, amount); } - /** Looks up `amount` entries after `sortKey` in the timeline for `roomId` within the same fragment. - * The entry for `sortKey` is not included. + /** Looks up `amount` entries after `eventKey` in the timeline for `roomId` within the same fragment. + * The entry for `eventKey` is not included. * @param {string} roomId - * @param {SortKey} sortKey + * @param {EventKey} eventKey * @param {number} amount * @return {Promise} a promise resolving to an array with 0 or more entries, in ascending order. */ - eventsAfter(roomId, sortKey, amount) { - const idbRange = this.lowerBoundRange(sortKey, true).asIDBKeyRange(roomId); + eventsAfter(roomId, eventKey, amount) { + const idbRange = this.lowerBoundRange(eventKey, true).asIDBKeyRange(roomId); return this._timelineStore.selectLimit(idbRange, amount); } - /** Looks up `amount` entries before `sortKey` in the timeline for `roomId` within the same fragment. - * The entry for `sortKey` is not included. + /** Looks up `amount` entries before `eventKey` in the timeline for `roomId` within the same fragment. + * The entry for `eventKey` is not included. * @param {string} roomId - * @param {SortKey} sortKey + * @param {EventKey} eventKey * @param {number} amount * @return {Promise} a promise resolving to an array with 0 or more entries, in ascending order. */ - async eventsBefore(roomId, sortKey, amount) { - const range = this.upperBoundRange(sortKey, true).asIDBKeyRange(roomId); + async eventsBefore(roomId, eventKey, amount) { + const range = this.upperBoundRange(eventKey, true).asIDBKeyRange(roomId); const events = await this._timelineStore.selectLimitReverse(range, amount); events.reverse(); // because we fetched them backwards return events; @@ -196,7 +196,7 @@ export default class RoomTimelineStore { return firstFoundEventId; } - /** Inserts a new entry into the store. The combination of roomId and sortKey should not exist yet, or an error is thrown. + /** Inserts a new entry into the store. The combination of roomId and eventKey should not exist yet, or an error is thrown. * @param {Entry} entry the entry to insert * @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not. * @throws {StorageError} ... @@ -206,7 +206,7 @@ export default class RoomTimelineStore { return this._timelineStore.add(entry); } - /** Updates the entry into the store with the given [roomId, sortKey] combination. + /** Updates the entry into the store with the given [roomId, eventKey] combination. * If not yet present, will insert. Might be slower than add. * @param {Entry} entry the entry to update. * @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not. @@ -215,12 +215,16 @@ export default class RoomTimelineStore { return this._timelineStore.put(entry); } - get(roomId, sortKey) { - return this._timelineStore.get([roomId, sortKey]); + get(roomId, eventKey) { + return this._timelineStore.get([roomId, eventKey.fragmentId, eventKey.eventIndex]); } // returns the entries as well!! (or not always needed? I guess not always needed, so extra method) removeRange(roomId, range) { // TODO: read the entries! return this._timelineStore.delete(range.asIDBKeyRange(roomId)); } + + getByEventId(roomId, eventId) { + return this._timelineStore.index("byEventId").get([roomId, eventId]); + } } diff --git a/src/matrix/storage/idb/transaction.js b/src/matrix/storage/idb/transaction.js index 78883964..1a16d08d 100644 --- a/src/matrix/storage/idb/transaction.js +++ b/src/matrix/storage/idb/transaction.js @@ -2,8 +2,9 @@ import {txnAsPromise} from "./utils.js"; import Store from "./store.js"; import SessionStore from "./stores/SessionStore.js"; import RoomSummaryStore from "./stores/RoomSummaryStore.js"; -import RoomTimelineStore from "./stores/RoomTimelineStore.js"; +import TimelineEventStore from "./stores/TimelineEventStore.js"; import RoomStateStore from "./stores/RoomStateStore.js"; +import TimelineFragmentStore from "./stores/TimelineFragmentStore.js"; export default class Transaction { constructor(txn, allowedStoreNames) { @@ -41,8 +42,12 @@ export default class Transaction { return this._store("roomSummary", idbStore => new RoomSummaryStore(idbStore)); } - get roomTimeline() { - return this._store("roomTimeline", idbStore => new RoomTimelineStore(idbStore)); + get timelineFragments() { + return this._store("timelineFragments", idbStore => new TimelineFragmentStore(idbStore)); + } + + get timelineEvents() { + return this._store("timelineEvents", idbStore => new TimelineEventStore(idbStore)); } get roomState() { @@ -56,4 +61,4 @@ export default class Transaction { abort() { this._txn.abort(); } -} \ No newline at end of file +}