diff --git a/doc/FRAGMENTS.md b/doc/FRAGMENTS.md index 63a20ee5..e68b5051 100644 --- a/doc/FRAGMENTS.md +++ b/doc/FRAGMENTS.md @@ -57,5 +57,15 @@ thoughts: in case of a gap fill, we need to return what was changed to the fragment (was it joined with another fragment, what's the new token), and which events were actually added. +we return entries! fragmentboundaryentry(start or end) or evententry. so looks much like the gaps we had before, but now they are not stored in the timeline store, but based on fragments. + - where do we translate from fragments to gap entries? and back? in the timeline object? that would make sense, that seems to be the only place we need that translation + +# SortKey + +so, it feels simpler to store fragmentId and eventIndex as fields on the entry instead of an array/arraybuffer in the field sortKey. Currently, the tiles code somewhat relies on having sortKeys but nothing too hard to change. + +so, what we could do: + - we create EventKey(fragmentId, eventIndex) that has the nextKey methods. + - we create a class EventEntry that wraps what is stored in the timeline store. This has a reference to the fragmentindex and has an opaque compare method. Tiles delegate to this method. EventEntry could later on also contain methods like MatrixEvent has in the riot js-sdk, e.g. something to safely dig into the event object. diff --git a/src/matrix/room/persister.js b/src/matrix/room/persister.js index b837f469..94d811dc 100644 --- a/src/matrix/room/persister.js +++ b/src/matrix/room/persister.js @@ -1,5 +1,7 @@ -import SortKey from "./timeline/SortKey.js"; +import EventKey from "./timeline/EventKey.js"; import FragmentIdIndex from "./timeline/FragmentIdIndex.js"; +import EventEntry from "./timeline/entries/EventEntry.js"; +import FragmentBoundaryEntry from "./timeline/entries/FragmentBoundaryEntry.js"; function gapEntriesAreEqual(a, b) { if (!a || !b || !a.gap || !b.gap) { @@ -28,14 +30,14 @@ function replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, } export default class RoomPersister { - constructor({roomId, storage}) { - this._roomId = roomId; + constructor({roomId, storage}) { + this._roomId = roomId; this._storage = storage; - this._lastLiveKey = null; + this._lastLiveKey = null; this._fragmentIdIndex = new FragmentIdIndex([]); //only used when timeline is loaded ... e.g. "certain" methods on this class... split up? - } + } - async load(txn) { + async load(txn) { const liveFragment = await txn.roomFragments.liveFragment(this._roomId); if (liveFragment) { const [lastEvent] = await txn.roomTimeline.lastEvents(this._roomId, liveFragment.id, 1); @@ -43,15 +45,12 @@ export default class RoomPersister { // we could split it up into a SortKey (only with compare) and // a EventKey (no compare or fragment index) with nextkey methods and getters/setters for eventIndex/fragmentId // we probably need to convert from one to the other though, so bother? - const lastLiveKey = new SortKey(this._fragmentIdIndex); - lastLiveKey.fragmentId = liveFragment.id; - lastLiveKey.eventIndex = lastEvent.eventIndex; - this._lastLiveKey = lastLiveKey; + this._lastLiveKey = new EventKey(liveFragment.id, lastEvent.eventIndex); } // if there is no live fragment, we don't create it here because load gets a readonly txn. // this is on purpose, load shouldn't modify the store console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString()); - } + } async persistGapFill(gapEntry, response) { const backwards = !!gapEntry.prev_batch; @@ -124,24 +123,24 @@ export default class RoomPersister { return {newEntries, eventFound}; } - async _getLiveFragment(txn, previousToken) { + async _createLiveFragment(txn, previousToken) { const liveFragment = await txn.roomFragments.liveFragment(this._roomId); if (!liveFragment) { if (!previousToken) { previousToken = null; } - let defaultId = SortKey.firstLiveFragmentId; - txn.roomFragments.add({ + const fragment = { roomId: this._roomId, - id: defaultId, + id: EventKey.defaultLiveKey.fragmentId, previousId: null, nextId: null, previousToken: previousToken, nextToken: null - }); - return defaultId; + }; + txn.roomFragments.add(fragment); + return fragment; } else { - return liveFragment.id; + return liveFragment; } } @@ -152,71 +151,72 @@ export default class RoomPersister { } oldFragment.nextId = newFragmentId; txn.roomFragments.update(oldFragment); - txn.roomFragments.add({ + const newFragment = { roomId: this._roomId, id: newFragmentId, previousId: oldFragmentId, nextId: null, previousToken: previousToken, nextToken: null - }); + }; + txn.roomFragments.add(newFragment); + return newFragment; } - async persistSync(roomResponse, txn) { - // means we haven't synced this room yet (just joined or did initial sync) + async persistSync(roomResponse, txn) { + const entries = []; if (!this._lastLiveKey) { + // means we haven't synced this room yet (just joined or did initial sync) + // as this is probably a limited sync, prev_batch should be there // (but don't fail if it isn't, we won't be able to back-paginate though) - const fragmentId = await this._getLiveFragment(txn, timeline.prev_batch); - this._lastLiveKey = new SortKey(this._fragmentIdIndex); - this._lastLiveKey.fragmentId = fragmentId; - this._lastLiveKey.eventIndex = SortKey.firstLiveEventIndex; - } - // replace live fragment for limited sync, *only* if we had a live fragment already - else if (timeline.limited) { + let liveFragment = await this._createLiveFragment(txn, timeline.prev_batch); + this._lastLiveKey = new EventKey(liveFragment.id, EventKey.defaultLiveKey.eventIndex); + entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdIndex)); + } else if (timeline.limited) { + // replace live fragment for limited sync, *only* if we had a live fragment already const oldFragmentId = this._lastLiveKey.fragmentId; - this._lastLiveKey = this._lastLiveKey.nextFragmentKey(); - this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn); + this._lastLiveKey = this._lastLiveKey.nextFragmentKey(); + const [oldFragment, newFragment] = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn); + entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdIndex)); + entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdIndex)); } - let nextKey = this._lastLiveKey; - const timeline = roomResponse.timeline; - const entries = []; + let currentKey = this._lastLiveKey; + const timeline = roomResponse.timeline; if (timeline.events) { for(const event of timeline.events) { - nextKey = nextKey.nextKey(); - entries.push(this._createEventEntry(nextKey, event)); - } - } - // write to store - for(const entry of entries) { - txn.roomTimeline.insert(entry); + currentKey = currentKey.nextKey(); + const entry = this._createEventEntry(currentKey, event); + txn.roomTimeline.insert(entry); + entries.push(new EventEntry(entry, this._fragmentIdIndex)); + } } - // right thing to do? if the txn fails, not sure we'll continue anyways ... - // only advance the key once the transaction has - // succeeded - txn.complete().then(() => { - console.log("txn complete, setting key"); - this._lastLiveKey = nextKey; - }); + // right thing to do? if the txn fails, not sure we'll continue anyways ... + // only advance the key once the transaction has + // succeeded + txn.complete().then(() => { + console.log("txn complete, setting key"); + this._lastLiveKey = currentKey; + }); - // persist state - const state = roomResponse.state; - if (state.events) { - for (const event of state.events) { - txn.roomState.setStateEvent(this._roomId, event) - } - } + // persist state + const state = roomResponse.state; + if (state.events) { + for (const event of state.events) { + txn.roomState.setStateEvent(this._roomId, event) + } + } // persist live state events in timeline - if (timeline.events) { - for (const event of timeline.events) { - if (typeof event.state_key === "string") { - txn.roomState.setStateEvent(this._roomId, event); - } - } - } + if (timeline.events) { + for (const event of timeline.events) { + if (typeof event.state_key === "string") { + txn.roomState.setStateEvent(this._roomId, event); + } + } + } return entries; - } + } _createEventEntry(key, event) { return { diff --git a/src/matrix/room/timeline/EventKey.js b/src/matrix/room/timeline/EventKey.js new file mode 100644 index 00000000..83621060 --- /dev/null +++ b/src/matrix/room/timeline/EventKey.js @@ -0,0 +1,145 @@ +const DEFAULT_LIVE_FRAGMENT_ID = 0; +const MIN_EVENT_INDEX = Number.MIN_SAFE_INTEGER + 1; +const MAX_EVENT_INDEX = Number.MAX_SAFE_INTEGER - 1; +const MID_EVENT_INDEX = 0; + +export default class EventKey { + constructor(fragmentId, eventIndex) { + this.fragmentId = fragmentId; + this.eventIndex = eventIndex; + } + + nextFragmentKey() { + // could take MIN_EVENT_INDEX here if it can't be paged back + return new EventKey(this.fragmentId + 1, MID_EVENT_INDEX); + } + + nextKey() { + return new EventKey(this.fragmentId, this.eventIndex + 1); + } + + static get maxKey() { + return new EventKey(Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER); + } + + static get minKey() { + return new EventKey(Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER); + } + + static get defaultLiveKey() { + return new EventKey(DEFAULT_LIVE_FRAGMENT_ID, MID_EVENT_INDEX); + } + + toString() { + return `[${this.fragmentId}/${this.eventIndex}]`; + } +} + +//#ifdef TESTS +export function xtests() { + const fragmentIdComparer = {compare: (a, b) => a - b}; + + return { + test_no_fragment_index(assert) { + const min = EventKey.minKey; + const max = EventKey.maxKey; + const a = new EventKey(); + a.eventIndex = 1; + a.fragmentId = 1; + + assert(min.compare(min) === 0); + assert(max.compare(max) === 0); + assert(a.compare(a) === 0); + + assert(min.compare(max) < 0); + assert(max.compare(min) > 0); + + assert(min.compare(a) < 0); + assert(a.compare(min) > 0); + + assert(max.compare(a) > 0); + assert(a.compare(max) < 0); + }, + + test_default_key(assert) { + const k = new EventKey(fragmentIdComparer); + assert.equal(k.fragmentId, MID); + assert.equal(k.eventIndex, MID); + }, + + test_inc(assert) { + const a = new EventKey(fragmentIdComparer); + const b = a.nextKey(); + assert.equal(a.fragmentId, b.fragmentId); + assert.equal(a.eventIndex + 1, b.eventIndex); + const c = b.previousKey(); + assert.equal(b.fragmentId, c.fragmentId); + assert.equal(c.eventIndex + 1, b.eventIndex); + assert.equal(a.eventIndex, c.eventIndex); + }, + + test_min_key(assert) { + const minKey = EventKey.minKey; + const k = new EventKey(fragmentIdComparer); + assert(minKey.fragmentId <= k.fragmentId); + assert(minKey.eventIndex <= k.eventIndex); + assert(k.compare(minKey) > 0); + assert(minKey.compare(k) < 0); + }, + + test_max_key(assert) { + const maxKey = EventKey.maxKey; + const k = new EventKey(fragmentIdComparer); + assert(maxKey.fragmentId >= k.fragmentId); + assert(maxKey.eventIndex >= k.eventIndex); + assert(k.compare(maxKey) < 0); + assert(maxKey.compare(k) > 0); + }, + + test_immutable(assert) { + const a = new EventKey(fragmentIdComparer); + const fragmentId = a.fragmentId; + const eventIndex = a.eventIndex; + a.nextFragmentKey(); + assert.equal(a.fragmentId, fragmentId); + assert.equal(a.eventIndex, eventIndex); + }, + + test_cmp_fragmentid_first(assert) { + const a = new EventKey(fragmentIdComparer); + const b = new EventKey(fragmentIdComparer); + a.fragmentId = 2; + a.eventIndex = 1; + b.fragmentId = 1; + b.eventIndex = 100000; + assert(a.compare(b) > 0); + }, + + test_cmp_eventindex_second(assert) { + const a = new EventKey(fragmentIdComparer); + const b = new EventKey(fragmentIdComparer); + a.fragmentId = 1; + a.eventIndex = 100000; + b.fragmentId = 1; + b.eventIndex = 2; + assert(a.compare(b) > 0); + assert(b.compare(a) < 0); + }, + + test_cmp_max_larger_than_min(assert) { + assert(EventKey.minKey.compare(EventKey.maxKey) < 0); + }, + + test_cmp_fragmentid_first_large(assert) { + const a = new EventKey(fragmentIdComparer); + const b = new EventKey(fragmentIdComparer); + a.fragmentId = MAX; + a.eventIndex = MIN; + b.fragmentId = MIN; + b.eventIndex = MAX; + assert(b < a); + assert(a > b); + } + }; +} +//#endif diff --git a/src/matrix/room/timeline/SortKey.js b/src/matrix/room/timeline/SortKey.js deleted file mode 100644 index b045ce1c..00000000 --- a/src/matrix/room/timeline/SortKey.js +++ /dev/null @@ -1,227 +0,0 @@ -const MIN_INT32 = -2147483648; -const MID_INT32 = 0; -const MAX_INT32 = 2147483647; - -const MIN_UINT32 = 0; -const MID_UINT32 = 2147483647; -const MAX_UINT32 = 4294967295; - -const MIN = MIN_UINT32; -const MID = MID_UINT32; -const MAX = MAX_UINT32; - -export default class SortKey { - constructor(fragmentIdComparer, buffer) { - if (buffer) { - this._keys = new DataView(buffer); - } else { - this._keys = new DataView(new ArrayBuffer(8)); - // start default key right at the middle fragment key, min event key - // so we have the same amount of key address space either way - this.fragmentId = MID; - this.eventIndex = MID; - } - this._fragmentIdComparer = fragmentIdComparer; - } - - get fragmentId() { - return this._keys.getUint32(0, false); - } - - set fragmentId(value) { - return this._keys.setUint32(0, value, false); - } - - get eventIndex() { - return this._keys.getUint32(4, false); - } - - set eventIndex(value) { - return this._keys.setUint32(4, value, false); - } - - get buffer() { - return this._keys.buffer; - } - - nextFragmentKey() { - const k = new SortKey(this._fragmentIdComparer); - k.fragmentId = this.fragmentId + 1; - k.eventIndex = MIN; - return k; - } - - nextKey() { - const k = new SortKey(this._fragmentIdComparer); - k.fragmentId = this.fragmentId; - k.eventIndex = this.eventIndex + 1; - return k; - } - - previousKey() { - const k = new SortKey(this._fragmentIdComparer); - k.fragmentId = this.fragmentId; - k.eventIndex = this.eventIndex - 1; - return k; - } - - clone() { - const k = new SortKey(); - k.fragmentId = this.fragmentId; - k.eventIndex = this.eventIndex; - return k; - } - - static get maxKey() { - const maxKey = new SortKey(null); - maxKey.fragmentId = MAX; - maxKey.eventIndex = MAX; - return maxKey; - } - - static get minKey() { - const minKey = new SortKey(null); - minKey.fragmentId = MIN; - minKey.eventIndex = MIN; - return minKey; - } - - static get firstLiveFragmentId() { - return MID; - } - - static get firstLiveEventIndex() { - return MID; - } - - compare(otherKey) { - const fragmentDiff = this.fragmentId - otherKey.fragmentId; - if (fragmentDiff === 0) { - return this.eventIndex - otherKey.eventIndex; - } else { - // minKey and maxKey might not have fragmentIdComparer, so short-circuit this first ... - if ((this.fragmentId === MIN && otherKey.fragmentId !== MIN) || (this.fragmentId !== MAX && otherKey.fragmentId === MAX)) { - return -1; - } - if ((this.fragmentId === MAX && otherKey.fragmentId !== MAX) || (this.fragmentId !== MIN && otherKey.fragmentId === MIN)) { - return 1; - } - // ... then delegate to fragmentIdComparer. - // This might throw if the relation of two fragments is unknown. - return this._fragmentIdComparer.compare(this.fragmentId, otherKey.fragmentId); - } - } - - toString() { - return `[${this.fragmentId}/${this.eventIndex}]`; - } -} - -//#ifdef TESTS -export function tests() { - const fragmentIdComparer = {compare: (a, b) => a - b}; - - return { - test_no_fragment_index(assert) { - const min = SortKey.minKey; - const max = SortKey.maxKey; - const a = new SortKey(); - a.eventIndex = 1; - a.fragmentId = 1; - - assert(min.compare(min) === 0); - assert(max.compare(max) === 0); - assert(a.compare(a) === 0); - - assert(min.compare(max) < 0); - assert(max.compare(min) > 0); - - assert(min.compare(a) < 0); - assert(a.compare(min) > 0); - - assert(max.compare(a) > 0); - assert(a.compare(max) < 0); - }, - - test_default_key(assert) { - const k = new SortKey(fragmentIdComparer); - assert.equal(k.fragmentId, MID); - assert.equal(k.eventIndex, MID); - }, - - test_inc(assert) { - const a = new SortKey(fragmentIdComparer); - const b = a.nextKey(); - assert.equal(a.fragmentId, b.fragmentId); - assert.equal(a.eventIndex + 1, b.eventIndex); - const c = b.previousKey(); - assert.equal(b.fragmentId, c.fragmentId); - assert.equal(c.eventIndex + 1, b.eventIndex); - assert.equal(a.eventIndex, c.eventIndex); - }, - - test_min_key(assert) { - const minKey = SortKey.minKey; - const k = new SortKey(fragmentIdComparer); - assert(minKey.fragmentId <= k.fragmentId); - assert(minKey.eventIndex <= k.eventIndex); - assert(k.compare(minKey) > 0); - assert(minKey.compare(k) < 0); - }, - - test_max_key(assert) { - const maxKey = SortKey.maxKey; - const k = new SortKey(fragmentIdComparer); - assert(maxKey.fragmentId >= k.fragmentId); - assert(maxKey.eventIndex >= k.eventIndex); - assert(k.compare(maxKey) < 0); - assert(maxKey.compare(k) > 0); - }, - - test_immutable(assert) { - const a = new SortKey(fragmentIdComparer); - const fragmentId = a.fragmentId; - const eventIndex = a.eventIndex; - a.nextFragmentKey(); - assert.equal(a.fragmentId, fragmentId); - assert.equal(a.eventIndex, eventIndex); - }, - - test_cmp_fragmentid_first(assert) { - const a = new SortKey(fragmentIdComparer); - const b = new SortKey(fragmentIdComparer); - a.fragmentId = 2; - a.eventIndex = 1; - b.fragmentId = 1; - b.eventIndex = 100000; - assert(a.compare(b) > 0); - }, - - test_cmp_eventindex_second(assert) { - const a = new SortKey(fragmentIdComparer); - const b = new SortKey(fragmentIdComparer); - a.fragmentId = 1; - a.eventIndex = 100000; - b.fragmentId = 1; - b.eventIndex = 2; - assert(a.compare(b) > 0); - assert(b.compare(a) < 0); - }, - - test_cmp_max_larger_than_min(assert) { - assert(SortKey.minKey.compare(SortKey.maxKey) < 0); - }, - - test_cmp_fragmentid_first_large(assert) { - const a = new SortKey(fragmentIdComparer); - const b = new SortKey(fragmentIdComparer); - a.fragmentId = MAX; - a.eventIndex = MIN; - b.fragmentId = MIN; - b.eventIndex = MAX; - assert(b < a); - assert(a > b); - } - }; -} -//#endif diff --git a/src/matrix/room/timeline/entries/BaseEntry.js b/src/matrix/room/timeline/entries/BaseEntry.js new file mode 100644 index 00000000..56566efc --- /dev/null +++ b/src/matrix/room/timeline/entries/BaseEntry.js @@ -0,0 +1,20 @@ +//entries can be sorted, first by fragment, then by entry index. + +export default class BaseEntry { + get fragmentId() { + throw new Error("unimplemented"); + } + + get entryIndex() { + throw new Error("unimplemented"); + } + + compare(otherEntry) { + if (this.fragmentId === otherEntry.fragmentId) { + return this.entryIndex - otherEntry.entryIndex; + } else { + // This might throw if the relation of two fragments is unknown. + return this._fragmentIdComparer.compare(this.fragmentId, otherEntry.fragmentId); + } + } +} diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js new file mode 100644 index 00000000..07d00a76 --- /dev/null +++ b/src/matrix/room/timeline/entries/EventEntry.js @@ -0,0 +1,28 @@ +import BaseEntry from "./BaseEntry.js"; + +export default class EventEntry extends BaseEntry { + constructor(eventEntry, fragmentIdComparator) { + super(fragmentIdComparator); + this._eventEntry = eventEntry; + } + + get fragmentId() { + return this._eventEntry.fragmentId; + } + + get entryIndex() { + return this._eventEntry.eventIndex; + } + + get content() { + return this._eventEntry.event.content; + } + + get type() { + return this._eventEntry.event.type; + } + + get id() { + return this._eventEntry.event.event_id; + } +} diff --git a/src/matrix/room/timeline/entries/FragmentBoundaryEntry.js b/src/matrix/room/timeline/entries/FragmentBoundaryEntry.js new file mode 100644 index 00000000..610d32c6 --- /dev/null +++ b/src/matrix/room/timeline/entries/FragmentBoundaryEntry.js @@ -0,0 +1,49 @@ +import BaseEntry from "./BaseEntry.js"; + +export default class FragmentBoundaryEntry extends BaseEntry { + constructor(fragment, isFragmentStart, fragmentIdComparator) { + super(fragmentIdComparator); + this._fragment = fragment; + this._isFragmentStart = isFragmentStart; + } + + static start(fragment, fragmentIdComparator) { + return new FragmentBoundaryEntry(fragment, true, fragmentIdComparator); + } + + static end(fragment, fragmentIdComparator) { + return new FragmentBoundaryEntry(fragment, false, fragmentIdComparator); + } + + get hasStarted() { + return this._isFragmentStart; + } + + get hasEnded() { + return !this.hasStarted; + } + + get fragment() { + return this._fragment; + } + + get fragmentId() { + return this._fragment.id; + } + + get entryIndex() { + if (this.hasStarted) { + return Number.MIN_SAFE_INTEGER; + } else { + return Number.MAX_SAFE_INTEGER; + } + } + + get isGap() { + if (this.hasStarted) { + return !!this.fragment.nextToken; + } else { + return !!this.fragment.previousToken; + } + } +}