create Entry classes and return fragment boundaries as entries as well

they can then be used for gap tiles.

doc/FRAGMENTS.md

@@ -57,5 +57,15 @@ thoughts:
 
     in case of a gap fill, we need to return what was changed to the fragment (was it joined with another fragment, what's the new token), and which events were actually added.
 
+we return entries! fragmentboundaryentry(start or end) or evententry. so looks much like the gaps we had before, but now they are not stored in the timeline store, but based on fragments.
+
 - where do we translate from fragments to gap entries? and back? in the timeline object?
     that would make sense, that seems to be the only place we need that translation
+
+# SortKey
+
+so, it feels simpler to store fragmentId and eventIndex as fields on the entry instead of an array/arraybuffer in the field sortKey. Currently, the tiles code somewhat relies on having sortKeys but nothing too hard to change.
+
+so, what we could do:
+    - we create EventKey(fragmentId, eventIndex) that has the nextKey methods.
+    - we create a class EventEntry that wraps what is stored in the timeline store. This has a reference to the fragmentindex and has an opaque compare method. Tiles delegate to this method. EventEntry could later on also contain methods like MatrixEvent has in the riot js-sdk, e.g. something to safely dig into the event object.

src/matrix/room/persister.js

@@ -1,5 +1,7 @@
-import SortKey from "./timeline/SortKey.js";
+import EventKey from "./timeline/EventKey.js";
 import FragmentIdIndex from "./timeline/FragmentIdIndex.js";
+import EventEntry from "./timeline/entries/EventEntry.js";
+import FragmentBoundaryEntry from "./timeline/entries/FragmentBoundaryEntry.js";
 
 function gapEntriesAreEqual(a, b) {
     if (!a || !b || !a.gap || !b.gap) {
@@ -28,14 +30,14 @@ function replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey,
 }
 
 export default class RoomPersister {
-	constructor({roomId, storage}) {
+    constructor({roomId, storage}) {
-		this._roomId = roomId;
+        this._roomId = roomId;
         this._storage = storage;
-		this._lastLiveKey = null;
+        this._lastLiveKey = null;
         this._fragmentIdIndex = new FragmentIdIndex([]);   //only used when timeline is loaded ... e.g. "certain" methods on this class... split up?
-	}
+    }
 
-	async load(txn) {
+    async load(txn) {
         const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
         if (liveFragment) {
             const [lastEvent] = await txn.roomTimeline.lastEvents(this._roomId, liveFragment.id, 1);
@@ -43,15 +45,12 @@ export default class RoomPersister {
             // we could split it up into a SortKey (only with compare) and
             // a EventKey (no compare or fragment index) with nextkey methods and getters/setters for eventIndex/fragmentId
             // we probably need to convert from one to the other though, so bother?
-            const lastLiveKey = new SortKey(this._fragmentIdIndex);
+            this._lastLiveKey = new EventKey(liveFragment.id, lastEvent.eventIndex);
-            lastLiveKey.fragmentId = liveFragment.id;
-            lastLiveKey.eventIndex = lastEvent.eventIndex;
-            this._lastLiveKey = lastLiveKey;
         }
         // if there is no live fragment, we don't create it here because load gets a readonly txn.
         // this is on purpose, load shouldn't modify the store
         console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString());
-	}
+    }
 
     async persistGapFill(gapEntry, response) {
         const backwards = !!gapEntry.prev_batch;
@@ -124,24 +123,24 @@ export default class RoomPersister {
         return {newEntries, eventFound};
     }
 
-    async _getLiveFragment(txn, previousToken) {
+    async _createLiveFragment(txn, previousToken) {
         const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
         if (!liveFragment) {
             if (!previousToken) {
                 previousToken = null;
             }
-            let defaultId = SortKey.firstLiveFragmentId;
+            const fragment = {
-            txn.roomFragments.add({
                 roomId: this._roomId,
-                id: defaultId,
+                id: EventKey.defaultLiveKey.fragmentId,
                 previousId: null,
                 nextId: null,
                 previousToken: previousToken,
                 nextToken: null
-            });
+            };
-            return defaultId;
+            txn.roomFragments.add(fragment);
+            return fragment;
         } else {
-            return liveFragment.id;
+            return liveFragment;
         }
     }
 
@@ -152,71 +151,72 @@ export default class RoomPersister {
         }
         oldFragment.nextId = newFragmentId;
         txn.roomFragments.update(oldFragment);
-        txn.roomFragments.add({
+        const newFragment = {
             roomId: this._roomId,
             id: newFragmentId,
             previousId: oldFragmentId,
             nextId: null,
             previousToken: previousToken,
             nextToken: null
-        });
+        };
+        txn.roomFragments.add(newFragment);
+        return newFragment;
     }
 
-	async persistSync(roomResponse, txn) {
+    async persistSync(roomResponse, txn) {
-        // means we haven't synced this room yet (just joined or did initial sync)
+        const entries = [];
         if (!this._lastLiveKey) {
+            // means we haven't synced this room yet (just joined or did initial sync)
+            
             // as this is probably a limited sync, prev_batch should be there
             // (but don't fail if it isn't, we won't be able to back-paginate though)
-            const fragmentId = await this._getLiveFragment(txn, timeline.prev_batch);
+            let liveFragment = await this._createLiveFragment(txn, timeline.prev_batch);
-            this._lastLiveKey = new SortKey(this._fragmentIdIndex);
+            this._lastLiveKey = new EventKey(liveFragment.id, EventKey.defaultLiveKey.eventIndex);
-            this._lastLiveKey.fragmentId = fragmentId;
+            entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdIndex));
-            this._lastLiveKey.eventIndex = SortKey.firstLiveEventIndex;
+        } else if (timeline.limited) {
-        }
+            // replace live fragment for limited sync, *only* if we had a live fragment already
-        // replace live fragment for limited sync, *only* if we had a live fragment already
-        else if (timeline.limited) {
             const oldFragmentId = this._lastLiveKey.fragmentId;
-			this._lastLiveKey = this._lastLiveKey.nextFragmentKey();
+            this._lastLiveKey = this._lastLiveKey.nextFragmentKey();
-            this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
+            const [oldFragment, newFragment] = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
+            entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdIndex));
+            entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdIndex));
         }
-		let nextKey = this._lastLiveKey;
+        let currentKey = this._lastLiveKey;
-		const timeline = roomResponse.timeline;
+        const timeline = roomResponse.timeline;
-        const entries = [];
         if (timeline.events) {
             for(const event of timeline.events) {
-                nextKey = nextKey.nextKey();
+                currentKey = currentKey.nextKey();
-                entries.push(this._createEventEntry(nextKey, event));
+                const entry = this._createEventEntry(currentKey, event);
-			}
+                txn.roomTimeline.insert(entry);
-		}
+                entries.push(new EventEntry(entry, this._fragmentIdIndex));
-        // write to store
+            }
-        for(const entry of entries) {
-            txn.roomTimeline.insert(entry);
         }
-		// right thing to do? if the txn fails, not sure we'll continue anyways ...
+        // right thing to do? if the txn fails, not sure we'll continue anyways ...
-		// only advance the key once the transaction has
+        // only advance the key once the transaction has
-		// succeeded 
+        // succeeded 
-		txn.complete().then(() => {
+        txn.complete().then(() => {
-			console.log("txn complete, setting key");
+            console.log("txn complete, setting key");
-			this._lastLiveKey = nextKey;
+            this._lastLiveKey = currentKey;
-		});
+        });
 
-		// persist state
+        // persist state
-		const state = roomResponse.state;
+        const state = roomResponse.state;
-		if (state.events) {
+        if (state.events) {
-			for (const event of state.events) {
+            for (const event of state.events) {
-				txn.roomState.setStateEvent(this._roomId, event)
+                txn.roomState.setStateEvent(this._roomId, event)
-			}
+            }
-		}
+        }
         // persist live state events in timeline
-		if (timeline.events) {
+        if (timeline.events) {
-			for (const event of timeline.events) {
+            for (const event of timeline.events) {
-				if (typeof event.state_key === "string") {
+                if (typeof event.state_key === "string") {
-					txn.roomState.setStateEvent(this._roomId, event);
+                    txn.roomState.setStateEvent(this._roomId, event);
-				}
+                }
-			}
+            }
-		}
+        }
 
         return entries;
-	}
+    }
 
     _createEventEntry(key, event) {
         return {

src/matrix/room/timeline/EventKey.js

@@ -0,0 +1,145 @@
+const DEFAULT_LIVE_FRAGMENT_ID = 0;
+const MIN_EVENT_INDEX = Number.MIN_SAFE_INTEGER + 1;
+const MAX_EVENT_INDEX = Number.MAX_SAFE_INTEGER - 1;
+const MID_EVENT_INDEX = 0;
+
+export default class EventKey {
+    constructor(fragmentId, eventIndex) {
+        this.fragmentId = fragmentId;
+        this.eventIndex = eventIndex;
+    }
+
+    nextFragmentKey() {
+        // could take MIN_EVENT_INDEX here if it can't be paged back
+        return new EventKey(this.fragmentId + 1, MID_EVENT_INDEX);
+    }
+
+    nextKey() {
+        return new EventKey(this.fragmentId, this.eventIndex + 1);
+    }
+
+    static get maxKey() {
+        return new EventKey(Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER);
+    }
+
+    static get minKey() {
+        return new EventKey(Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER);
+    }
+
+    static get defaultLiveKey() {
+        return new EventKey(DEFAULT_LIVE_FRAGMENT_ID, MID_EVENT_INDEX);
+    }
+
+    toString() {
+        return `[${this.fragmentId}/${this.eventIndex}]`;
+    }
+}
+
+//#ifdef TESTS
+export function xtests() {
+    const fragmentIdComparer = {compare: (a, b) => a - b};
+
+    return {
+        test_no_fragment_index(assert) {
+            const min = EventKey.minKey;
+            const max = EventKey.maxKey;
+            const a = new EventKey();
+            a.eventIndex = 1;
+            a.fragmentId = 1;
+
+            assert(min.compare(min) === 0);
+            assert(max.compare(max) === 0);
+            assert(a.compare(a) === 0);
+
+            assert(min.compare(max) < 0);
+            assert(max.compare(min) > 0);
+
+            assert(min.compare(a) < 0);
+            assert(a.compare(min) > 0);
+            
+            assert(max.compare(a) > 0);
+            assert(a.compare(max) < 0);
+        },
+
+        test_default_key(assert) {
+            const k = new EventKey(fragmentIdComparer);
+            assert.equal(k.fragmentId, MID);
+            assert.equal(k.eventIndex, MID);
+        },
+
+        test_inc(assert) {
+            const a = new EventKey(fragmentIdComparer);
+            const b = a.nextKey();
+            assert.equal(a.fragmentId, b.fragmentId);
+            assert.equal(a.eventIndex + 1, b.eventIndex);
+            const c = b.previousKey();
+            assert.equal(b.fragmentId, c.fragmentId);
+            assert.equal(c.eventIndex + 1, b.eventIndex);
+            assert.equal(a.eventIndex, c.eventIndex);
+        },
+
+        test_min_key(assert) {
+            const minKey = EventKey.minKey;
+            const k = new EventKey(fragmentIdComparer);
+            assert(minKey.fragmentId <= k.fragmentId);
+            assert(minKey.eventIndex <= k.eventIndex);
+            assert(k.compare(minKey) > 0);
+            assert(minKey.compare(k) < 0);
+        },
+
+        test_max_key(assert) {
+            const maxKey = EventKey.maxKey;
+            const k = new EventKey(fragmentIdComparer);
+            assert(maxKey.fragmentId >= k.fragmentId);
+            assert(maxKey.eventIndex >= k.eventIndex);
+            assert(k.compare(maxKey) < 0);
+            assert(maxKey.compare(k) > 0);
+        },
+
+        test_immutable(assert) {
+            const a = new EventKey(fragmentIdComparer);
+            const fragmentId = a.fragmentId;
+            const eventIndex = a.eventIndex;
+            a.nextFragmentKey();
+            assert.equal(a.fragmentId, fragmentId);
+            assert.equal(a.eventIndex, eventIndex);
+        },
+
+        test_cmp_fragmentid_first(assert) {
+            const a = new EventKey(fragmentIdComparer);
+            const b = new EventKey(fragmentIdComparer);
+            a.fragmentId = 2;
+            a.eventIndex = 1;
+            b.fragmentId = 1;
+            b.eventIndex = 100000;
+            assert(a.compare(b) > 0);
+        },
+
+        test_cmp_eventindex_second(assert) {
+            const a = new EventKey(fragmentIdComparer);
+            const b = new EventKey(fragmentIdComparer);
+            a.fragmentId = 1;
+            a.eventIndex = 100000;
+            b.fragmentId = 1;
+            b.eventIndex = 2;
+            assert(a.compare(b) > 0);
+            assert(b.compare(a) < 0);
+        },
+
+        test_cmp_max_larger_than_min(assert) {
+            assert(EventKey.minKey.compare(EventKey.maxKey) < 0);
+        },
+
+        test_cmp_fragmentid_first_large(assert) {
+            const a = new EventKey(fragmentIdComparer);
+            const b = new EventKey(fragmentIdComparer);
+            a.fragmentId = MAX;
+            a.eventIndex = MIN;
+            b.fragmentId = MIN;
+            b.eventIndex = MAX;
+            assert(b < a);
+            assert(a > b);
+        }
+    };
+}
+//#endif

src/matrix/room/timeline/SortKey.js

@@ -1,227 +0,0 @@
-const MIN_INT32 = -2147483648;
-const MID_INT32 = 0;
-const MAX_INT32 = 2147483647;
-
-const MIN_UINT32 = 0;
-const MID_UINT32 = 2147483647;
-const MAX_UINT32 = 4294967295;
-
-const MIN = MIN_UINT32;
-const MID = MID_UINT32;
-const MAX = MAX_UINT32;
-
-export default class SortKey {
-    constructor(fragmentIdComparer, buffer) {
-        if (buffer) {
-            this._keys = new DataView(buffer);
-        } else {
-            this._keys = new DataView(new ArrayBuffer(8));
-            // start default key right at the middle fragment key, min event key
-            // so we have the same amount of key address space either way
-            this.fragmentId = MID;
-            this.eventIndex = MID;
-        }
-        this._fragmentIdComparer = fragmentIdComparer;
-    }
-
-    get fragmentId() {
-        return this._keys.getUint32(0, false);
-    }
-
-    set fragmentId(value) {
-        return this._keys.setUint32(0, value, false);
-    }
-
-    get eventIndex() {
-        return this._keys.getUint32(4, false);
-    }
-
-    set eventIndex(value) {
-        return this._keys.setUint32(4, value, false);
-    }
-
-    get buffer() {
-        return this._keys.buffer;
-    }
-
-    nextFragmentKey() {
-        const k = new SortKey(this._fragmentIdComparer);
-        k.fragmentId = this.fragmentId + 1;
-        k.eventIndex = MIN;
-        return k;
-    }
-
-    nextKey() {
-        const k = new SortKey(this._fragmentIdComparer);
-        k.fragmentId = this.fragmentId;
-        k.eventIndex = this.eventIndex + 1;
-        return k;
-    }
-
-    previousKey() {
-        const k = new SortKey(this._fragmentIdComparer);
-        k.fragmentId = this.fragmentId;
-        k.eventIndex = this.eventIndex - 1;
-        return k;
-    }
-
-    clone() {
-        const k = new SortKey();
-        k.fragmentId = this.fragmentId;
-        k.eventIndex = this.eventIndex; 
-        return k;
-    }
-
-    static get maxKey() {
-        const maxKey = new SortKey(null);
-        maxKey.fragmentId = MAX;
-        maxKey.eventIndex = MAX;
-        return maxKey;
-    }
-
-    static get minKey() {
-        const minKey = new SortKey(null);
-        minKey.fragmentId = MIN;
-        minKey.eventIndex = MIN;
-        return minKey;
-    }
-
-    static get firstLiveFragmentId() {
-        return MID;
-    }
-
-    static get firstLiveEventIndex() {
-        return MID;
-    }
-
-    compare(otherKey) {
-        const fragmentDiff = this.fragmentId - otherKey.fragmentId;
-        if (fragmentDiff === 0) {
-            return this.eventIndex - otherKey.eventIndex;
-        } else {
-            // minKey and maxKey might not have fragmentIdComparer, so short-circuit this first ...
-            if ((this.fragmentId === MIN && otherKey.fragmentId !== MIN) || (this.fragmentId !== MAX && otherKey.fragmentId === MAX)) {
-                return -1;
-            }
-            if ((this.fragmentId === MAX && otherKey.fragmentId !== MAX) || (this.fragmentId !== MIN && otherKey.fragmentId === MIN)) {
-                return 1;
-            }
-            // ... then delegate to fragmentIdComparer.
-            // This might throw if the relation of two fragments is unknown.
-            return this._fragmentIdComparer.compare(this.fragmentId, otherKey.fragmentId);
-        }
-    }
-
-    toString() {
-        return `[${this.fragmentId}/${this.eventIndex}]`;
-    }
-}
-
-//#ifdef TESTS
-export function tests() {
-    const fragmentIdComparer = {compare: (a, b) => a - b};
-
-    return {
-        test_no_fragment_index(assert) {
-            const min = SortKey.minKey;
-            const max = SortKey.maxKey;
-            const a = new SortKey();
-            a.eventIndex = 1;
-            a.fragmentId = 1;
-
-            assert(min.compare(min) === 0);
-            assert(max.compare(max) === 0);
-            assert(a.compare(a) === 0);
-
-            assert(min.compare(max) < 0);
-            assert(max.compare(min) > 0);
-
-            assert(min.compare(a) < 0);
-            assert(a.compare(min) > 0);
-            
-            assert(max.compare(a) > 0);
-            assert(a.compare(max) < 0);
-        },
-
-        test_default_key(assert) {
-            const k = new SortKey(fragmentIdComparer);
-            assert.equal(k.fragmentId, MID);
-            assert.equal(k.eventIndex, MID);
-        },
-
-        test_inc(assert) {
-            const a = new SortKey(fragmentIdComparer);
-            const b = a.nextKey();
-            assert.equal(a.fragmentId, b.fragmentId);
-            assert.equal(a.eventIndex + 1, b.eventIndex);
-            const c = b.previousKey();
-            assert.equal(b.fragmentId, c.fragmentId);
-            assert.equal(c.eventIndex + 1, b.eventIndex);
-            assert.equal(a.eventIndex, c.eventIndex);
-        },
-
-        test_min_key(assert) {
-            const minKey = SortKey.minKey;
-            const k = new SortKey(fragmentIdComparer);
-            assert(minKey.fragmentId <= k.fragmentId);
-            assert(minKey.eventIndex <= k.eventIndex);
-            assert(k.compare(minKey) > 0);
-            assert(minKey.compare(k) < 0);
-        },
-
-        test_max_key(assert) {
-            const maxKey = SortKey.maxKey;
-            const k = new SortKey(fragmentIdComparer);
-            assert(maxKey.fragmentId >= k.fragmentId);
-            assert(maxKey.eventIndex >= k.eventIndex);
-            assert(k.compare(maxKey) < 0);
-            assert(maxKey.compare(k) > 0);
-        },
-
-        test_immutable(assert) {
-            const a = new SortKey(fragmentIdComparer);
-            const fragmentId = a.fragmentId;
-            const eventIndex = a.eventIndex;
-            a.nextFragmentKey();
-            assert.equal(a.fragmentId, fragmentId);
-            assert.equal(a.eventIndex, eventIndex);
-        },
-
-        test_cmp_fragmentid_first(assert) {
-            const a = new SortKey(fragmentIdComparer);
-            const b = new SortKey(fragmentIdComparer);
-            a.fragmentId = 2;
-            a.eventIndex = 1;
-            b.fragmentId = 1;
-            b.eventIndex = 100000;
-            assert(a.compare(b) > 0);
-        },
-
-        test_cmp_eventindex_second(assert) {
-            const a = new SortKey(fragmentIdComparer);
-            const b = new SortKey(fragmentIdComparer);
-            a.fragmentId = 1;
-            a.eventIndex = 100000;
-            b.fragmentId = 1;
-            b.eventIndex = 2;
-            assert(a.compare(b) > 0);
-            assert(b.compare(a) < 0);
-        },
-
-        test_cmp_max_larger_than_min(assert) {
-            assert(SortKey.minKey.compare(SortKey.maxKey) < 0);
-        },
-
-        test_cmp_fragmentid_first_large(assert) {
-            const a = new SortKey(fragmentIdComparer);
-            const b = new SortKey(fragmentIdComparer);
-            a.fragmentId = MAX;
-            a.eventIndex = MIN;
-            b.fragmentId = MIN;
-            b.eventIndex = MAX;
-            assert(b < a);
-            assert(a > b);
-        }
-    };
-}
-//#endif

src/matrix/room/timeline/entries/BaseEntry.js

@@ -0,0 +1,20 @@
+//entries can be sorted, first by fragment, then by entry index.
+
+export default class BaseEntry {
+    get fragmentId() {
+        throw new Error("unimplemented");
+    }
+
+    get entryIndex() {
+        throw new Error("unimplemented");
+    }
+
+    compare(otherEntry) {
+        if (this.fragmentId === otherEntry.fragmentId) {
+            return this.entryIndex - otherEntry.entryIndex;
+        } else {
+            // This might throw if the relation of two fragments is unknown.
+            return this._fragmentIdComparer.compare(this.fragmentId, otherEntry.fragmentId);
+        }
+    }
+}

src/matrix/room/timeline/entries/EventEntry.js

@@ -0,0 +1,28 @@
+import BaseEntry from "./BaseEntry.js";
+
+export default class EventEntry extends BaseEntry {
+    constructor(eventEntry, fragmentIdComparator) {
+        super(fragmentIdComparator);
+        this._eventEntry = eventEntry;
+    }
+
+    get fragmentId() {
+        return this._eventEntry.fragmentId;
+    }
+
+    get entryIndex() {
+        return this._eventEntry.eventIndex;
+    }
+
+    get content() {
+        return this._eventEntry.event.content;
+    }
+
+    get type() {
+        return this._eventEntry.event.type;
+    }
+
+    get id() {
+        return this._eventEntry.event.event_id;
+    }
+}

src/matrix/room/timeline/entries/FragmentBoundaryEntry.js

@@ -0,0 +1,49 @@
+import BaseEntry from "./BaseEntry.js";
+
+export default class FragmentBoundaryEntry extends BaseEntry {
+    constructor(fragment, isFragmentStart, fragmentIdComparator) {
+        super(fragmentIdComparator);
+        this._fragment = fragment;
+        this._isFragmentStart = isFragmentStart;
+    }
+
+    static start(fragment, fragmentIdComparator) {
+        return new FragmentBoundaryEntry(fragment, true, fragmentIdComparator);
+    }
+
+    static end(fragment, fragmentIdComparator) {
+        return new FragmentBoundaryEntry(fragment, false, fragmentIdComparator);
+    }
+    
+    get hasStarted() {
+        return this._isFragmentStart;
+    }
+
+    get hasEnded() {
+        return !this.hasStarted;
+    }
+
+    get fragment() {
+        return this._fragment;
+    }
+
+    get fragmentId() {
+        return this._fragment.id;
+    }
+
+    get entryIndex() {
+        if (this.hasStarted) {
+            return Number.MIN_SAFE_INTEGER;
+        } else {
+            return Number.MAX_SAFE_INTEGER;
+        }
+    }
+
+    get isGap() {
+        if (this.hasStarted) {
+            return !!this.fragment.nextToken;
+        } else {
+            return !!this.fragment.previousToken;
+        }
+    }
+}