From 784588440c09797b71a3d80fcca04560569c6362 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Sun, 19 May 2019 20:49:46 +0200 Subject: [PATCH] WIP for fragment support --- doc/FRAGMENTS.md | 18 ++++-- .../session/room/timeline/TilesCollection.js | 1 + .../room/timeline/FragmentIdComparer.js | 10 +-- src/matrix/room/timeline/Timeline.js | 7 +++ .../room/timeline/persistence/GapPersister.js | 10 +-- .../timeline/persistence/SyncPersister.js | 17 +++-- .../timeline/persistence/TimelineReader.js | 63 +++++++++++++++++++ .../room/timeline/persistence/common.js | 18 +++++- src/matrix/session.js | 1 + src/matrix/sync.js | 1 + src/observable/list/SortedArray.js | 38 +++++++++++ 11 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 src/matrix/room/timeline/persistence/TimelineReader.js create mode 100644 src/observable/list/SortedArray.js diff --git a/doc/FRAGMENTS.md b/doc/FRAGMENTS.md index a63a96ea..af944ed7 100644 --- a/doc/FRAGMENTS.md +++ b/doc/FRAGMENTS.md @@ -5,21 +5,31 @@ - SortKey - FragmentId - EventIndex - - write fragmentStore + - DONE: write fragmentStore - load all fragments - add a fragment (live on limited sync, or /context) - connect two fragments - update token on fragment (when filling gap or connecting two fragments) fragments can need connecting when filling a gap or creating a new /context fragment - - adapt timelineStore + - DONE: adapt timelineStore how will fragments be exposed in timeline store? - all read operations are passed a fragment id - adapt persister - DONE: persist fragments in /sync - - load n items before and after key - - fill gaps / fragment filling + - DONE: fill gaps / fragment filling + - load n items before and after key, + - need to add fragments as we come across boundaries + - also cache fragments? not for now ... + - not doing any of the above, just reloading and rebuilding for now + + - adapt Timeline + - turn ObservableArray into ObservableSortedArray + - upsert already sorted sections + - upsert single entry + - adapt TilesCollection & Tile to entry changes + - add live fragment id optimization if we haven't done so already - lets try to not have to have the fragmentindex in memory if the timeline isn't loaded - could do this by only loading all fragments into index when filling gaps, backpaginating, ... and on persister load only load the last fragment. This wouldn't even need a FragmentIndex? diff --git a/src/domain/session/room/timeline/TilesCollection.js b/src/domain/session/room/timeline/TilesCollection.js index 2b15bb97..36678764 100644 --- a/src/domain/session/room/timeline/TilesCollection.js +++ b/src/domain/session/room/timeline/TilesCollection.js @@ -22,6 +22,7 @@ export default class TilesCollection extends BaseObservableList { for (let entry of this._entries) { if (!currentTile || !currentTile.tryIncludeEntry(entry)) { currentTile = this._tileCreator(entry); + // if (currentTile) here? this._tiles.push(currentTile); } } diff --git a/src/matrix/room/timeline/FragmentIdComparer.js b/src/matrix/room/timeline/FragmentIdComparer.js index 7ffb9422..df5c3ac7 100644 --- a/src/matrix/room/timeline/FragmentIdComparer.js +++ b/src/matrix/room/timeline/FragmentIdComparer.js @@ -163,7 +163,7 @@ export default class FragmentIdComparer { export function tests() { return { test_1_island_3_fragments(assert) { - const index = new FragmentIdIndex([ + const index = new FragmentIdComparer([ {id: 3, previousId: 2}, {id: 1, nextId: 2}, {id: 2, nextId: 3, previousId: 1}, @@ -180,7 +180,7 @@ export function tests() { assert.equal(index.compare(1, 1), 0); }, test_2_island_dont_compare(assert) { - const index = new FragmentIdIndex([ + const index = new FragmentIdComparer([ {id: 1}, {id: 2}, ]); @@ -188,7 +188,7 @@ export function tests() { assert.throws(() => index.compare(2, 1)); }, test_2_island_compare_internally(assert) { - const index = new FragmentIdIndex([ + const index = new FragmentIdComparer([ {id: 1, nextId: 2}, {id: 2, previousId: 1}, {id: 11, nextId: 12}, @@ -203,12 +203,12 @@ export function tests() { assert.throws(() => index.compare(12, 2)); }, test_unknown_id(assert) { - const index = new FragmentIdIndex([{id: 1}]); + const index = new FragmentIdComparer([{id: 1}]); assert.throws(() => index.compare(1, 2)); assert.throws(() => index.compare(2, 1)); }, test_rebuild_flushes_old_state(assert) { - const index = new FragmentIdIndex([ + const index = new FragmentIdComparer([ {id: 1, nextId: 2}, {id: 2, previousId: 1}, ]); diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js index fcadeadf..2eb8d19b 100644 --- a/src/matrix/room/timeline/Timeline.js +++ b/src/matrix/room/timeline/Timeline.js @@ -1,6 +1,7 @@ import { ObservableArray } from "../../../observable/index.js"; import sortedIndex from "../../../utils/sortedIndex.js"; import GapPersister from "./persistence/GapPersister.js"; +import TimelineReader from "./persistence/TimelineReader.js"; export default class Timeline { constructor({roomId, storage, closeCallback, fragmentIdComparer}) { @@ -9,6 +10,11 @@ export default class Timeline { this._closeCallback = closeCallback; this._entriesList = new ObservableArray(); this._fragmentIdComparer = fragmentIdComparer; + this._timelineReader = new TimelineReader({ + roomId: this._roomId, + storage: this._storage, + fragmentIdComparer: this._fragmentIdComparer + }); } /** @package */ @@ -53,6 +59,7 @@ export default class Timeline { } async loadAtTop(amount) { + // TODO: use TimelineReader::readFrom here, and insert returned array at location for first and last entry. const firstEntry = this._entriesList.at(0); if (firstEntry) { const txn = await this._storage.readTxn([this._storage.storeNames.timelineEvents]); diff --git a/src/matrix/room/timeline/persistence/GapPersister.js b/src/matrix/room/timeline/persistence/GapPersister.js index 8eabb226..9b92409f 100644 --- a/src/matrix/room/timeline/persistence/GapPersister.js +++ b/src/matrix/room/timeline/persistence/GapPersister.js @@ -1,14 +1,6 @@ import EventKey from "../EventKey.js"; import EventEntry from "../entries/EventEntry.js"; -import {createEventEntry} from "./common.js"; - -function directionalAppend(array, value, direction) { - if (direction.isForward) { - array.push(value); - } else { - array.splice(0, 0, value); - } -} +import {createEventEntry, directionalAppend} from "./common.js"; export default class GapPersister { constructor({roomId, storage, fragmentIdComparer}) { diff --git a/src/matrix/room/timeline/persistence/SyncPersister.js b/src/matrix/room/timeline/persistence/SyncPersister.js index eaa3bfdd..97b21952 100644 --- a/src/matrix/room/timeline/persistence/SyncPersister.js +++ b/src/matrix/room/timeline/persistence/SyncPersister.js @@ -11,7 +11,7 @@ export default class SyncPersister { } async load(txn) { - const liveFragment = await txn.roomFragments.liveFragment(this._roomId); + const liveFragment = await txn.timelineFragments.liveFragment(this._roomId); if (liveFragment) { const [lastEvent] = await txn.roomTimeline.lastEvents(this._roomId, liveFragment.id, 1); // sorting and identifying (e.g. sort key and pk to insert) are a bit intertwined here @@ -26,7 +26,7 @@ export default class SyncPersister { } async _createLiveFragment(txn, previousToken) { - const liveFragment = await txn.roomFragments.liveFragment(this._roomId); + const liveFragment = await txn.timelineFragments.liveFragment(this._roomId); if (!liveFragment) { if (!previousToken) { previousToken = null; @@ -39,7 +39,7 @@ export default class SyncPersister { previousToken: previousToken, nextToken: null }; - txn.roomFragments.add(fragment); + txn.timelineFragments.add(fragment); return fragment; } else { return liveFragment; @@ -47,12 +47,12 @@ export default class SyncPersister { } async _replaceLiveFragment(oldFragmentId, newFragmentId, previousToken, txn) { - const oldFragment = await txn.roomFragments.get(oldFragmentId); + const oldFragment = await txn.timelineFragments.get(oldFragmentId); if (!oldFragment) { throw new Error(`old live fragment doesn't exist: ${oldFragmentId}`); } oldFragment.nextId = newFragmentId; - txn.roomFragments.update(oldFragment); + txn.timelineFragments.update(oldFragment); const newFragment = { roomId: this._roomId, id: newFragmentId, @@ -61,7 +61,7 @@ export default class SyncPersister { previousToken: previousToken, nextToken: null }; - txn.roomFragments.add(newFragment); + txn.timelineFragments.add(newFragment); return {oldFragment, newFragment}; } @@ -117,6 +117,11 @@ export default class SyncPersister { } } + if (timeline.limited) { + const fragments = await txn.timelineFragments.all(this._roomId); + this._fragmentIdComparer.rebuild(fragments); + } + return entries; } } diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js new file mode 100644 index 00000000..75402aee --- /dev/null +++ b/src/matrix/room/timeline/persistence/TimelineReader.js @@ -0,0 +1,63 @@ +import {directionalConcat, directionalAppend} from "./common.js"; +import EventKey from "../EventKey.js"; +import EventEntry from "../entries/EventEntry.js"; +import FragmentBoundaryEntry from "../entries/FragmentBoundaryEntry.js"; + +export default class TimelineReader { + constructor({roomId, storage, fragmentIdComparer}) { + this._roomId = roomId; + this._storage = storage; + this._fragmentIdComparer = fragmentIdComparer; + } + + async readFrom(eventKey, direction, amount) { + const txn = this._storage.readTxn([ + this._storage.storeNames.timelineEvents, + this._storage.storeNames.timelineFragments, + ]); + let entries = []; + let loadedFragment = false; + + const timelineStore = txn.timelineEvents; + const fragmentStore = txn.timelineFragments; + + while (entries.length < amount && eventKey) { + let eventsWithinFragment; + if (direction.isForward) { + eventsWithinFragment = timelineStore.eventsAfter(eventKey, amount); + } else { + eventsWithinFragment = timelineStore.eventsBefore(eventKey, amount); + } + const eventEntries = eventsWithinFragment.map(e => new EventEntry(e, this._fragmentIdComparer)); + entries = directionalConcat(entries, eventEntries, direction); + // prepend or append eventsWithinFragment to entries, and wrap them in EventEntry + + if (entries.length < amount) { + const fragment = await fragmentStore.get(this._roomId, eventKey.fragmentId); + // this._fragmentIdComparer.addFragment(fragment); + let fragmentEntry = new FragmentBoundaryEntry(fragment, direction.isBackward, this._fragmentIdComparer); + // append or prepend fragmentEntry, reuse func from GapPersister? + directionalAppend(entries, fragmentEntry, direction); + // don't count it in amount perhaps? or do? + if (fragmentEntry.linkedFragmentId) { + const nextFragment = await fragmentStore.get(this._roomId, fragmentEntry.linkedFragmentId); + // this._fragmentIdComparer.addFragment(nextFragment); + const nextFragmentEntry = new FragmentBoundaryEntry(nextFragment, direction.isForward, this._fragmentIdComparer); + directionalAppend(entries, nextFragmentEntry, direction); + eventKey = new EventKey(nextFragmentEntry.fragmentId, nextFragmentEntry.eventIndex); + loadedFragment = true; + } else { + eventKey = null; + } + } + } + + // reload fragments + if (loadedFragment) { + const fragments = await fragmentStore.all(this._roomId); + this._fragmentIdComparer.rebuild(fragments); + } + + return entries; + } +} diff --git a/src/matrix/room/timeline/persistence/common.js b/src/matrix/room/timeline/persistence/common.js index ac5a6c9d..5bec1a82 100644 --- a/src/matrix/room/timeline/persistence/common.js +++ b/src/matrix/room/timeline/persistence/common.js @@ -4,4 +4,20 @@ export function createEventEntry(key, event) { eventIndex: key.eventIndex, event: event, }; -} \ No newline at end of file +} + +export function directionalAppend(array, value, direction) { + if (direction.isForward) { + array.push(value); + } else { + array.unshift(value); + } +} + +export function directionalConcat(array, otherArray, direction) { + if (direction.isForward) { + return array.concat(otherArray); + } else { + return otherArray.concat(array); + } +} diff --git a/src/matrix/session.js b/src/matrix/session.js index d86065dc..763d0265 100644 --- a/src/matrix/session.js +++ b/src/matrix/session.js @@ -18,6 +18,7 @@ export default class Session { this._storage.storeNames.roomSummary, this._storage.storeNames.roomState, this._storage.storeNames.timelineEvents, + this._storage.storeNames.timelineFragments, ]); // restore session object this._session = await txn.session.get(); diff --git a/src/matrix/sync.js b/src/matrix/sync.js index b96f2110..b12dd73e 100644 --- a/src/matrix/sync.js +++ b/src/matrix/sync.js @@ -74,6 +74,7 @@ export default class Sync extends EventEmitter { storeNames.session, storeNames.roomSummary, storeNames.timelineEvents, + storeNames.timelineFragments, storeNames.roomState, ]); const roomChanges = []; diff --git a/src/observable/list/SortedArray.js b/src/observable/list/SortedArray.js new file mode 100644 index 00000000..e01bae4f --- /dev/null +++ b/src/observable/list/SortedArray.js @@ -0,0 +1,38 @@ +import BaseObservableList from "./BaseObservableList.js"; +import sortedIndex from "../../utils/sortedIndex"; + +export default class SortedArray extends BaseObservableList { + constructor(comparator) { + super(); + this._comparator = comparator; + this._items = []; + } + + setSortedMany(items) { + + } + + set(item) { + const idx = sortedIndex(this._items, item, this._comparator); + if (idx < this._items.length || this._comparator(this._items[idx], item) !== 0) { + this._items.splice(idx, 0, item); + //emitAdd + } else { + this._items[idx] = item; + //emitRemove + //emitAdd + } + } + + get array() { + return this._items; + } + + get length() { + return this._items.length; + } + + [Symbol.iterator]() { + return this._items.values(); + } +}