Arsharth/hydrogen-web
1
0
Fork 0
forked from mystiq/hydrogen-web
Code Issues Pull requests Projects Releases Packages Wiki Activity

split up RoomPersister to SyncPersister

Browse source 
also rename stores to timelineEvents and timelineFragments
This commit is contained in:
Bruno Windels 2019-05-12 20:24:06 +02:00
parent bf835ac01d
commit 89bc0e1696
8 changed files with 87 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/FRAGMENTS.md
View file

@ -17,7 +17,7 @@
how will fragments be exposed in timeline store?
- all read operations are passed a fragment id
- adapt persister
- persist fragments in /sync
- DONE: persist fragments in /sync
- load n items before and after key
- fill gaps / fragment filling
- add live fragment id optimization if we haven't done so already

11
src/matrix/room/room.js
View file

@ -1,7 +1,8 @@
import EventEmitter from "../../EventEmitter.js";
import RoomSummary from "./summary.js";
import RoomPersister from "./persister.js";
import Timeline from "./timeline.js";
import SyncPersister from "./timeline/persistence/SyncPersister.js";
import FragmentIdComparer from "./timeline/FragmentIdComparer.js";
export default class Room extends EventEmitter {
constructor({roomId, storage, hsApi, emitCollectionChange}) {
@ -10,14 +11,15 @@ export default class Room extends EventEmitter {
this._storage = storage;
this._hsApi = hsApi;
this._summary = new RoomSummary(roomId);
this._persister = new RoomPersister({roomId, storage});
this._fragmentIdComparer = new FragmentIdComparer([]);
this._syncPersister = new SyncPersister({roomId, storage, fragmentIdComparer: this._fragmentIdComparer});
this._emitCollectionChange = emitCollectionChange;
this._timeline = null;
}
persistSync(roomResponse, membership, txn) {
const summaryChanged = this._summary.applySync(roomResponse, membership, txn);
const newTimelineEntries = this._persister.persistSync(roomResponse, txn);
const newTimelineEntries = this._syncPersister.persistSync(roomResponse, txn);
return {summaryChanged, newTimelineEntries};
}
@ -33,7 +35,7 @@ export default class Room extends EventEmitter {
load(summary, txn) {
this._summary.load(summary);
return this._persister.load(txn);
return this._syncPersister.load(txn);
}
get name() {
@ -51,7 +53,6 @@ export default class Room extends EventEmitter {
this._timeline = new Timeline({
roomId: this.id,
storage: this._storage,
persister: this._persister,
hsApi: this._hsApi,
closeCallback: () => this._timeline = null,
});

2
src/matrix/room/timeline/FragmentIdIndex.js → src/matrix/room/timeline/FragmentIdComparer.js
View file

@ -106,7 +106,7 @@ class Island {
/*
index for fast lookup of how two fragments can be sorted
*/
export default class FragmentIdIndex {
export default class FragmentIdComparer {
constructor(fragments) {
this.rebuild(fragments);
}

134
src/matrix/room/persister.js → src/matrix/room/timeline/persistence/SyncPersister.js
View file

@ -1,40 +1,13 @@
import EventKey from "./timeline/EventKey.js";
import FragmentIdIndex from "./timeline/FragmentIdIndex.js";
import EventEntry from "./timeline/entries/EventEntry.js";
import FragmentBoundaryEntry from "./timeline/entries/FragmentBoundaryEntry.js";
import EventKey from "../EventKey.js";
import EventEntry from "../entries/EventEntry.js";
import FragmentBoundaryEntry from "../entries/FragmentBoundaryEntry.js";
import {createEventEntry} from "./common.js";
function gapEntriesAreEqual(a, b) {
if (!a || !b || !a.gap || !b.gap) {
return false;
}
const gapA = a.gap, gapB = b.gap;
return gapA.prev_batch === gapB.prev_batch && gapA.next_batch === gapB.next_batch;
}
function replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards) {
let replacedRange;
if (neighbourEventKey) {
replacedRange = backwards ?
roomTimeline.boundRange(neighbourEventKey, gapKey, false, true) :
roomTimeline.boundRange(gapKey, neighbourEventKey, true, false);
} else {
replacedRange = roomTimeline.onlyRange(gapKey);
}
const removedEntries = roomTimeline.getAndRemoveRange(this._roomId, replacedRange);
for (let entry of newEntries) {
roomTimeline.add(entry);
}
return removedEntries;
}
export default class RoomPersister {
constructor({roomId, storage}) {
export default class SyncPersister {
constructor({roomId, storage, fragmentIdComparer}) {
this._roomId = roomId;
this._storage = storage;
this._lastLiveKey = null;
this._fragmentIdIndex = new FragmentIdIndex([]); //only used when timeline is loaded ... e.g. "certain" methods on this class... split up?
this._fragmentIdComparer = fragmentIdComparer;
}
async load(txn) {
@ -52,77 +25,6 @@ export default class RoomPersister {
console.log("room persister load", this._roomId, this._lastLiveKey && this._lastLiveKey.toString());
}
async persistGapFill(gapEntry, response) {
const backwards = !!gapEntry.prev_batch;
const {chunk, start, end} = response;
if (!Array.isArray(chunk)) {
throw new Error("Invalid chunk in response");
}
if (typeof end !== "string") {
throw new Error("Invalid end token in response");
}
if ((backwards && start !== gapEntry.prev_batch) || (!backwards && start !== gapEntry.next_batch)) {
throw new Error("start is not equal to prev_batch or next_batch");
}
const gapKey = gapEntry.sortKey;
const txn = await this._storage.readWriteTxn([this._storage.storeNames.roomTimeline]);
let result;
try {
const roomTimeline = txn.roomTimeline;
// make sure what we've been given is actually persisted
// in the timeline, otherwise we're replacing something
// that doesn't exist (maybe it has been replaced already, or ...)
const persistedEntry = await roomTimeline.get(this._roomId, gapKey);
if (!gapEntriesAreEqual(gapEntry, persistedEntry)) {
throw new Error("Gap is not present in the timeline");
}
// find the previous event before the gap we could merge with
const neighbourEventEntry = await (backwards ?
roomTimeline.previousEvent(this._roomId, gapKey) :
roomTimeline.nextEvent(this._roomId, gapKey));
const neighbourEventId = neighbourEventEntry ? neighbourEventEntry.event.event_id : undefined;
const {newEntries, eventFound} = this._createNewGapEntries(chunk, end, gapKey, neighbourEventId, backwards);
const neighbourEventKey = eventFound ? neighbourEventEntry.sortKey : undefined;
const replacedEntries = replaceGapEntries(roomTimeline, newEntries, gapKey, neighbourEventKey, backwards);
result = {newEntries, replacedEntries};
} catch (err) {
txn.abort();
throw err;
}
await txn.complete();
return result;
}
_createNewGapEntries(chunk, nextPaginationToken, gapKey, neighbourEventId, backwards) {
if (backwards) {
// if backwards, the last events are the ones closest to the gap,
// and need to be assigned a key derived from the gap first,
// so swap order to only need one loop for both directions
chunk.reverse();
}
let sortKey = gapKey;
const {newEntries, eventFound} = chunk.reduce((acc, event) => {
acc.eventFound = acc.eventFound || event.event_id === neighbourEventId;
if (!acc.eventFound) {
acc.newEntries.push(this._createEventEntry(sortKey, event));
sortKey = backwards ? sortKey.previousKey() : sortKey.nextKey();
}
}, {newEntries: [], eventFound: false});
if (!eventFound) {
// as we're replacing an existing gap, no need to increment the gap index
newEntries.push(this._createGapEntry(sortKey, nextPaginationToken, backwards));
}
if (backwards) {
// swap resulting array order again if needed
newEntries.reverse();
}
return {newEntries, eventFound};
}
async _createLiveFragment(txn, previousToken) {
const liveFragment = await txn.roomFragments.liveFragment(this._roomId);
if (!liveFragment) {
@ -160,7 +62,7 @@ export default class RoomPersister {
nextToken: null
};
txn.roomFragments.add(newFragment);
return newFragment;
return {oldFragment, newFragment};
}
async persistSync(roomResponse, txn) {
@ -172,23 +74,23 @@ export default class RoomPersister {
// (but don't fail if it isn't, we won't be able to back-paginate though)
let liveFragment = await this._createLiveFragment(txn, timeline.prev_batch);
this._lastLiveKey = new EventKey(liveFragment.id, EventKey.defaultLiveKey.eventIndex);
entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdIndex));
entries.push(FragmentBoundaryEntry.start(liveFragment, this._fragmentIdComparer));
} else if (timeline.limited) {
// replace live fragment for limited sync, *only* if we had a live fragment already
const oldFragmentId = this._lastLiveKey.fragmentId;
this._lastLiveKey = this._lastLiveKey.nextFragmentKey();
const [oldFragment, newFragment] = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdIndex));
entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdIndex));
const {oldFragment, newFragment} = this._replaceLiveFragment(oldFragmentId, this._lastLiveKey.fragmentId, timeline.prev_batch, txn);
entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdComparer));
entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdComparer));
}
let currentKey = this._lastLiveKey;
const timeline = roomResponse.timeline;
if (timeline.events) {
for(const event of timeline.events) {
currentKey = currentKey.nextKey();
const entry = this._createEventEntry(currentKey, event);
const entry = createEventEntry(currentKey, event);
txn.roomTimeline.insert(entry);
entries.push(new EventEntry(entry, this._fragmentIdIndex));
entries.push(new EventEntry(entry, this._fragmentIdComparer));
}
}
// right thing to do? if the txn fails, not sure we'll continue anyways ...
@ -217,14 +119,6 @@ export default class RoomPersister {
return entries;
}
_createEventEntry(key, event) {
return {
fragmentId: key.fragmentId,
eventIndex: key.eventIndex,
event: event,
};
}
}
//#ifdef TESTS

7
src/matrix/room/timeline/persistence/common.js Normal file
View file

@ -0,0 +1,7 @@
export function createEventEntry(key, event) {
return {
fragmentId: key.fragmentId,
eventIndex: key.eventIndex,
event: event,
};
}

5
src/matrix/storage/idb/storage.js
View file

@ -1,6 +1,5 @@
import Transaction from "./transaction.js";
export const STORE_NAMES = ["session", "roomState", "roomSummary", "roomTimeline"];
import { STORE_NAMES } from "../common.js";
export default class Storage {
constructor(idbDatabase) {
@ -30,4 +29,4 @@ export default class Storage {
const txn = this._db.transaction(storeNames, "readwrite");
return new Transaction(txn, storeNames);
}
}
}

90
src/matrix/storage/idb/stores/RoomTimelineStore.js → src/matrix/storage/idb/stores/TimelineEventStore.js
View file

@ -1,4 +1,4 @@
import SortKey from "../../../room/timeline/SortKey.js";
import EventKey from "../../../room/timeline/EventKey.js";
class Range {
constructor(only, lower, upper, lowerOpen, upperOpen) {
@ -12,14 +12,14 @@ class Range {
asIDBKeyRange(roomId) {
// only
if (this._only) {
return IDBKeyRange.only([roomId, this._only.buffer]);
return IDBKeyRange.only([roomId, this._only.fragmentId, this._only.eventIndex]);
}
// lowerBound
// also bound as we don't want to move into another roomId
if (this._lower && !this._upper) {
return IDBKeyRange.bound(
[roomId, this._lower.buffer],
[roomId, SortKey.maxKey.buffer],
[roomId, this._lower.fragmentId, this._lower.eventIndex],
[roomId, EventKey.maxKey.fragmentId, EventKey.maxKey.eventIndex],
this._lowerOpen,
false
);
@ -28,8 +28,8 @@ class Range {
// also bound as we don't want to move into another roomId
if (!this._lower && this._upper) {
return IDBKeyRange.bound(
[roomId, SortKey.minKey.buffer],
[roomId, this._upper.buffer],
[roomId, EventKey.minKey.fragmentId, EventKey.minKey.eventIndex],
[roomId, this._upper.fragmentId, this._upper.eventIndex],
false,
this._upperOpen
);
@ -37,8 +37,8 @@ class Range {
// bound
if (this._lower && this._upper) {
return IDBKeyRange.bound(
[roomId, this._lower.buffer],
[roomId, this._upper.buffer],
[roomId, this._lower.fragmentId, this._lower.eventIndex],
[roomId, this._upper.fragmentId, this._upper.eventIndex],
this._lowerOpen,
this._upperOpen
);
@ -57,44 +57,44 @@ class Range {
*
* @typedef {Object} Entry
* @property {string} roomId
* @property {SortKey} sortKey
* @property {EventKey} eventKey
* @property {?Event} event if an event entry, the event
* @property {?Gap} gap if a gap entry, the gap
*/
export default class RoomTimelineStore {
export default class TimelineEventStore {
constructor(timelineStore) {
this._timelineStore = timelineStore;
}
/** Creates a range that only includes the given key
* @param {SortKey} sortKey the key
* @param {EventKey} eventKey the key
* @return {Range} the created range
*/
onlyRange(sortKey) {
return new Range(sortKey);
onlyRange(eventKey) {
return new Range(eventKey);
}
/** Creates a range that includes all keys before sortKey, and optionally also the key itself.
* @param {SortKey} sortKey the key
/** Creates a range that includes all keys before eventKey, and optionally also the key itself.
* @param {EventKey} eventKey the key
* @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the upper end.
* @return {Range} the created range
*/
upperBoundRange(sortKey, open=false) {
return new Range(undefined, undefined, sortKey, undefined, open);
upperBoundRange(eventKey, open=false) {
return new Range(undefined, undefined, eventKey, undefined, open);
}
/** Creates a range that includes all keys after sortKey, and optionally also the key itself.
* @param {SortKey} sortKey the key
/** Creates a range that includes all keys after eventKey, and optionally also the key itself.
* @param {EventKey} eventKey the key
* @param {boolean} [open=false] whether the key is included (false) or excluded (true) from the range at the lower end.
* @return {Range} the created range
*/
lowerBoundRange(sortKey, open=false) {
return new Range(undefined, sortKey, undefined, open);
lowerBoundRange(eventKey, open=false) {
return new Range(undefined, eventKey, undefined, open);
}
/** Creates a range that includes all keys between `lower` and `upper`, and optionally the given keys as well.
* @param {SortKey} lower the lower key
* @param {SortKey} upper the upper key
* @param {EventKey} lower the lower key
* @param {EventKey} upper the upper key
* @param {boolean} [lowerOpen=false] whether the lower key is included (false) or excluded (true) from the range.
* @param {boolean} [upperOpen=false] whether the upper key is included (false) or excluded (true) from the range.
* @return {Range} the created range
@ -110,9 +110,9 @@ export default class RoomTimelineStore {
* @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
*/
async lastEvents(roomId, fragmentId, amount) {
const sortKey = SortKey.maxKey;
sortKey.fragmentId = fragmentId;
return this.eventsBefore(roomId, sortKey, amount);
const eventKey = EventKey.maxKey;
eventKey.fragmentId = fragmentId;
return this.eventsBefore(roomId, eventKey, amount);
}
/** Looks up the first `amount` entries in the timeline for `roomId`.
@ -122,32 +122,32 @@ export default class RoomTimelineStore {
* @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
*/
async firstEvents(roomId, fragmentId, amount) {
const sortKey = SortKey.minKey;
sortKey.fragmentId = fragmentId;
return this.eventsAfter(roomId, sortKey, amount);
const eventKey = EventKey.minKey;
eventKey.fragmentId = fragmentId;
return this.eventsAfter(roomId, eventKey, amount);
}
/** Looks up `amount` entries after `sortKey` in the timeline for `roomId` within the same fragment.
* The entry for `sortKey` is not included.
/** Looks up `amount` entries after `eventKey` in the timeline for `roomId` within the same fragment.
* The entry for `eventKey` is not included.
* @param {string} roomId
* @param {SortKey} sortKey
* @param {EventKey} eventKey
* @param {number} amount
* @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
*/
eventsAfter(roomId, sortKey, amount) {
const idbRange = this.lowerBoundRange(sortKey, true).asIDBKeyRange(roomId);
eventsAfter(roomId, eventKey, amount) {
const idbRange = this.lowerBoundRange(eventKey, true).asIDBKeyRange(roomId);
return this._timelineStore.selectLimit(idbRange, amount);
}
/** Looks up `amount` entries before `sortKey` in the timeline for `roomId` within the same fragment.
* The entry for `sortKey` is not included.
/** Looks up `amount` entries before `eventKey` in the timeline for `roomId` within the same fragment.
* The entry for `eventKey` is not included.
* @param {string} roomId
* @param {SortKey} sortKey
* @param {EventKey} eventKey
* @param {number} amount
* @return {Promise<Entry[]>} a promise resolving to an array with 0 or more entries, in ascending order.
*/
async eventsBefore(roomId, sortKey, amount) {
const range = this.upperBoundRange(sortKey, true).asIDBKeyRange(roomId);
async eventsBefore(roomId, eventKey, amount) {
const range = this.upperBoundRange(eventKey, true).asIDBKeyRange(roomId);
const events = await this._timelineStore.selectLimitReverse(range, amount);
events.reverse(); // because we fetched them backwards
return events;
@ -196,7 +196,7 @@ export default class RoomTimelineStore {
return firstFoundEventId;
}
/** Inserts a new entry into the store. The combination of roomId and sortKey should not exist yet, or an error is thrown.
/** Inserts a new entry into the store. The combination of roomId and eventKey should not exist yet, or an error is thrown.
* @param {Entry} entry the entry to insert
* @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not.
* @throws {StorageError} ...
@ -206,7 +206,7 @@ export default class RoomTimelineStore {
return this._timelineStore.add(entry);
}
/** Updates the entry into the store with the given [roomId, sortKey] combination.
/** Updates the entry into the store with the given [roomId, eventKey] combination.
* If not yet present, will insert. Might be slower than add.
* @param {Entry} entry the entry to update.
* @return {Promise<>} a promise resolving to undefined if the operation was successful, or a StorageError if not.
@ -215,12 +215,16 @@ export default class RoomTimelineStore {
return this._timelineStore.put(entry);
}
get(roomId, sortKey) {
return this._timelineStore.get([roomId, sortKey]);
get(roomId, eventKey) {
return this._timelineStore.get([roomId, eventKey.fragmentId, eventKey.eventIndex]);
}
// returns the entries as well!! (or not always needed? I guess not always needed, so extra method)
removeRange(roomId, range) {
// TODO: read the entries!
return this._timelineStore.delete(range.asIDBKeyRange(roomId));
}
getByEventId(roomId, eventId) {
return this._timelineStore.index("byEventId").get([roomId, eventId]);
}
}

13
src/matrix/storage/idb/transaction.js
View file

@ -2,8 +2,9 @@ import {txnAsPromise} from "./utils.js";
import Store from "./store.js";
import SessionStore from "./stores/SessionStore.js";
import RoomSummaryStore from "./stores/RoomSummaryStore.js";
import RoomTimelineStore from "./stores/RoomTimelineStore.js";
import TimelineEventStore from "./stores/TimelineEventStore.js";
import RoomStateStore from "./stores/RoomStateStore.js";
import TimelineFragmentStore from "./stores/TimelineFragmentStore.js";
export default class Transaction {
constructor(txn, allowedStoreNames) {
@ -41,8 +42,12 @@ export default class Transaction {
return this._store("roomSummary", idbStore => new RoomSummaryStore(idbStore));
}
get roomTimeline() {
return this._store("roomTimeline", idbStore => new RoomTimelineStore(idbStore));
get timelineFragments() {
return this._store("timelineFragments", idbStore => new TimelineFragmentStore(idbStore));
}
get timelineEvents() {
return this._store("timelineEvents", idbStore => new TimelineEventStore(idbStore));
}
get roomState() {
@ -56,4 +61,4 @@ export default class Transaction {
abort() {
this._txn.abort();
}
}
}