From f5d3092031c6139719f25897b3e44d79957fb215 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 26 Jun 2020 23:26:24 +0200 Subject: [PATCH] WIP --- doc/impl-thoughts/MEMBERS.md | 8 ++ doc/impl-thoughts/timeline-member.md | 7 ++ src/domain/SessionLoadViewModel.js | 1 + src/domain/SessionPickerViewModel.js | 19 ++-- src/matrix/ServerFeatures.js | 19 ++++ src/matrix/SessionContainer.js | 9 +- src/matrix/Sync.js | 2 + src/matrix/room/Room.js | 36 ++++++- src/matrix/room/RoomMember.js | 69 +++++++++++++ src/matrix/room/RoomSummary.js | 14 ++- .../room/timeline/persistence/SyncWriter.js | 98 ++++++++++++++----- src/matrix/storage/common.js | 1 + src/matrix/storage/idb/StorageFactory.js | 29 ++---- src/matrix/storage/idb/Transaction.js | 5 + src/matrix/storage/idb/schema.js | 65 ++++++++++++ src/matrix/storage/idb/stores/MemberStore.js | 18 ---- .../storage/idb/stores/RoomMemberStore.js | 24 +++++ .../storage/idb/stores/RoomStateStore.js | 8 +- src/matrix/storage/idb/utils.js | 8 +- 19 files changed, 358 insertions(+), 82 deletions(-) create mode 100644 doc/impl-thoughts/timeline-member.md create mode 100644 src/matrix/ServerFeatures.js create mode 100644 src/matrix/room/RoomMember.js create mode 100644 src/matrix/storage/idb/schema.js delete mode 100644 src/matrix/storage/idb/stores/MemberStore.js create mode 100644 src/matrix/storage/idb/stores/RoomMemberStore.js diff --git a/doc/impl-thoughts/MEMBERS.md b/doc/impl-thoughts/MEMBERS.md index d2b1db3e..0103b879 100644 --- a/doc/impl-thoughts/MEMBERS.md +++ b/doc/impl-thoughts/MEMBERS.md @@ -1,3 +1,11 @@ +# TODO + +## Member list + + - support migrations in StorageFactory + - migrate all stores from key to key_path + - how to deal with members coming from backfill? do we even need to store them? + # How to store members? All of this is assuming we'll use lazy loading of members. diff --git a/doc/impl-thoughts/timeline-member.md b/doc/impl-thoughts/timeline-member.md new file mode 100644 index 00000000..a0c4eccb --- /dev/null +++ b/doc/impl-thoughts/timeline-member.md @@ -0,0 +1,7 @@ +## Get member for timeline event + +so when writing sync, we persist the display name and avatar + +the server might or might not support lazy loading + +if it is a room we just joined \ No newline at end of file diff --git a/src/domain/SessionLoadViewModel.js b/src/domain/SessionLoadViewModel.js index 2a7de3be..1e51a322 100644 --- a/src/domain/SessionLoadViewModel.js +++ b/src/domain/SessionLoadViewModel.js @@ -51,6 +51,7 @@ export class SessionLoadViewModel extends ViewModel { this._error = err; } finally { this._loading = false; + // loadLabel in case of sc.loadError also gets updated through this this.emitChange("loading"); } } diff --git a/src/domain/SessionPickerViewModel.js b/src/domain/SessionPickerViewModel.js index 78cb5bac..3c0bde03 100644 --- a/src/domain/SessionPickerViewModel.js +++ b/src/domain/SessionPickerViewModel.js @@ -159,13 +159,18 @@ export class SessionPickerViewModel extends ViewModel { } async import(json) { - const data = JSON.parse(json); - const {sessionInfo} = data; - sessionInfo.comment = `Imported on ${new Date().toLocaleString()} from id ${sessionInfo.id}.`; - sessionInfo.id = this._createSessionContainer().createNewSessionId(); - await this._storageFactory.import(sessionInfo.id, data.stores); - await this._sessionInfoStorage.add(sessionInfo); - this._sessions.set(new SessionItemViewModel(sessionInfo, this)); + try { + const data = JSON.parse(json); + const {sessionInfo} = data; + sessionInfo.comment = `Imported on ${new Date().toLocaleString()} from id ${sessionInfo.id}.`; + sessionInfo.id = this._createSessionContainer().createNewSessionId(); + await this._storageFactory.import(sessionInfo.id, data.stores); + await this._sessionInfoStorage.add(sessionInfo); + this._sessions.set(new SessionItemViewModel(sessionInfo, this)); + } catch (err) { + alert(err.message); + console.error(err); + } } async delete(id) { diff --git a/src/matrix/ServerFeatures.js b/src/matrix/ServerFeatures.js new file mode 100644 index 00000000..c2875942 --- /dev/null +++ b/src/matrix/ServerFeatures.js @@ -0,0 +1,19 @@ +const R0_5_0 = "r0.5.0"; + +export class ServerFeatures { + constructor(versionResponse) { + this._versionResponse = versionResponse; + } + + _supportsVersion(version) { + if (!this._versionResponse) { + return false; + } + const {versions} = this._versionResponse; + return Array.isArray(versions) && versions.includes(version); + } + + get lazyLoadMembers() { + return this._supportsVersion(R0_5_0); + } +} diff --git a/src/matrix/SessionContainer.js b/src/matrix/SessionContainer.js index f7e33c61..e5cd684c 100644 --- a/src/matrix/SessionContainer.js +++ b/src/matrix/SessionContainer.js @@ -94,7 +94,7 @@ export class SessionContainer { this._status.set(LoadStatus.LoginFailed); } else if (err instanceof ConnectionError) { this._loginFailure = LoginFailure.Connection; - this._status.set(LoadStatus.LoginFailure); + this._status.set(LoadStatus.LoginFailed); } else { this._status.set(LoadStatus.Error); } @@ -175,9 +175,14 @@ export class SessionContainer { } } // only transition into Ready once the first sync has succeeded - this._waitForFirstSyncHandle = this._sync.status.waitFor(s => s === SyncStatus.Syncing); + this._waitForFirstSyncHandle = this._sync.status.waitFor(s => s === SyncStatus.Syncing || s === SyncStatus.Stopped); try { await this._waitForFirstSyncHandle.promise; + if (this._sync.status.get() === SyncStatus.Stopped) { + if (this._sync.error) { + throw this._sync.error; + } + } } catch (err) { // if dispose is called from stop, bail out if (err instanceof AbortError) { diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index de079e4d..aba1ae30 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -92,6 +92,7 @@ export class Sync { storeNames.session, storeNames.roomSummary, storeNames.roomState, + storeNames.roomMembers, storeNames.timelineEvents, storeNames.timelineFragments, storeNames.pendingEvents, @@ -116,6 +117,7 @@ export class Sync { } } catch(err) { console.warn("aborting syncTxn because of error"); + console.error(err); // avoid corrupting state by only // storing the sync up till the point // the exception occurred diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index cc8e60d5..f7e9b335 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -34,7 +34,7 @@ export class Room extends EventEmitter { afterSync({summaryChanges, newTimelineEntries, newLiveKey, removedPendingEvents}) { this._syncWriter.afterSync(newLiveKey); if (summaryChanges) { - this._summary.afterSync(summaryChanges); + this._summary.applyChanges(summaryChanges); this.emit("change"); this._emitCollectionChange(this); } @@ -59,6 +59,40 @@ export class Room extends EventEmitter { return this._sendQueue.enqueueEvent(eventType, content); } + async loadMemberList() { + let members; + if (!this._summary.hasFetchedMembers) { + // we need to get the syncToken here! + const memberResponse = await this._hsApi.members(this._roomId, syncToken).response; + + const txn = await this._storage.readWriteTxn([ + this._storage.storeNames.roomSummary, + this._storage.storeNames.roomMembers, + ]); + const summaryChanges = this._summary.writeHasFetchedMembers(true, txn); + const {roomMembers} = txn; + const memberEvents = memberResponse.chunk; + if (!Array.isArray(memberEvents)) { + throw new Error("malformed"); + } + members = await Promise.all(memberEvents.map(async memberEvent => { + const userId = memberEvent && memberEvent.state_key; + if (!userId) { + throw new Error("malformed"); + } + const memberData = await roomMembers.get(this._roomId, userId); + const member = updateOrCreateMember(this._roomId, memberData, event); + if (member) { + roomMembers.set(member.serialize()); + } + return member; + })); + await txn.complete(); + this._summary.applyChanges(summaryChanges); + } + return new MemberList(this._roomId, members, this._storage); + } + /** @public */ async fillGap(fragmentEntry, amount) { diff --git a/src/matrix/room/RoomMember.js b/src/matrix/room/RoomMember.js new file mode 100644 index 00000000..f6e5237d --- /dev/null +++ b/src/matrix/room/RoomMember.js @@ -0,0 +1,69 @@ +export const EVENT_TYPE = "m.room.member"; + +export class RoomMember { + constructor(data) { + this._data = data; + } + + static async updateOrCreateMember(roomId, memberData, memberEvent) { + if (!memberEvent) { + return; + } + + const userId = memberEvent.state_key; + const {content} = memberEvent; + + if (!userId || !content) { + return; + } + + let member; + if (memberData) { + member = new RoomMember(memberData); + member.updateWithMemberEvent(memberEvent); + } else { + member = RoomMember.fromMemberEvent(this._roomId, memberEvent); + } + return member; + } + + static fromMemberEvent(roomId, memberEvent) { + const userId = memberEvent && memberEvent.state_key; + if (!userId) { + return; + } + + const member = new RoomMember({ + roomId: roomId, + userId: userId, + avatarUrl: null, + displayName: null, + membership: null, + deviceTrackingStatus: 0, + }); + member.updateWithMemberEvent(memberEvent); + return member; + } + + get roomId() { + return this._data.roomId; + } + + get userId() { + return this._data.userId; + } + + updateWithMemberEvent(event) { + if (!event || !event.content) { + return; + } + const {content} = event; + this._data.membership = content.membership; + this._data.avatarUrl = content.avatar_url; + this._data.displayName = content.displayname; + } + + serialize() { + return this.data; + } +} diff --git a/src/matrix/room/RoomSummary.js b/src/matrix/room/RoomSummary.js index 177c1bc4..b15d9998 100644 --- a/src/matrix/room/RoomSummary.js +++ b/src/matrix/room/RoomSummary.js @@ -82,6 +82,7 @@ class SummaryData { this.heroes = copy ? copy.heroes : null; this.canonicalAlias = copy ? copy.canonicalAlias : null; this.altAliases = copy ? copy.altAliases : null; + this.hasFetchedMembers = copy ? copy.hasFetchedMembers : false; this.cloned = copy ? true : false; } @@ -132,6 +133,17 @@ export class RoomSummary { return this._data.joinCount; } + get hasFetchedMembers() { + return this._data.hasFetchedMembers; + } + + writeHasFetchedMembers(value, txn) { + const data = new SummaryData(this._data); + data.hasFetchedMembers = value; + txn.roomSummary.set(data.serialize()); + return data; + } + writeSync(roomResponse, membership, txn) { // clear cloned flag, so cloneIfNeeded makes a copy and // this._data is not modified if any field is changed. @@ -149,7 +161,7 @@ export class RoomSummary { } } - afterSync(data) { + applyChanges(data) { this._data = data; } diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js index 92907b5a..6400aa9d 100644 --- a/src/matrix/room/timeline/persistence/SyncWriter.js +++ b/src/matrix/room/timeline/persistence/SyncWriter.js @@ -2,6 +2,7 @@ import {EventKey} from "../EventKey.js"; import {EventEntry} from "../entries/EventEntry.js"; import {FragmentBoundaryEntry} from "../entries/FragmentBoundaryEntry.js"; import {createEventEntry} from "./common.js"; +import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../RoomMember.js"; // Synapse bug? where the m.room.create event appears twice in sync response // when first syncing the room @@ -81,9 +82,76 @@ export class SyncWriter { return {oldFragment, newFragment}; } + async _writeMember(event, txn) { + if (!event) { + return; + } + + const userId = event.state_key; + const {content} = event; + + if (!userId || !content) { + return; + } + + let member; + if (memberData) { + member = new RoomMember(memberData); + member.updateWithMemberEvent(event); + } else { + member = RoomMember.fromMemberEvent(this._roomId, event); + } + } + + async _writeStateEvent(event, txn) { + if (event.type === MEMBER_EVENT_TYPE) { + const userId = event && event.state_key; + if (userId) { + const memberData = await txn.roomMembers.get(this._roomId, userId); + const member = updateOrCreateMember(this._roomId, memberData, event); + if (member) { + txn.roomMembers.set(member.serialize()); + } + } + } else { + txn.roomState.set(this._roomId, event); + } + } + + async _writeStateEvents(roomResponse, txn) { + // persist state + const {state, timeline} = roomResponse; + if (state.events) { + for (const event of state.events) { + await this._writeStateEvent(event, txn); + } + } + // persist live state events in timeline + if (timeline.events) { + for (const event of timeline.events) { + if (typeof event.state_key === "string") { + this._writeStateEvent(event, txn); + } + } + } + } + + _writeTimeline(entries, timeline, currentKey, txn) { + if (timeline.events) { + const events = deduplicateEvents(timeline.events); + for(const event of events) { + currentKey = currentKey.nextKey(); + const entry = createEventEntry(currentKey, this._roomId, event); + txn.timelineEvents.insert(entry); + entries.push(new EventEntry(entry, this._fragmentIdComparer)); + } + } + return currentKey; + } + async writeSync(roomResponse, txn) { const entries = []; - const timeline = roomResponse.timeline; + const {timeline} = roomResponse; let currentKey = this._lastLiveKey; if (!currentKey) { // means we haven't synced this room yet (just joined or did initial sync) @@ -101,30 +169,10 @@ export class SyncWriter { entries.push(FragmentBoundaryEntry.end(oldFragment, this._fragmentIdComparer)); entries.push(FragmentBoundaryEntry.start(newFragment, this._fragmentIdComparer)); } - if (timeline.events) { - const events = deduplicateEvents(timeline.events); - for(const event of events) { - currentKey = currentKey.nextKey(); - const entry = createEventEntry(currentKey, this._roomId, event); - txn.timelineEvents.insert(entry); - entries.push(new EventEntry(entry, this._fragmentIdComparer)); - } - } - // persist state - const state = roomResponse.state; - if (state.events) { - for (const event of state.events) { - txn.roomState.setStateEvent(this._roomId, event); - } - } - // persist live state events in timeline - if (timeline.events) { - for (const event of timeline.events) { - if (typeof event.state_key === "string") { - txn.roomState.setStateEvent(this._roomId, event); - } - } - } + + await this._writeStateEvents(roomResponse, txn); + + currentKey = this._writeTimeline(entries, timeline, currentKey, txn); return {entries, newLiveKey: currentKey}; } diff --git a/src/matrix/storage/common.js b/src/matrix/storage/common.js index 4ae55e65..fe0eebba 100644 --- a/src/matrix/storage/common.js +++ b/src/matrix/storage/common.js @@ -2,6 +2,7 @@ export const STORE_NAMES = Object.freeze([ "session", "roomState", "roomSummary", + "roomMembers", "timelineEvents", "timelineFragments", "pendingEvents", diff --git a/src/matrix/storage/idb/StorageFactory.js b/src/matrix/storage/idb/StorageFactory.js index 8f5babcd..00c8ead9 100644 --- a/src/matrix/storage/idb/StorageFactory.js +++ b/src/matrix/storage/idb/StorageFactory.js @@ -1,9 +1,10 @@ import {Storage} from "./Storage.js"; import { openDatabase, reqAsPromise } from "./utils.js"; import { exportSession, importSession } from "./export.js"; +import { schema } from "./schema.js"; const sessionName = sessionId => `brawl_session_${sessionId}`; -const openDatabaseWithSessionId = sessionId => openDatabase(sessionName(sessionId), createStores, 1); +const openDatabaseWithSessionId = sessionId => openDatabase(sessionName(sessionId), createStores, schema.length); export class StorageFactory { async create(sessionId) { @@ -28,26 +29,10 @@ export class StorageFactory { } } -function createStores(db) { - db.createObjectStore("session", {keyPath: "key"}); - // any way to make keys unique here? (just use put?) - db.createObjectStore("roomSummary", {keyPath: "roomId"}); +async function createStores(db, txn, oldVersion, version) { + const startIdx = oldVersion || 0; - // need index to find live fragment? prooobably ok without for now - //key = room_id | fragment_id - db.createObjectStore("timelineFragments", {keyPath: "key"}); - //key = room_id | fragment_id | event_index - const timelineEvents = db.createObjectStore("timelineEvents", {keyPath: "key"}); - //eventIdKey = room_id | event_id - timelineEvents.createIndex("byEventId", "eventIdKey", {unique: true}); - //key = room_id | event.type | event.state_key, - db.createObjectStore("roomState", {keyPath: "key"}); - db.createObjectStore("pendingEvents", {keyPath: "key"}); - - // const roomMembers = db.createObjectStore("roomMembers", {keyPath: [ - // "event.room_id", - // "event.content.membership", - // "event.state_key" - // ]}); - // roomMembers.createIndex("byName", ["room_id", "content.name"]); + for(let i = startIdx; i < version; ++i) { + await schema[i](db, txn); + } } diff --git a/src/matrix/storage/idb/Transaction.js b/src/matrix/storage/idb/Transaction.js index dc18321b..8e0dcc57 100644 --- a/src/matrix/storage/idb/Transaction.js +++ b/src/matrix/storage/idb/Transaction.js @@ -5,6 +5,7 @@ import {SessionStore} from "./stores/SessionStore.js"; import {RoomSummaryStore} from "./stores/RoomSummaryStore.js"; import {TimelineEventStore} from "./stores/TimelineEventStore.js"; import {RoomStateStore} from "./stores/RoomStateStore.js"; +import {RoomMemberStore} from "./stores/RoomMemberStore.js"; import {TimelineFragmentStore} from "./stores/TimelineFragmentStore.js"; import {PendingEventStore} from "./stores/PendingEventStore.js"; @@ -56,6 +57,10 @@ export class Transaction { return this._store("roomState", idbStore => new RoomStateStore(idbStore)); } + get roomMembers() { + return this._store("roomMembers", idbStore => new RoomMemberStore(idbStore)); + } + get pendingEvents() { return this._store("pendingEvents", idbStore => new PendingEventStore(idbStore)); } diff --git a/src/matrix/storage/idb/schema.js b/src/matrix/storage/idb/schema.js new file mode 100644 index 00000000..fc88b42f --- /dev/null +++ b/src/matrix/storage/idb/schema.js @@ -0,0 +1,65 @@ +import {iterateCursor} from "./utils.js"; +import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/RoomMember.js"; + +// FUNCTIONS SHOULD ONLY BE APPENDED!! +// the index in the array is the database version +export const schema = [ + createInitialStores, + createMemberStore, +]; +// TODO: how to deal with git merge conflicts of this array? + + +// how do we deal with schema updates vs existing data migration in a way that +//v1 +function createInitialStores(db) { + db.createObjectStore("session", {keyPath: "key"}); + // any way to make keys unique here? (just use put?) + db.createObjectStore("roomSummary", {keyPath: "roomId"}); + + // need index to find live fragment? prooobably ok without for now + //key = room_id | fragment_id + db.createObjectStore("timelineFragments", {keyPath: "key"}); + //key = room_id | fragment_id | event_index + const timelineEvents = db.createObjectStore("timelineEvents", {keyPath: "key"}); + //eventIdKey = room_id | event_id + timelineEvents.createIndex("byEventId", "eventIdKey", {unique: true}); + //key = room_id | event.type | event.state_key, + db.createObjectStore("roomState", {keyPath: "key"}); + db.createObjectStore("pendingEvents", {keyPath: "key"}); +} +//v2 +async function createMemberStore(db, txn) { + const roomMembers = db.createObjectStore("roomMembers", {keyPath: [ + "roomId", + "userId" + ]}); + // migrate existing member state events over + const roomState = txn.objectStore("roomState"); + await iterateCursor(roomState.openCursor(), entry => { + if (entry.event.type === MEMBER_EVENT_TYPE) { + roomState.delete(entry.key); + const member = RoomMember.fromMemberEvent(entry.roomId, entry.event); + if (member) { + roomMembers.add(member.serialize()); + } + } + }); +} + +function migrateKeyPathToArray(db, isNew) { + if (isNew) { + // create the new stores with the final name + } else { + // create the new stores with a tmp name + // migrate the data over + // change the name + } + + // maybe it is ok to just run all the migration steps? + // it might be a bit slower to create a store twice ... + // but at least the path of migration or creating a new store + // will go through the same code + // + // might not even be slower, as this is all happening within one transaction +} diff --git a/src/matrix/storage/idb/stores/MemberStore.js b/src/matrix/storage/idb/stores/MemberStore.js deleted file mode 100644 index e05da3e3..00000000 --- a/src/matrix/storage/idb/stores/MemberStore.js +++ /dev/null @@ -1,18 +0,0 @@ -// no historical members for now -class MemberStore { - async getMember(roomId, userId) { - - } - - /* async getMemberAtSortKey(roomId, userId, sortKey) { - - } */ - // multiple members here? does it happen at same sort key? - async setMembers(roomId, members) { - - } - - async getSortedMembers(roomId, offset, amount) { - - } -} diff --git a/src/matrix/storage/idb/stores/RoomMemberStore.js b/src/matrix/storage/idb/stores/RoomMemberStore.js new file mode 100644 index 00000000..67527ce2 --- /dev/null +++ b/src/matrix/storage/idb/stores/RoomMemberStore.js @@ -0,0 +1,24 @@ +// no historical members for now +export class RoomMemberStore { + constructor(roomMembersStore) { + this._roomMembersStore = roomMembersStore; + } + + get(roomId, userId) { + return this._roomMembersStore.get([roomId, userId]); + } + + async set(member) { + return this._roomMembersStore.put(member); + } + + /* + async getMemberAtSortKey(roomId, userId, sortKey) { + + } + + async getSortedMembers(roomId, offset, amount) { + + } + */ +} diff --git a/src/matrix/storage/idb/stores/RoomStateStore.js b/src/matrix/storage/idb/stores/RoomStateStore.js index 21c17c5a..65a973b0 100644 --- a/src/matrix/storage/idb/stores/RoomStateStore.js +++ b/src/matrix/storage/idb/stores/RoomStateStore.js @@ -3,15 +3,15 @@ export class RoomStateStore { this._roomStateStore = idbStore; } - async getEvents(type) { + async getAllForType(type) { } - async getEventsForKey(type, stateKey) { - + async get(type, stateKey) { + } - async setStateEvent(roomId, event) { + async set(roomId, event) { const key = `${roomId}|${event.type}|${event.state_key}`; const entry = {roomId, event, key}; return this._roomStateStore.put(entry); diff --git a/src/matrix/storage/idb/utils.js b/src/matrix/storage/idb/utils.js index 26f16eba..f80fe582 100644 --- a/src/matrix/storage/idb/utils.js +++ b/src/matrix/storage/idb/utils.js @@ -15,8 +15,9 @@ export function openDatabase(name, createObjectStore, version) { const req = window.indexedDB.open(name, version); req.onupgradeneeded = (ev) => { const db = ev.target.result; + const txn = ev.target.transaction; const oldVersion = ev.oldVersion; - createObjectStore(db, oldVersion, version); + createObjectStore(db, txn, oldVersion, version); }; return reqAsPromise(req); } @@ -52,7 +53,10 @@ export function iterateCursor(cursorRequest, processValue) { resolve(false); return; // end of results } - const {done, jumpTo} = processValue(cursor.value, cursor.key); + const result = processValue(cursor.value, cursor.key); + const done = result && result.done; + const jumpTo = result && result.jumpTo; + if (done) { resolve(true); } else if(jumpTo) {