mystiq/hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity

Remove IDBValidKey references

Browse source
This commit is contained in:
Danila Fedorin 2021-08-31 14:48:38 -07:00
commit f5943ced97
32 changed files with 463 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/FAQ.md
View file

@ -37,3 +37,5 @@ There are no npm modules yet published for Hydrogen. The easiest is probably to
For example, for a single room chat, you could create an instance of `Platform`, you create a new `SessionContainer` with it, call `startWithLogin` on it, observe `sessionContainer.loadStatus` to know when initial sync is done, then do `sessionContainer.session.rooms.get('roomid')` and you create a `RoomViewModel` with it and pass that to a `RoomView`. Then you call `document.appendChild(roomView.mount())` and you should see a syncing room.
Feel free to ask for pointers in #hydrogen:matrix.org as the documentation is still lacking considerably. Note that at this early, pre 1.0 stage of the project, there is no promise of API stability yet.
Also, to make end-to-end encryption work, you'll likely need some tweaks to your build system, see [this issue](https://github.com/vector-im/hydrogen-web/issues/467).

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "hydrogen-web",
"version": "0.2.7",
"version": "0.2.8",
"description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB",
"main": "index.js",
"directories": {

2
src/matrix/SessionContainer.js
View file

@ -204,7 +204,7 @@ export class SessionContainer {
reconnector: this._reconnector,
});
this._sessionId = sessionInfo.id;
this._storage = await this._platform.storageFactory.create(sessionInfo.id);
this._storage = await this._platform.storageFactory.create(sessionInfo.id, log);
// no need to pass access token to session
const filteredSessionInfo = {
id: sessionInfo.id,

30
src/matrix/e2ee/DeviceTracker.js
View file

@ -19,6 +19,22 @@ import {verifyEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js";
const TRACKING_STATUS_OUTDATED = 0;
const TRACKING_STATUS_UPTODATE = 1;
export function addRoomToIdentity(identity, userId, roomId) {
if (!identity) {
identity = {
userId: userId,
roomIds: [roomId],
deviceTrackingStatus: TRACKING_STATUS_OUTDATED,
};
return identity;
} else {
if (!identity.roomIds.includes(roomId)) {
identity.roomIds.push(roomId);
return identity;
}
}
}
// map 1 device from /keys/query response to DeviceIdentity
function deviceKeysAsDeviceIdentity(deviceSection) {
const deviceId = deviceSection["device_id"];
@ -107,17 +123,9 @@ export class DeviceTracker {
async _writeMember(member, txn) {
const {userIdentities} = txn;
const identity = await userIdentities.get(member.userId);
if (!identity) {
userIdentities.set({
userId: member.userId,
roomIds: [member.roomId],
deviceTrackingStatus: TRACKING_STATUS_OUTDATED,
});
} else {
if (!identity.roomIds.includes(member.roomId)) {
identity.roomIds.push(member.roomId);
userIdentities.set(identity);
}
const updatedIdentity = addRoomToIdentity(identity, member.userId, member.roomId);
if (updatedIdentity) {
userIdentities.set(updatedIdentity);
}
}

4
src/matrix/room/Room.js
View file

@ -30,6 +30,7 @@ const EVENT_ENCRYPTED_TYPE = "m.room.encrypted";
export class Room extends BaseRoom {
constructor(options) {
super(options);
// TODO: pass pendingEvents to start like pendingOperations?
const {pendingEvents} = options;
const relationWriter = new RelationWriter({
roomId: this.id,
@ -120,7 +121,8 @@ export class Room extends BaseRoom {
txn.roomMembers.removeAllForRoom(this.id);
}
const {entries: newEntries, updatedEntries, newLiveKey, memberChanges} =
await log.wrap("syncWriter", log => this._syncWriter.writeSync(roomResponse, isRejoin, txn, log), log.level.Detail);
await log.wrap("syncWriter", log => this._syncWriter.writeSync(
roomResponse, isRejoin, summaryChanges.hasFetchedMembers, txn, log), log.level.Detail);
if (decryptChanges) {
const decryption = await log.wrap("decryptChanges", log => decryptChanges.write(txn, log));
log.set("decryptionResults", decryption.results.size);

10
src/matrix/room/members/RoomMember.js
View file

@ -141,6 +141,16 @@ export class MemberChange {
return this.previousMembership === "join" && this.membership !== "join";
}
/** The result can be a false negative when all of these apply:
* - the complete set of room members hasn't been fetched yet.
* - the member event for this change was received in the
* state section and wasn't present in the timeline section.
* - the room response was limited, e.g. there was a gap.
*
* This is because during sync, in this case it is not possible
* to distinguish between a new member that joined the room
* during a gap and a lazy-loading member.
* */
get hasJoined() {
return this.previousMembership !== "join" && this.membership === "join";
}

350
src/matrix/room/timeline/persistence/MemberWriter.js
View file

@ -23,57 +23,27 @@ export class MemberWriter {
this._cache = new LRUCache(5, member => member.userId);
}
writeTimelineMemberEvent(event, txn) {
return this._writeMemberEvent(event, false, txn);
prepareMemberSync(stateEvents, timelineEvents, hasFetchedMembers) {
return new MemberSync(this, stateEvents, timelineEvents, hasFetchedMembers);
}
writeStateMemberEvent(event, isLimited, txn) {
// member events in the state section when the room response
// is not limited must always be lazy loaded members.
// If they are not, they will be repeated in the timeline anyway.
return this._writeMemberEvent(event, !isLimited, txn);
}
async _writeMemberEvent(event, isLazyLoadingMember, txn) {
const userId = event.state_key;
if (!userId) {
return;
}
const member = RoomMember.fromMemberEvent(this._roomId, event);
if (!member) {
return;
}
let existingMember = this._cache.get(userId);
async _writeMember(member, txn) {
let existingMember = this._cache.get(member.userId);
if (!existingMember) {
const memberData = await txn.roomMembers.get(this._roomId, userId);
const memberData = await txn.roomMembers.get(this._roomId, member.userId);
if (memberData) {
existingMember = new RoomMember(memberData);
}
}
// either never heard of the member, or something changed
if (!existingMember || !existingMember.equals(member)) {
txn.roomMembers.set(member.serialize());
this._cache.set(member);
// we also return a member change for lazy loading members if something changed,
// so when the dupe timeline event comes and it doesn't see a diff
// with the cache, we already returned the event here.
//
// it's just important that we don't consider the first LL event
// for a user we see as a membership change, or we'll share keys with
// them, etc...
if (isLazyLoadingMember && !existingMember) {
// we don't have a previous member, but we know this is not a
// membership change as it's a lazy loaded
// member so take the membership from the member
return new MemberChange(member, member.membership);
}
return new MemberChange(member, existingMember?.membership);
}
}
async lookupMember(userId, event, timelineEvents, txn) {
async lookupMember(userId, txn) {
let member = this._cache.get(userId);
if (!member) {
const memberData = await txn.roomMembers.get(this._roomId, userId);
@ -82,61 +52,154 @@ export class MemberWriter {
this._cache.set(member);
}
}
if (!member) {
// sometimes the member event isn't included in state, but rather in the timeline,
// even if it is not the first event in the timeline. In this case, go look for
// the last one before the event, or if none is found,
// the least recent matching member event in the timeline.
// The latter is needed because of new joins picking up their own display name
let foundEvent = false;
let memberEventBefore;
let firstMemberEvent;
for (let i = timelineEvents.length - 1; i >= 0; i -= 1) {
const e = timelineEvents[i];
let matchingEvent;
if (e.type === MEMBER_EVENT_TYPE && e.state_key === userId) {
matchingEvent = e;
firstMemberEvent = matchingEvent;
return member;
}
}
class MemberSync {
constructor(memberWriter, stateEvents, timelineEvents, hasFetchedMembers) {
this._memberWriter = memberWriter;
this._timelineEvents = timelineEvents;
this._hasFetchedMembers = hasFetchedMembers;
this._newStateMembers = null;
if (stateEvents) {
this._newStateMembers = this._stateEventsToMembers(stateEvents);
}
}
get _roomId() {
return this._memberWriter._roomId;
}
_stateEventsToMembers(stateEvents) {
let members;
for (const event of stateEvents) {
if (event.type === MEMBER_EVENT_TYPE) {
const member = RoomMember.fromMemberEvent(this._roomId, event);
if (member) {
if (!members) {
members = new Map();
}
members.set(member.userId, member);
}
}
}
return members;
}
_timelineEventsToMembers(timelineEvents) {
let members;
// iterate backwards to only add the last member in the timeline
for (let i = timelineEvents.length - 1; i >= 0; i--) {
const e = timelineEvents[i];
const userId = e.state_key;
if (e.type === MEMBER_EVENT_TYPE && !members?.has(userId)) {
const member = RoomMember.fromMemberEvent(this._roomId, e);
if (member) {
if (!members) {
members = new Map();
}
members.set(member.userId, member);
}
}
}
return members;
}
async lookupMemberAtEvent(userId, event, txn) {
let member;
if (this._timelineEvents) {
member = this._findPrecedingMemberEventInTimeline(userId, event);
if (member) {
return member;
}
}
member = this._newStateMembers?.get(userId);
if (member) {
return member;
}
return await this._memberWriter.lookupMember(userId, txn);
}
async write(txn) {
const memberChanges = new Map();
let newTimelineMembers;
if (this._timelineEvents) {
newTimelineMembers = this._timelineEventsToMembers(this._timelineEvents);
}
if (this._newStateMembers) {
for (const member of this._newStateMembers.values()) {
if (!newTimelineMembers?.has(member.userId)) {
const memberChange = await this._memberWriter._writeMember(member, txn);
if (memberChange) {
// if the member event appeared only in the state section,
// AND we haven't heard about it AND we haven't fetched all members yet (to avoid #470),
// this may be a lazy loading member (if it's not in a gap, we are certain
// it is a ll member, in a gap, we can't tell), so we pass in our own membership as
// as the previous one so we won't consider it a join to not have false positives (to avoid #192).
// see also MemberChange.hasJoined
const maybeLazyLoadingMember = !this._hasFetchedMembers && !memberChange.previousMembership;
if (maybeLazyLoadingMember) {
memberChange.previousMembership = member.membership;
}
memberChanges.set(memberChange.userId, memberChange);
}
}
}
}
if (newTimelineMembers) {
for (const member of newTimelineMembers.values()) {
const memberChange = await this._memberWriter._writeMember(member, txn);
if (memberChange) {
memberChanges.set(memberChange.userId, memberChange);
}
}
}
return memberChanges;
}
// try to find the first member event before the given event,
// so we respect historical display names within the chunk of timeline
_findPrecedingMemberEventInTimeline(userId, event) {
let eventIndex = -1;
for (let i = this._timelineEvents.length - 1; i >= 0; i--) {
const e = this._timelineEvents[i];
if (e.event_id === event.event_id) {
eventIndex = i;
break;
}
}
for (let i = eventIndex - 1; i >= 0; i--) {
const e = this._timelineEvents[i];
if (e.type === MEMBER_EVENT_TYPE && e.state_key === userId) {
const member = RoomMember.fromMemberEvent(this._roomId, e);
if (member) {
return member;
}
if (!foundEvent) {
if (e.event_id === event.event_id) {
foundEvent = true;
}
} else if (matchingEvent) {
memberEventBefore = matchingEvent;
break;
}
}
// first see if we found a member event before the event we're looking up the sender for
if (memberEventBefore) {
member = RoomMember.fromMemberEvent(this._roomId, memberEventBefore);
}
// and only if we didn't, fall back to the first member event,
// regardless of where it is positioned relative to the lookup event
else if (firstMemberEvent) {
member = RoomMember.fromMemberEvent(this._roomId, firstMemberEvent);
}
}
return member;
}
}
export function tests() {
let idCounter = 0;
function createMemberEvent(membership, userId, displayName, avatarUrl) {
idCounter += 1;
return {
content: {
membership,
"displayname": displayName,
"avatar_url": avatarUrl
},
event_id: `$${idCounter}`,
sender: userId,
"state_key": userId,
type: "m.room.member"
};
}
function createStorage(initialMembers = []) {
const members = new Map();
for (const m of initialMembers) {
@ -164,102 +227,195 @@ export function tests() {
const avatar = "mxc://hs.tld/def";
return {
"new join through state": async assert => {
"new join": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage();
const change = await writer.writeStateMemberEvent(createMemberEvent("join", alice), true, txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("join", alice)], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasJoined);
assert.equal(txn.members.get(alice).membership, "join");
},
"accept invite through state": async assert => {
"accept invite": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("invite", alice)]);
const change = await writer.writeStateMemberEvent(createMemberEvent("join", alice), true, txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("join", alice)], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert.equal(change.previousMembership, "invite");
assert(change.hasJoined);
assert.equal(txn.members.get(alice).membership, "join");
},
"change display name through timeline": async assert => {
"change display name": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("join", alice, "Alies"), txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("join", alice, "Alies")], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(!change.hasJoined);
assert.equal(change.member.displayName, "Alies");
assert.equal(txn.members.get(alice).displayName, "Alies");
},
"set avatar through timeline": async assert => {
"set avatar": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("join", alice, "Alice", avatar), txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("join", alice, "Alice", avatar)], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(!change.hasJoined);
assert.equal(change.member.avatarUrl, avatar);
assert.equal(txn.members.get(alice).avatarUrl, avatar);
},
"ignore redundant member event": async assert => {
"ignore redundant member event in timeline": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice", avatar)]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("join", alice, "Alice", avatar), txn);
assert(!change);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("join", alice, "Alice", avatar)], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 0);
},
"ignore redundant member event in state": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice", avatar)]);
const memberSync = writer.prepareMemberSync([createMemberEvent("join", alice, "Alice", avatar)], [], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 0);
},
"leave": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("leave", alice, "Alice"), txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("leave", alice, "Alice")], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasLeft);
assert(!change.hasJoined);
},
"ban": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("ban", alice, "Alice"), txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("ban", alice, "Alice")], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasLeft);
assert(!change.hasJoined);
},
"reject invite": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("invite", alice, "Alice")]);
const change = await writer.writeTimelineMemberEvent(createMemberEvent("leave", alice, "Alice"), txn);
const memberSync = writer.prepareMemberSync([], [createMemberEvent("leave", alice, "Alice")], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(!change.hasLeft);
assert(!change.hasJoined);
},
"lazy loaded member we already know about doens't return change": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeStateMemberEvent(createMemberEvent("join", alice, "Alice"), false, txn);
assert(!change);
const memberSync = writer.prepareMemberSync([createMemberEvent("join", alice, "Alice")], [], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 0);
},
"lazy loaded member we already know about changes display name": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage([member("join", alice, "Alice")]);
const change = await writer.writeStateMemberEvent(createMemberEvent("join", alice, "Alies"), false, txn);
const memberSync = writer.prepareMemberSync([createMemberEvent("join", alice, "Alies")], [], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(!change.hasJoined);
assert.equal(change.member.displayName, "Alies");
},
"unknown lazy loaded member returns change, but not considered a membership change": async assert => {
"unknown lazy loaded member returns change, but not considered a join": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage();
const change = await writer.writeStateMemberEvent(createMemberEvent("join", alice, "Alice"), false, txn);
const memberSync = writer.prepareMemberSync([createMemberEvent("join", alice, "Alice")], [], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(!change.hasJoined);
assert(!change.hasLeft);
assert.equal(change.member.membership, "join");
assert.equal(txn.members.get(alice).displayName, "Alice");
},
"newly joined member causes a change with lookup done first": async assert => {
const event = createMemberEvent("join", alice, "Alice");
"new join through both timeline and state": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage();
const member = await writer.lookupMember(event.sender, event, [event], txn);
assert(member);
const change = await writer.writeTimelineMemberEvent(event, txn);
assert(change);
const aliceJoin = createMemberEvent("join", alice, "Alice");
const memberSync = writer.prepareMemberSync([aliceJoin], [aliceJoin], false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasJoined);
assert(!change.hasLeft);
},
"lookupMember returns closest member in the past": async assert => {
"change display name in timeline with lazy loaded member in state": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage();
const memberSync = writer.prepareMemberSync(
[createMemberEvent("join", alice, "Alice")],
[createMemberEvent("join", alice, "Alies")],
false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasJoined);
assert(!change.hasLeft);
assert.equal(change.member.displayName, "Alies");
},
"lookupMemberAtEvent returns closest member in the past": async assert => {
const event1 = createMemberEvent("join", alice, "Alice");
const event2 = createMemberEvent("join", alice, "Alies");
const event3 = createMemberEvent("join", alice, "Alys");
const events = [event1, event2, event3];
// we write first because the MemberWriter assumes it is called before
// the SyncWriter does any lookups
const writer = new MemberWriter(roomId);
const txn = createStorage();
const member = await writer.lookupMember(event3.sender, event3, [event1, event2, event3], txn);
const memberSync = await writer.prepareMemberSync([], events, false);
let member = await memberSync.lookupMemberAtEvent(event1.sender, event1, txn);
assert.equal(member, undefined);
member = await memberSync.lookupMemberAtEvent(event2.sender, event2, txn);
assert.equal(member.displayName, "Alice");
member = await memberSync.lookupMemberAtEvent(event3.sender, event3, txn);
assert.equal(member.displayName, "Alies");
assert.equal(txn.members.size, 0);
const changes = await memberSync.write(txn);
assert.equal(txn.members.size, 1);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasJoined);
},
"lookupMemberAtEvent falls back on state event": async assert => {
const event1 = createMemberEvent("join", alice, "Alice");
const event2 = createMemberEvent("join", alice, "Alies");
// we write first because the MemberWriter assumes it is called before
// the SyncWriter does any lookups
const writer = new MemberWriter(roomId);
const txn = createStorage();
const memberSync = await writer.prepareMemberSync([event1], [event2], false);
const member = await memberSync.lookupMemberAtEvent(event2.sender, event2, txn);
assert.equal(member.displayName, "Alice");
assert.equal(txn.members.size, 0);
const changes = await memberSync.write(txn);
assert.equal(txn.members.size, 1);
assert.equal(changes.size, 1);
const change = changes.get(alice);
assert(change.hasJoined);
},
"write works without event arrays": async assert => {
const writer = new MemberWriter(roomId);
const txn = createStorage();
const memberSync = await writer.prepareMemberSync(undefined, undefined, false);
const changes = await memberSync.write(txn);
assert.equal(changes.size, 0);
},
};
}

71
src/matrix/room/timeline/persistence/SyncWriter.js
View file

@ -133,38 +133,31 @@ export class SyncWriter {
return currentKey;
}
async _writeStateEvents(roomResponse, memberChanges, isLimited, txn, log) {
// persist state
const {state} = roomResponse;
if (Array.isArray(state?.events)) {
log.set("stateEvents", state.events.length);
for (const event of state.events) {
if (event.type === MEMBER_EVENT_TYPE) {
const memberChange = await this._memberWriter.writeStateMemberEvent(event, isLimited, txn);
if (memberChange) {
memberChanges.set(memberChange.userId, memberChange);
}
} else {
txn.roomState.set(this._roomId, event);
}
async _writeStateEvents(stateEvents, txn, log) {
let nonMemberStateEvents = 0;
for (const event of stateEvents) {
// member events are written prior by MemberWriter
if (event.type !== MEMBER_EVENT_TYPE) {
txn.roomState.set(this._roomId, event);
nonMemberStateEvents += 1;
}
}
log.set("stateEvents", nonMemberStateEvents);
}
async _writeTimeline(timeline, currentKey, memberChanges, txn, log) {
async _writeTimeline(timelineEvents, timeline, memberSync, currentKey, txn, log) {
const entries = [];
const updatedEntries = [];
if (Array.isArray(timeline?.events) && timeline.events.length) {
if (timelineEvents?.length) {
// only create a fragment when we will really write an event
currentKey = await this._ensureLiveFragment(currentKey, entries, timeline, txn, log);
const events = deduplicateEvents(timeline.events);
log.set("timelineEvents", events.length);
log.set("timelineEvents", timelineEvents.length);
let timelineStateEventCount = 0;
for(const event of events) {
for(const event of timelineEvents) {
// store event in timeline
currentKey = currentKey.nextKey();
const storageEntry = createEventEntry(currentKey, this._roomId, event);
let member = await this._memberWriter.lookupMember(event.sender, event, events, txn);
let member = await memberSync.lookupMemberAtEvent(event.sender, event, txn);
if (member) {
storageEntry.displayName = member.displayName;
storageEntry.avatarUrl = member.avatarUrl;
@ -178,17 +171,11 @@ export class SyncWriter {
}
// update state events after writing event, so for a member event,
// we only update the member info after having written the member event
// to the timeline, as we want that event to have the old profile info
if (typeof event.state_key === "string") {
// to the timeline, as we want that event to have the old profile info.
// member events are written prior by MemberWriter.
if (typeof event.state_key === "string" && event.type !== MEMBER_EVENT_TYPE) {
timelineStateEventCount += 1;
if (event.type === MEMBER_EVENT_TYPE) {
const memberChange = await this._memberWriter.writeTimelineMemberEvent(event, txn);
if (memberChange) {
memberChanges.set(memberChange.userId, memberChange);
}
} else {
txn.roomState.set(this._roomId, event);
}
txn.roomState.set(this._roomId, event);
}
}
log.set("timelineStateEventCount", timelineStateEventCount);
@ -224,14 +211,13 @@ export class SyncWriter {
* @type {SyncWriterResult}
* @property {Array<BaseEntry>} entries new timeline entries written
* @property {EventKey} newLiveKey the advanced key to write events at
* @property {Map<string, MemberChange>} memberChanges member changes in the processed sync ny user id
*
* @param {Object} roomResponse [description]
* @param {boolean} isRejoin whether the room was rejoined in the sync being processed
* @param {Transaction} txn
* @return {SyncWriterResult}
*/
async writeSync(roomResponse, isRejoin, txn, log) {
async writeSync(roomResponse, isRejoin, hasFetchedMembers, txn, log) {
let {timeline} = roomResponse;
// we have rejoined the room after having synced it before,
// check for overlap with the last synced event
@ -239,13 +225,22 @@ export class SyncWriter {
if (isRejoin) {
timeline = await this._handleRejoinOverlap(timeline, txn, log);
}
const memberChanges = new Map();
// important this happens before _writeTimeline so
// members are available in the transaction
await this._writeStateEvents(roomResponse, memberChanges, timeline?.limited, txn, log);
let timelineEvents;
if (Array.isArray(timeline?.events)) {
timelineEvents = deduplicateEvents(timeline.events);
}
const {state} = roomResponse;
let stateEvents;
if (Array.isArray(state?.events)) {
stateEvents = state.events;
}
const memberSync = this._memberWriter.prepareMemberSync(stateEvents, timelineEvents, hasFetchedMembers);
if (stateEvents) {
await this._writeStateEvents(stateEvents, txn, log);
}
const {currentKey, entries, updatedEntries} =
await this._writeTimeline(timeline, this._lastLiveKey, memberChanges, txn, log);
log.set("memberChanges", memberChanges.size);
await this._writeTimeline(timelineEvents, timeline, memberSync, this._lastLiveKey, txn, log);
const memberChanges = await memberSync.write(txn);
return {entries, updatedEntries, newLiveKey: currentKey, memberChanges};
}

20
src/matrix/storage/idb/StorageFactory.js
View file

@ -21,8 +21,9 @@ import { schema } from "./schema.js";
import { detectWebkitEarlyCloseTxnBug } from "./quirks.js";
const sessionName = sessionId => `hydrogen_session_${sessionId}`;
const openDatabaseWithSessionId = function(sessionId, idbFactory) {
return openDatabase(sessionName(sessionId), createStores, schema.length, idbFactory);
const openDatabaseWithSessionId = function(sessionId, idbFactory, log) {
const create = (db, txn, oldVersion, version) => createStores(db, txn, oldVersion, version, log);
return openDatabase(sessionName(sessionId), create, schema.length, idbFactory);
}
async function requestPersistedStorage() {
@ -49,7 +50,7 @@ export class StorageFactory {
this._IDBKeyRange = IDBKeyRange;
}
async create(sessionId) {
async create(sessionId, log) {
await this._serviceWorkerHandler?.preventConcurrentSessionAccess(sessionId);
requestPersistedStorage().then(persisted => {
// Firefox lies here though, and returns true even if the user denied the request
@ -59,7 +60,7 @@ export class StorageFactory {
});
const hasWebkitEarlyCloseTxnBug = await detectWebkitEarlyCloseTxnBug(this._idbFactory);
const db = await openDatabaseWithSessionId(sessionId, this._idbFactory);
const db = await openDatabaseWithSessionId(sessionId, this._idbFactory, log);
return new Storage(db, this._IDBKeyRange, hasWebkitEarlyCloseTxnBug);
}
@ -80,10 +81,11 @@ export class StorageFactory {
}
}
async function createStores(db, txn, oldVersion, version) {
async function createStores(db, txn, oldVersion, version, log) {
const startIdx = oldVersion || 0;
for(let i = startIdx; i < version; ++i) {
await schema[i](db, txn);
}
return log.wrap({l: "storage migration", oldVersion, version}, async log => {
for(let i = startIdx; i < version; ++i) {
await log.wrap(`v${i + 1}`, log => schema[i](db, txn, log));
}
});
}

8
src/matrix/storage/idb/Store.ts
View file

@ -148,7 +148,7 @@ export class Store<T> extends QueryTarget<T> {
return new QueryTarget<T>(new QueryTargetWrapper<T>(this._idbStore.index(indexName)));
}
put(value: T): Promise<IDBValidKey> {
put(value: T): void {
// If this request fails, the error will bubble up to the transaction and abort it,
// which is the behaviour we want. Therefore, it is ok to not create a promise for this
// request and await it.
@ -159,12 +159,12 @@ export class Store<T> extends QueryTarget<T> {
//
// Note that this can still throw synchronously, like it does for TransactionInactiveError,
// see https://www.w3.org/TR/IndexedDB-2/#transaction-lifetime-concept
return reqAsPromise(this._idbStore.put(value));
this._idbStore.put(value);
}
add(value: T): Promise<IDBValidKey> {
add(value: T): void {
// ok to not monitor result of request, see comment in `put`.
return reqAsPromise(this._idbStore.add(value));
this._idbStore.add(value);
}
delete(keyOrKeyRange: IDBValidKey | IDBKeyRange): Promise<undefined> {

17
src/matrix/storage/idb/error.ts
View file

@ -25,23 +25,18 @@ function _sourceName(source: IDBIndex | IDBObjectStore): string {
function _sourceDatabase(source: IDBIndex | IDBObjectStore): string {
return "objectStore" in source ?
source.objectStore.transaction.db.name :
source.transaction.db.name;
source.objectStore?.transaction?.db?.name :
source.transaction?.db?.name;
}
export class IDBError extends StorageError {
storeName: string;
databaseName: string;
constructor(message: string, source: IDBIndex | IDBCursor | IDBObjectStore, cause: DOMException | null = null) {
let storeName: string, databaseName: string;
if (source instanceof IDBCursor) {
storeName = _sourceName(source.source);
databaseName = _sourceDatabase(source.source);
} else {
storeName = _sourceName(source);
databaseName = _sourceDatabase(source);
}
constructor(message: string, sourceOrCursor: IDBIndex | IDBCursor | IDBObjectStore, cause: DOMException | null = null) {
const source = "source" in sourceOrCursor ? sourceOrCursor.source : sourceOrCursor;
const storeName = _sourceName(source);
const databaseName = _sourceDatabase(source);
let fullMessage = `${message} on ${databaseName}.${storeName}`;
if (cause) {
fullMessage += ": ";

47
src/matrix/storage/idb/schema.js
View file

@ -1,8 +1,10 @@
import {iterateCursor, reqAsPromise} from "./utils";
import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js";
import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js";
import {RoomMemberStore} from "./stores/RoomMemberStore";
import {SessionStore} from "./stores/SessionStore";
import {encodeScopeTypeKey} from "./stores/OperationStore";
import {MAX_UNICODE} from "./stores/common";
// FUNCTIONS SHOULD ONLY BE APPENDED!!
// the index in the array is the database version
@ -17,6 +19,7 @@ export const schema = [
createArchivedRoomSummaryStore,
migrateOperationScopeIndex,
createTimelineRelationsStore,
fixMissingRoomsInUserIdentities
];
// TODO: how to deal with git merge conflicts of this array?
@ -142,3 +145,47 @@ async function migrateOperationScopeIndex(db, txn) {
function createTimelineRelationsStore(db) {
db.createObjectStore("timelineRelations", {keyPath: "key"});
}
//v11 doesn't change the schema, but ensures all userIdentities have all the roomIds they should (see #470)
async function fixMissingRoomsInUserIdentities(db, txn, log) {
const roomSummaryStore = txn.objectStore("roomSummary");
const trackedRoomIds = [];
await iterateCursor(roomSummaryStore.openCursor(), roomSummary => {
if (roomSummary.isTrackingMembers) {
trackedRoomIds.push(roomSummary.roomId);
}
});
const outboundGroupSessionsStore = txn.objectStore("outboundGroupSessions");
const userIdentitiesStore = txn.objectStore("userIdentities");
const roomMemberStore = txn.objectStore("roomMembers");
for (const roomId of trackedRoomIds) {
let foundMissing = false;
const joinedUserIds = [];
const memberRange = IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true);
await log.wrap({l: "room", id: roomId}, async log => {
await iterateCursor(roomMemberStore.openCursor(memberRange), member => {
if (member.membership === "join") {
joinedUserIds.push(member.userId);
}
});
log.set("joinedUserIds", joinedUserIds.length);
for (const userId of joinedUserIds) {
const identity = await reqAsPromise(userIdentitiesStore.get(userId));
const originalRoomCount = identity?.roomIds?.length;
const updatedIdentity = addRoomToIdentity(identity, userId, roomId);
if (updatedIdentity) {
log.log({l: `fixing up`, id: userId,
roomsBefore: originalRoomCount, roomsAfter: updatedIdentity.roomIds.length});
userIdentitiesStore.put(updatedIdentity);
foundMissing = true;
}
}
log.set("foundMissing", foundMissing);
if (foundMissing) {
// clear outbound megolm session,
// so we'll create a new one on the next message that will be properly shared
outboundGroupSessionsStore.delete(roomId);
}
});
}
}

4
src/matrix/storage/idb/stores/AccountDataStore.ts
View file

@ -32,7 +32,7 @@ export class AccountDataStore {
return await this._store.get(type);
}
set(event: AccountDataEntry): Promise<IDBValidKey> {
return this._store.put(event);
set(event: AccountDataEntry): void {
this._store.put(event);
}
}

4
src/matrix/storage/idb/stores/DeviceIdentityStore.ts
View file

@ -69,9 +69,9 @@ export class DeviceIdentityStore {
return this._store.get(encodeKey(userId, deviceId));
}
set(deviceIdentity: DeviceIdentity): Promise<IDBValidKey> {
set(deviceIdentity: DeviceIdentity): void {
deviceIdentity.key = encodeKey(deviceIdentity.userId, deviceIdentity.deviceId);
return this._store.put(deviceIdentity);
this._store.put(deviceIdentity);
}
getByCurve25519Key(curve25519Key: string): Promise<DeviceIdentity | null> {

4
src/matrix/storage/idb/stores/GroupSessionDecryptionStore.ts
View file

@ -38,9 +38,9 @@ export class GroupSessionDecryptionStore {
return this._store.get(encodeKey(roomId, sessionId, messageIndex));
}
set(roomId: string, sessionId: string, messageIndex: number, decryption: GroupSessionDecryption): Promise<IDBValidKey> {
set(roomId: string, sessionId: string, messageIndex: number, decryption: GroupSessionDecryption): void {
decryption.key = encodeKey(roomId, sessionId, messageIndex);
return this._store.put(decryption);
this._store.put(decryption);
}
removeAllForRoom(roomId: string): Promise<undefined> {

4
src/matrix/storage/idb/stores/InboundGroupSessionStore.ts
View file

@ -48,9 +48,9 @@ export class InboundGroupSessionStore {
return this._store.get(encodeKey(roomId, senderKey, sessionId));
}
set(session: InboundGroupSession): Promise<IDBValidKey> {
set(session: InboundGroupSession): void {
session.key = encodeKey(session.roomId, session.senderKey, session.sessionId);
return this._store.put(session);
this._store.put(session);
}
removeAllForRoom(roomId: string): Promise<undefined> {

4
src/matrix/storage/idb/stores/InviteStore.ts
View file

@ -41,8 +41,8 @@ export class InviteStore {
return this._inviteStore.selectAll();
}
set(invite: InviteData): Promise<IDBValidKey> {
return this._inviteStore.put(invite);
set(invite: InviteData): void {
this._inviteStore.put(invite);
}
remove(roomId: string): void {

4
src/matrix/storage/idb/stores/OlmSessionStore.ts
View file

@ -65,9 +65,9 @@ export class OlmSessionStore {
return this._store.get(encodeKey(senderKey, sessionId));
}
set(session: OlmSession): Promise<IDBValidKey> {
set(session: OlmSession): void {
session.key = encodeKey(session.senderKey, session.sessionId);
return this._store.put(session);
this._store.put(session);
}
remove(senderKey: string, sessionId: string): Promise<undefined> {

8
src/matrix/storage/idb/stores/OperationStore.ts
View file

@ -61,13 +61,13 @@ export class OperationStore {
return results;
}
add(operation: Operation): Promise<IDBValidKey> {
add(operation: Operation): void {
operation.scopeTypeKey = encodeScopeTypeKey(operation.scope, operation.type);
return this._store.add(operation);
this._store.add(operation);
}
update(operation: Operation): Promise<IDBValidKey> {
return this._store.put(operation);
update(operation: Operation): void {
this._store.put(operation);
}
remove(id: string): Promise<undefined> {

4
src/matrix/storage/idb/stores/OutboundGroupSessionStore.ts
View file

@ -36,7 +36,7 @@ export class OutboundGroupSessionStore {
return this._store.get(roomId);
}
set(session: OutboundSession): Promise<IDBValidKey> {
return this._store.put(session);
set(session: OutboundSession): void {
this._store.put(session);
}
}

8
src/matrix/storage/idb/stores/PendingEventStore.ts
View file

@ -73,13 +73,13 @@ export class PendingEventStore {
return !!key;
}
add(pendingEvent: PendingEntry): Promise<IDBValidKey> {
add(pendingEvent: PendingEntry): void {
pendingEvent.key = encodeKey(pendingEvent.roomId, pendingEvent.queueIndex);
return this._eventStore.add(pendingEvent);
this._eventStore.add(pendingEvent);
}
update(pendingEvent: PendingEntry): Promise<IDBValidKey> {
return this._eventStore.put(pendingEvent);
update(pendingEvent: PendingEntry): void {
this._eventStore.put(pendingEvent);
}
getAll(): Promise<PendingEntry[]> {

8
src/matrix/storage/idb/stores/RoomMemberStore.ts
View file

@ -50,10 +50,10 @@ export class RoomMemberStore {
return this._roomMembersStore.get(encodeKey(roomId, userId));
}
async set(member: MemberData): Promise<IDBValidKey> {
set(member: MemberData): void {
// Object.assign would be more typesafe, but small objects
(member as any).key = encodeKey(member.roomId, member.userId);
return this._roomMembersStore.put(member as MemberStorageEntry);
this._roomMembersStore.put(member as MemberStorageEntry);
}
getAll(roomId: string): Promise<MemberData[]> {
@ -78,10 +78,10 @@ export class RoomMemberStore {
return userIds;
}
removeAllForRoom(roomId: string): Promise<undefined> {
removeAllForRoom(roomId: string): void {
// exclude both keys as they are theoretical min and max,
// but we should't have a match for just the room id, or room id with max
const range = this._roomMembersStore.IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true);
return this._roomMembersStore.delete(range);
this._roomMembersStore.delete(range);
}
}

4
src/matrix/storage/idb/stores/RoomStateStore.ts
View file

@ -41,10 +41,10 @@ export class RoomStateStore {
return this._roomStateStore.get(key);
}
set(roomId: string, event: StateEvent): Promise<IDBValidKey> {
set(roomId: string, event: StateEvent): void {
const key = encodeKey(roomId, event.type, event.state_key);
const entry = {roomId, event, key};
return this._roomStateStore.put(entry);
this._roomStateStore.put(entry);
}
removeAllForRoom(roomId: string): Promise<undefined> {

4
src/matrix/storage/idb/stores/RoomSummaryStore.ts
View file

@ -42,8 +42,8 @@ export class RoomSummaryStore {
return this._summaryStore.selectAll();
}
set(summary: SummaryData): Promise<IDBValidKey> {
return this._summaryStore.put(summary);
set(summary: SummaryData): void {
this._summaryStore.put(summary);
}
get(roomId: string): Promise<SummaryData | null> {

14
src/matrix/storage/idb/stores/SessionStore.ts
View file

@ -27,22 +27,22 @@ export class SessionStore {
this._sessionStore = sessionStore;
}
async get(key: IDBValidKey): Promise<any> {
async get(key: string): Promise<any> {
const entry = await this._sessionStore.get(key);
if (entry) {
return entry.value;
}
}
set(key: string, value: any): Promise<IDBValidKey> {
return this._sessionStore.put({key, value});
set(key: string, value: any): void {
this._sessionStore.put({key, value});
}
add(key: string, value: any): Promise<IDBValidKey> {
return this._sessionStore.add({key, value});
add(key: string, value: any): void {
this._sessionStore.add({key, value});
}
remove(key: IDBValidKey): Promise<undefined> {
return this._sessionStore.delete(key);
remove(key: string): void {
this._sessionStore.delete(key);
}
}

26
src/matrix/storage/idb/stores/TimelineEventStore.ts
View file

@ -19,7 +19,7 @@ import { StorageError } from "../../common";
import { encodeUint32 } from "../utils";
import {KeyLimits} from "../../common";
import {Store} from "../Store";
import {RoomEvent, StateEvent} from "../../types";
import {TimelineEvent, StateEvent} from "../../types";
interface Annotation {
count: number;
@ -31,7 +31,7 @@ interface StorageEntry {
roomId: string;
fragmentId: number;
eventIndex: number;
event: RoomEvent | StateEvent;
event: TimelineEvent | StateEvent;
displayName?: string;
avatarUrl?: string;
annotations?: { [key : string]: Annotation };
@ -53,15 +53,15 @@ function decodeEventIdKey(eventIdKey: string): { roomId: string, eventId: string
}
class Range {
private _IDBKeyRange: any; // TODO what's the appropriate representation here?
private _IDBKeyRange: typeof IDBKeyRange;
private _only?: EventKey;
private _lower?: EventKey;
private _upper?: EventKey;
private _lowerOpen: boolean;
private _upperOpen: boolean;
constructor(IDBKeyRange: any, only?: EventKey, lower?: EventKey, upper?: EventKey, lowerOpen: boolean = false, upperOpen: boolean = false) {
this._IDBKeyRange = IDBKeyRange;
constructor(_IDBKeyRange: any, only?: EventKey, lower?: EventKey, upper?: EventKey, lowerOpen: boolean = false, upperOpen: boolean = false) {
this._IDBKeyRange = _IDBKeyRange;
this._only = only;
this._lower = lower;
this._upper = upper;
@ -262,23 +262,23 @@ export class TimelineEventStore {
/** Inserts a new entry into the store. The combination of roomId and eventKey should not exist yet, or an error is thrown.
* @param entry the entry to insert
* @return a promise resolving to undefined if the operation was successful, or a StorageError if not.
* @return nothing. To wait for the operation to finish, await the transaction it's part of.
* @throws {StorageError} ...
*/
insert(entry: StorageEntry): Promise<IDBValidKey> {
insert(entry: StorageEntry): void {
entry.key = encodeKey(entry.roomId, entry.fragmentId, entry.eventIndex);
entry.eventIdKey = encodeEventIdKey(entry.roomId, entry.event.event_id);
// TODO: map error? or in idb/store?
return this._timelineStore.add(entry);
this._timelineStore.add(entry);
}
/** Updates the entry into the store with the given [roomId, eventKey] combination.
* If not yet present, will insert. Might be slower than add.
* @param entry the entry to update.
* @return a promise resolving to undefined if the operation was successful, or a StorageError if not.
* @return nothing. To wait for the operation to finish, await the transaction it's part of.
*/
update(entry: StorageEntry): Promise<IDBValidKey> {
return this._timelineStore.put(entry);
update(entry: StorageEntry): void {
this._timelineStore.put(entry);
}
get(roomId: string, eventKey: EventKey): Promise<StorageEntry | null> {
@ -289,10 +289,10 @@ export class TimelineEventStore {
return this._timelineStore.index("byEventId").get(encodeEventIdKey(roomId, eventId));
}
removeAllForRoom(roomId: string): Promise<undefined> {
removeAllForRoom(roomId: string): void {
const minKey = encodeKey(roomId, KeyLimits.minStorageKey, KeyLimits.minStorageKey);
const maxKey = encodeKey(roomId, KeyLimits.maxStorageKey, KeyLimits.maxStorageKey);
const range = this._timelineStore.IDBKeyRange.bound(minKey, maxKey);
return this._timelineStore.delete(range);
this._timelineStore.delete(range);
}
}

8
src/matrix/storage/idb/stores/TimelineFragmentStore.ts
View file

@ -74,13 +74,13 @@ export class TimelineFragmentStore {
// should generate an id an return it?
// depends if we want to do anything smart with fragment ids,
// like give them meaning depending on range. not for now probably ...
add(fragment: Fragment): Promise<IDBValidKey> {
add(fragment: Fragment): void {
(fragment as any).key = encodeKey(fragment.roomId, fragment.id);
return this._store.add(fragment as FragmentEntry);
this._store.add(fragment as FragmentEntry);
}
update(fragment: FragmentEntry): Promise<IDBValidKey> {
return this._store.put(fragment);
update(fragment: FragmentEntry): void {
this._store.put(fragment);
}
get(roomId: string, fragmentId: number): Promise<FragmentEntry | null> {

4
src/matrix/storage/idb/stores/TimelineRelationStore.ts
View file

@ -39,8 +39,8 @@ export class TimelineRelationStore {
this._store = store;
}
add(roomId: string, targetEventId: string, relType: string, sourceEventId: string): Promise<IDBValidKey> {
return this._store.add({key: encodeKey(roomId, targetEventId, relType, sourceEventId)});
add(roomId: string, targetEventId: string, relType: string, sourceEventId: string): void {
this._store.add({key: encodeKey(roomId, targetEventId, relType, sourceEventId)});
}
remove(roomId: string, targetEventId: string, relType: string, sourceEventId: string): Promise<undefined> {

4
src/matrix/storage/idb/stores/UserIdentityStore.ts
View file

@ -32,8 +32,8 @@ export class UserIdentityStore {
return this._store.get(userId);
}
set(userIdentity: UserIdentity): Promise<IDBValidKey> {
return this._store.put(userIdentity);
set(userIdentity: UserIdentity): void {
this._store.put(userIdentity);
}
remove(userId: string): Promise<undefined> {

13
src/matrix/storage/idb/utils.ts
View file

@ -71,12 +71,19 @@ type CreateObjectStore = (db : IDBDatabase, txn: IDBTransaction | null, oldVersi
export function openDatabase(name: string, createObjectStore: CreateObjectStore, version: number, idbFactory: IDBFactory = window.indexedDB): Promise<IDBDatabase> {
const req = idbFactory.open(name, version);
req.onupgradeneeded = (ev : IDBVersionChangeEvent) => {
req.onupgradeneeded = async (ev : IDBVersionChangeEvent) => {
const req = ev.target as IDBRequest<IDBDatabase>;
const db = req.result;
const txn = req.transaction;
const txn = req.transaction!;
const oldVersion = ev.oldVersion;
createObjectStore(db, txn, oldVersion, version);
try {
await createObjectStore(db, txn, oldVersion, version);
} catch (err) {
// try aborting on error, if that hasn't been done already
try {
txn.abort();
} catch (err) {}
}
};
return reqAsPromise(req);
}

4
src/matrix/storage/types.ts
View file

@ -16,7 +16,7 @@ limitations under the License.
export type Content = { [key: string]: any }
export interface RoomEvent {
export interface TimelineEvent {
content: Content;
type: string;
event_id: string;
@ -25,4 +25,4 @@ export interface RoomEvent {
unsigned?: Content;
}
export type StateEvent = RoomEvent & { prev_content?: Content, state_key: string }
export type StateEvent = TimelineEvent & { prev_content?: Content, state_key: string }

5
src/mocks/Storage.js
View file

@ -16,7 +16,8 @@ limitations under the License.
import {FDBFactory, FDBKeyRange} from "../../lib/fake-indexeddb/index.js";
import {StorageFactory} from "../matrix/storage/idb/StorageFactory.js";
import {NullLogItem} from "../logging/NullLogger.js";
export function createMockStorage() {
return new StorageFactory(null, new FDBFactory(), FDBKeyRange).create(1);
}
return new StorageFactory(null, new FDBFactory(), FDBKeyRange).create(1, new NullLogItem());
}