Merge pull request #811 from vector-im/bwindels/sharekeyswithinvitees

Key sharing based on room history visibility

src/matrix/e2ee/DeviceTracker.js

@@ -15,11 +15,13 @@ limitations under the License.
 */
 
 import {verifyEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js";
+import {HistoryVisibility, shouldShareKey} from "./common.js";
+import {RoomMember} from "../room/members/RoomMember.js";
 
 const TRACKING_STATUS_OUTDATED = 0;
 const TRACKING_STATUS_UPTODATE = 1;
 
-export function addRoomToIdentity(identity, userId, roomId) {
+function addRoomToIdentity(identity, userId, roomId) {
     if (!identity) {
         identity = {
             userId: userId,
@@ -79,28 +81,57 @@ export class DeviceTracker {
         }));
     }
 
-    writeMemberChanges(room, memberChanges, txn) {
+    /** @return Promise<{added: string[], removed: string[]}> the user ids for who the room was added or removed to the userIdentity,
-        return Promise.all(Array.from(memberChanges.values()).map(async memberChange => {
+     *                                                        and with who a key should be now be shared
-            return this._applyMemberChange(memberChange, txn);
+     **/
+    async writeMemberChanges(room, memberChanges, historyVisibility, txn) {
+        const added = [];
+        const removed = [];
+        await Promise.all(Array.from(memberChanges.values()).map(async memberChange => {
+            // keys should now be shared with this member?
+            // add the room to the userIdentity if so
+            if (shouldShareKey(memberChange.membership, historyVisibility)) {
+                if (await this._addRoomToUserIdentity(memberChange.roomId, memberChange.userId, txn)) {
+                    added.push(memberChange.userId);
+                }
+            } else if (shouldShareKey(memberChange.previousMembership, historyVisibility)) {
+                // try to remove room we were previously sharing the key with the member but not anymore
+                const {roomId} = memberChange;
+                // if we left the room, remove room from all user identities in the room
+                if (memberChange.userId === this._ownUserId) {
+                    const userIds = await txn.roomMembers.getAllUserIds(roomId);
+                    await Promise.all(userIds.map(userId => {
+                        return this._removeRoomFromUserIdentity(roomId, userId, txn);
                     }));
+                } else {
+                    await this._removeRoomFromUserIdentity(roomId, memberChange.userId, txn);
+                }
+                removed.push(memberChange.userId);
+            }
+        }));
+        return {added, removed};
     }
 
-    async trackRoom(room, log) {
+    async trackRoom(room, historyVisibility, log) {
         if (room.isTrackingMembers || !room.isEncrypted) {
             return;
         }
-        const memberList = await room.loadMemberList(log);
+        const memberList = await room.loadMemberList(undefined, log);
-        try {
         const txn = await this._storage.readWriteTxn([
             this._storage.storeNames.roomSummary,
             this._storage.storeNames.userIdentities,
         ]);
+        try {
             let isTrackingChanges;
             try {
                 isTrackingChanges = room.writeIsTrackingMembers(true, txn);
                 const members = Array.from(memberList.members.values());
                 log.set("members", members.length);
-                await this._writeJoinedMembers(members, txn);
+                await Promise.all(members.map(async member => {
+                    if (shouldShareKey(member.membership, historyVisibility)) {
+                        await this._addRoomToUserIdentity(member.roomId, member.userId, txn);
+                    }
+                }));
             } catch (err) {
                 txn.abort();
                 throw err;
@@ -112,21 +143,43 @@ export class DeviceTracker {
         }
     }
 
-    async _writeJoinedMembers(members, txn) {
+    async writeHistoryVisibility(room, historyVisibility, syncTxn, log) {
+        const added = [];
+        const removed = [];
+        if (room.isTrackingMembers && room.isEncrypted) {
+            await log.wrap("rewriting userIdentities", async log => {
+                const memberList = await room.loadMemberList(syncTxn, log);
+                try {
+                    const members = Array.from(memberList.members.values());
+                    log.set("members", members.length);
                     await Promise.all(members.map(async member => {
-            if (member.membership === "join") {
+                        if (shouldShareKey(member.membership, historyVisibility)) {
-                await this._writeMember(member, txn);
+                            if (await this._addRoomToUserIdentity(member.roomId, member.userId, syncTxn)) {
+                                added.push(member.userId);
+                            }
+                        } else {
+                            if (await this._removeRoomFromUserIdentity(member.roomId, member.userId, syncTxn)) {
+                                removed.push(member.userId);
+                            }
                         }
                     }));
+                } finally {
+                    memberList.release();
+                }
+            });
+        }
+        return {added, removed};
     }
 
-    async _writeMember(member, txn) {
+    async _addRoomToUserIdentity(roomId, userId, txn) {
         const {userIdentities} = txn;
-        const identity = await userIdentities.get(member.userId);
+        const identity = await userIdentities.get(userId);
-        const updatedIdentity = addRoomToIdentity(identity, member.userId, member.roomId);
+        const updatedIdentity = addRoomToIdentity(identity, userId, roomId);
         if (updatedIdentity) {
             userIdentities.set(updatedIdentity);
+            return true;
         }
+        return false;
     }
 
     async _removeRoomFromUserIdentity(roomId, userId, txn) {
@@ -141,28 +194,9 @@ export class DeviceTracker {
             } else {
                 userIdentities.set(identity);
             }
+            return true;
         }
-    }
+        return false;
-
-    async _applyMemberChange(memberChange, txn) {
-        // TODO: depends whether we encrypt for invited users??
-        // add room
-        if (memberChange.hasJoined) {
-            await this._writeMember(memberChange.member, txn);
-        }
-        // remove room
-        else if (memberChange.hasLeft) {
-            const {roomId} = memberChange;
-            // if we left the room, remove room from all user identities in the room
-            if (memberChange.userId === this._ownUserId) {
-                const userIds = await txn.roomMembers.getAllUserIds(roomId);
-                await Promise.all(userIds.map(userId => {
-                    return this._removeRoomFromUserIdentity(roomId, userId, txn);
-                }));
-            } else {
-                await this._removeRoomFromUserIdentity(roomId, memberChange.userId, txn);
-            }
-        }
     }
 
     async _queryKeys(userIds, hsApi, log) {
@@ -367,16 +401,18 @@ export class DeviceTracker {
 
 import {createMockStorage} from "../../mocks/Storage";
 import {Instance as NullLoggerInstance} from "../../logging/NullLogger";
+import {MemberChange} from "../room/members/RoomMember";
 
 export function tests() {
 
     function createUntrackedRoomMock(roomId, joinedUserIds, invitedUserIds = []) {
         return {
+            id: roomId,
             isTrackingMembers: false,
             isEncrypted: true,
             loadMemberList: () => {
-                const joinedMembers = joinedUserIds.map(userId => {return {membership: "join", roomId, userId};});
+                const joinedMembers = joinedUserIds.map(userId => {return RoomMember.fromUserId(roomId, userId, "join");});
-                const invitedMembers = invitedUserIds.map(userId => {return {membership: "invite", roomId, userId};});
+                const invitedMembers = invitedUserIds.map(userId => {return RoomMember.fromUserId(roomId, userId, "invite");});
                 const members = joinedMembers.concat(invitedMembers);
                 const memberMap = members.reduce((map, member) => {
                     map.set(member.userId, member);
@@ -440,10 +476,29 @@ export function tests() {
             }
         };
     }
+
+    async function writeMemberListToStorage(room, storage) {
+        const txn = await storage.readWriteTxn([
+            storage.storeNames.roomMembers,
+        ]);
+        const memberList = await room.loadMemberList(txn);
+        try {
+            for (const member of memberList.members.values()) {
+                txn.roomMembers.set(member.serialize());
+            }
+        } catch (err) {
+            txn.abort();
+            throw err;
+        } finally {
+            memberList.release();
+        }
+        await txn.complete();
+    }
+
     const roomId = "!abc:hs.tld";
 
     return {
-        "trackRoom only writes joined members": async assert => {
+        "trackRoom only writes joined members with history visibility of joined": async assert => {
             const storage = await createMockStorage();
             const tracker = new DeviceTracker({
                 storage,
@@ -453,7 +508,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"], ["@charly:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const txn = await storage.readTxn([storage.storeNames.userIdentities]);
             assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), {
                 userId: "@alice:hs.tld",
@@ -477,7 +532,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const hsApi = createQueryKeysHSApiMock();
             const devices = await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApi, NullLoggerInstance.item);
             assert.equal(devices.length, 2);
@@ -494,7 +549,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const hsApi = createQueryKeysHSApiMock();
             // query devices first time
             await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApi, NullLoggerInstance.item);
@@ -512,6 +567,169 @@ export function tests() {
             const txn2 = await storage.readTxn([storage.storeNames.deviceIdentities]);
             // also check the modified key was not stored
             assert.equal((await txn2.deviceIdentities.get("@alice:hs.tld", "device1")).ed25519Key, "ed25519:@alice:hs.tld:device1:key");
-        }
+        },
+        "change history visibility from joined to invited adds invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, 
+                ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+            const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Invited, txn, NullLoggerInstance.item);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+            assert.deepEqual(added, ["@bob:hs.tld"]);
+            assert.deepEqual(removed, []);
+        },
+        "change history visibility from invited to joined removes invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, 
+                ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+            const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Joined, txn, NullLoggerInstance.item);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, ["@bob:hs.tld"]);
+        },
+        "adding invitee with history visibility of invited adds room to userIdentities": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // inviting a new member
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite"));
+            const {added, removed} = await tracker.writeMemberChanges(room, [inviteChange], HistoryVisibility.Invited, txn);
+            assert.deepEqual(added, ["@bob:hs.tld"]);
+            assert.deepEqual(removed, []);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+        },
+        "adding invitee with history visibility of joined doesn't add room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // inviting a new member
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite"));
+            const memberChanges = new Map([[inviteChange.userId, inviteChange]]);
+            const {added, removed} = await tracker.writeMemberChanges(room, memberChanges, HistoryVisibility.Joined, txn);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, []);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+        },
+        "getting all devices after changing history visibility now includes invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const hsApi = createQueryKeysHSApiMock();
+            // write memberlist from room mock to mock storage,
+            // as devicesForTrackedRoom reads directly from roomMembers store.
+            await writeMemberListToStorage(room, storage);
+            const devices = await tracker.devicesForTrackedRoom(roomId, hsApi, NullLoggerInstance.item);
+            assert.equal(devices.length, 2);
+            assert.equal(devices.find(d => d.userId === "@alice:hs.tld").ed25519Key, "ed25519:@alice:hs.tld:device1:key");
+            assert.equal(devices.find(d => d.userId === "@bob:hs.tld").ed25519Key, "ed25519:@bob:hs.tld:device1:key");
+        },
+        "rejecting invite with history visibility of invited removes room from user identity": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // reject invite
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "leave"), "invite");
+            const memberChanges = new Map([[inviteChange.userId, inviteChange]]);
+            const {added, removed} = await tracker.writeMemberChanges(room, memberChanges, HistoryVisibility.Invited, txn);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, ["@bob:hs.tld"]);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+        },
+        "remove room from user identity sharing multiple rooms with us preserves other room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room1 = await createUntrackedRoomMock("!abc:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            const room2 = await createUntrackedRoomMock("!def:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item);
+            await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn1 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]);
+            const leaveChange = new MemberChange(RoomMember.fromUserId(room2.id, "@bob:hs.tld", "leave"), "join");
+            const memberChanges = new Map([[leaveChange.userId, leaveChange]]);
+            const txn2 = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            await tracker.writeMemberChanges(room2, memberChanges, HistoryVisibility.Joined, txn2);
+            await txn2.complete();
+            const txn3 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn3.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]);
+        },
+        "add room to user identity sharing multiple rooms with us preserves other room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room1 = await createUntrackedRoomMock("!abc:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            const room2 = await createUntrackedRoomMock("!def:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn1 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]);
+            await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn2 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn2.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]);
+        },
     }
 }

src/matrix/e2ee/RoomEncryption.js

@@ -19,8 +19,10 @@ import {groupEventsBySession} from "./megolm/decryption/utils";
 import {mergeMap} from "../../utils/mergeMap";
 import {groupBy} from "../../utils/groupBy";
 import {makeTxnId} from "../common.js";
+import {iterateResponseStateEvents} from "../room/common";
 
 const ENCRYPTED_TYPE = "m.room.encrypted";
+const ROOM_HISTORY_VISIBILITY_TYPE = "m.room.history_visibility";
 // how often ensureMessageKeyIsShared can check if it needs to
 // create a new outbound session
 // note that encrypt could still create a new session
@@ -45,6 +47,7 @@ export class RoomEncryption {
         this._isFlushingRoomKeyShares = false;
         this._lastKeyPreShareTime = null;
         this._keySharePromise = null;
+        this._historyVisibility = undefined;
         this._disposed = false;
     }
 
@@ -77,22 +80,68 @@ export class RoomEncryption {
         this._senderDeviceCache = new Map();    // purge the sender device cache
     }
 
-    async writeMemberChanges(memberChanges, txn, log) {
+    async writeSync(roomResponse, memberChanges, txn, log) {
-        let shouldFlush = false;
+        let historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility, txn);
-        const memberChangesArray = Array.from(memberChanges.values());
+        const addedMembers = [];
-        // this also clears our session if we leave the room ourselves
+        const removedMembers = [];
-        if (memberChangesArray.some(m => m.hasLeft)) {
+        // update the historyVisibility if needed
+        await iterateResponseStateEvents(roomResponse, event => {
+            // TODO: can the same state event appear twice? Hence we would be rewriting the useridentities twice...
+            // we'll see in the logs
+            if(event.state_key === "" && event.type === ROOM_HISTORY_VISIBILITY_TYPE) {
+                const newHistoryVisibility = event?.content?.history_visibility;
+                if (newHistoryVisibility !== historyVisibility) {
+                    return log.wrap({
+                        l: "history_visibility changed",
+                        from: historyVisibility,
+                        to: newHistoryVisibility
+                    }, async log => {
+                        historyVisibility = newHistoryVisibility;
+                        const result = await this._deviceTracker.writeHistoryVisibility(this._room, historyVisibility, txn, log);
+                        addedMembers.push(...result.added);
+                        removedMembers.push(...result.removed);
+                    });
+                }
+            }
+        });
+        // process member changes
+        if (memberChanges.size) {
+            const result = await this._deviceTracker.writeMemberChanges(
+                this._room, memberChanges, historyVisibility, txn);
+            addedMembers.push(...result.added);
+            removedMembers.push(...result.removed);
+        }
+        // discard key if somebody (including ourselves) left
+        if (removedMembers.length) {
             log.log({
                 l: "discardOutboundSession",
-                leftUsers: memberChangesArray.filter(m => m.hasLeft).map(m => m.userId),
+                leftUsers: removedMembers,
             });
             this._megolmEncryption.discardOutboundSession(this._room.id, txn);
         }
-        if (memberChangesArray.some(m => m.hasJoined)) {
+        let shouldFlush = false;
-            shouldFlush = await this._addShareRoomKeyOperationForNewMembers(memberChangesArray, txn, log);
+        // add room to userIdentities if needed, and share the current key with them
+        if (addedMembers.length) {
+            shouldFlush = await this._addShareRoomKeyOperationForMembers(addedMembers, txn, log);
         }
-        await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
+        return {shouldFlush, historyVisibility};
-        return shouldFlush;
+    }
+
+    afterSync({historyVisibility}) {
+        this._historyVisibility = historyVisibility;
+    }
+
+    async _loadHistoryVisibilityIfNeeded(historyVisibility, txn = undefined) {
+        if (!historyVisibility) {
+            if (!txn) {
+                txn = await this._storage.readTxn([this._storage.storeNames.roomState]);
+            }
+            const visibilityEntry = await txn.roomState.get(this._room.id, ROOM_HISTORY_VISIBILITY_TYPE, "");
+            if (visibilityEntry) {
+                return visibilityEntry.event?.content?.history_visibility;
+            }
+        }
+        return historyVisibility;
     }
 
     async prepareDecryptAll(events, newKeys, source, txn) {
@@ -274,10 +323,15 @@ export class RoomEncryption {
     }
 
     async _shareNewRoomKey(roomKeyMessage, hsApi, log) {
+        this._historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility);
+        await this._deviceTracker.trackRoom(this._room, this._historyVisibility, log);
+        const devices = await this._deviceTracker.devicesForTrackedRoom(this._room.id, hsApi, log);
+        const userIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
+            
         let writeOpTxn = await this._storage.readWriteTxn([this._storage.storeNames.operations]);
         let operation;
         try {
-            operation = this._writeRoomKeyShareOperation(roomKeyMessage, null, writeOpTxn);
+            operation = this._writeRoomKeyShareOperation(roomKeyMessage, userIds, writeOpTxn);
         } catch (err) {
             writeOpTxn.abort();
             throw err;
@@ -288,8 +342,7 @@ export class RoomEncryption {
         await this._processShareRoomKeyOperation(operation, hsApi, log);
     }
 
-    async _addShareRoomKeyOperationForNewMembers(memberChangesArray, txn, log) {
+    async _addShareRoomKeyOperationForMembers(userIds, txn, log) {
-        const userIds = memberChangesArray.filter(m => m.hasJoined).map(m => m.userId);
         const roomKeyMessage = await this._megolmEncryption.createRoomKeyMessage(
             this._room.id, txn);
         if (roomKeyMessage) {
@@ -342,18 +395,9 @@ export class RoomEncryption {
 
     async _processShareRoomKeyOperation(operation, hsApi, log) {
         log.set("id", operation.id);
-
+        this._historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility);
-        await this._deviceTracker.trackRoom(this._room, log);
+        await this._deviceTracker.trackRoom(this._room, this._historyVisibility, log);
-        let devices;
+        const devices = await this._deviceTracker.devicesForRoomMembers(this._room.id, operation.userIds, hsApi, log);
-        if (operation.userIds === null) {
-            devices = await this._deviceTracker.devicesForTrackedRoom(this._room.id, hsApi, log);
-            const userIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
-            operation.userIds = userIds;
-            await this._updateOperationsStore(operations => operations.update(operation));
-        } else {
-            devices = await this._deviceTracker.devicesForRoomMembers(this._room.id, operation.userIds, hsApi, log);
-        }
-        
         const messages = await log.wrap("olm encrypt", log => this._olmEncryption.encrypt(
             "m.room_key", operation.roomKeyMessage, devices, hsApi, log));
         const missingDevices = devices.filter(d => !messages.some(m => m.device === d));
@@ -507,3 +551,143 @@ class BatchDecryptionResult {
         }));
     }
 }
+
+import {createMockStorage} from "../../mocks/Storage";
+import {Clock as MockClock} from "../../mocks/Clock";
+import {poll} from "../../mocks/poll";
+import {Instance as NullLoggerInstance} from "../../logging/NullLogger";
+import {ConsoleLogger} from "../../logging/ConsoleLogger";
+import {HomeServer as MockHomeServer} from "../../mocks/HomeServer.js";
+
+export function tests() {
+    const roomId = "!abc:hs.tld";
+    return {
+        "ensureMessageKeyIsShared tracks room and passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            const megolmMock = {
+                async ensureOutboundSession() { return { }; }
+            };
+            const olmMock = {
+                async encrypt() { return []; }
+            }
+            let isRoomTracked = false;
+            let isDevicesRequested = false;
+            const deviceTracker = {
+                async trackRoom(room, historyVisibility) {
+                    // only assert on first call
+                    if (isRoomTracked) { return; }
+                    assert(!isDevicesRequested);
+                    assert.equal(room.id, roomId);
+                    assert.equal(historyVisibility, "invited");
+                    isRoomTracked = true;
+                },
+                async devicesForTrackedRoom() {
+                    assert(isRoomTracked);
+                    isDevicesRequested = true;
+                    return [];
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            await writeTxn.complete();
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                megolmEncryption: megolmMock,
+                olmEncryption: olmMock,
+                storage,
+                deviceTracker,
+                clock: new MockClock()
+            });
+            const homeServer = new MockHomeServer();
+            const promise = roomEncryption.ensureMessageKeyIsShared(homeServer.api, NullLoggerInstance.item);
+            // need to poll because sendToDevice isn't first async step
+            const request = await poll(() => homeServer.requests.sendToDevice?.[0]);
+            request.respond({});
+            await promise;
+            assert(isRoomTracked);
+            assert(isDevicesRequested);
+        },
+        "encrypt tracks room and passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            const megolmMock = {
+                async encrypt() { return { roomKeyMessage: {} }; }
+            };
+            const olmMock = {
+                async encrypt() { return []; }
+            }
+            let isRoomTracked = false;
+            let isDevicesRequested = false;
+            const deviceTracker = {
+                async trackRoom(room, historyVisibility) {
+                    // only assert on first call
+                    if (isRoomTracked) { return; }
+                    assert(!isDevicesRequested);
+                    assert.equal(room.id, roomId);
+                    assert.equal(historyVisibility, "invited");
+                    isRoomTracked = true;
+                },
+                async devicesForTrackedRoom() {
+                    assert(isRoomTracked);
+                    isDevicesRequested = true;
+                    return [];
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            await writeTxn.complete();
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                megolmEncryption: megolmMock,
+                olmEncryption: olmMock,
+                storage,
+                deviceTracker
+            });
+            const homeServer = new MockHomeServer();
+            const promise = roomEncryption.encrypt("m.room.message", {body: "hello"}, homeServer.api, NullLoggerInstance.item);
+            // need to poll because sendToDevice isn't first async step
+            const request = await poll(() => homeServer.requests.sendToDevice?.[0]);
+            request.respond({});
+            await promise;
+            assert(isRoomTracked);
+            assert(isDevicesRequested);
+        },
+        "writeSync passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            let isMemberChangesCalled = false;
+            const deviceTracker = {
+                async writeMemberChanges(room, memberChanges, historyVisibility, txn) {
+                    assert.equal(historyVisibility, "invited");
+                    isMemberChangesCalled = true;
+                    return {removed: [], added: []};
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            const memberChanges = new Map([["@alice:hs.tld", {}]]);
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                storage,
+                deviceTracker
+            });
+            const roomResponse = {};
+            const txn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            await roomEncryption.writeSync(roomResponse, memberChanges, txn, NullLoggerInstance.item);
+            assert(isMemberChangesCalled);
+        },
+    }
+}

src/matrix/e2ee/common.js

@@ -69,3 +69,28 @@ export function createRoomEncryptionEvent() {
         }
     }
 }
+
+
+// Use enum when converting to TS
+export const HistoryVisibility = Object.freeze({
+    Joined: "joined",
+    Invited: "invited",
+    WorldReadable: "world_readable",
+    Shared: "shared",
+});
+
+export function shouldShareKey(membership, historyVisibility) {
+    switch (historyVisibility) {
+        case HistoryVisibility.WorldReadable:
+            return true;
+        case HistoryVisibility.Shared:
+            // was part of room at some time
+            return membership !== undefined;
+        case HistoryVisibility.Joined:
+            return membership === "join";
+        case HistoryVisibility.Invited:
+            return membership === "invite" || membership === "join";
+        default:
+            return false;
+    }
+}

src/matrix/room/BaseRoom.js

@@ -243,7 +243,7 @@ export class BaseRoom extends EventEmitter {
 
 
     /** @public */
-    async loadMemberList(log = null) {
+    async loadMemberList(txn = undefined, log = null) {
         if (this._memberList) {
             // TODO: also await fetchOrLoadMembers promise here
             this._memberList.retain();
@@ -254,6 +254,9 @@ export class BaseRoom extends EventEmitter {
                 roomId: this._roomId,
                 hsApi: this._hsApi,
                 storage: this._storage,
+                // pass in a transaction if we know we won't need to fetch (which would abort the transaction)
+                // and we want to make this operation part of the larger transaction
+                txn,
                 syncToken: this._getSyncToken(),
                 // to handle race between /members and /sync
                 setChangedMembersMap: map => this._changedMembersDuringSync = map,

src/matrix/room/Room.js

@@ -139,11 +139,11 @@ export class Room extends BaseRoom {
         }
         log.set("newEntries", newEntries.length);
         log.set("updatedEntries", updatedEntries.length);
-        let shouldFlushKeyShares = false;
+        let encryptionChanges;
         // pass member changes to device tracker
-        if (roomEncryption && this.isTrackingMembers && memberChanges?.size) {
+        if (roomEncryption) {
-            shouldFlushKeyShares = await roomEncryption.writeMemberChanges(memberChanges, txn, log);
+            encryptionChanges = await roomEncryption.writeSync(roomResponse, memberChanges, txn, log);
-            log.set("shouldFlushKeyShares", shouldFlushKeyShares);
+            log.set("shouldFlushKeyShares", encryptionChanges.shouldFlush);
         }
         const allEntries = newEntries.concat(updatedEntries);
         // also apply (decrypted) timeline entries to the summary changes
@@ -188,7 +188,7 @@ export class Room extends BaseRoom {
             memberChanges,
             heroChanges,
             powerLevelsEvent,
-            shouldFlushKeyShares,
+            encryptionChanges,
         };
     }
 
@@ -201,11 +201,14 @@ export class Room extends BaseRoom {
         const {
             summaryChanges, newEntries, updatedEntries, newLiveKey,
             removedPendingEvents, memberChanges, powerLevelsEvent,
-            heroChanges, roomEncryption
+            heroChanges, roomEncryption, encryptionChanges
         } = changes;
         log.set("id", this.id);
         this._syncWriter.afterSync(newLiveKey);
         this._setEncryption(roomEncryption);
+        if (this._roomEncryption) {
+            this._roomEncryption.afterSync(encryptionChanges);
+        }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
                 for (const [userId, memberChange] of memberChanges.entries()) {
@@ -288,8 +291,8 @@ export class Room extends BaseRoom {
         }
     }
 
-    needsAfterSyncCompleted({shouldFlushKeyShares}) {
+    needsAfterSyncCompleted({encryptionChanges}) {
-        return shouldFlushKeyShares;
+        return encryptionChanges?.shouldFlush;
     }
 
     /**

src/matrix/room/common.ts

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import type {StateEvent} from "../storage/types";
+
 export function getPrevContentFromStateEvent(event) {
     // where to look for prev_content is a bit of a mess,
     // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz
@@ -40,3 +42,83 @@ export enum RoomType {
     Private,
     Public
 }
+
+type RoomResponse = {
+    state?: {
+        events?: Array<StateEvent>
+    },
+    timeline?: {
+        events?: Array<StateEvent>
+    }
+}
+
+/** iterates over any state events in a sync room response, in the order that they should be applied (from older to younger events) */
+export function iterateResponseStateEvents(roomResponse: RoomResponse, callback: (StateEvent) => Promise<void> | void): Promise<void> | void {
+    let promises: Promise<void>[] | undefined = undefined;
+    const callCallback = stateEvent => {
+        const result = callback(stateEvent);
+        if (result instanceof Promise) {
+            promises = promises ?? [];
+            promises.push(result);
+        }
+    };
+    // first iterate over state events, they precede the timeline
+    const stateEvents = roomResponse.state?.events;
+    if (stateEvents) {
+        for (let i = 0; i < stateEvents.length; i++) {
+            callCallback(stateEvents[i]);
+        }
+    }
+    // now see if there are any state events within the timeline
+    let timelineEvents = roomResponse.timeline?.events;
+    if (timelineEvents) {
+        for (let i = 0; i < timelineEvents.length; i++) {
+            const event = timelineEvents[i];
+            if (typeof event.state_key === "string") {
+                callCallback(event);
+            }
+        }
+    }
+    if (promises) {
+        return Promise.all(promises).then(() => undefined);
+    }
+}
+
+export function tests() {
+    return {
+        "test iterateResponseStateEvents with both state and timeline sections": assert => {
+            const roomResponse = {
+                state: {
+                    events: [
+                        {type: "m.room.member", state_key: "1"},
+                        {type: "m.room.member", state_key: "2", content: {a: 1}},
+                    ]
+                },
+                timeline: {
+                    events: [
+                        {type: "m.room.message"},
+                        {type: "m.room.member", state_key: "3"},
+                        {type: "m.room.message"},
+                        {type: "m.room.member", state_key: "2", content: {a: 2}},
+                    ]
+                }
+            } as unknown as RoomResponse;
+            const expectedStateKeys = ["1", "2", "3", "2"];
+            const expectedAForMember2 = [1, 2];
+            iterateResponseStateEvents(roomResponse, event => {
+                assert.strictEqual(event.type, "m.room.member");
+                assert.strictEqual(expectedStateKeys.shift(), event.state_key);
+                if (event.state_key === "2") {
+                    assert.strictEqual(expectedAForMember2.shift(), event.content.a);
+                }
+            });
+            assert.strictEqual(expectedStateKeys.length, 0);
+            assert.strictEqual(expectedAForMember2.length, 0);
+        },
+        "test iterateResponseStateEvents with empty response": assert => {
+            iterateResponseStateEvents({}, () => {
+                assert.fail("no events expected");
+            });
+        }
+    }
+}

src/matrix/room/members/RoomMember.js

@@ -137,6 +137,10 @@ export class MemberChange {
         return this.member.membership;
     }
 
+    get wasInvited() {
+        return this.previousMembership === "invite" && this.membership !== "invite";
+    }
+
     get hasLeft() {
         return this.previousMembership === "join" && this.membership !== "join";
     }

src/matrix/room/members/load.js

@@ -17,10 +17,12 @@ limitations under the License.
 
 import {RoomMember} from "./RoomMember.js";
 
-async function loadMembers({roomId, storage}) {
+async function loadMembers({roomId, storage, txn}) {
-    const txn = await storage.readTxn([
+    if (!txn) {
+        txn = await storage.readTxn([
             storage.storeNames.roomMembers,
         ]);
+    }
     const memberDatas = await txn.roomMembers.getAll(roomId);
     return memberDatas.map(d => new RoomMember(d));
 }

src/matrix/storage/idb/schema.ts

@@ -2,7 +2,6 @@ import {IDOMStorage} from "./types";
 import {ITransaction} from "./QueryTarget";
 import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils";
 import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js";
-import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js";
 import {SESSION_E2EE_KEY_PREFIX} from "../../e2ee/common.js";
 import {SummaryData} from "../../room/RoomSummary";
 import {RoomMemberStore, MemberData} from "./stores/RoomMemberStore";
@@ -183,51 +182,12 @@ function createTimelineRelationsStore(db: IDBDatabase) : void {
     db.createObjectStore("timelineRelations", {keyPath: "key"});
 }
 
-//v11 doesn't change the schema, but ensures all userIdentities have all the roomIds they should (see #470)
+//v11 doesn't change the schema,
-async function fixMissingRoomsInUserIdentities(db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: ILogItem) {
+// but ensured all userIdentities have all the roomIds they should (see #470)
-    const roomSummaryStore = txn.objectStore("roomSummary");
+
-    const trackedRoomIds: string[] = [];
+// 2022-07-20: The fix dated from August 2021, and have removed it now because of a
-    await iterateCursor<SummaryData>(roomSummaryStore.openCursor(), roomSummary => {
+// refactoring needed in the device tracker, which made it inconvenient to expose addRoomToIdentity
-        if (roomSummary.isTrackingMembers) {
+function fixMissingRoomsInUserIdentities() {}
-            trackedRoomIds.push(roomSummary.roomId);
-        }
-        return NOT_DONE;
-    });
-    const outboundGroupSessionsStore = txn.objectStore("outboundGroupSessions");
-    const userIdentitiesStore: IDBObjectStore = txn.objectStore("userIdentities");
-    const roomMemberStore = txn.objectStore("roomMembers");
-    for (const roomId of trackedRoomIds) {
-        let foundMissing = false;
-        const joinedUserIds: string[] = [];
-        const memberRange = IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true);
-        await log.wrap({l: "room", id: roomId}, async log => {
-            await iterateCursor<MemberData>(roomMemberStore.openCursor(memberRange), member => {
-                if (member.membership === "join") {
-                    joinedUserIds.push(member.userId);
-                }
-                return NOT_DONE;
-            });
-            log.set("joinedUserIds", joinedUserIds.length);
-            for (const userId of joinedUserIds) {
-                const identity = await reqAsPromise(userIdentitiesStore.get(userId));
-                const originalRoomCount = identity?.roomIds?.length;
-                const updatedIdentity = addRoomToIdentity(identity, userId, roomId);
-                if (updatedIdentity) {
-                    log.log({l: `fixing up`, id: userId,
-                        roomsBefore: originalRoomCount, roomsAfter: updatedIdentity.roomIds.length});
-                    userIdentitiesStore.put(updatedIdentity);
-                    foundMissing = true;
-                }
-            }
-            log.set("foundMissing", foundMissing);
-            if (foundMissing) {
-                // clear outbound megolm session,
-                // so we'll create a new one on the next message that will be properly shared
-                outboundGroupSessionsStore.delete(roomId);
-            }
-        });
-    }
-}
 
 // v12 move ssssKey to e2ee:ssssKey so it will get backed up in the next step
 async function changeSSSSKeyPrefix(db: IDBDatabase, txn: IDBTransaction) {