support multiple devices in call per user

src/matrix/calls/CallHandler.ts

@@ -55,7 +55,6 @@ export class CallHandler {
     async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise<GroupCall> {
         const logItem = this.options.logger.child({l: "call", incoming: false});
         const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem);
-        console.log("created call with id", call.id);
         this._calls.set(call.id, call);
         try {
             await call.create(localMedia, name);
@@ -67,9 +66,7 @@ export class CallHandler {
             }
             throw err;
         }
-        console.log("joining call I just created");
         await call.join(localMedia);
-        console.log("joined!");
         return call;
     }
 
@@ -79,7 +76,6 @@ export class CallHandler {
 
     /** @internal */
     handleRoomState(room: Room, events: StateEvent[], log: ILogItem) {
-        console.log("handling room state");
         // first update call events
         for (const event of events) {
             if (event.type === EventType.GroupCall) {
@@ -135,7 +131,7 @@ export class CallHandler {
             const callId = call["m.call_id"];
             const groupCall = this._calls.get(callId);
             // TODO: also check the member when receiving the m.call event
-            groupCall?.addMember(userId, call, log);
+            groupCall?.updateMember(userId, call, log);
         };
         const newCallIdsMemberOf = new Set<string>(calls.map(call => call["m.call_id"]));
         let previousCallIdsMemberOf = this.memberToCallIds.get(userId);

src/matrix/calls/LocalMedia.ts

@@ -61,6 +61,11 @@ export class LocalMedia {
         return metadata;
     }
 
+    clone() {
+        // TODO: implement
+        return this;
+    }
+
     dispose() {
         this.cameraTrack?.stop();
         this.microphoneTrack?.stop();

src/matrix/calls/TODO.md

@@ -108,6 +108,7 @@ Expose call objects
 expose volume events from audiotrack to group call
 Write view model
 write view
+ - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare
 
 ## Calls questions
  - how do we handle glare between group calls (e.g. different state events with different call ids?)

src/matrix/calls/callEventTypes.ts

@@ -22,6 +22,19 @@ export enum EventType {
 // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
 export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
 
+export interface CallDeviceMembership {
+    device_id: string
+}
+
+export interface CallMembership {
+    ["m.call_id"]: string,
+    ["m.devices"]: CallDeviceMembership[]
+}
+
+export interface CallMemberContent {
+    ["m.calls"]: CallMembership[];
+}
+
 export interface SessionDescription {
     sdp?: string;
     type: RTCSdpType

src/matrix/calls/group/GroupCall.ts

@@ -24,7 +24,7 @@ import {EventEmitter} from "../../../utils/EventEmitter";
 import type {Options as MemberOptions} from "./Member";
 import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap";
 import type {Track} from "../../../platform/types/MediaDevices";
-import type {SignallingMessage, MGroupCallBase} from "../callEventTypes";
+import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes";
 import type {Room} from "../../room/Room";
 import type {StateEvent} from "../../storage/types";
 import type {Platform} from "../../../platform/web/Platform";
@@ -43,11 +43,22 @@ export enum GroupCallState {
     Joined = "joined",
 }
 
+function getMemberKey(userId: string, deviceId: string) {
+    return JSON.stringify(userId)+`,`+JSON.stringify(deviceId);
+}
+
+function memberKeyIsForUser(key: string, userId: string) {
+    return key.startsWith(JSON.stringify(userId)+`,`);
+}
+
+function getDeviceFromMemberKey(key: string): string {
+    return JSON.parse(`[${key}]`)[1];
+}
+
 export type Options = Omit<MemberOptions, "emitUpdate" | "confId" | "encryptDeviceMessage"> & {
     emitUpdate: (call: GroupCall, params?: any) => void;
     encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
     storage: Storage,
-    ownDeviceId: string
 };
 
 export class GroupCall extends EventEmitter<{change: never}> {
@@ -70,7 +81,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
         this._state = id ? GroupCallState.Created : GroupCallState.Fledgling;
         this._memberOptions = Object.assign({}, options, {
             confId: this.id,
-            emitUpdate: member => this._members.update(member.member.userId, member),
+            emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member),
             encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log) => {
                 return this.options.encryptDeviceMessage(this.roomId, userId, message, log);
             }
@@ -173,26 +184,42 @@ export class GroupCall extends EventEmitter<{change: never}> {
     }
 
     /** @internal */
-    addMember(userId: string, memberCallInfo, syncLog: ILogItem) {
+    updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) {
-        this.logItem.wrap({l: "addMember", id: userId}, log => {
+        this.logItem.wrap({l: "updateMember", id: userId}, log => {
             syncLog.refDetached(log);
-
+            const devices = callMembership["m.devices"];
-            if (userId === this.options.ownUserId) {
+            const previousDeviceIds = this.getDeviceIdsForUserId(userId);
-                if (this._state === GroupCallState.Joining) {
+            for (const device of devices) {
-                    this._state = GroupCallState.Joined;
+                const deviceId = device.device_id;
-                    this.emitChange();
+                const memberKey = getMemberKey(userId, deviceId);
+                if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
+                    if (this._state === GroupCallState.Joining) {
+                        this._state = GroupCallState.Joined;
+                        this.emitChange();
+                    }
+                    return;
+                }
+                let member = this._members.get(memberKey);
+                if (member) {
+                    member.updateCallInfo(device);
+                } else {
+                    const logItem = this.logItem.child("member");
+                    member = new Member(
+                        RoomMember.fromUserId(this.roomId, userId, "join"),
+                        device, this._memberOptions, logItem
+                    );
+                    this._members.add(memberKey, member);
+                    if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
+                        member.connect(this._localMedia!.clone());
+                    }
                 }
-                return;
             }
-            let member = this._members.get(userId);
+
-            if (member) {
+            const newDeviceIds = new Set<string>(devices.map(call => call.device_id));
-                member.updateCallInfo(memberCallInfo);
+            // remove user as member of any calls not present anymore
-            } else {
+            for (const previousDeviceId of previousDeviceIds) {
-                const logItem = this.logItem.child("member");
+                if (!newDeviceIds.has(previousDeviceId)) {
-                member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem);
+                    this.removeMemberDevice(userId, previousDeviceId, syncLog);
-                this._members.add(userId, member);
-                if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
-                    member.connect(this._localMedia!);
                 }
             }
         });
@@ -200,9 +227,24 @@ export class GroupCall extends EventEmitter<{change: never}> {
 
     /** @internal */
     removeMember(userId: string, syncLog: ILogItem) {
-        this.logItem.wrap({l: "removeMember", id: userId}, log => {
+        const deviceIds = this.getDeviceIdsForUserId(userId);
+        for (const deviceId of deviceIds) {
+            this.removeMemberDevice(userId, deviceId, syncLog);
+        }
+    }
+
+    private getDeviceIdsForUserId(userId: string): string[] {
+        return Array.from(this._members.keys())
+            .filter(key => memberKeyIsForUser(key, userId))
+            .map(key => getDeviceFromMemberKey(key));
+    }
+
+    /** @internal */
+    private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) {
+        const memberKey = getMemberKey(userId, deviceId);
+        this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => {
             syncLog.refDetached(log);
-            if (userId === this.options.ownUserId) {
+            if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
                 if (this._state === GroupCallState.Joined) {
                     this._localMedia?.dispose();
                     this._localMedia = undefined;
@@ -212,9 +254,9 @@ export class GroupCall extends EventEmitter<{change: never}> {
                     this._state = GroupCallState.Created;
                 }
             } else {
-                const member = this._members.get(userId);
+                const member = this._members.get(memberKey);
                 if (member) {
-                    this._members.remove(userId);
+                    this._members.remove(memberKey);
                     member.disconnect();
                 }
             }
@@ -225,7 +267,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
     /** @internal */
     handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, userId: string, deviceId: string, syncLog: ILogItem) {
         // TODO: return if we are not membering to the call
-        let member = this._members.get(userId);
+        let member = this._members.get(getMemberKey(userId, deviceId));
         if (member) {
             member.handleDeviceMessage(message, deviceId, syncLog);
         } else {

src/matrix/calls/group/Member.ts

@@ -23,7 +23,7 @@ import type {Options as PeerCallOptions} from "../PeerCall";
 import type {LocalMedia} from "../LocalMedia";
 import type {HomeServerApi} from "../../net/HomeServerApi";
 import type {Track} from "../../../platform/types/MediaDevices";
-import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes";
+import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes";
 import type {GroupCall} from "./GroupCall";
 import type {RoomMember} from "../../room/members/RoomMember";
 import type {EncryptedMessage} from "../../e2ee/olm/Encryption";
@@ -32,6 +32,7 @@ import type {ILogItem} from "../../../logging/types";
 export type Options = Omit<PeerCallOptions, "emitUpdate" | "sendSignallingMessage"> & {
     confId: string,
     ownUserId: string,
+    ownDeviceId: string,
     hsApi: HomeServerApi,
     encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
     emitUpdate: (participant: Member, params?: any) => void,
@@ -43,7 +44,7 @@ export class Member {
 
     constructor(
         public readonly member: RoomMember,
-        private memberCallInfo: Record<string, any>,
+        private callDeviceMembership: CallDeviceMembership,
         private readonly options: Options,
         private readonly logItem: ILogItem,
     ) {
@@ -58,15 +59,30 @@ export class Member {
         return this.peerCall?.state === CallState.Connected;
     }
 
+    get userId(): string {
+        return this.member.userId;
+    }
+
+    get deviceId(): string {
+        return this.callDeviceMembership.device_id;
+    }
+
     /** @internal */
     connect(localMedia: LocalMedia) {
-        this.logItem.log("connect");
+        this.logItem.wrap("connect", () => {
-        this.localMedia = localMedia;
+            this.localMedia = localMedia;
-        // otherwise wait for it to connect
+            // otherwise wait for it to connect
-        if (this.member.userId < this.options.ownUserId) {
+            let shouldInitiateCall;
-            this.peerCall = this._createPeerCall(makeId("c"));
+            if (this.member.userId === this.options.ownUserId) {
-            this.peerCall.call(localMedia);
+                shouldInitiateCall = this.deviceId < this.options.ownDeviceId;
-        }
+            } else {
+                shouldInitiateCall = this.member.userId < this.options.ownUserId;
+            }
+            if (shouldInitiateCall) {
+                this.peerCall = this._createPeerCall(makeId("c"));
+                this.peerCall.call(localMedia);
+            }
+        });
     }
 
     /** @internal */
@@ -80,8 +96,8 @@ export class Member {
     }
 
     /** @internal */
-    updateCallInfo(memberCallInfo) {
+    updateCallInfo(callDeviceMembership: CallDeviceMembership) {
-        // m.calls object from the m.call.member event
+        this.callDeviceMembership = callDeviceMembership;
     }
 
     /** @internal */

src/observable/map/ObservableMap.ts

@@ -80,15 +80,15 @@ export class ObservableMap<K, V> extends BaseObservableMap<K, V> {
         return this._values.size;
     }
 
-    [Symbol.iterator](): Iterator<[K, V]> {
+    [Symbol.iterator](): IterableIterator<[K, V]> {
         return this._values.entries();
     }
 
-    values(): Iterator<V> {
+    values(): IterableIterator<V> {
         return this._values.values();
     }
 
-    keys(): Iterator<K> {
+    keys(): IterableIterator<K> {
         return this._values.keys();
     }
 }