diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index ef617cd2..678f4f44 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -55,7 +55,6 @@ export class CallHandler { async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem); - console.log("created call with id", call.id); this._calls.set(call.id, call); try { await call.create(localMedia, name); @@ -67,9 +66,7 @@ export class CallHandler { } throw err; } - console.log("joining call I just created"); await call.join(localMedia); - console.log("joined!"); return call; } @@ -79,7 +76,6 @@ export class CallHandler { /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { - console.log("handling room state"); // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { @@ -135,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.addMember(userId, call, log); + groupCall?.updateMember(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index b148dd92..b64bdee5 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -61,6 +61,11 @@ export class LocalMedia { return metadata; } + clone() { + // TODO: implement + return this; + } + dispose() { this.cameraTrack?.stop(); this.microphoneTrack?.stop(); diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a07da60e..4faf4f4e 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -108,6 +108,7 @@ Expose call objects expose volume events from audiotrack to group call Write view model write view + - handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare ## Calls questions - how do we handle glare between group calls (e.g. different state events with different call ids?) diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 0e9eb8f8..4416087b 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -22,6 +22,19 @@ export enum EventType { // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface CallDeviceMembership { + device_id: string +} + +export interface CallMembership { + ["m.call_id"]: string, + ["m.devices"]: CallDeviceMembership[] +} + +export interface CallMemberContent { + ["m.calls"]: CallMembership[]; +} + export interface SessionDescription { sdp?: string; type: RTCSdpType diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 86c93ba3..3ef6a5ff 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -24,7 +24,7 @@ import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; @@ -43,11 +43,22 @@ export enum GroupCallState { Joined = "joined", } +function getMemberKey(userId: string, deviceId: string) { + return JSON.stringify(userId)+`,`+JSON.stringify(deviceId); +} + +function memberKeyIsForUser(key: string, userId: string) { + return key.startsWith(JSON.stringify(userId)+`,`); +} + +function getDeviceFromMemberKey(key: string): string { + return JSON.parse(`[${key}]`)[1]; +} + export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, - ownDeviceId: string }; export class GroupCall extends EventEmitter<{change: never}> { @@ -70,7 +81,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { confId: this.id, - emitUpdate: member => this._members.update(member.member.userId, member), + emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member), encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } @@ -173,26 +184,42 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - addMember(userId: string, memberCallInfo, syncLog: ILogItem) { - this.logItem.wrap({l: "addMember", id: userId}, log => { + updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); - - if (userId === this.options.ownUserId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + const devices = callMembership["m.devices"]; + const previousDeviceIds = this.getDeviceIdsForUserId(userId); + for (const device of devices) { + const deviceId = device.device_id; + const memberKey = getMemberKey(userId, deviceId); + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + this.emitChange(); + } + return; + } + let member = this._members.get(memberKey); + if (member) { + member.updateCallInfo(device); + } else { + const logItem = this.logItem.child("member"); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } } - return; } - let member = this._members.get(userId); - if (member) { - member.updateCallInfo(memberCallInfo); - } else { - const logItem = this.logItem.child("member"); - member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem); - this._members.add(userId, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!); + + const newDeviceIds = new Set(devices.map(call => call.device_id)); + // remove user as member of any calls not present anymore + for (const previousDeviceId of previousDeviceIds) { + if (!newDeviceIds.has(previousDeviceId)) { + this.removeMemberDevice(userId, previousDeviceId, syncLog); } } }); @@ -200,9 +227,24 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ removeMember(userId: string, syncLog: ILogItem) { - this.logItem.wrap({l: "removeMember", id: userId}, log => { + const deviceIds = this.getDeviceIdsForUserId(userId); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, syncLog); + } + } + + private getDeviceIdsForUserId(userId: string): string[] { + return Array.from(this._members.keys()) + .filter(key => memberKeyIsForUser(key, userId)) + .map(key => getDeviceFromMemberKey(key)); + } + + /** @internal */ + private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + const memberKey = getMemberKey(userId, deviceId); + this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { syncLog.refDetached(log); - if (userId === this.options.ownUserId) { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { if (this._state === GroupCallState.Joined) { this._localMedia?.dispose(); this._localMedia = undefined; @@ -212,9 +254,9 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Created; } } else { - const member = this._members.get(userId); + const member = this._members.get(memberKey); if (member) { - this._members.remove(userId); + this._members.remove(memberKey); member.disconnect(); } } @@ -225,7 +267,7 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call - let member = this._members.get(userId); + let member = this._members.get(getMemberKey(userId, deviceId)); if (member) { member.handleDeviceMessage(message, deviceId, syncLog); } else { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 89e0db4c..d31837f4 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -23,7 +23,7 @@ import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes"; import type {GroupCall} from "./GroupCall"; import type {RoomMember} from "../../room/members/RoomMember"; import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; @@ -32,6 +32,7 @@ import type {ILogItem} from "../../../logging/types"; export type Options = Omit & { confId: string, ownUserId: string, + ownDeviceId: string, hsApi: HomeServerApi, encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, @@ -43,7 +44,7 @@ export class Member { constructor( public readonly member: RoomMember, - private memberCallInfo: Record, + private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, ) { @@ -58,15 +59,30 @@ export class Member { return this.peerCall?.state === CallState.Connected; } + get userId(): string { + return this.member.userId; + } + + get deviceId(): string { + return this.callDeviceMembership.device_id; + } + /** @internal */ connect(localMedia: LocalMedia) { - this.logItem.log("connect"); - this.localMedia = localMedia; - // otherwise wait for it to connect - if (this.member.userId < this.options.ownUserId) { - this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); - } + this.logItem.wrap("connect", () => { + this.localMedia = localMedia; + // otherwise wait for it to connect + let shouldInitiateCall; + if (this.member.userId === this.options.ownUserId) { + shouldInitiateCall = this.deviceId < this.options.ownDeviceId; + } else { + shouldInitiateCall = this.member.userId < this.options.ownUserId; + } + if (shouldInitiateCall) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + }); } /** @internal */ @@ -80,8 +96,8 @@ export class Member { } /** @internal */ - updateCallInfo(memberCallInfo) { - // m.calls object from the m.call.member event + updateCallInfo(callDeviceMembership: CallDeviceMembership) { + this.callDeviceMembership = callDeviceMembership; } /** @internal */ diff --git a/src/observable/map/ObservableMap.ts b/src/observable/map/ObservableMap.ts index d604ab0a..79662e29 100644 --- a/src/observable/map/ObservableMap.ts +++ b/src/observable/map/ObservableMap.ts @@ -80,15 +80,15 @@ export class ObservableMap extends BaseObservableMap { return this._values.size; } - [Symbol.iterator](): Iterator<[K, V]> { + [Symbol.iterator](): IterableIterator<[K, V]> { return this._values.entries(); } - values(): Iterator { + values(): IterableIterator { return this._values.values(); } - keys(): Iterator { + keys(): IterableIterator { return this._values.keys(); } }