From d7360e774103feae4225c7be1afb67760e7a1489 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:18:46 +0200 Subject: [PATCH] fix multiple device support --- src/matrix/DeviceMessageHandler.js | 6 +- src/matrix/calls/CallHandler.ts | 4 +- src/matrix/calls/PeerCall.ts | 14 +-- src/matrix/calls/group/GroupCall.ts | 133 +++++++++++++++++---------- src/matrix/calls/group/Member.ts | 4 +- src/platform/types/MediaDevices.ts | 1 + src/platform/web/dom/MediaDevices.ts | 4 + 7 files changed, 103 insertions(+), 63 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..ee10716e 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,11 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + if (dr.device) { + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); + } else { + console.error("could not deliver message because don't have device for sender key", dr.event); + } } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 678f4f44..89730391 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -131,7 +131,7 @@ export class CallHandler { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event - groupCall?.updateMember(userId, call, log); + groupCall?.updateMembership(userId, call, log); }; const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); @@ -140,7 +140,7 @@ export class CallHandler { for (const previousCallId of previousCallIdsMemberOf) { if (!newCallIdsMemberOf.has(previousCallId)) { const groupCall = this._calls.get(previousCallId); - groupCall?.removeMember(userId, log); + groupCall?.removeMembership(userId, log); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 3e23b3a6..b85cee61 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -412,7 +412,8 @@ export class PeerCall implements IDisposable { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); } @@ -772,21 +773,14 @@ export class PeerCall implements IDisposable { this.hangupParty = hangupParty; // this.hangupReason = hangupReason; this.setState(CallState.Ended, log); - this.stopAllMedia(); + //this.localMedia?.dispose(); + //this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } - private stopAllMedia(): void { - if (this.localMedia) { - for (const track of this.localMedia.tracks) { - track.stop(); - } - } - } - private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 3ef6a5ff..98e3381d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -95,10 +95,18 @@ export class GroupCall extends EventEmitter<{change: never}> { return this.callContent?.["m.terminated"] === true; } + get isRinging(): boolean { + return this._state === GroupCallState.Created && this.intent === "m.ring" && !this.isMember(this.options.ownUserId); + } + get name(): string { return this.callContent?.["m.name"]; } + get intent(): string { + return this.callContent?.["m.intent"]; + } + join(localMedia: LocalMedia): Promise { return this.logItem.wrap("join", async log => { if (this._state !== GroupCallState.Created) { @@ -134,6 +142,8 @@ export class GroupCall extends EventEmitter<{change: never}> { if (this._members.size === 0) { await this.terminate(); } + } else { + log.set("already_left", true); } }); } @@ -184,7 +194,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) { + updateMembership(userId: string, callMembership: CallMembership, syncLog: ILogItem) { this.logItem.wrap({l: "updateMember", id: userId}, log => { syncLog.refDetached(log); const devices = callMembership["m.devices"]; @@ -192,45 +202,62 @@ export class GroupCall extends EventEmitter<{change: never}> { for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joining) { - this._state = GroupCallState.Joined; - this.emitChange(); + log.wrap({l: "update device member", id: memberKey}, log => { + if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { + if (this._state === GroupCallState.Joining) { + log.set("update_own", true); + this._state = GroupCallState.Joined; + this.emitChange(); + } + } else { + let member = this._members.get(memberKey); + if (member) { + log.set("update", true); + member!.updateCallInfo(device); + } else { + const logItem = this.logItem.child({l: "member", id: memberKey}); + log.set("add", true); + log.refDetached(logItem); + member = new Member( + RoomMember.fromUserId(this.roomId, userId, "join"), + device, this._memberOptions, logItem + ); + this._members.add(memberKey, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!.clone()); + } + } } - return; - } - let member = this._members.get(memberKey); - if (member) { - member.updateCallInfo(device); - } else { - const logItem = this.logItem.child("member"); - member = new Member( - RoomMember.fromUserId(this.roomId, userId, "join"), - device, this._memberOptions, logItem - ); - this._members.add(memberKey, member); - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - member.connect(this._localMedia!.clone()); - } - } + }); } const newDeviceIds = new Set(devices.map(call => call.device_id)); // remove user as member of any calls not present anymore for (const previousDeviceId of previousDeviceIds) { if (!newDeviceIds.has(previousDeviceId)) { - this.removeMemberDevice(userId, previousDeviceId, syncLog); + log.wrap({l: "remove device member", id: getMemberKey(userId, previousDeviceId)}, log => { + this.removeMemberDevice(userId, previousDeviceId, log); + }); } } + if (userId === this.options.ownUserId && !newDeviceIds.has(this.options.ownDeviceId)) { + this.removeOwnDevice(log); + } }); } /** @internal */ - removeMember(userId: string, syncLog: ILogItem) { + removeMembership(userId: string, syncLog: ILogItem) { const deviceIds = this.getDeviceIdsForUserId(userId); - for (const deviceId of deviceIds) { - this.removeMemberDevice(userId, deviceId, syncLog); - } + this.logItem.wrap("removeMember", log => { + syncLog.refDetached(log); + for (const deviceId of deviceIds) { + this.removeMemberDevice(userId, deviceId, log); + } + if (userId === this.options.ownUserId) { + this.removeOwnDevice(log); + } + }); } private getDeviceIdsForUserId(userId: string): string[] { @@ -239,26 +266,32 @@ export class GroupCall extends EventEmitter<{change: never}> { .map(key => getDeviceFromMemberKey(key)); } + private isMember(userId: string): boolean { + return Array.from(this._members.keys()).some(key => memberKeyIsForUser(key, userId)); + } + + private removeOwnDevice(log: ILogItem) { + if (this._state === GroupCallState.Joined) { + log.set("leave_own", true); + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } + this._state = GroupCallState.Created; + this.emitChange(); + } + } + /** @internal */ - private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) { + private removeMemberDevice(userId: string, deviceId: string, log: ILogItem) { const memberKey = getMemberKey(userId, deviceId); - this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => { - syncLog.refDetached(log); - if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { - if (this._state === GroupCallState.Joined) { - this._localMedia?.dispose(); - this._localMedia = undefined; - for (const [,member] of this._members) { - member.disconnect(); - } - this._state = GroupCallState.Created; - } - } else { - const member = this._members.get(memberKey); - if (member) { - this._members.remove(memberKey); - member.disconnect(); - } + log.wrap({l: "removeMemberDevice", id: memberKey}, log => { + const member = this._members.get(memberKey); + if (member) { + log.set("leave", true); + this._members.remove(memberKey); + member.disconnect(); } this.emitChange(); }); @@ -315,9 +348,15 @@ export class GroupCall extends EventEmitter<{change: never}> { const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); if (stateEvent) { const content = stateEvent.event.content; - const callsInfo = content["m.calls"]; - content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); - return content; + const callInfo = content["m.calls"]?.find(c => c["m.call_id"] === this.id); + if (callInfo) { + const devicesInfo = callInfo["m.devices"]; + const deviceIndex = devicesInfo.findIndex(d => d["device_id"] === this.options.ownDeviceId); + if (deviceIndex !== -1) { + devicesInfo.splice(deviceIndex, 1); + return content; + } + } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index d31837f4..cd26fba2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -47,9 +47,7 @@ export class Member { private callDeviceMembership: CallDeviceMembership, private readonly options: Options, private readonly logItem: ILogItem, - ) { - logItem.set("id", member.userId); - } + ) {} get remoteTracks(): Track[] { return this.peerCall?.remoteTracks ?? []; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index ed9015bf..db267871 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -37,6 +37,7 @@ export interface Track { get muted(): boolean; setMuted(muted: boolean): void; stop(): void; + clone(): Track; } export interface AudioTrack extends Track { diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 22f3d634..49fd7fa5 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -111,6 +111,10 @@ export class TrackWrapper implements Track { stop() { this.track.stop(); } + + clone() { + return this.track.clone(); + } } export class AudioTrackWrapper extends TrackWrapper {