From beeb191588caf0d7f876a9fe0fed85b0d5a01b83 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 26 Apr 2022 21:11:41 +0200 Subject: [PATCH 1/9] reset member when seeing a new session id also buffer to_device messages for members we don't have a member event for already. --- src/matrix/calls/group/GroupCall.ts | 41 ++++++++++++++++++++++++----- src/matrix/calls/group/Member.ts | 32 +++++++++++++++++----- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9f8eb88c..29be893d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -65,6 +65,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _memberOptions: MemberOptions; private _state: GroupCallState; private localMuteSettings: MuteSettings = new MuteSettings(false, false); + private bufferedDeviceMessages = new Map>>(); constructor( public readonly id: string, @@ -234,7 +235,7 @@ export class GroupCall extends EventEmitter<{change: never}> { let member = this._members.get(memberKey); if (member) { log.set("update", true); - member!.updateCallInfo(device); + member!.updateCallInfo(device, log); } else { const logItem = this.logItem.child({l: "member", id: memberKey}); log.set("add", true); @@ -249,6 +250,9 @@ export class GroupCall extends EventEmitter<{change: never}> { member.connect(this._localMedia!.clone(), this.localMuteSettings); } } + // flush pending messages, either after having created the member, + // or updated the session id with updateCallInfo + this.flushPendingDeviceMessages(member, log); } }); } @@ -282,6 +286,23 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + private flushPendingDeviceMessages(member: Member, log: ILogItem) { + const memberKey = getMemberKey(member.userId, member.deviceId); + const bufferedMessages = this.bufferedDeviceMessages.get(memberKey); + // check if we have any pending message for the member with (userid, deviceid, sessionid) + if (bufferedMessages) { + for (const message of bufferedMessages) { + if (message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, log); + bufferedMessages.delete(message); + } + } + if (bufferedMessages.size === 0) { + this.bufferedDeviceMessages.delete(memberKey); + } + } + } + private getDeviceIdsForUserId(userId: string): string[] { return Array.from(this._members.keys()) .filter(key => memberKeyIsForUser(key, userId)) @@ -322,13 +343,21 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call - let member = this._members.get(getMemberKey(userId, deviceId)); - if (member) { - member.handleDeviceMessage(message, deviceId, syncLog); + const key = getMemberKey(userId, deviceId); + let member = this._members.get(key); + if (member && message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, syncLog); } else { - const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId}); + const item = this.logItem.log({l: "member not found, buffering", userId, deviceId, sessionId: message.content.sender_session_id}); syncLog.refDetached(item); - // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? + // we haven't received the m.call.member yet for this caller (or with this session id). + // buffer the device messages or create the member/call as it should arrive in a moment + let messages = this.bufferedDeviceMessages.get(key); + if (!messages) { + messages = new Set(); + this.bufferedDeviceMessages.set(key, messages); + } + messages.add(message); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e3f50c06..bef9bdb2 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -33,6 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, @@ -81,6 +82,11 @@ export class Member { return this.callDeviceMembership.device_id; } + /** session id of the member */ + get sessionId(): string { + return this.callDeviceMembership.session_id; + } + get dataChannel(): any | undefined { return this.peerCall?.dataChannel; } @@ -106,8 +112,8 @@ export class Member { } /** @internal */ - disconnect(hangup: boolean) { - this.logItem.wrap("disconnect", log => { + disconnect(hangup: boolean, log?: ILogItem) { + (log ?? this.logItem).wrap("disconnect", log => { if (hangup) { this.peerCall?.hangup(CallErrorCode.UserHangup); } else { @@ -121,8 +127,20 @@ export class Member { } /** @internal */ - updateCallInfo(callDeviceMembership: CallDeviceMembership) { - this.callDeviceMembership = callDeviceMembership; + updateCallInfo(callDeviceMembership: CallDeviceMembership, log: ILogItem) { + log.wrap({l: "updateing device membership", deviceId: this.deviceId}, log => { + // session id is changing, disconnect so we start with a new slate for the new session + if (callDeviceMembership.session_id !== this.sessionId) { + log.wrap({ + l: "member event changes session id", + oldSessionId: this.sessionId, + newSessionId: callDeviceMembership.session_id + }, log => { + this.disconnect(false, log); + }); + } + this.callDeviceMembership = callDeviceMembership; + }); } /** @internal */ @@ -151,7 +169,7 @@ export class Member { groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; groupMessage.content.sender_session_id = this.options.sessionId; - groupMessage.content.dest_session_id = this.callDeviceMembership.session_id; + groupMessage.content.dest_session_id = this.sessionId; // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); // const payload = formatToDeviceMessagesPayload(encryptedMessages); const payload = { @@ -174,7 +192,7 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { + handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { syncLog.refDetached(this.logItem); const destSessionId = message.content.dest_session_id; if (destSessionId !== this.options.sessionId) { @@ -185,7 +203,7 @@ export class Member { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - this.peerCall.handleIncomingSignallingMessage(message, deviceId); + this.peerCall.handleIncomingSignallingMessage(message, this.deviceId); } else { // TODO: need to buffer events until invite comes? } From 6b22078140ba98c7d77365625ae7241a9abf25cc Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:34:01 +0100 Subject: [PATCH 2/9] prevent localMedia being disposed when disconnecting on session change this would cause us to not send any media anymore and a black screen on the other side that just refreshed --- src/matrix/calls/group/Member.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index bef9bdb2..fb8c2164 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -112,8 +112,8 @@ export class Member { } /** @internal */ - disconnect(hangup: boolean, log?: ILogItem) { - (log ?? this.logItem).wrap("disconnect", log => { + disconnect(hangup: boolean) { + this.logItem.wrap("disconnect", log => { if (hangup) { this.peerCall?.hangup(CallErrorCode.UserHangup); } else { @@ -136,7 +136,13 @@ export class Member { oldSessionId: this.sessionId, newSessionId: callDeviceMembership.session_id }, log => { - this.disconnect(false, log); + // prevent localMedia from being stopped + // as connect won't be called again when reconnecting + // to the new session + const localMedia = this.localMedia; + this.localMedia = undefined; + this.disconnect(false); + this.localMedia = localMedia; }); } this.callDeviceMembership = callDeviceMembership; From 230ccd95abde45fbaee0dc852457c68116c35da2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:33:12 +0200 Subject: [PATCH 3/9] reset retryCount when disconnecting --- src/matrix/calls/group/Member.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index fb8c2164..3f7730a5 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -123,6 +123,7 @@ export class Member { this.peerCall = undefined; this.localMedia?.dispose(); this.localMedia = undefined; + this.retryCount = 0; }); } From be04eeded0509bfde4159642bc6b35c3329aae51 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:33:27 +0200 Subject: [PATCH 4/9] always reevaluate remote media when receiving a new remote track not just when we don't know the stream already this caused the video track to not appear when the other party sends the invite. Also added more logging --- src/matrix/calls/PeerCall.ts | 63 ++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 6aff72c1..15331ab5 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -902,6 +902,9 @@ export class PeerCall implements IDisposable { private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { + log.set("kind", track.kind); + log.set("id", track.id); + log.set("streams", streams.map(s => s.id)); if (streams.length === 0) { log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); return; @@ -932,39 +935,49 @@ export class PeerCall implements IDisposable { disposeListener, stream }); - this.updateRemoteMedia(log); } + this.updateRemoteMedia(log); } private updateRemoteMedia(log: ILogItem): void { - this._remoteMedia.userMedia = undefined; - this._remoteMedia.screenShare = undefined; - if (this.remoteSDPStreamMetadata) { - for (const streamDetails of this._remoteStreams.values()) { - const {stream} = streamDetails; - const metaData = this.remoteSDPStreamMetadata[stream.id]; - if (metaData) { - if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { - this._remoteMedia.userMedia = stream; - const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); - if (audioReceiver) { - audioReceiver.track.enabled = !metaData.audio_muted; + log.wrap("reevaluating remote media", log => { + this._remoteMedia.userMedia = undefined; + this._remoteMedia.screenShare = undefined; + if (this.remoteSDPStreamMetadata) { + for (const streamDetails of this._remoteStreams.values()) { + const {stream} = streamDetails; + const metaData = this.remoteSDPStreamMetadata[stream.id]; + if (metaData) { + if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { + this._remoteMedia.userMedia = stream; + const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); + if (audioReceiver) { + audioReceiver.track.enabled = !metaData.audio_muted; + } + const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); + if (videoReceiver) { + videoReceiver.track.enabled = !metaData.video_muted; + } + this._remoteMuteSettings = new MuteSettings( + metaData.audio_muted || !audioReceiver?.track, + metaData.video_muted || !videoReceiver?.track + ); + log.log({ + l: "setting userMedia", + micMuted: this._remoteMuteSettings.microphone, + cameraMuted: this._remoteMuteSettings.camera + }); + } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { + this._remoteMedia.screenShare = stream; + log.log("setting screenShare"); } - const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); - if (videoReceiver) { - videoReceiver.track.enabled = !metaData.video_muted; - } - this._remoteMuteSettings = new MuteSettings( - metaData.audio_muted || !audioReceiver?.track, - metaData.video_muted || !videoReceiver?.track - ); - } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { - this._remoteMedia.screenShare = stream; + } else { + log.log({l: "no metadata yet for stream, ignoring for now", id: stream.id}); } } } - } - this.options.emitUpdate(this, undefined); + this.options.emitUpdate(this, undefined); + }); } private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { From 6394138c4ab0ac99225d9df5f64444a16458b6f3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 19:40:13 +0200 Subject: [PATCH 5/9] fix isMuted logic in view model --- src/domain/session/room/CallViewModel.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 3db1feb8..da526fcd 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -67,14 +67,6 @@ export class CallViewModel extends ViewModel { } } - get isCameraMuted(): boolean { - return this.call.muteSettings.camera; - } - - get isMicrophoneMuted(): boolean { - return this.call.muteSettings.microphone; - } - async toggleVideo() { this.call.setMuted(this.call.muteSettings.toggleCamera()); } @@ -95,11 +87,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } get isCameraMuted(): boolean { - return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream); + return isMuted(this.call.muteSettings.camera, !!getStreamVideoTrack(this.stream)); } get isMicrophoneMuted(): boolean { - return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream); + return isMuted(this.call.muteSettings.microphone, !!getStreamAudioTrack(this.stream)); } get avatarLetter(): string { @@ -135,11 +127,11 @@ export class CallMemberViewModel extends ViewModel implements ISt } get isCameraMuted(): boolean { - return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream); + return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream)); } get isMicrophoneMuted(): boolean { - return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream); + return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream)); } get avatarLetter(): string { @@ -178,3 +170,11 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; } + +function isMuted(muted: boolean | undefined, hasTrack: boolean) { + if (muted) { + return true; + } else { + return !hasTrack; + } +} From bffce7fafe28e9f85570c65e4e47023c01f84ffa Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 19:40:49 +0200 Subject: [PATCH 6/9] more logging --- src/matrix/calls/group/GroupCall.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 29be893d..fa7f4f9d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -224,7 +224,7 @@ export class GroupCall extends EventEmitter<{change: never}> { for (const device of devices) { const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "update device member", id: memberKey}, log => { + log.wrap({l: "update device member", id: memberKey, sessionId: device.session_id}, log => { if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { if (this._state === GroupCallState.Joining) { log.set("update_own", true); @@ -348,7 +348,13 @@ export class GroupCall extends EventEmitter<{change: never}> { if (member && message.content.sender_session_id === member.sessionId) { member.handleDeviceMessage(message, syncLog); } else { - const item = this.logItem.log({l: "member not found, buffering", userId, deviceId, sessionId: message.content.sender_session_id}); + const item = this.logItem.log({ + l: "buffering to_device message, member not found", + userId, + deviceId, + sessionId: message.content.sender_session_id, + type: message.type + }); syncLog.refDetached(item); // we haven't received the m.call.member yet for this caller (or with this session id). // buffer the device messages or create the member/call as it should arrive in a moment From 4f2999f8d8a06f964d4c1526cd890b5b21c8823b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 19:41:02 +0200 Subject: [PATCH 7/9] reconnect when detecting session id change, so we send invite if needed --- src/matrix/calls/group/Member.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 3f7730a5..3d773b7e 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -143,7 +143,9 @@ export class Member { const localMedia = this.localMedia; this.localMedia = undefined; this.disconnect(false); - this.localMedia = localMedia; + // connect again, as the other side might be waiting for our invite + // after refreshing + this.connect(localMedia!, this.localMuteSettings!); }); } this.callDeviceMembership = callDeviceMembership; From b03b296391dd1d7520607937edecc43eaf63d7cd Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 19:41:25 +0200 Subject: [PATCH 8/9] comments, todo housekeeping --- src/matrix/calls/PeerCall.ts | 1 + src/matrix/calls/TODO.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 15331ab5..e5e70a46 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -184,6 +184,7 @@ export class PeerCall implements IDisposable { } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead + // TODO: we don't actually use this await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); }); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a7f4e82c..91840af7 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -14,7 +14,7 @@ - implement muting tracks with m.call.sdp_stream_metadata_changed - implement renegotiation - making logging better - - finish session id support + - DONE: finish session id support - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - reeable crypto & implement fetching olm keys before sending encrypted signalling message From aa709ee6e9616adc36990e1403ccdc5698506dc9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 27 Apr 2022 19:41:42 +0200 Subject: [PATCH 9/9] make text white for now --- src/platform/web/ui/css/themes/element/theme.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index a0ad1ef0..db0ad66b 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1197,6 +1197,7 @@ button.RoomDetailsView_row::after { .StreamView_muteStatus { align-self: end; justify-self: start; + color: var(--text-color--lighter-80); } .StreamView_muteStatus.microphoneMuted::before {