diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 3db1feb8..da526fcd 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -67,14 +67,6 @@ export class CallViewModel extends ViewModel { } } - get isCameraMuted(): boolean { - return this.call.muteSettings.camera; - } - - get isMicrophoneMuted(): boolean { - return this.call.muteSettings.microphone; - } - async toggleVideo() { this.call.setMuted(this.call.muteSettings.toggleCamera()); } @@ -95,11 +87,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamV } get isCameraMuted(): boolean { - return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream); + return isMuted(this.call.muteSettings.camera, !!getStreamVideoTrack(this.stream)); } get isMicrophoneMuted(): boolean { - return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream); + return isMuted(this.call.muteSettings.microphone, !!getStreamAudioTrack(this.stream)); } get avatarLetter(): string { @@ -135,11 +127,11 @@ export class CallMemberViewModel extends ViewModel implements ISt } get isCameraMuted(): boolean { - return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream); + return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream)); } get isMicrophoneMuted(): boolean { - return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream); + return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream)); } get avatarLetter(): string { @@ -178,3 +170,11 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; } + +function isMuted(muted: boolean | undefined, hasTrack: boolean) { + if (muted) { + return true; + } else { + return !hasTrack; + } +} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 6aff72c1..e5e70a46 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -184,6 +184,7 @@ export class PeerCall implements IDisposable { } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead + // TODO: we don't actually use this await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); }); } @@ -902,6 +903,9 @@ export class PeerCall implements IDisposable { private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { + log.set("kind", track.kind); + log.set("id", track.id); + log.set("streams", streams.map(s => s.id)); if (streams.length === 0) { log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); return; @@ -932,39 +936,49 @@ export class PeerCall implements IDisposable { disposeListener, stream }); - this.updateRemoteMedia(log); } + this.updateRemoteMedia(log); } private updateRemoteMedia(log: ILogItem): void { - this._remoteMedia.userMedia = undefined; - this._remoteMedia.screenShare = undefined; - if (this.remoteSDPStreamMetadata) { - for (const streamDetails of this._remoteStreams.values()) { - const {stream} = streamDetails; - const metaData = this.remoteSDPStreamMetadata[stream.id]; - if (metaData) { - if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { - this._remoteMedia.userMedia = stream; - const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); - if (audioReceiver) { - audioReceiver.track.enabled = !metaData.audio_muted; + log.wrap("reevaluating remote media", log => { + this._remoteMedia.userMedia = undefined; + this._remoteMedia.screenShare = undefined; + if (this.remoteSDPStreamMetadata) { + for (const streamDetails of this._remoteStreams.values()) { + const {stream} = streamDetails; + const metaData = this.remoteSDPStreamMetadata[stream.id]; + if (metaData) { + if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { + this._remoteMedia.userMedia = stream; + const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); + if (audioReceiver) { + audioReceiver.track.enabled = !metaData.audio_muted; + } + const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); + if (videoReceiver) { + videoReceiver.track.enabled = !metaData.video_muted; + } + this._remoteMuteSettings = new MuteSettings( + metaData.audio_muted || !audioReceiver?.track, + metaData.video_muted || !videoReceiver?.track + ); + log.log({ + l: "setting userMedia", + micMuted: this._remoteMuteSettings.microphone, + cameraMuted: this._remoteMuteSettings.camera + }); + } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { + this._remoteMedia.screenShare = stream; + log.log("setting screenShare"); } - const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); - if (videoReceiver) { - videoReceiver.track.enabled = !metaData.video_muted; - } - this._remoteMuteSettings = new MuteSettings( - metaData.audio_muted || !audioReceiver?.track, - metaData.video_muted || !videoReceiver?.track - ); - } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { - this._remoteMedia.screenShare = stream; + } else { + log.log({l: "no metadata yet for stream, ignoring for now", id: stream.id}); } } } - } - this.options.emitUpdate(this, undefined); + this.options.emitUpdate(this, undefined); + }); } private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a7f4e82c..91840af7 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -14,7 +14,7 @@ - implement muting tracks with m.call.sdp_stream_metadata_changed - implement renegotiation - making logging better - - finish session id support + - DONE: finish session id support - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - reeable crypto & implement fetching olm keys before sending encrypted signalling message diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9b5df521..55efa6f9 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -65,6 +65,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _memberOptions: MemberOptions; private _state: GroupCallState; private localMuteSettings: MuteSettings = new MuteSettings(false, false); + private bufferedDeviceMessages = new Map>>(); private _deviceIndex?: number; private _eventTimestamp?: number; @@ -235,7 +236,7 @@ export class GroupCall extends EventEmitter<{change: never}> { const device = devices[deviceIndex]; const deviceId = device.device_id; const memberKey = getMemberKey(userId, deviceId); - log.wrap({l: "update device member", id: memberKey}, log => { + log.wrap({l: "update device member", id: memberKey, sessionId: device.session_id}, log => { if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) { this._deviceIndex = deviceIndex; @@ -250,7 +251,7 @@ export class GroupCall extends EventEmitter<{change: never}> { let member = this._members.get(memberKey); if (member) { log.set("update", true); - member!.updateCallInfo(device, deviceIndex, eventTimestamp); + member!.updateCallInfo(device, deviceIndex, eventTimestamp, log); } else { const logItem = this.logItem.child({l: "member", id: memberKey}); log.set("add", true); @@ -265,6 +266,9 @@ export class GroupCall extends EventEmitter<{change: never}> { member.connect(this._localMedia!.clone(), this.localMuteSettings); } } + // flush pending messages, either after having created the member, + // or updated the session id with updateCallInfo + this.flushPendingDeviceMessages(member, log); } }); } @@ -298,6 +302,23 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + private flushPendingDeviceMessages(member: Member, log: ILogItem) { + const memberKey = getMemberKey(member.userId, member.deviceId); + const bufferedMessages = this.bufferedDeviceMessages.get(memberKey); + // check if we have any pending message for the member with (userid, deviceid, sessionid) + if (bufferedMessages) { + for (const message of bufferedMessages) { + if (message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, log); + bufferedMessages.delete(message); + } + } + if (bufferedMessages.size === 0) { + this.bufferedDeviceMessages.delete(memberKey); + } + } + } + private getDeviceIdsForUserId(userId: string): string[] { return Array.from(this._members.keys()) .filter(key => memberKeyIsForUser(key, userId)) @@ -338,13 +359,27 @@ export class GroupCall extends EventEmitter<{change: never}> { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, syncLog: ILogItem) { // TODO: return if we are not membering to the call - let member = this._members.get(getMemberKey(userId, deviceId)); - if (member) { - member.handleDeviceMessage(message, deviceId, syncLog); + const key = getMemberKey(userId, deviceId); + let member = this._members.get(key); + if (member && message.content.sender_session_id === member.sessionId) { + member.handleDeviceMessage(message, syncLog); } else { - const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId}); + const item = this.logItem.log({ + l: "buffering to_device message, member not found", + userId, + deviceId, + sessionId: message.content.sender_session_id, + type: message.type + }); syncLog.refDetached(item); - // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? + // we haven't received the m.call.member yet for this caller (or with this session id). + // buffer the device messages or create the member/call as it should arrive in a moment + let messages = this.bufferedDeviceMessages.get(key); + if (!messages) { + messages = new Set(); + this.bufferedDeviceMessages.set(key, messages); + } + messages.add(message); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 47db7674..72a6ee55 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -33,6 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, @@ -83,6 +84,11 @@ export class Member { return this.callDeviceMembership.device_id; } + /** session id of the member */ + get sessionId(): string { + return this.callDeviceMembership.session_id; + } + get dataChannel(): any | undefined { return this.peerCall?.dataChannel; } @@ -127,14 +133,35 @@ export class Member { this.peerCall = undefined; this.localMedia?.dispose(); this.localMedia = undefined; + this.retryCount = 0; }); } /** @internal */ - updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number) { - this.callDeviceMembership = callDeviceMembership; - this._deviceIndex = deviceIndex; - this._eventTimestamp = eventTimestamp; + updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number, log: ILogItem) { + log.wrap({l: "updateing device membership", deviceId: this.deviceId}, log => { + // session id is changing, disconnect so we start with a new slate for the new session + if (callDeviceMembership.session_id !== this.sessionId) { + log.wrap({ + l: "member event changes session id", + oldSessionId: this.sessionId, + newSessionId: callDeviceMembership.session_id + }, log => { + // prevent localMedia from being stopped + // as connect won't be called again when reconnecting + // to the new session + const localMedia = this.localMedia; + this.localMedia = undefined; + this.disconnect(false); + // connect again, as the other side might be waiting for our invite + // after refreshing + this.connect(localMedia!, this.localMuteSettings!); + }); + } + this.callDeviceMembership = callDeviceMembership; + this._deviceIndex = deviceIndex; + this._eventTimestamp = eventTimestamp; + }); } /** @internal */ @@ -163,7 +190,7 @@ export class Member { groupMessage.content.device_id = this.options.ownDeviceId; groupMessage.content.party_id = this.options.ownDeviceId; groupMessage.content.sender_session_id = this.options.sessionId; - groupMessage.content.dest_session_id = this.callDeviceMembership.session_id; + groupMessage.content.dest_session_id = this.sessionId; // const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); // const payload = formatToDeviceMessagesPayload(encryptedMessages); const payload = { @@ -186,7 +213,7 @@ export class Member { } /** @internal */ - handleDeviceMessage(message: SignallingMessage, deviceId: string, syncLog: ILogItem) { + handleDeviceMessage(message: SignallingMessage, syncLog: ILogItem): void { syncLog.refDetached(this.logItem); const destSessionId = message.content.dest_session_id; if (destSessionId !== this.options.sessionId) { @@ -197,7 +224,7 @@ export class Member { this.peerCall = this._createPeerCall(message.content.call_id); } if (this.peerCall) { - this.peerCall.handleIncomingSignallingMessage(message, deviceId); + this.peerCall.handleIncomingSignallingMessage(message, this.deviceId); } else { // TODO: need to buffer events until invite comes? } diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index a0ad1ef0..db0ad66b 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -1197,6 +1197,7 @@ button.RoomDetailsView_row::after { .StreamView_muteStatus { align-self: end; justify-self: start; + color: var(--text-color--lighter-80); } .StreamView_muteStatus.microphoneMuted::before {