From 82ffb557e5cfa1082b6bdd5c799bd5635346ebd0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:09:31 +0200 Subject: [PATCH 1/4] update TODO --- src/matrix/calls/TODO.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a0a64752..a7f4e82c 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -9,14 +9,14 @@ ## TODO - DONE: implement receiving hangup - - making logging better + - DONE: implement cloning the localMedia so it works in safari? + - DONE: implement 3 retries per peer + - implement muting tracks with m.call.sdp_stream_metadata_changed - implement renegotiation + - making logging better - finish session id support - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - - implement muting tracks with m.call.sdp_stream_metadata_changed - - DONE: implement cloning the localMedia so it works in safari? - - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? - make UI pretsy From 99769eb84e0f7cd577fdee5a60def920ecb50a4f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:10:49 +0200 Subject: [PATCH 2/4] implement basic renegotiation --- src/matrix/calls/PeerCall.ts | 53 ++++++++++++++++++++++++------ src/matrix/calls/callEventTypes.ts | 7 ++++ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 515e1b65..98747e40 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -18,24 +18,24 @@ import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; import {Disposables, IDisposable} from "../../utils/Disposables"; -import type {Room} from "../room/Room"; -import type {StateEvent} from "../storage/types"; -import type {ILogItem} from "../../logging/types"; - -import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; -import type {LocalMedia} from "./LocalMedia"; - import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, CallErrorCode, } from "./callEventTypes"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; +import type {TimeoutCreator, Timeout} from "../../platform/types/types"; +import type {LocalMedia} from "./LocalMedia"; import type { MCallBase, MCallInvite, + MCallNegotiate, MCallAnswer, MCallSDPStreamMetadataChanged, MCallCandidates, @@ -265,6 +265,9 @@ export class PeerCall implements IDisposable { case EventType.Answer: await this.handleAnswer(message.content, partyId, log); break; + case EventType.Negotiate: + await this.handleRemoteNegotiate(message.content, partyId, log); + break; case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; @@ -341,10 +344,10 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { - log.log("would send renegotiation now but not implemented"); // send Negotiate message - //await this.sendSignallingMessage({type: EventType.Invite, content}); - //this.setState(CallState.InviteSent); + content.description = content.offer; + delete content.offer; + await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { this.makingOffer = false; @@ -497,6 +500,36 @@ export class PeerCall implements IDisposable { } } + + private async handleRemoteNegotiate(content: MCallNegotiate, partyId: PartyId, log: ILogItem): Promise { + if (this._state !== CallState.Connected) { + log.log({l: `Ignoring renegotiate because not connected`, status: this._state}); + return; + } + + if (this.opponentPartyId !== partyId) { + log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); + return; + } + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); + } else { + log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.description); + } catch (e) { + await log.wrap(`Failed to set remote description`, log => { + log.catch(e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } + } + private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 4726175c..449bd469 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -97,6 +97,12 @@ export type MCallInvite = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } +export type MCallNegotiate = Base & { + description: SessionDescription; + lifetime: number; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } @@ -213,6 +219,7 @@ export enum CallErrorCode { export type SignallingMessage = {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Negotiate, content: MCallNegotiate} | {type: EventType.Answer, content: MCallAnswer} | {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | {type: EventType.Candidates, content: MCallCandidates} | From 55c6dcf613ad8c3c4272bd09bf679b1acff74dba Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:11:24 +0200 Subject: [PATCH 3/4] don't re-clone streams when not needed --- src/matrix/calls/LocalMedia.ts | 36 ++++++++++++++++++++++++++--- src/matrix/calls/group/GroupCall.ts | 8 +++---- src/matrix/calls/group/Member.ts | 8 +++---- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 7dbc9e17..933ae89f 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -43,13 +43,43 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } + replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { + let userMedia; + let screenShare; + const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { + let stream; + if (oldOriginalStream?.id === newStream?.id) { + stream = oldCloneStream; + } else { + stream = newStream?.clone(); + oldCloneStream?.audioTrack?.stop(); + oldCloneStream?.videoTrack?.stop(); + } + return stream; + } + return new LocalMedia( + cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), + this.microphoneMuted, this.cameraMuted, + cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare), + this.dataChannelOptions + ); + } + clone(): LocalMedia { return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.userMedia?.audioTrack?.stop(); - this.userMedia?.videoTrack?.stop(); - this.screenShare?.videoTrack?.stop(); + this.stopExcept(undefined); + } + + stopExcept(newMedia: LocalMedia | undefined) { + if(newMedia?.userMedia?.id !== this.userMedia?.id) { + this.userMedia?.audioTrack?.stop(); + this.userMedia?.videoTrack?.stop(); + } + if(newMedia?.screenShare?.id !== this.screenShare?.id) { + this.screenShare?.videoTrack?.stop(); + } } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9fc8d1ee..cb962b86 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -124,13 +124,13 @@ export class GroupCall extends EventEmitter<{change: never}> { } async setMedia(localMedia: LocalMedia): Promise { - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - const oldMedia = this._localMedia; + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) { + const oldMedia = this._localMedia!; this._localMedia = localMedia; await Promise.all(Array.from(this._members.values()).map(m => { - return m.setMedia(localMedia!.clone()); + return m.setMedia(localMedia, oldMedia); })); - oldMedia?.dispose(); + oldMedia?.stopExcept(localMedia); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 22a3d1fa..714cd821 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -185,11 +185,9 @@ export class Member { } /** @internal */ - async setMedia(localMedia: LocalMedia): Promise { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - await this.peerCall?.setMedia(localMedia); - oldMedia?.dispose(); + async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { + this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia); + await this.peerCall?.setMedia(this.localMedia); } private _createPeerCall(callId: string): PeerCall { From 10a6269147df9b00ff322a793c07e31dad129cf9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:15:57 +0200 Subject: [PATCH 4/4] always send new metadata after calling setMedia --- src/matrix/calls/PeerCall.ts | 39 ++++++++++-------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 98747e40..79fb04fb 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -217,24 +217,14 @@ export class PeerCall implements IDisposable { log.set("userMedia_video_muted", localMedia.cameraMuted); log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); log.set("datachannel", !!localMedia.dataChannelOptions); - - const oldMetaData = this.getSDPMetadata(); - const willRenegotiate = await this.updateLocalMedia(localMedia, log); - // TODO: if we will renegotiate, we don't bother sending the metadata changed event - // because the renegotiate event will send new metadata anyway, but is that the right - // call? - if (!willRenegotiate) { - const newMetaData = this.getSDPMetadata(); - if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { - const content: MCallSDPStreamMetadataChanged = { - call_id: this.callId, - version: 1, - seq: this.seq++, - [SDPStreamMetadataKey]: newMetaData - }; - await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); - } - } + await this.updateLocalMedia(localMedia, log); + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); }); } @@ -856,8 +846,8 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), - video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), + audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack, + video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack, }; } if (this.localMedia?.screenShare) { @@ -889,9 +879,8 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } - private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { return logItem.wrap("updateLocalMedia", async log => { - let willRenegotiate = false; const oldMedia = this.localMedia; this.localMedia = localMedia; const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { @@ -920,13 +909,11 @@ export class PeerCall implements IDisposable { log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { this.peerConnection.removeTrack(sender); this.peerConnection.addTrack(track); - willRenegotiate = true; }); } } else { log.wrap(`adding ${logLabel} ${track.kind} track`, log => { this.peerConnection.addTrack(track); - willRenegotiate = true; }); } } else { @@ -937,20 +924,17 @@ export class PeerCall implements IDisposable { log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { sender.track.enabled = false; this.peerConnection.removeTrack(sender); - willRenegotiate = true; }); } } } else if (track) { log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id}); if (sender && muted !== wasMuted) { - // TODO: why does unmuting not work? wasMuted is false log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { // sender.track.enabled = !muted; // This doesn't always seem to trigger renegotiation?? // We should probably always send the new metadata first ... sender.enable(!muted); - willRenegotiate = true; }); } else { log.log(`${logLabel} ${track.kind} track hasn't changed`); @@ -964,7 +948,6 @@ export class PeerCall implements IDisposable { await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); - return willRenegotiate; // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); }