From 382fba88bdf757887ac978712fc8004788006bfb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 23:19:44 +0200 Subject: [PATCH] WIP for muting --- src/domain/session/room/CallViewModel.ts | 6 + src/matrix/calls/LocalMedia.ts | 14 +- src/matrix/calls/PeerCall.ts | 256 +++++++++---------- src/matrix/calls/TODO.md | 2 +- src/matrix/calls/group/GroupCall.ts | 11 + src/matrix/calls/group/Member.ts | 8 + src/platform/types/MediaDevices.ts | 3 + src/platform/web/dom/MediaDevices.ts | 34 ++- src/platform/web/dom/WebRTC.ts | 8 + src/platform/web/index.html | 2 +- src/platform/web/ui/general/TemplateView.ts | 6 +- src/platform/web/ui/session/room/CallView.ts | 23 +- 12 files changed, 210 insertions(+), 163 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index bc7b54ff..05245cd3 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel { this.call.leave(); } } + + async toggleVideo() { + const localMedia = this.call.localMedia!; + const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); + await this.call.setMedia(toggledMedia); + } } type MemberOptions = BaseOptions & {member: Member}; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6564adac..7dbc9e17 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( public readonly userMedia?: Stream, + public readonly microphoneMuted: boolean = false, + public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} + withMuted(microphone: boolean, camera: boolean) { + return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); + } + withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5b6e2ca8..b450ac3f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -31,6 +31,7 @@ import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, + CallErrorCode, } from "./callEventTypes"; import type { MCallBase, @@ -66,6 +67,7 @@ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; + // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; private seq: number = 0; // A queue for candidates waiting to go out. @@ -151,7 +153,6 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - // we should keep an object with streams by purpose ... e.g. RemoteMedia? get remoteMedia(): Readonly { return this._remoteMedia; } @@ -163,7 +164,7 @@ export class PeerCall implements IDisposable { } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.setMedia(localMedia); + this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } @@ -179,7 +180,7 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateAnswer, log); - this.setMedia(localMedia, log); + this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -208,48 +209,32 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { - return logItem.wrap("setMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { - const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; + setMedia(localMedia: LocalMedia): Promise { + return this.logItem.wrap("setMedia", async log => { + log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); + log.set("userMedia_audio_muted", localMedia.microphoneMuted); + log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); + log.set("userMedia_video_muted", localMedia.cameraMuted); + log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("datachannel", !!localMedia.dataChannelOptions); - const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { - if (track) { - if (oldTrack && sender) { - log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - sender.replaceTrack(track); - }); - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } - } + const oldMetaData = this.getSDPMetadata(); + const willRenegotiate = await this.updateLocalMedia(localMedia, log); + if (!willRenegotiate) { + const newMetaData = this.getSDPMetadata(); + if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: newMetaData + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log); } - - applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); - applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); } - - applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); - // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } - /** group calls would handle reject at the group call level, not at the peer call level */ - async reject() { - - } - hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); @@ -280,7 +265,14 @@ export class PeerCall implements IDisposable { case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; default: log.log(`Unknown event type for call: ${message.type}`); break; @@ -444,12 +436,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - // if (this.peerConnection.remoteTracks.length === 0) { - // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - // }); - // return; - // } + if (this.peerConnection.remoteStreams.size === 0) { + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } this.setState(CallState.Ringing, log); @@ -719,7 +711,7 @@ export class PeerCall implements IDisposable { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); - // TODO: apply muting + } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -828,10 +820,9 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled), - video_muted: !(streamSender?.videoSender?.enabled), + audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), + video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), }; - console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; @@ -851,6 +842,8 @@ export class PeerCall implements IDisposable { if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { this._remoteMedia.userMedia = streamReceiver.stream; + streamReceiver.audioReceiver?.enable(!metaData.audio_muted); + streamReceiver.videoReceiver?.enable(!metaData.video_muted); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = streamReceiver.stream; } @@ -860,6 +853,77 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + return logItem.wrap("updateLocalMedia", async log => { + let willRenegotiate = false; + const oldMedia = this.localMedia; + this.localMedia = localMedia; + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { + const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined; + + const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { + const changed = (!track && oldTrack) || + (track && !oldTrack) || + (track && oldTrack && !track.equals(oldTrack)); + if (changed) { + if (track) { + if (oldTrack && sender && !track.equals(oldTrack)) { + try { + await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + return sender.replaceTrack(track); + }); + } catch (err) { + // can't replace the track without renegotiating + log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + if (sender) { + // this will be used for muting, do we really want to trigger renegotiation here? + // we want to disable the sender, but also remove the track as we don't want to keep + // using the webcam if we don't need to + log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { + sender.track.enabled = false; + this.peerConnection.removeTrack(sender); + willRenegotiate = true; + }); + } + } + } else if (track) { + console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted}); + if (sender && muted !== wasMuted) { + // TODO: why does unmuting not work? wasMuted is false + log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { + sender.track.enabled = !muted; + sender.enable(!muted); + willRenegotiate = true; + }); + } else { + log.log(`${logLabel} ${track.kind} track hasn't changed`); + } + } + } + + await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); + await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); + } + + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + return willRenegotiate; + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + }); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); @@ -915,100 +979,11 @@ export enum CallDirection { Outbound = 'outbound', } -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; @@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || - eventType === EventType.Hangup; + eventType === EventType.Hangup || + eventType === EventType.SDPStreamMetadataChanged || + eventType === EventType.SDPStreamMetadataChangedPrefix || + eventType === EventType.Negotiate; } export function tests() { diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 8c041fcc..a0a64752 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -15,7 +15,7 @@ - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - - implement cloning the localMedia so it works in safari? + - DONE: implement cloning the localMedia so it works in safari? - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 892ed67c..9fc8d1ee 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + async setMedia(localMedia: LocalMedia): Promise { + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + const oldMedia = this._localMedia; + this._localMedia = localMedia; + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMedia(localMedia!.clone()); + })); + oldMedia?.dispose(); + } + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 06693030..e821e3da 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -182,6 +182,14 @@ export class Member { } } + /** @internal */ + async setMedia(localMedia: LocalMedia): Promise { + const oldMedia = this.localMedia; + this.localMedia = localMedia; + await this.peerCall?.setMedia(localMedia); + oldMedia?.dispose(); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index d0edbbea..85f64f95 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -39,6 +39,9 @@ export interface Track { readonly label: string; readonly id: string; readonly settings: MediaTrackSettings; + get enabled(): boolean; + set enabled(value: boolean); + equals(track: Track): boolean; stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 3723162a..a04fca91 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -75,27 +75,37 @@ export class StreamWrapper implements Stream { public audioTrack: AudioTrackWrapper | undefined = undefined; public videoTrack: TrackWrapper | undefined = undefined; - constructor(public readonly stream: MediaStream) { - for (const track of stream.getTracks()) { - this.update(track); + constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { + if (clonedTracks) { + this.audioTrack = clonedTracks.audioTrack; + this.videoTrack = clonedTracks.videoTrack; + } else { + for (const track of stream.getTracks()) { + this.update(track); + } } } get id(): string { return this.stream.id; } clone(): Stream { - return new StreamWrapper(this.stream.clone()); + const clonedStream = this.stream.clone(); + const clonedTracks = { + audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, + videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, + }; + return new StreamWrapper(clonedStream, clonedTracks); } update(track: MediaStreamTrack): TrackWrapper | undefined { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream); + this.videoTrack = new TrackWrapper(track, this.stream, track.id); } return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream); + this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); } return this.audioTrack; } @@ -105,14 +115,18 @@ export class StreamWrapper implements Stream { export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream + public readonly stream: MediaStream, + public readonly originalId: string, ) {} get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - + get enabled(): boolean { return this.track.enabled; } + set enabled(enabled: boolean) { this.track.enabled = enabled; } + // test equality across clones + equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } stop() { this.track.stop(); } } @@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream) { - super(track, stream); + constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { + super(track, stream, originalId); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 672c14d4..22096699 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { this.transceiver.direction === this.exclusiveValue; } + enableWithoutRenegotiation(enabled: boolean) { + this.track.track.enabled = enabled; + } + enable(enabled: boolean) { if (enabled !== this.enabled) { + // do this first, so we stop sending track data immediately. + // this will still consume bandwidth though, so also disable the transceiver, + // which will trigger a renegotiation though. + this.enableWithoutRenegotiation(enabled); if (enabled) { if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; diff --git a/src/platform/web/index.html b/src/platform/web/index.html index 5950d89f..064c61a1 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -11,7 +11,7 @@ - +