From cdb2a79b62f27e7e1e25d36a200adcff3bfa96f3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 22 Apr 2022 14:48:14 +0100 Subject: [PATCH] add muting again, separate from changing media --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/PeerCall.ts | 80 +++++++++++++++++++++--- src/matrix/calls/common.ts | 8 +++ src/matrix/calls/group/GroupCall.ts | 19 +++++- src/matrix/calls/group/Member.ts | 16 +++-- 5 files changed, 111 insertions(+), 14 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index e6a6914f..df61d334 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -57,7 +57,7 @@ export class CallViewModel extends ViewModel { } async toggleVideo() { - //this.call.setMuted(); + this.call.setMuted(this.call.muteSettings.toggleCamera()); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 799fd9b8..81828c32 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,7 +17,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; -import {WebRTC, PeerConnection, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { @@ -166,13 +166,14 @@ export class PeerCall implements IDisposable { return this._remoteMedia; } - call(localMedia: LocalMedia): Promise { + call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("call", async log => { if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); + this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); @@ -183,12 +184,13 @@ export class PeerCall implements IDisposable { }); } - answer(localMedia: LocalMedia): Promise { + answer(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("answer", async log => { if (this._state !== CallState.Ringing) { return; } this.setState(CallState.CreateAnswer, log); + this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { @@ -235,12 +237,54 @@ export class PeerCall implements IDisposable { }); } + setMuted(localMuteSettings: MuteSettings) { + return this.logItem.wrap("setMuted", async log => { + this.localMuteSettings = localMuteSettings; + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + + if (this.localMedia) { + const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); + if (userMediaAudio) { + this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); + } + const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); + if (userMediaVideo) { + this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); + } + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); + } + }); + } + hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); }); } + private muteTrack(track: Track, muted: boolean, log: ILogItem): void { + log.wrap({l: "track", kind: track.kind, id: track.id}, log => { + const enabled = !muted; + log.set("enabled", enabled); + const transceiver = this.findTransceiverForTrack(track); + if (transceiver) { + if (transceiver.sender.track) { + transceiver.sender.track.enabled = enabled; + } + log.set("fromDirection", transceiver.direction); + enableSenderOnTransceiver(transceiver, enabled); + log.set("toDirection", transceiver.direction); + } + }); + } + private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; @@ -872,10 +916,17 @@ export class PeerCall implements IDisposable { private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { return this.peerConnection.getReceivers().find(r => { - return r.track.kind === "audio" && this._remoteTrackToStreamId.get(r.track.id) === streamId; + return r.track.kind === kind && this._remoteTrackToStreamId.get(r.track.id) === streamId; }); } + private findTransceiverForTrack(track: Track): Transceiver | undefined { + return this.peerConnection.getTransceivers().find(t => { + return t.sender.track?.id === track.id; + }); + } + + private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { if (streams.length === 0) { log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); @@ -1072,7 +1123,22 @@ export function handlesEventType(eventType: string): boolean { eventType === EventType.Negotiate; } - -export function tests() { - +function enableSenderOnTransceiver(transceiver: Transceiver, enabled: boolean) { + return enableTransceiver(transceiver, enabled, "sendonly", "recvonly"); +} + +function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusiveValue: TransceiverDirection, excludedValue: TransceiverDirection) { + if (enabled) { + if (transceiver.direction === "inactive") { + transceiver.direction = exclusiveValue; + } else { + transceiver.direction = "sendrecv"; + } + } else { + if (transceiver.direction === "sendrecv") { + transceiver.direction = excludedValue; + } else { + transceiver.direction = "inactive"; + } + } } diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index c5970d4b..6770aec1 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -26,4 +26,12 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin export class MuteSettings { constructor (public readonly microphone: boolean, public readonly camera: boolean) {} + + toggleCamera(): MuteSettings { + return new MuteSettings(this.microphone, !this.camera); + } + + toggleMicrophone(): MuteSettings { + return new MuteSettings(!this.microphone, this.camera); + } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 4992f6b6..57d08cd3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,6 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {MuteSettings} from "../common"; import {RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -63,6 +64,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; + private localMuteSettings: MuteSettings = new MuteSettings(false, false); constructor( public readonly id: string, @@ -118,7 +120,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } }); } @@ -134,6 +136,19 @@ export class GroupCall extends EventEmitter<{change: never}> { } } + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + for (const [,member] of this._members) { + member.setMuted(this.localMuteSettings); + } + } + } + + get muteSettings(): MuteSettings { + return this.localMuteSettings; + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } @@ -230,7 +245,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._members.add(memberKey, member); if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { // Safari can't send a MediaStream to multiple sources, so clone it - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 714cd821..e7eefd4d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; +import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; @@ -50,6 +51,7 @@ const errorCodesWithoutRetry = [ export class Member { private peerCall?: PeerCall; private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private retryCount: number = 0; constructor( @@ -80,9 +82,10 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia) { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings) { this.logItem.wrap("connect", () => { this.localMedia = localMedia; + this.localMuteSettings = localMuteSettings; // otherwise wait for it to connect let shouldInitiateCall; // the lexicographically lower side initiates the call @@ -93,7 +96,7 @@ export class Member { } if (shouldInitiateCall) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(localMedia, localMuteSettings); } }); } @@ -121,7 +124,7 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(this.localMedia!, this.localMuteSettings!); } else if (peerCall.state === CallState.Ended) { const hangupReason = peerCall.hangupReason; @@ -130,7 +133,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { this.retryCount += 1; if (this.retryCount <= 3) { - this.connect(this.localMedia!); + this.connect(this.localMedia!, this.localMuteSettings!); } } } @@ -190,6 +193,11 @@ export class Member { await this.peerCall?.setMedia(this.localMedia); } + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + this.peerCall?.setMuted(muteSettings); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate,