From 3198ca6a9252b022f4d316fdf6e848f1ed7662d2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 26 Apr 2022 14:20:44 +0200 Subject: [PATCH] expose remote mute settings --- src/matrix/calls/PeerCall.ts | 141 ++++++++++++----------------- src/matrix/calls/callEventTypes.ts | 7 +- src/matrix/calls/group/Member.ts | 4 + src/platform/types/WebRTC.ts | 1 - 4 files changed, 65 insertions(+), 88 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 81828c32..bfe0b349 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -98,7 +98,7 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; private _remoteMedia: RemoteMedia; - private remoteMuteSettings?: MuteSettings; + private _remoteMuteSettings = new MuteSettings(); constructor( private callId: string, @@ -166,6 +166,10 @@ export class PeerCall implements IDisposable { return this._remoteMedia; } + get remoteMuteSettings(): MuteSettings { + return this._remoteMuteSettings; + } + call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("call", async log => { if (this._state !== CallState.Fledgling) { @@ -279,7 +283,7 @@ export class PeerCall implements IDisposable { transceiver.sender.track.enabled = enabled; } log.set("fromDirection", transceiver.direction); - enableSenderOnTransceiver(transceiver, enabled); + // enableSenderOnTransceiver(transceiver, enabled); log.set("toDirection", transceiver.direction); } }); @@ -307,7 +311,7 @@ export class PeerCall implements IDisposable { await this.handleAnswer(message.content, partyId, log); break; case EventType.Negotiate: - await this.handleRemoteNegotiate(message.content, partyId, log); + await this.onNegotiateReceived(message.content, log); break; case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); @@ -344,7 +348,7 @@ export class PeerCall implements IDisposable { } // calls are serialized and deduplicated by responsePromiseChain - private handleNegotiation = async (log: ILogItem): Promise => { + private async handleNegotiation(log: ILogItem): Promise { this.makingOffer = true; try { try { @@ -429,7 +433,7 @@ export class PeerCall implements IDisposable { } await this.handleInvite(content, partyId, log); // TODO: need to skip state check - await this.answer(this.localMedia!); + await this.answer(this.localMedia!, this.localMuteSettings!); } else { log.log( "Glare detected: rejecting incoming call " + newCallId + @@ -546,36 +550,6 @@ export class PeerCall implements IDisposable { } } - - private async handleRemoteNegotiate(content: MCallNegotiate, partyId: PartyId, log: ILogItem): Promise { - if (this._state !== CallState.Connected) { - log.log({l: `Ignoring renegotiate because not connected`, status: this._state}); - return; - } - - if (this.opponentPartyId !== partyId) { - log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); - return; - } - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); - } else { - log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); - } - - try { - await this.peerConnection.setRemoteDescription(content.description); - } catch (e) { - await log.wrap(`Failed to set remote description`, log => { - log.catch(e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - }); - return; - } - } - private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, @@ -645,55 +619,55 @@ export class PeerCall implements IDisposable { await this.addIceCandidates(candidates, log); } - // private async onNegotiateReceived(event: MatrixEvent): Promise { - // const content = event.getContent(); - // const description = content.description; - // if (!description || !description.sdp || !description.type) { - // this.logger.info(`Ignoring invalid m.call.negotiate event`); - // return; - // } - // // Politeness always follows the direction of the call: in a glare situation, - // // we pick either the inbound or outbound call, so one side will always be - // // inbound and one outbound - // const polite = this.direction === CallDirection.Inbound; + private async onNegotiateReceived(content: MCallNegotiate, log: ILogItem): Promise { + const description = content.description; + if (!description || !description.sdp || !description.type) { + log.log(`Ignoring invalid m.call.negotiate event`); + return; + } + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + const polite = this.direction === CallDirection.Inbound; - // // Here we follow the perfect negotiation logic from - // // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - // const offerCollision = ( - // (description.type === 'offer') && - // (this.makingOffer || this.peerConnection.signalingState !== 'stable') - // ); + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + const offerCollision = ( + (description.type === 'offer') && + (this.makingOffer || this.peerConnection.signalingState !== 'stable') + ); - // this.ignoreOffer = !polite && offerCollision; - // if (this.ignoreOffer) { - // this.logger.info(`Ignoring colliding negotiate event because we're impolite`); - // return; - // } + this.ignoreOffer = !polite && offerCollision; + if (this.ignoreOffer) { + log.log(`Ignoring colliding negotiate event because we're impolite`); + return; + } - // const sdpStreamMetadata = content[SDPStreamMetadataKey]; - // if (sdpStreamMetadata) { - // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - // } else { - // this.logger.warn(`Received negotiation event without SDPStreamMetadata!`); - // } + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); + } else { + log.log(`Received negotiation event without SDPStreamMetadata!`); + } - // try { - // await this.peerConnection.setRemoteDescription(description); - - // if (description.type === 'offer') { - // await this.peerConnection.setLocalDescription(); - // await this.sendSignallingMessage({ - // type: EventType.CallNegotiate, - // content: { - // description: this.peerConnection.localDescription!, - // [SDPStreamMetadataKey]: this.getSDPMetadata(), - // } - // }); - // } - // } catch (err) { - // this.logger.warn(`Failed to complete negotiation`, err); - // } - // } + try { + await this.peerConnection.setRemoteDescription(description); + if (description.type === 'offer') { + await this.peerConnection.setLocalDescription(); + const content = { + call_id: this.callId, + description: this.peerConnection.localDescription!, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; + await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); + } + } catch (err) { + log.log(`Failed to complete negotiation`, err); + } + } private async sendAnswer(log: ILogItem): Promise { const localDescription = this.peerConnection.localDescription!; @@ -980,6 +954,10 @@ export class PeerCall implements IDisposable { if (videoReceiver) { videoReceiver.track.enabled = !metaData.audio_muted; } + this._remoteMuteSettings = new MuteSettings( + metaData.audio_muted || !audioReceiver?.track, + metaData.video_muted || !videoReceiver?.track + ); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = stream; } @@ -1012,7 +990,8 @@ export class PeerCall implements IDisposable { // can't replace the track without renegotiating{ log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { this.peerConnection.removeTrack(sender); - this.peerConnection.addTrack(newTrack); + const newSender = this.peerConnection.addTrack(newTrack); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); }); } } else if (!newTrack) { diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 449bd469..0490b44f 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,7 +1,7 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ import type {StateEvent} from "../storage/types"; - +import type {SessionDescription} from "../../platform/types/WebRTC"; export enum EventType { GroupCall = "org.matrix.msc3401.call", GroupCallMember = "org.matrix.msc3401.call.member", @@ -36,11 +36,6 @@ export interface CallMemberContent { ["m.calls"]: CallMembership[]; } -export interface SessionDescription { - sdp?: string; - type: RTCSdpType -} - export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e7eefd4d..e3f50c06 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,6 +65,10 @@ export class Member { return this.peerCall?.remoteMedia; } + get remoteMuteSettings(): MuteSettings | undefined { + return this.peerCall?.remoteMuteSettings; + } + get isConnected(): boolean { return this.peerCall?.state === CallState.Connected; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index ca2ca646..9f2e1e0e 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -104,7 +104,6 @@ export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendr export interface SessionDescription { readonly sdp: string; readonly type: SdpType; - toJSON(): any; } export interface AnswerOptions {}