/* Copyright 2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, CallErrorCode, } from "./callEventTypes"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import type {LocalMedia} from "./LocalMedia"; import type { MCallBase, MCallInvite, MCallNegotiate, MCallAnswer, MCallSDPStreamMetadataChanged, MCallCandidates, MCallHangupReject, SDPStreamMetadata, SignallingMessage } from "./callEventTypes"; export type Options = { webRTC: WebRTC, forceTURN: boolean, turnServers: RTCIceServer[], createTimeout: TimeoutCreator, emitUpdate: (peerCall: PeerCall, params: any) => void; sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; }; export class RemoteMedia { constructor(public userMedia?: Stream | undefined, public screenShare?: Stream | undefined) {} } // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; private localMuteSettings?: MuteSettings; private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible private candidateSendQueue: Array = []; // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick private remoteCandidateBuffer? = new Map(); private remoteSDPStreamMetadata?: SDPStreamMetadata; private responsePromiseChain?: Promise; private opponentPartyId?: PartyId; private hangupParty: CallParty; private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); private _remoteTrackToStreamId = new Map(); private _remoteStreams = new Map(); // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; private sentEndOfCandidates: boolean = false; private iceDisconnectedTimeout?: Timeout; private _dataChannel?: any; private _hangupReason?: CallErrorCode; private _remoteMedia: RemoteMedia; private _remoteMuteSettings = new MuteSettings(); constructor( private callId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { this._remoteMedia = new RemoteMedia(); this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { this.peerConnection.addEventListener(type, listener); const dispose = () => { this.peerConnection.removeEventListener(type, listener); }; this.disposables.track(dispose); }; listen("iceconnectionstatechange", () => { const state = this.peerConnection.iceConnectionState; this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { this.onIceConnectionStateChange(state, log); }); }); listen("icecandidate", event => { this.logItem.wrap("onLocalIceCandidate", log => { if (event.candidate) { this.handleLocalIceCandidate(event.candidate, log); } }); }); listen("icegatheringstatechange", () => { const state = this.peerConnection.iceGatheringState; this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { this.handleIceGatheringState(state, log); }); }); listen("track", event => { this.logItem.wrap("onRemoteTrack", log => { this.onRemoteTrack(event.track, event.streams, log); }); }); listen("datachannel", event => { this.logItem.wrap("onRemoteDataChannel", log => { this._dataChannel = event.channel; this.options.emitUpdate(this, undefined); }); }); listen("negotiationneeded", () => { const promiseCreator = () => { return this.logItem.wrap("onNegotiationNeeded", log => { return this.handleNegotiation(log); }); }; this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); }); } get dataChannel(): any | undefined { return this._dataChannel; } get state(): CallState { return this._state; } get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } get remoteMedia(): Readonly { return this._remoteMedia; } get remoteMuteSettings(): MuteSettings { return this._remoteMuteSettings; } call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("call", async log => { if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); }); } answer(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("answer", async log => { if (this._state !== CallState.Ringing) { return; } this.setState(CallState.CreateAnswer, log); this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); } catch (err) { await log.wrap(`Failed to create answer`, log => { log.catch(err); this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, log); }); return; } try { await this.peerConnection.setLocalDescription(myAnswer); this.setState(CallState.Connecting, log); } catch (err) { await log.wrap(`Error setting local description!`, log => { log.catch(err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); }); return; } // Allow a short time for initial candidates to be gathered try { await this.delay(200); } catch (err) { return; } await this.sendAnswer(log); }); } setMedia(localMedia: LocalMedia): Promise { return this.logItem.wrap("setMedia", async log => { log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia)); log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia)); log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare)); log.set("datachannel", !!localMedia.dataChannelOptions); await this.updateLocalMedia(localMedia, log); const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); }); } setMuted(localMuteSettings: MuteSettings) { return this.logItem.wrap("setMuted", async log => { this.localMuteSettings = localMuteSettings; log.set("cameraMuted", localMuteSettings.camera); log.set("microphoneMuted", localMuteSettings.microphone); if (this.localMedia) { const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); if (userMediaAudio) { this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); } const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); if (userMediaVideo) { this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); } const content: MCallSDPStreamMetadataChanged = { call_id: this.callId, version: 1, seq: this.seq++, [SDPStreamMetadataKey]: this.getSDPMetadata() }; await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); } }); } hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); }); } private muteTrack(track: Track, muted: boolean, log: ILogItem): void { log.wrap({l: "track", kind: track.kind, id: track.id}, log => { const enabled = !muted; log.set("enabled", enabled); const transceiver = this.findTransceiverForTrack(track); if (transceiver) { if (transceiver.sender.track) { transceiver.sender.track.enabled = enabled; } log.set("fromDirection", transceiver.direction); // enableSenderOnTransceiver(transceiver, enabled); log.set("toDirection", transceiver.direction); } }); } private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; } this.terminate(CallParty.Local, errorCode, log); await this.sendHangupWithCallId(this.callId, errorCode, log); } handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { return this.logItem.wrap({l: "receive", id: message.type, partyId}, async log => { switch (message.type) { case EventType.Invite: if (this.callId !== message.content.call_id) { await this.handleInviteGlare(message.content, partyId, log); } else { await this.handleFirstInvite(message.content, partyId, log); } break; case EventType.Answer: await this.handleAnswer(message.content, partyId, log); break; case EventType.Negotiate: await this.onNegotiateReceived(message.content, log); break; case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; case EventType.SDPStreamMetadataChanged: case EventType.SDPStreamMetadataChangedPrefix: this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); break; case EventType.Hangup: // TODO: this is a bit hacky, double check its what we need this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); break; default: log.log(`Unknown event type for call: ${message.type}`); break; } }); } private sendHangupWithCallId(callId: string, reason: CallErrorCode | undefined, log: ILogItem): Promise { const content = { call_id: callId, version: 1, seq: this.seq++, }; // TODO: Don't send UserHangup reason to older clients if (reason) { content["reason"] = reason; } return this.sendSignallingMessage({ type: EventType.Hangup, content }, log); } // calls are serialized and deduplicated by responsePromiseChain private async handleNegotiation(log: ILogItem): Promise { this.makingOffer = true; try { try { await this.peerConnection.setLocalDescription(); } catch (err) { log.log(`Error setting local description!`).catch(err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, log); return; } if (this.peerConnection.iceGatheringState === 'gathering') { // Allow a short time for initial candidates to be gathered try { await this.delay(200); } catch (err) { return; } } if (this._state === CallState.Ended) { return; } const offer = this.peerConnection.localDescription!; // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in offer`); this.candidateSendQueue = []; // need to queue this if (this._state === CallState.CreateOffer) { const content = { call_id: this.callId, offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { const content = { call_id: this.callId, description: offer, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { this.makingOffer = false; } this.sendCandidateQueue(log); await log.wrap("invite timeout", async log => { if (this._state === CallState.InviteSent) { try { await this.delay(CALL_TIMEOUT_MS); } catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.InviteSent) { this._hangup(CallErrorCode.InviteTimeout, log); } } }); }; private async handleInviteGlare(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // this is only called when the ids are different const newCallId = content.call_id; if (this.callId! > newCallId) { log.log( "Glare detected: answering incoming call " + newCallId + " and canceling outgoing call ", ); // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer) { // TODO: don't send invite! } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced, log); } await this.handleInvite(content, partyId, log); // TODO: need to skip state check await this.answer(this.localMedia!, this.localMuteSettings!); } else { log.log( "Glare detected: rejecting incoming call " + newCallId + " and keeping outgoing call ", ); await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced, log); } } private handleHangupReceived(content: MCallHangupReject, log: ILogItem) { // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen // a partner yet but we're treating the hangup as a reject as per VoIP v0) // if (this.state === CallState.Ringing) { // default reason is user_hangup this.terminate(CallParty.Remote, content.reason || CallErrorCode.UserHangup, log); // } else { // log.set("ignored", true); // } }; private async handleFirstInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } await this.handleInvite(content, partyId, log); } private async handleInvite(content: MCallInvite, partyId: PartyId, log: ILogItem): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if // we haven't set the party ID, we'll ignore them. this.opponentPartyId = partyId; this.direction = CallDirection.Inbound; const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Call did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { // Q: Why do we set the remote description before accepting the call? To start creating ICE candidates? await this.peerConnection.setRemoteDescription(content.offer); await this.addBufferedIceCandidates(log); } catch (e) { await log.wrap(`Call failed to set remote description`, async log => { log.catch(e); return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (this.peerConnection.getReceivers().length === 0) { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } this.setState(CallState.Ringing, log); try { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); } catch (err) { return; } // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this._state === CallState.Ringing) { log.log(`Invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended, log); //this.localMedia?.dispose(); //this.localMedia = undefined; if (this.peerConnection.signalingState != 'closed') { this.peerConnection.close(); } } } private async handleAnswer(content: MCallAnswer, partyId: PartyId, log: ILogItem): Promise { if (this._state === CallState.Ended) { log.log(`Ignoring answer because call has ended`); return; } if (this.opponentPartyId !== undefined) { log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); return; } this.opponentPartyId = partyId; await this.addBufferedIceCandidates(log); this.setState(CallState.Connecting, log); const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { await this.peerConnection.setRemoteDescription(content.answer); } catch (e) { await log.wrap(`Failed to set remote description`, log => { log.catch(e); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); return; } } private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. // We cast because the interface lists all the properties as required but we // only want to send 'candidate' // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly // correct to have a candidate that lacks both of these. We'd have to figure out what // previous candidates had been sent with and copy them. const c = { candidate: '', } as RTCIceCandidate; this.queueCandidate(c, log); this.sentEndOfCandidates = true; } } private handleLocalIceCandidate(candidate: RTCIceCandidate, log: ILogItem) { log.set("sdpMid", candidate.sdpMid); log.set("candidate", candidate.candidate); if (this._state === CallState.Ended) { return; } // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (candidate.candidate !== '' || !this.sentEndOfCandidates) { this.queueCandidate(candidate, log); if (candidate.candidate === '') { this.sentEndOfCandidates = true; } } } private async handleRemoteIceCandidates(content: MCallCandidates, partyId: PartyId, log: ILogItem) { if (this.state === CallState.Ended) { log.log("Ignoring remote ICE candidate because call has ended"); return; } const candidates = content.candidates; if (!candidates) { log.log(`Ignoring candidates event with no candidates!`); return; } const fromPartyId = content.version === 0 ? null : partyId || null; if (this.opponentPartyId === undefined) { // we haven't picked an opponent yet so save the candidates log.log(`Buffering ${candidates.length} candidates until we pick an opponent`); const bufferedCandidates = this.remoteCandidateBuffer!.get(fromPartyId) || []; bufferedCandidates.push(...candidates); this.remoteCandidateBuffer!.set(fromPartyId, bufferedCandidates); return; } if (this.opponentPartyId !== partyId) { log.log( `Ignoring candidates from party ID ${partyId}: ` + `we have chosen party ID ${this.opponentPartyId}`, ); return; } await this.addIceCandidates(candidates, log); } private async onNegotiateReceived(content: MCallNegotiate, log: ILogItem): Promise { const description = content.description; if (!description || !description.sdp || !description.type) { log.log(`Ignoring invalid m.call.negotiate event`); return; } // Politeness always follows the direction of the call: in a glare situation, // we pick either the inbound or outbound call, so one side will always be // inbound and one outbound const polite = this.direction === CallDirection.Inbound; // Here we follow the perfect negotiation logic from // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation const offerCollision = ( (description.type === 'offer') && (this.makingOffer || this.peerConnection.signalingState !== 'stable') ); this.ignoreOffer = !polite && offerCollision; if (this.ignoreOffer) { log.log(`Ignoring colliding negotiate event because we're impolite`); return; } const sdpStreamMetadata = content[SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); } else { log.log(`Received negotiation event without SDPStreamMetadata!`); } try { await this.peerConnection.setRemoteDescription(description); if (description.type === 'offer') { await this.peerConnection.setLocalDescription(); const content = { call_id: this.callId, description: this.peerConnection.localDescription!, [SDPStreamMetadataKey]: this.getSDPMetadata(), version: 1, seq: this.seq++, lifetime: CALL_TIMEOUT_MS }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } catch (err) { log.log(`Failed to complete negotiation`, err); } } private async sendAnswer(log: ILogItem): Promise { const localDescription = this.peerConnection.localDescription!; const answerContent: MCallAnswer = { call_id: this.callId, version: 1, seq: this.seq++, answer: { sdp: localDescription.sdp, type: localDescription.type, }, [SDPStreamMetadataKey]: this.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. log.log(`Discarding ${ this.candidateSendQueue.length} candidates that will be sent in answer`); this.candidateSendQueue = []; try { await this.sendSignallingMessage({type: EventType.Answer, content: answerContent}, log); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, log); throw error; } // error handler re-throws so this won't happen on error, but // we don't want the same error handling on the candidate queue this.sendCandidateQueue(log); } private queueCandidate(content: RTCIceCandidate, log: ILogItem): void { // We partially de-trickle candidates by waiting for `delay` before sending them // amalgamated, in order to avoid sending too many m.call.candidates events and hitting // rate limits in Matrix. // In practice, it'd be better to remove rate limits for m.call.* // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 // currently proposes as the way to indicate that candidate gathering is complete. // This will hopefully be changed to an explicit rather than implicit notification // shortly. this.candidateSendQueue.push(content); // Don't send the ICE candidates yet if the call is in the ringing state if (this._state === CallState.Ringing) return; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) const sendLogItem = this.logItem.child("wait to send candidates"); log.refDetached(sendLogItem); this.delay(this.direction === CallDirection.Inbound ? 500 : 2000) .then(() => { return this.sendCandidateQueue(sendLogItem); }, err => {}) // swallow delay AbortError .finally(() => { sendLogItem.finish(); }); } private async sendCandidateQueue(log: ILogItem): Promise { return log.wrap("send candidates queue", async log => { log.set("queueLength", this.candidateSendQueue.length); if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { return; } const candidates = this.candidateSendQueue; this.candidateSendQueue = []; try { await this.sendSignallingMessage({ type: EventType.Candidates, content: { call_id: this.callId, version: 1, seq: this.seq++, candidates }, }, log); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(log); } catch (error) { log.catch(error); // don't retry this event: we'll send another one later as we might // have more candidates by then. // put all the candidates we failed to send back in the queue // TODO: terminate doesn't seem to vibe with the comment above? this.terminate(CallParty.Local, CallErrorCode.SignallingFailed, log); } }); } private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata, log: ILogItem): void { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); } private async addBufferedIceCandidates(log: ILogItem): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); if (bufferedCandidates) { log.log(`Adding ${ bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); await this.addIceCandidates(bufferedCandidates, log); } this.remoteCandidateBuffer = undefined; } } private async addIceCandidates(candidates: RTCIceCandidate[], log: ILogItem): Promise { for (const candidate of candidates) { if ( (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { log.log(`Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } log.log(`Got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConnection.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { log.log(`Failed to add remote ICE candidate`, err); } } } } private onIceConnectionStateChange = (state: RTCIceConnectionState, log: ILogItem): void => { if (this._state === CallState.Ended) { return; // because ICE can still complete as we're ending the call } // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet if (state == 'connected') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this._hangup(CallErrorCode.IceFailed, log); } else if (state == 'disconnected') { this.iceDisconnectedTimeout = this.options.createTimeout(30 * 1000); this.iceDisconnectedTimeout.elapsed().then(() => { this._hangup(CallErrorCode.IceFailed, log); }, () => { /* ignore AbortError */ }); } }; private setState(state: CallState, log: ILogItem): void { if (state !== this._state) { log.log({l: "change state", status: state, oldState: this._state}); const oldState = this._state; this._state = state; let deferred = this.statePromiseMap.get(state); if (deferred) { deferred.resolve(); this.statePromiseMap.delete(state); } this.options.emitUpdate(this, undefined); } } private waitForState(states: CallState[]): Promise { // TODO: rework this, do we need to clean up the promises? return Promise.race(states.map(state => { let deferred = this.statePromiseMap.get(state); if (!deferred) { let resolve; const promise = new Promise(r => { resolve = r; }); deferred = {resolve, promise}; this.statePromiseMap.set(state, deferred); } return deferred.promise; })); } private terminate(hangupParty: CallParty, hangupReason: CallErrorCode, log: ILogItem): void { if (this._state === CallState.Ended) { return; } this.hangupParty = hangupParty; this._hangupReason = hangupReason; this.setState(CallState.Ended, log); this.localMedia = undefined; if (this.peerConnection && this.peerConnection.signalingState !== 'closed') { this.peerConnection.close(); } } private getSDPMetadata(): SDPStreamMetadata { const metadata = {}; if (this.localMedia?.userMedia) { const streamId = this.localMedia.userMedia.id; metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), }; } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Screenshare }; } return metadata; } private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { return this.peerConnection.getReceivers().find(r => { return r.track.kind === kind && this._remoteTrackToStreamId.get(r.track.id) === streamId; }); } private findTransceiverForTrack(track: Track): Transceiver | undefined { return this.peerConnection.getTransceivers().find(t => { return t.sender.track?.id === track.id; }); } private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { if (streams.length === 0) { log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); return; } const stream = streams[0]; this._remoteTrackToStreamId.set(track.id, stream.id); if (!this._remoteStreams.has(stream.id)) { const listener = (event: StreamTrackEvent): void => { this.logItem.wrap({l: "removetrack", id: event.track.id}, log => { const streamId = this._remoteTrackToStreamId.get(event.track.id); if (streamId) { this._remoteTrackToStreamId.delete(event.track.id); const streamDetails = this._remoteStreams.get(streamId); if (streamDetails && streamDetails.stream.getTracks().length === 0) { this.disposables.disposeTracked(disposeListener); this._remoteStreams.delete(stream.id); this.updateRemoteMedia(log); } } }) }; stream.addEventListener("removetrack", listener); const disposeListener = () => { stream.removeEventListener("removetrack", listener); }; this.disposables.track(disposeListener); this._remoteStreams.set(stream.id, { disposeListener, stream }); this.updateRemoteMedia(log); } } private updateRemoteMedia(log: ILogItem): void { this._remoteMedia.userMedia = undefined; this._remoteMedia.screenShare = undefined; if (this.remoteSDPStreamMetadata) { for (const streamDetails of this._remoteStreams.values()) { const {stream} = streamDetails; const metaData = this.remoteSDPStreamMetadata[stream.id]; if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { this._remoteMedia.userMedia = stream; const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); if (audioReceiver) { audioReceiver.track.enabled = !metaData.audio_muted; } const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); if (videoReceiver) { videoReceiver.track.enabled = !metaData.audio_muted; } this._remoteMuteSettings = new MuteSettings( metaData.audio_muted || !audioReceiver?.track, metaData.video_muted || !videoReceiver?.track ); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = stream; } } } } this.options.emitUpdate(this, undefined); } private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { return logItem.wrap("updateLocalMedia", async log => { const oldMedia = this.localMedia; this.localMedia = localMedia; const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { if (!oldTrack && newTrack) { log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { const sender = this.peerConnection.addTrack(newTrack, stream!); this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); }); } else if (oldTrack) { const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); if (sender) { if (newTrack && oldTrack.id !== newTrack.id) { try { await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { return sender.replaceTrack(newTrack); }); } catch (err) { // can't replace the track without renegotiating{ log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { this.peerConnection.removeTrack(sender); const newSender = this.peerConnection.addTrack(newTrack); this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose); }); } } else if (!newTrack) { log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { this.peerConnection.removeTrack(sender); }); } else { log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); } } // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia } } await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); }; await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); await timeout.elapsed(); this.disposables.untrack(timeout); } private sendSignallingMessage(message: SignallingMessage, log: ILogItem) { return log.wrap({l: "send", id: message.type}, async log => { return this.options.sendSignallingMessage(message, log); }); } public dispose(): void { this.disposables.dispose(); this.peerConnection.close(); } public close(reason: CallErrorCode | undefined, log: ILogItem): void { if (reason === undefined) { reason = CallErrorCode.UserHangup; } this.terminate(CallParty.Local, reason, log); } } //import { randomString } from '../randomstring'; // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. type PartyId = string | null; export enum CallParty { Local = 'local', Remote = 'remote', } export enum CallState { Fledgling = 'fledgling', CreateOffer = 'create_offer', InviteSent = 'invite_sent', CreateAnswer = 'create_answer', Connecting = 'connecting', Connected = 'connected', Ringing = 'ringing', Ended = 'ended', } export enum CallDirection { Inbound = 'inbound', Outbound = 'outbound', } /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; //const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export class CallError extends Error { code: string; constructor(code: CallErrorCode, msg: string, err: Error) { // Still don't think there's any way to have proper nested errors super(msg + ": " + err); this.code = code; } } export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || eventType === EventType.Hangup || eventType === EventType.SDPStreamMetadataChanged || eventType === EventType.SDPStreamMetadataChangedPrefix || eventType === EventType.Negotiate; } function enableSenderOnTransceiver(transceiver: Transceiver, enabled: boolean) { return enableTransceiver(transceiver, enabled, "sendonly", "recvonly"); } function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusiveValue: TransceiverDirection, excludedValue: TransceiverDirection) { if (enabled) { if (transceiver.direction === "inactive") { transceiver.direction = exclusiveValue; } else { transceiver.direction = "sendrecv"; } } else { if (transceiver.direction === "sendrecv") { transceiver.direction = excludedValue; } else { transceiver.direction = "inactive"; } } }