diff --git a/package.json b/package.json index 12c73994..e356347b 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "postcss-flexbugs-fixes": "^5.0.2", "regenerator-runtime": "^0.13.7", "text-encoding": "^0.7.0", - "typescript": "^4.3.5", + "typescript": "^4.4", "vite": "^2.6.14", "xxhashjs": "^0.2.2" }, diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 20f4abd3..470559a9 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/calls/Call.ts b/src/matrix/calls/Call.ts deleted file mode 100644 index d6ac4612..00000000 --- a/src/matrix/calls/Call.ts +++ /dev/null @@ -1,2493 +0,0 @@ -/* -Copyright 2015, 2016 OpenMarket Ltd -Copyright 2017 New Vector Ltd -Copyright 2019, 2020 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; -import { CallFeed } from './CallFeed'; - -// events: hangup, error(err), replaced(call), state(state, oldState) - -/** - * Fires whenever an error occurs when call.js encounters an issue with setting up the call. - *

- * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or - * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client - * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access - * to their audio/video hardware. - * - * @event module:webrtc/call~MatrixCall#"error" - * @param {Error} err The error raised by MatrixCall. - * @example - * matrixCall.on("error", function(err){ - * console.error(err.code, err); - * }); - */ - -interface CallOpts { - roomId?: string; - invitee?: string; - client?: any; // Fix when client is TSified - forceTURN?: boolean; - turnServers?: Array; - opponentDeviceId?: string; - opponentSessionId?: string; - groupCallId?: string; -} - -interface TurnServer { - urls: Array; - username?: string; - password?: string; - ttl?: number; -} - -interface AssertedIdentity { - id: string; - displayName: string; -} - -export enum CallState { - Fledgling = 'fledgling', - InviteSent = 'invite_sent', - WaitLocalMedia = 'wait_local_media', - CreateOffer = 'create_offer', - CreateAnswer = 'create_answer', - Connecting = 'connecting', - Connected = 'connected', - Ringing = 'ringing', - Ended = 'ended', -} - -export enum CallType { - Voice = 'voice', - Video = 'video', -} - -export enum CallDirection { - Inbound = 'inbound', - Outbound = 'outbound', -} - -export enum CallParty { - Local = 'local', - Remote = 'remote', -} - -export enum CallEvent { - Hangup = 'hangup', - State = 'state', - Error = 'error', - Replaced = 'replaced', - - // The value of isLocalOnHold() has changed - LocalHoldUnhold = 'local_hold_unhold', - // The value of isRemoteOnHold() has changed - RemoteHoldUnhold = 'remote_hold_unhold', - // backwards compat alias for LocalHoldUnhold: remove in a major version bump - HoldUnhold = 'hold_unhold', - // Feeds have changed - FeedsChanged = 'feeds_changed', - - AssertedIdentityChanged = 'asserted_identity_changed', - - LengthChanged = 'length_changed', - - DataChannel = 'datachannel', - - SendVoipEvent = "send_voip_event", -} - -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - -/** - * The version field that we set in m.call.* events - */ -const VOIP_PROTO_VERSION = 1; - -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - -/** The length of time a call can be ringing for. */ -const CALL_TIMEOUT_MS = 60000; - -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - -export class CallError extends Error { - code: string; - - constructor(code: CallErrorCode, msg: string, err: Error) { - // Still don't think there's any way to have proper nested errors - super(msg + ": " + err); - - this.code = code; - } -} - -export function genCallID(): string { - return Date.now().toString() + randomString(16); -} - -/** - * Construct a new Matrix Call. - * @constructor - * @param {Object} opts Config options. - * @param {string} opts.roomId The room ID for this call. - * @param {Object} opts.webRtc The WebRTC globals from the browser. - * @param {boolean} opts.forceTURN whether relay through TURN should be forced. - * @param {Object} opts.URL The URL global. - * @param {Array} opts.turnServers Optional. A list of TURN servers. - * @param {MatrixClient} opts.client The Matrix Client instance to send events to. - */ -export class MatrixCall extends EventEmitter { - public roomId: string; - public callId: string; - public invitee?: string; - public state = CallState.Fledgling; - public hangupParty: CallParty; - public hangupReason: string; - public direction: CallDirection; - public ourPartyId: string; - public peerConn?: RTCPeerConnection; - - private client: MatrixClient; - private forceTURN: boolean; - private turnServers: Array; - // A queue for candidates waiting to go out. - // We try to amalgamate candidates into a single candidate message where - // possible - private candidateSendQueue: Array = []; - private candidateSendTries = 0; - private sentEndOfCandidates = false; - private feeds: Array = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; - private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; - private successor: MatrixCall; - private opponentMember: RoomMember; - private opponentVersion: number | string; - // The party ID of the other side: undefined if we haven't chosen a partner - // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string; - private opponentCaps: CallCapabilities; - private inviteTimeout: number; - private iceDisconnectedTimeout: number; - - // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold - // This flag represents whether we want the other party to be on hold - private remoteOnHold = false; - - // the stats for the call at the point it ended. We can't get these after we - // tear the call down, so we just grab a snapshot before we stop the call. - // The typescript definitions have this type as 'any' :( - private callStatsAtEnd: any[]; - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - private makingOffer = false; - private ignoreOffer: boolean; - - private responsePromiseChain?: Promise; - - // If candidates arrive before we've picked an opponent (which, in particular, - // will happen if the opponent sends candidates eagerly before the user answers - // the call) we buffer them up here so we can then add the ones from the party we pick - private remoteCandidateBuffer = new Map(); - - private remoteAssertedIdentity: AssertedIdentity; - - private remoteSDPStreamMetadata: SDPStreamMetadata; - - private callLengthInterval: number; - private callLength = 0; - - private opponentDeviceId: string; - private opponentSessionId: string; - public groupCallId: string; - - constructor(opts: CallOpts) { - super(); - this.roomId = opts.roomId; - this.invitee = opts.invitee; - this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId; - this.opponentDeviceId = opts.opponentDeviceId; - this.opponentSessionId = opts.opponentSessionId; - this.groupCallId = opts.groupCallId; - // Array of Objects with urls, username, credential keys - this.turnServers = opts.turnServers || []; - if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { - this.turnServers.push({ - urls: [FALLBACK_ICE_SERVER], - }); - } - for (const server of this.turnServers) { - utils.checkObjectHasKeys(server, ["urls"]); - } - this.callId = genCallID(); - } - - /** - * Place a voice call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVoiceCall(): Promise { - await this.placeCall(true, false); - } - - /** - * Place a video call to this room. - * @throws If you have not specified a listener for 'error' events. - */ - public async placeVideoCall(): Promise { - await this.placeCall(true, true); - } - - /** - * Create a datachannel using this call's peer connection. - * @param label A human readable label for this datachannel - * @param options An object providing configuration options for the data channel. - */ - public createDataChannel(label: string, options: RTCDataChannelInit) { - const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); - return dataChannel; - } - - public getOpponentMember(): RoomMember { - return this.opponentMember; - } - - public getOpponentSessionId(): string { - return this.opponentSessionId; - } - - public opponentCanBeTransferred(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); - } - - public opponentSupportsDTMF(): boolean { - return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); - } - - public getRemoteAssertedIdentity(): AssertedIdentity { - return this.remoteAssertedIdentity; - } - - public get type(): CallType { - return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack) - ? CallType.Video - : CallType.Voice; - } - - public get hasLocalUserMediaVideoTrack(): boolean { - return this.localUsermediaStream?.getVideoTracks().length > 0; - } - - public get hasRemoteUserMediaVideoTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getVideoTracks().length > 0 - ); - }); - } - - public get hasLocalUserMediaAudioTrack(): boolean { - return this.localUsermediaStream?.getAudioTracks().length > 0; - } - - public get hasRemoteUserMediaAudioTrack(): boolean { - return this.getRemoteFeeds().some((feed) => { - return ( - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - feed.stream.getAudioTracks().length > 0 - ); - }); - } - - public get localUsermediaFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get localScreensharingFeed(): CallFeed { - return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get localUsermediaStream(): MediaStream { - return this.localUsermediaFeed?.stream; - } - - public get localScreensharingStream(): MediaStream { - return this.localScreensharingFeed?.stream; - } - - public get remoteUsermediaFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get remoteScreensharingFeed(): CallFeed { - return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - } - - public get remoteUsermediaStream(): MediaStream { - return this.remoteUsermediaFeed?.stream; - } - - public get remoteScreensharingStream(): MediaStream { - return this.remoteScreensharingFeed?.stream; - } - - private getFeedByStreamId(streamId: string): CallFeed { - return this.getFeeds().find((feed) => feed.stream.id === streamId); - } - - /** - * Returns an array of all CallFeeds - * @returns {Array} CallFeeds - */ - public getFeeds(): Array { - return this.feeds; - } - - /** - * Returns an array of all local CallFeeds - * @returns {Array} local CallFeeds - */ - public getLocalFeeds(): Array { - return this.feeds.filter((feed) => feed.isLocal()); - } - - /** - * Returns an array of all remote CallFeeds - * @returns {Array} remote CallFeeds - */ - public getRemoteFeeds(): Array { - return this.feeds.filter((feed) => !feed.isLocal()); - } - - /** - * Generates and returns localSDPStreamMetadata - * @returns {SDPStreamMetadata} localSDPStreamMetadata - */ - private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { - const metadata: SDPStreamMetadata = {}; - for (const localFeed of this.getLocalFeeds()) { - if (updateStreamIds) { - localFeed.sdpMetadataStreamId = localFeed.stream.id; - } - - metadata[localFeed.sdpMetadataStreamId] = { - purpose: localFeed.purpose, - audio_muted: localFeed.isAudioMuted(), - video_muted: localFeed.isVideoMuted(), - }; - } - return metadata; - } - - /** - * Returns true if there are no incoming feeds, - * otherwise returns false - * @returns {boolean} no incoming feeds - */ - public noIncomingFeeds(): boolean { - return !this.feeds.some((feed) => !feed.isLocal()); - } - - private pushRemoteFeed(stream: MediaStream): void { - // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.opponentSupportsSDPStreamMetadata()) { - this.pushRemoteFeedWithoutMetadata(stream); - return; - } - - const userId = this.getOpponentMember().userId; - const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; - const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; - - if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); - return; - } - - // Try to find a feed with the same purpose as the new stream, - // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); - } - - /** - * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata - */ - private pushRemoteFeedWithoutMetadata(stream: MediaStream): void { - const userId = this.getOpponentMember().userId; - // We can guess the purpose here since the other client can only send one stream - const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when transceiver directionality is changed and the 'active' - // status of streams change - Dave - // If we already have a stream, check this stream has the same id - if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); - return; - } - - // Try to find a feed with the same stream id as the new stream, - // if we find it replace the old stream with the new one - const feed = this.getFeedByStreamId(stream.id); - if (feed) { - feed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); - } - - private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { - const userId = this.client.getUserId(); - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - // We try to replace an existing feed if there already is one with the same purpose - const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - } - - /** - * Pushes supplied feed to the call - * @param {CallFeed} callFeed to push - * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection - */ - public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { - if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { - logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); - return; - } - - this.feeds.push(callFeed); - - if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - for (const track of callFeed.stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); - } - } - - logger.info( - `Pushed local stream `+ - `(id="${callFeed.stream.id}", `+ - `active="${callFeed.stream.active}", `+ - `purpose="${callFeed.purpose}")`, - ); - - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - /** - * Removes local call feed from the call and its tracks from the peer - * connection - * @param callFeed to remove - */ - public removeLocalFeed(callFeed: CallFeed): void { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia - ? this.usermediaSenders - : this.screensharingSenders; - - for (const sender of senderArray) { - this.peerConn.removeTrack(sender); - } - - if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { - this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); - } - - // Empty the array - senderArray.splice(0, senderArray.length); - this.deleteFeed(callFeed); - } - - private deleteAllFeeds(): void { - for (const feed of this.feeds) { - if (!feed.isLocal() || !this.groupCallId) { - feed.dispose(); - } - } - - this.feeds = []; - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - private deleteFeedByStream(stream: MediaStream): void { - const feed = this.getFeedByStreamId(stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - this.deleteFeed(feed); - } - - private deleteFeed(feed: CallFeed): void { - feed.dispose(); - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - } - - // The typescript definitions have this type as 'any' :( - public async getCurrentCallStats(): Promise { - if (this.callHasEnded()) { - return this.callStatsAtEnd; - } - - return this.collectCallStats(); - } - - private async collectCallStats(): Promise { - // This happens when the call fails before it starts. - // For example when we fail to get capture sources - if (!this.peerConn) return; - - const statsReport = await this.peerConn.getStats(); - const stats = []; - for (const item of statsReport) { - stats.push(item[1]); - } - - return stats; - } - - /** - * Configure this call from an invite event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.invite event - */ - public async initWithInvite(event: MatrixEvent): Promise { - const invite = event.getContent(); - this.direction = CallDirection.Inbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - const sdpStreamMetadata = invite[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - this.peerConn = this.createPeerConnection(); - // we must set the party ID before await-ing on anything: the call event - // handler will start giving us more call events (eg. candidates) so if - // we haven't set the party ID, we'll ignore them. - this.chooseOpponent(event); - try { - await this.peerConn.setRemoteDescription(invite.offer); - await this.addBufferedIceCandidates(); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; - - // According to previous comments in this file, firefox at some point did not - // add streams until media started arriving on them. Testing latest firefox - // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - this.setState(CallState.Ringing); - - if (event.getLocalAge()) { - setTimeout(() => { - if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup); - } - }, invite.lifetime - event.getLocalAge()); - } - } - - /** - * Configure this call from a hangup or reject event. Used by MatrixClient. - * @param {MatrixEvent} event The m.call.hangup event - */ - public initWithHangup(event: MatrixEvent): void { - // perverse as it may seem, sometimes we want to instantiate a call with a - // hangup message (because when getting the state of the room on load, events - // come in reverse order and we want to remember that a call has been hung up) - this.setState(CallState.Ended); - } - - private shouldAnswerWithMediaType( - wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video", - ): boolean { - if (wantedValue && !valueOfTheOtherSide) { - // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); - return false; - } else if ( - !utils.isNullOrUndefined(wantedValue) && - wantedValue !== valueOfTheOtherSide && - !this.opponentSupportsSDPStreamMetadata() - ) { - logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, - ); - return valueOfTheOtherSide; - } - return wantedValue ?? valueOfTheOtherSide; - } - - /** - * Answer a call. - */ - public async answer(audio?: boolean, video?: boolean): Promise { - if (this.inviteOrAnswerSent) return; - // TODO: Figure out how to do this - if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - const prevState = this.state; - const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); - const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); - - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream( - answerWithAudio, answerWithVideo, - ); - this.waitForLocalAVStream = false; - const usermediaFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - - const feeds = [usermediaFeed]; - - if (this.localScreensharingFeed) { - feeds.push(this.localScreensharingFeed); - } - - this.answerWithCallFeeds(feeds); - } catch (e) { - if (answerWithVideo) { - // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); - this.setState(prevState); - this.waitForLocalAVStream = false; - await this.answer(answerWithAudio, false); - } else { - this.getUserMediaFailed(e); - return; - } - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } - } - - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - if (this.inviteOrAnswerSent) return; - - logger.debug(`Answering call ${this.callId}`); - - this.queueGotCallFeedsForAnswer(callFeeds); - } - - /** - * Replace this call with a new call, e.g. for glare resolution. Used by - * MatrixClient. - * @param {MatrixCall} newCall The new call. - */ - public replacedBy(newCall: MatrixCall): void { - if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); - newCall.waitForLocalAVStream = true; - } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - if (newCall.direction === CallDirection.Outbound) { - newCall.queueGotCallFeedsForAnswer([]); - } else { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); - } - } - this.successor = newCall; - this.emit(CallEvent.Replaced, newCall); - this.hangup(CallErrorCode.Replaced, true); - } - - /** - * Hangup a call. - * @param {string} reason The reason why the call is being hung up. - * @param {boolean} suppressEvent True to suppress emitting an event. - */ - public hangup(reason: CallErrorCode, suppressEvent: boolean): void { - if (this.callHasEnded()) return; - - logger.debug("Ending call " + this.callId); - this.terminate(CallParty.Local, reason, !suppressEvent); - // We don't want to send hangup here if we didn't even get to sending an invite - if (this.state === CallState.WaitLocalMedia) return; - const content = {}; - // Don't send UserHangup reason to older clients - if ((this.opponentVersion && this.opponentVersion >= 1) || reason !== CallErrorCode.UserHangup) { - content["reason"] = reason; - } - this.sendVoipEvent(EventType.CallHangup, content); - } - - /** - * Reject a call - * This used to be done by calling hangup, but is a separate method and protocol - * event as of MSC2746. - */ - public reject(): void { - if (this.state !== CallState.Ringing) { - throw Error("Call must be in 'ringing' state to reject!"); - } - - if (this.opponentVersion < 1) { - logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, - ); - this.hangup(CallErrorCode.UserHangup, true); - return; - } - - logger.debug("Rejecting call: " + this.callId); - this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); - this.sendVoipEvent(EventType.CallReject, {}); - } - - /** - * Adds an audio and/or video track - upgrades the call - * @param {boolean} audio should add an audio track - * @param {boolean} video should add an video track - */ - private async upgradeCall( - audio: boolean, video: boolean, - ): Promise { - // We don't do call downgrades - if (!audio && !video) return; - if (!this.opponentSupportsSDPStreamMetadata()) return; - - try { - const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; - const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); - - const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); - if (upgradeAudio && upgradeVideo) { - if (this.hasLocalUserMediaAudioTrack) return; - if (this.hasLocalUserMediaVideoTrack) return; - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } else if (upgradeAudio) { - if (this.hasLocalUserMediaAudioTrack) return; - - const audioTrack = stream.getAudioTracks()[0]; - this.localUsermediaStream.addTrack(audioTrack); - this.peerConn.addTrack(audioTrack, this.localUsermediaStream); - } else if (upgradeVideo) { - if (this.hasLocalUserMediaVideoTrack) return; - - const videoTrack = stream.getVideoTracks()[0]; - this.localUsermediaStream.addTrack(videoTrack); - this.peerConn.addTrack(videoTrack, this.localUsermediaStream); - } - } catch (error) { - logger.error("Failed to upgrade the call", error); - this.emit(CallEvent.Error, - new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), - ); - } - } - - /** - * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false - * @returns {boolean} can screenshare - */ - public opponentSupportsSDPStreamMetadata(): boolean { - return Boolean(this.remoteSDPStreamMetadata); - } - - /** - * If there is a screensharing stream returns true, otherwise returns false - * @returns {boolean} is screensharing - */ - public isScreensharing(): boolean { - return Boolean(this.localScreensharingStream); - } - - /** - * Starts/stops screensharing - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - // Skip if there is nothing to do - if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - - // Fallback to replaceTrack() - if (!this.opponentSupportsSDPStreamMetadata()) { - return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); - } - - logger.debug(`Set screensharing enabled? ${enabled}`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - return false; - } - } - - /** - * Starts/stops screensharing - * Should be used ONLY if the opponent doesn't support SDPStreamMetadata - * @param enabled the desired screensharing state - * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use - * @returns {boolean} new screensharing state - */ - private async setScreensharingEnabledWithoutMetadataSupport( - enabled: boolean, desktopCapturerSourceId?: string, - ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); - if (enabled) { - try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); - if (!stream) return false; - - const track = stream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); - - return true; - } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); - return false; - } - } else { - const track = this.localUsermediaStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - sender.replaceTrack(track); - - this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); - this.deleteFeedByStream(this.localScreensharingStream); - - return false; - } - } - - /** - * Request a new local usermedia stream with the current device id. - */ - public async updateLocalUsermediaStream(stream: MediaStream) { - const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); - const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); - - const newSenders = []; - - for (const track of stream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - - /** - * Set whether our outbound video should be muted or not. - * @param {boolean} muted True to mute the outbound video. - * @returns the new mute state - */ - public async setLocalVideoMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasVideoDevice()) { - return this.isLocalVideoMuted(); - } - - if (!this.hasLocalUserMediaVideoTrack && !muted) { - await this.upgradeCall(false, true); - return this.isLocalVideoMuted(); - } - this.localUsermediaFeed?.setVideoMuted(muted); - this.updateMuteStatus(); - return this.isLocalVideoMuted(); - } - - /** - * Check if local video is muted. - * - * If there are multiple video tracks, all of the tracks need to be muted - * for this to return true. This means if there are no video tracks, this will - * return true. - * @return {Boolean} True if the local preview video is muted, else false - * (including if the call is not set up yet). - */ - public isLocalVideoMuted(): boolean { - return this.localUsermediaFeed?.isVideoMuted(); - } - - /** - * Set whether the microphone should be muted or not. - * @param {boolean} muted True to mute the mic. - * @returns the new mute state - */ - public async setMicrophoneMuted(muted: boolean): Promise { - if (!await this.client.getMediaHandler().hasAudioDevice()) { - return this.isMicrophoneMuted(); - } - - if (!this.hasLocalUserMediaAudioTrack && !muted) { - await this.upgradeCall(true, false); - return this.isMicrophoneMuted(); - } - this.localUsermediaFeed?.setAudioMuted(muted); - this.updateMuteStatus(); - return this.isMicrophoneMuted(); - } - - /** - * Check if the microphone is muted. - * - * If there are multiple audio tracks, all of the tracks need to be muted - * for this to return true. This means if there are no audio tracks, this will - * return true. - * @return {Boolean} True if the mic is muted, else false (including if the call - * is not set up yet). - */ - public isMicrophoneMuted(): boolean { - return this.localUsermediaFeed?.isAudioMuted(); - } - - /** - * @returns true if we have put the party on the other side of the call on hold - * (that is, we are signalling to them that we are not listening) - */ - public isRemoteOnHold(): boolean { - return this.remoteOnHold; - } - - public setRemoteOnHold(onHold: boolean): void { - if (this.isRemoteOnHold() === onHold) return; - this.remoteOnHold = onHold; - - for (const transceiver of this.peerConn.getTransceivers()) { - // We don't send hold music or anything so we're not actually - // sending anything, but sendrecv is fairly standard for hold and - // it makes it a lot easier to figure out who's put who on hold. - transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; - } - this.updateMuteStatus(); - - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). - * @returns true if the other party has put us on hold - */ - public isLocalOnHold(): boolean { - if (this.state !== CallState.Connected) return false; - - let callOnHold = true; - - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (const transceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); - - if (!trackOnHold) callOnHold = false; - } - - return callOnHold; - } - - /** - * Sends a DTMF digit to the other party - * @param digit The digit (nb. string - '#' and '*' are dtmf too) - */ - public sendDtmfDigit(digit: string): void { - for (const sender of this.peerConn.getSenders()) { - if (sender.track.kind === 'audio' && sender.dtmf) { - sender.dtmf.insertDTMF(digit); - return; - } - } - - throw new Error("Unable to find a track to send DTMF on"); - } - - private updateMuteStatus(): void { - this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), - }); - - const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; - const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - - setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); - } - - private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { - if (this.successor) { - this.successor.queueGotCallFeedsForAnswer(callFeeds); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - if (requestScreenshareFeed) { - this.peerConn.addTransceiver("video", { - direction: "recvonly", - }); - } - - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - } - - private async sendAnswer(): Promise { - const answerContent = { - answer: { - sdp: this.peerConn.localDescription.sdp, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: this.peerConn.localDescription.type, - }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - } as MCallAnswer; - - answerContent.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - // We have just taken the local description from the peerConn which will - // contain all the local candidates added so far, so we can discard any candidates - // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(EventType.CallAnswer, answerContent); - // If this isn't the first time we've tried to send the answer, - // we may have candidates queued up, so send them now. - this.inviteOrAnswerSent = true; - } catch (error) { - // We've failed to answer: back to the ringing state - this.setState(CallState.Ringing); - this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SendAnswer; - let message = "Failed to send answer"; - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - this.emit(CallEvent.Error, new CallError(code, message, error)); - throw error; - } - - // error handler re-throws so this won't happen on error, but - // we don't want the same error handling on the candidate queue - this.sendCandidateQueue(); - } - - private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); - } else { - this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); - } - } - - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { - if (this.callHasEnded()) return; - - this.waitForLocalAVStream = false; - - for (const feed of callFeeds) { - this.pushLocalFeed(feed); - } - - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } - } - - /** - * Internal - * @param {Object} event - */ - private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise => { - if (event.candidate) { - logger.debug( - "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + - event.candidate.candidate, - ); - - if (this.callHasEnded()) return; - - // As with the offer, note we need to make a copy of this object, not - // pass the original: that broke in Chrome ~m43. - if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) { - this.queueCandidate(event.candidate); - - if (event.candidate.candidate === '') this.sentEndOfCandidates = true; - } - } - }; - - private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; - } - }; - - public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise { - if (this.callHasEnded()) { - //debuglog("Ignoring remote ICE candidate because call has ended"); - return; - } - - const content = ev.getContent(); - const candidates = content.candidates; - if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); - return; - } - - const fromPartyId = content.version === 0 ? null : content.party_id || null; - - if (this.opponentPartyId === undefined) { - // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); - const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCandidates.push(...candidates); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); - return; - } - - if (!this.partyIdMatches(content)) { - logger.info( - `Ignoring candidates from party ID ${content.party_id}: ` + - `we have chosen party ID ${this.opponentPartyId}`, - ); - - return; - } - - await this.addIceCandidates(candidates); - } - - /** - * Used by MatrixClient. - * @param {Object} msg - */ - public async onAnswerReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); - - if (this.callHasEnded()) { - logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); - return; - } - - if (this.opponentPartyId !== undefined) { - logger.info( - `Ignoring answer from party ID ${content.party_id}: ` + - `we already have an answer/reject from ${this.opponentPartyId}`, - ); - return; - } - - this.chooseOpponent(event); - await this.addBufferedIceCandidates(); - - this.setState(CallState.Connecting); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); - } - - try { - await this.peerConn.setRemoteDescription(content.answer); - } catch (e) { - logger.debug("Failed to set remote description", e); - this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); - return; - } - - // If the answer we selected has a party_id, send a select_answer event - // We do this after setting the remote description since otherwise we'd block - // call setup on it - if (this.opponentPartyId !== null) { - try { - await this.sendVoipEvent(EventType.CallSelectAnswer, { - selected_party_id: this.opponentPartyId, - }); - } catch (err) { - // This isn't fatal, and will just mean that if another party has raced to answer - // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); - } - } - } - - public async onSelectAnswerReceived(event: MatrixEvent): Promise { - if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); - return; - } - - const selectedPartyId = event.getContent().selected_party_id; - - if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); - return; - } - - if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); - // The other party has picked somebody else's answer - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - } - } - - public async onNegotiateReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - const description = content.description; - if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); - return; - } - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - const polite = this.direction === CallDirection.Inbound; - - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - const offerCollision = ( - (description.type === 'offer') && - (this.makingOffer || this.peerConn.signalingState !== 'stable') - ); - - this.ignoreOffer = !polite && offerCollision; - if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); - return; - } - - const prevLocalOnHold = this.isLocalOnHold(); - - const sdpStreamMetadata = content[SDPStreamMetadataKey]; - if (sdpStreamMetadata) { - this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); - } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); - } - - try { - await this.peerConn.setRemoteDescription(description); - - if (description.type === 'offer') { - this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - - this.sendVoipEvent(EventType.CallNegotiate, { - description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), - }); - } - } catch (err) { - logger.warn("Failed to complete negotiation", err); - } - - const newLocalOnHold = this.isLocalOnHold(); - if (prevLocalOnHold !== newLocalOnHold) { - this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold); - // also this one for backwards compat - this.emit(CallEvent.HoldUnhold, newLocalOnHold); - } - } - - private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { - this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - for (const feed of this.getRemoteFeeds()) { - const streamId = feed.stream.id; - feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted); - feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted); - feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose; - } - } - - public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void { - const content = event.getContent(); - const metadata = content[SDPStreamMetadataKey]; - this.updateRemoteSDPStreamMetadata(metadata); - } - - public async onAssertedIdentityReceived(event: MatrixEvent): Promise { - const content = event.getContent(); - if (!content.asserted_identity) return; - - this.remoteAssertedIdentity = { - id: content.asserted_identity.id, - displayName: content.asserted_identity.display_name, - }; - this.emit(CallEvent.AssertedIdentityChanged); - } - - private callHasEnded(): boolean { - // This exists as workaround to typescript trying to be clever and erroring - // when putting if (this.state === CallState.Ended) return; twice in the same - // function, even though that function is async. - return this.state === CallState.Ended; - } - - private queueGotLocalOffer(): void { - // Ensure only one negotiate/answer event is being processed at a time. - if (this.responsePromiseChain) { - this.responsePromiseChain = - this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); - } else { - this.responsePromiseChain = this.wrappedGotLocalOffer(); - } - } - - private async wrappedGotLocalOffer(): Promise { - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } - } - - private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); - - if (this.callHasEnded()) { - logger.debug("Ignoring newly created offer on call ID " + this.callId + - " because the call has ended"); - return; - } - - try { - await this.peerConn.setLocalDescription(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - - if (this.peerConn.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - } - - if (this.callHasEnded()) return; - - const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate; - - const content = { - lifetime: CALL_TIMEOUT_MS, - } as MCallInviteNegotiate; - - if (eventType === EventType.CallInvite && this.invitee) { - content.invitee = this.invitee; - } - - // clunky because TypeScript can't follow the types through if we use an expression as the key - if (this.state === CallState.CreateOffer) { - content.offer = this.peerConn.localDescription; - } else { - content.description = this.peerConn.localDescription; - } - - content.capabilities = { - 'm.call.transferee': this.client.supportsCallTransfer, - 'm.call.dtmf': false, - }; - - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); - - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - try { - await this.sendVoipEvent(eventType, content); - } catch (error) { - logger.error("Failed to send invite", error); - if (error.event) this.client.cancelPendingEvent(error.event); - - let code = CallErrorCode.SignallingFailed; - let message = "Signalling failed"; - if (this.state === CallState.CreateOffer) { - code = CallErrorCode.SendInvite; - message = "Failed to send invite"; - } - if (error.name == 'UnknownDeviceError') { - code = CallErrorCode.UnknownDevices; - message = "Unknown devices present in the room"; - } - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.terminate(CallParty.Local, code, false); - - // no need to carry on & send the candidate queue, but we also - // don't want to rethrow the error - return; - } - - this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { - this.inviteOrAnswerSent = true; - this.setState(CallState.InviteSent); - this.inviteTimeout = setTimeout(() => { - this.inviteTimeout = null; - if (this.state === CallState.InviteSent) { - this.hangup(CallErrorCode.InviteTimeout, false); - } - }, CALL_TIMEOUT_MS); - } - } - - private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.LocalOfferFailed, - "Failed to get local offer!", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); - }; - - private getUserMediaFailed = (err: Error): void => { - if (this.successor) { - this.successor.getUserMediaFailed(err); - return; - } - - logger.warn("Failed to get user media - ending call", err); - - this.emit( - CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Couldn't start capturing media! Is your microphone set up and " + - "does this app have permission?", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - }; - - private onIceConnectionStateChanged = (): void => { - if (this.callHasEnded()) { - return; // because ICE can still complete as we're ending the call - } - logger.debug( - "Call ID " + this.callId + ": ICE connection state changed to: " + this.peerConn.iceConnectionState, - ); - // ideally we'd consider the call to be connected when we get media but - // chrome doesn't implement any of the 'onstarted' events yet - if (this.peerConn.iceConnectionState == 'connected') { - clearTimeout(this.iceDisconnectedTimeout); - this.setState(CallState.Connected); - - if (!this.callLengthInterval) { - this.callLengthInterval = setInterval(() => { - this.callLength++; - this.emit(CallEvent.LengthChanged, this.callLength); - }, 1000); - } - } else if (this.peerConn.iceConnectionState == 'failed') { - // Firefox for Android does not yet have support for restartIce() - if (this.peerConn.restartIce) { - this.peerConn.restartIce(); - } else { - this.hangup(CallErrorCode.IceFailed, false); - } - } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); - } - }; - - private onSignallingStateChanged = (): void => { - logger.debug( - "call " + this.callId + ": Signalling state changed to: " + - this.peerConn.signalingState, - ); - }; - - private onTrack = (ev: RTCTrackEvent): void => { - if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); - return; - } - - const stream = ev.streams[0]; - this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - }); - }; - - private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); - }; - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } - } - - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } - } - } - - private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); - - if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); - return; - } - - this.queueGotLocalOffer(); - }; - - public onHangupReceived = (msg: MCallHangupReject): void => { - logger.debug("Hangup received for call ID " + this.callId); - - // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen - // a partner yet but we're treating the hangup as a reject as per VoIP v0) - if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { - // default reason is user_hangup - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); - } - }; - - public onRejectReceived = (msg: MCallHangupReject): void => { - logger.debug("Reject received for call ID " + this.callId); - - // No need to check party_id for reject because if we'd received either - // an answer or reject, we wouldn't be in state InviteSent - - const shouldTerminate = ( - // reject events also end the call if it's ringing: it's another of - // our devices rejecting the call. - ([CallState.InviteSent, CallState.Ringing].includes(this.state)) || - // also if we're in the init state and it's an inbound call, since - // this means we just haven't entered the ringing state yet - this.state === CallState.Fledgling && this.direction === CallDirection.Inbound - ); - - if (shouldTerminate) { - this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); - } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); - } - }; - - public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); - this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); - }; - - private setState(state: CallState): void { - const oldState = this.state; - this.state = state; - this.emit(CallEvent.State, state, oldState); - } - - /** - * Internal - * @param {string} eventType - * @param {Object} content - * @return {Promise} - */ - private sendVoipEvent(eventType: string, content: object): Promise { - const realContent = Object.assign({}, content, { - version: VOIP_PROTO_VERSION, - call_id: this.callId, - party_id: this.ourPartyId, - conf_id: this.groupCallId, - }); - - if (this.opponentDeviceId) { - this.emit(CallEvent.SendVoipEvent, { - type: "toDevice", - eventType, - userId: this.invitee || this.getOpponentMember().userId, - opponentDeviceId: this.opponentDeviceId, - content: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }); - - return this.client.sendToDevice(eventType, { - [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: { - ...realContent, - device_id: this.client.deviceId, - sender_session_id: this.client.getSessionId(), - dest_session_id: this.opponentSessionId, - }, - }, - }); - } else { - this.emit(CallEvent.SendVoipEvent, { - type: "sendEvent", - eventType, - roomId: this.roomId, - content: realContent, - userId: this.invitee || this.getOpponentMember().userId, - }); - - return this.client.sendEvent(this.roomId, eventType, realContent); - } - } - - private queueCandidate(content: RTCIceCandidate): void { - // We partially de-trickle candidates by waiting for `delay` before sending them - // amalgamated, in order to avoid sending too many m.call.candidates events and hitting - // rate limits in Matrix. - // In practice, it'd be better to remove rate limits for m.call.* - - // N.B. this deliberately lets you queue and send blank candidates, which MSC2746 - // currently proposes as the way to indicate that candidate gathering is complete. - // This will hopefully be changed to an explicit rather than implicit notification - // shortly. - this.candidateSendQueue.push(content); - - // Don't send the ICE candidates yet if the call is in the ringing state: this - // means we tried to pick (ie. started generating candidates) and then failed to - // send the answer and went back to the ringing state. Queue up the candidates - // to send if we successfully send the answer. - // Equally don't send if we haven't yet sent the answer because we can send the - // first batch of candidates along with the answer - if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - - // MSC2746 recommends these values (can be quite long when calling because the - // callee will need a while to answer the call) - const delay = this.direction === CallDirection.Inbound ? 500 : 2000; - - if (this.candidateSendTries === 0) { - setTimeout(() => { - this.sendCandidateQueue(); - }, delay); - } - } - - /* - * Transfers this call to another user - */ - public async transfer(targetUserId: string): Promise { - // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we share with them. - const profileInfo = await this.client.getProfileInfo(targetUserId); - - const replacementId = genCallID(); - - const body = { - replacement_id: genCallID(), - target_user: { - id: targetUserId, - display_name: profileInfo.displayname, - avatar_url: profileInfo.avatar_url, - }, - create_call: replacementId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, body); - - await this.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - /* - * Transfers this call to the target call, effectively 'joining' the - * two calls (so the remote parties on each call are connected together). - */ - public async transferToCall(transferTargetCall?: MatrixCall): Promise { - const targetProfileInfo = await this.client.getProfileInfo(transferTargetCall.getOpponentMember().userId); - const transfereeProfileInfo = await this.client.getProfileInfo(this.getOpponentMember().userId); - - const newCallId = genCallID(); - - const bodyToTransferTarget = { - // the replacements on each side have their own ID, and it's distinct from the - // ID of the new call (but we can use the same function to generate it) - replacement_id: genCallID(), - target_user: { - id: this.getOpponentMember().userId, - display_name: transfereeProfileInfo.displayname, - avatar_url: transfereeProfileInfo.avatar_url, - }, - await_call: newCallId, - } as MCallReplacesEvent; - - await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget); - - const bodyToTransferee = { - replacement_id: genCallID(), - target_user: { - id: transferTargetCall.getOpponentMember().userId, - display_name: targetProfileInfo.displayname, - avatar_url: targetProfileInfo.avatar_url, - }, - create_call: newCallId, - } as MCallReplacesEvent; - - await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee); - - await this.terminate(CallParty.Local, CallErrorCode.Replaced, true); - await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transfered, true); - } - - private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { - if (this.callHasEnded()) return; - - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); - - if (this.inviteTimeout) { - clearTimeout(this.inviteTimeout); - this.inviteTimeout = null; - } - if (this.callLengthInterval) { - clearInterval(this.callLengthInterval); - this.callLengthInterval = null; - } - - this.callStatsAtEnd = await this.collectCallStats(); - - // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - this.stopAllMedia(); - this.deleteAllFeeds(); - - if (this.peerConn && this.peerConn.signalingState !== 'closed') { - this.peerConn.close(); - } - if (shouldEmit) { - this.emit(CallEvent.Hangup, this); - } - - this.client.callEventHandler.calls.delete(this.callId); - } - - private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); - for (const feed of this.feeds) { - if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Usermedia && - !this.groupCallId - ) { - this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if ( - feed.isLocal() && - feed.purpose === SDPStreamMetadataPurpose.Screenshare && - !this.groupCallId - ) { - this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || !this.groupCallId) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } - } - } - } - - private checkForErrorListener(): void { - if (this.listeners("error").length === 0) { - throw new Error( - "You MUST attach an error listener using call.on('error', function() {})", - ); - } - } - - private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { - return; - } - - const candidates = this.candidateSendQueue; - this.candidateSendQueue = []; - ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); - try { - await this.sendVoipEvent(EventType.CallCandidates, content); - // reset our retry count if we have successfully sent our candidates - // otherwise queueCandidate() will refuse to try to flush the queue - this.candidateSendTries = 0; - - // Try to send candidates again just in case we received more candidates while sending. - this.sendCandidateQueue(); - } catch (error) { - // don't retry this event: we'll send another one later as we might - // have more candidates by then. - if (error.event) this.client.cancelPendingEvent(error.event); - - // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...candidates); - - if (this.candidateSendTries > 5) { - logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, - ); - - const code = CallErrorCode.SignallingFailed; - const message = "Signalling failed"; - - this.emit(CallEvent.Error, new CallError(code, message, error)); - this.hangup(code, false); - - return; - } - - const delayMs = 500 * Math.pow(2, this.candidateSendTries); - ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); - setTimeout(() => { - this.sendCandidateQueue(); - }, delayMs); - } - } - - /** - * Place a call to this room. - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCall(audio: boolean, video: boolean): Promise { - if (!audio) { - throw new Error("You CANNOT start a call without audio"); - } - this.setState(CallState.WaitLocalMedia); - - try { - const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - const callFeed = new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.client.getUserId(), - stream, - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - }); - await this.placeCallWithCallFeeds([callFeed]); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } - - /** - * Place a call to this room with call feed. - * @param {CallFeed[]} callFeeds to use - * @throws if you have not specified a listener for 'error' events. - * @throws if have passed audio=false. - */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { - this.checkForErrorListener(); - this.direction = CallDirection.Outbound; - - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); - } - - private createPeerConnection(): RTCPeerConnection { - const pc = new window.RTCPeerConnection({ - iceTransportPolicy: this.forceTURN ? 'relay' : undefined, - iceServers: this.turnServers, - iceCandidatePoolSize: this.client.iceCandidatePoolSize, - }); - - // 'connectionstatechange' would be better, but firefox doesn't implement that. - pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChanged); - pc.addEventListener('signalingstatechange', this.onSignallingStateChanged); - pc.addEventListener('icecandidate', this.gotLocalIceCandidate); - pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); - pc.addEventListener('track', this.onTrack); - pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); - pc.addEventListener('datachannel', this.onDataChannel); - - return pc; - } - - private partyIdMatches(msg: MCallBase): boolean { - // They must either match or both be absent (in which case opponentPartyId will be null) - // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same - // here and use null if the version is 0 (woe betide any opponent sending messages in the - // same call with different versions) - const msgPartyId = msg.version === 0 ? null : msg.party_id || null; - return msgPartyId === this.opponentPartyId; - } - - // Commits to an opponent for the call - // ev: An invite or answer event - private chooseOpponent(ev: MatrixEvent): void { - // I choo-choo-choose you - const msg = ev.getContent(); - - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); - - this.opponentVersion = msg.version; - if (this.opponentVersion === 0) { - // set to null to indicate that we've chosen an opponent, but because - // they're v0 they have no party ID (even if they sent one, we're ignoring it) - this.opponentPartyId = null; - } else { - // set to their party ID, or if they're naughty and didn't send one despite - // not being v0, set it to null to indicate we picked an opponent with no - // party ID - this.opponentPartyId = msg.party_id || null; - } - this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - } - - private async addBufferedIceCandidates(): Promise { - const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCandidates); - } - this.remoteCandidateBuffer = null; - } - - private async addIceCandidates(candidates: RTCIceCandidate[]): Promise { - for (const candidate of candidates) { - if ( - (candidate.sdpMid === null || candidate.sdpMid === undefined) && - (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) - ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); - continue; - } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); - try { - await this.peerConn.addIceCandidate(candidate); - } catch (err) { - if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); - } - } - } - } - - public get hasPeerConnection(): boolean { - return Boolean(this.peerConn); - } -} - -export function setTracksEnabled(tracks: Array, enabled: boolean): void { - for (let i = 0; i < tracks.length; i++) { - tracks[i].enabled = enabled; - } -} - -/** - * DEPRECATED - * Use client.createCall() - * - * Create a new Matrix call for the browser. - * @param {MatrixClient} client The client instance to use. - * @param {string} roomId The room the call is in. - * @param {Object?} options DEPRECATED optional options map. - * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be - * forced. This option is deprecated - use opts.forceTURN when creating the matrix client - * since it's only possible to set this option on outbound calls. - * @return {MatrixCall} the call or null if the browser doesn't support calling. - */ -export function createNewMatrixCall(client: any, roomId: string, options?: CallOpts): MatrixCall { - // typeof prevents Node from erroring on an undefined reference - if (typeof(window) === 'undefined' || typeof(document) === 'undefined') { - // NB. We don't log here as apps try to create a call object as a test for - // whether calls are supported, so we shouldn't fill the logs up. - return null; - } - - // Firefox throws on so little as accessing the RTCPeerConnection when operating in - // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 - // though the concern is that the browser throwing a SecurityError will brick the - // client creation process. - try { - const supported = Boolean( - window.RTCPeerConnection || window.RTCSessionDescription || - window.RTCIceCandidate || navigator.mediaDevices, - ); - if (!supported) { - // Adds a lot of noise to test runs, so disable logging there. - if (process.env.NODE_ENV !== "test") { - logger.error("WebRTC is not supported in this browser / environment"); - } - return null; - } - } catch (e) { - logger.error("Exception thrown when trying to access WebRTC", e); - return null; - } - - const optionsForceTURN = options ? options.forceTURN : false; - - const opts: CallOpts = { - client: client, - roomId: roomId, - invitee: options?.invitee, - turnServers: client.getTurnServers(), - // call level options - forceTURN: client.forceTURN || optionsForceTURN, - opponentDeviceId: options?.opponentDeviceId, - opponentSessionId: options?.opponentSessionId, - groupCallId: options?.groupCallId, - }; - const call = new MatrixCall(opts); - - client.reEmitter.reEmit(call, Object.values(CallEvent)); - - return call; -} diff --git a/src/matrix/calls/CallFeed.ts b/src/matrix/calls/CallFeed.ts deleted file mode 100644 index c8cc8662..00000000 --- a/src/matrix/calls/CallFeed.ts +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2021 Šimon Brandner - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { SDPStreamMetadataPurpose } from "./callEventTypes"; - -const POLLING_INTERVAL = 200; // ms -export const SPEAKING_THRESHOLD = -60; // dB -const SPEAKING_SAMPLE_COUNT = 8; // samples - -export interface ICallFeedOpts { - client: MatrixClient; - roomId: string; - userId: string; - stream: MediaStream; - purpose: SDPStreamMetadataPurpose; - audioMuted: boolean; - videoMuted: boolean; -} - -export enum CallFeedEvent { - NewStream = "new_stream", - MuteStateChanged = "mute_state_changed", - VolumeChanged = "volume_changed", - Speaking = "speaking", -} - -export class CallFeed extends EventEmitter { - public stream: MediaStream; - public sdpMetadataStreamId: string; - public userId: string; - public purpose: SDPStreamMetadataPurpose; - public speakingVolumeSamples: number[]; - - private client: MatrixClient; - private roomId: string; - private audioMuted: boolean; - private videoMuted: boolean; - private measuringVolumeActivity = false; - private audioContext: AudioContext; - private analyser: AnalyserNode; - private frequencyBinCount: Float32Array; - private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; - private volumeLooperTimeout: number; - - constructor(opts: ICallFeedOpts) { - super(); - - this.client = opts.client; - this.roomId = opts.roomId; - this.userId = opts.userId; - this.purpose = opts.purpose; - this.audioMuted = opts.audioMuted; - this.videoMuted = opts.videoMuted; - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); - this.sdpMetadataStreamId = opts.stream.id; - - this.updateStream(null, opts.stream); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } - } - - private get hasAudioTrack(): boolean { - return this.stream.getAudioTracks().length > 0; - } - - private updateStream(oldStream: MediaStream, newStream: MediaStream): void { - if (newStream === oldStream) return; - - if (oldStream) { - oldStream.removeEventListener("addtrack", this.onAddTrack); - this.measureVolumeActivity(false); - } - if (newStream) { - this.stream = newStream; - newStream.addEventListener("addtrack", this.onAddTrack); - - if (this.hasAudioTrack) { - this.initVolumeMeasuring(); - } else { - this.measureVolumeActivity(false); - } - } - - this.emit(CallFeedEvent.NewStream, this.stream); - } - - private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); - - this.analyser = this.audioContext.createAnalyser(); - this.analyser.fftSize = 512; - this.analyser.smoothingTimeConstant = 0.1; - - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); - mediaStreamAudioSourceNode.connect(this.analyser); - - this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - } - - private onAddTrack = (): void => { - this.emit(CallFeedEvent.NewStream, this.stream); - }; - - /** - * Returns callRoom member - * @returns member of the callRoom - */ - public getMember(): RoomMember { - const callRoom = this.client.getRoom(this.roomId); - return callRoom.getMember(this.userId); - } - - /** - * Returns true if CallFeed is local, otherwise returns false - * @returns {boolean} is local? - */ - public isLocal(): boolean { - return this.userId === this.client.getUserId(); - } - - /** - * Returns true if audio is muted or if there are no audio - * tracks, otherwise returns false - * @returns {boolean} is audio muted? - */ - public isAudioMuted(): boolean { - return this.stream.getAudioTracks().length === 0 || this.audioMuted; - } - - /** - * Returns true video is muted or if there are no video - * tracks, otherwise returns false - * @returns {boolean} is video muted? - */ - public isVideoMuted(): boolean { - // We assume only one video track - return this.stream.getVideoTracks().length === 0 || this.videoMuted; - } - - public isSpeaking(): boolean { - return this.speaking; - } - - /** - * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. - * @param newStream new stream with which to replace the current one - */ - public setNewStream(newStream: MediaStream): void { - this.updateStream(this.stream, newStream); - } - - /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state - * @param muted is the feed's video muted? - */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Starts emitting volume_changed events where the emitter value is in decibels - * @param enabled emit volume changes - */ - public measureVolumeActivity(enabled: boolean): void { - if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - - this.measuringVolumeActivity = true; - this.volumeLooper(); - } else { - this.measuringVolumeActivity = false; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); - } - } - - public setSpeakingThreshold(threshold: number) { - this.speakingThreshold = threshold; - } - - private volumeLooper = () => { - if (!this.analyser) return; - - if (!this.measuringVolumeActivity) return; - - this.analyser.getFloatFrequencyData(this.frequencyBinCount); - - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } - } - - this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(maxVolume); - - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - - let newSpeaking = false; - - for (let i = 0; i < this.speakingVolumeSamples.length; i++) { - const volume = this.speakingVolumeSamples[i]; - - if (volume > this.speakingThreshold) { - newSpeaking = true; - break; - } - } - - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); - } - - this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); - }; - - public clone(): CallFeed { - const mediaHandler = this.client.getMediaHandler(); - const stream = this.stream.clone(); - - if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { - mediaHandler.userMediaStreams.push(stream); - } else { - mediaHandler.screensharingStreams.push(stream); - } - - return new CallFeed({ - client: this.client, - roomId: this.roomId, - userId: this.userId, - stream, - purpose: this.purpose, - audioMuted: this.audioMuted, - videoMuted: this.videoMuted, - }); - } - - public dispose(): void { - clearTimeout(this.volumeLooperTimeout); - } -} diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 1dad9ce8..d7d673ed 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -22,6 +22,8 @@ import type {ILogItem} from "../../logging/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import type {SignallingMessage} from "./PeerCall"; +import type {MGroupCallBase} from "./callEventTypes"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; @@ -33,7 +35,7 @@ enum CallSetupMessageType { Hangup = "m.call.hangup", } -const CALL_ID = "m.call_id"; +const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { @@ -69,7 +71,7 @@ export class GroupCallHandler { const participant = event.state_key; const sources = event.content["m.sources"]; for (const source of sources) { - const call = this.calls.get(source[CALL_ID]); + const call = this.calls.get(source[CONF_ID]); if (call && !call.isTerminated) { call.addParticipant(participant, source); } @@ -85,110 +87,9 @@ export class GroupCallHandler { eventType === CallSetupMessageType.Hangup; } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const callId = content[CALL_ID]; - const call = this.calls.get(callId); - call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log); + handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + const call = this.calls.get(event.content.conf_id); + call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } } -function participantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} - -class GroupParticipant implements PeerCallHandler { - private peerCall?: PeerCall; - - constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi - ) {} - - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); - } - - /** From PeerCallHandler - * @internal */ - override emitUpdate() { - - } - - /** From PeerCallHandler - * @internal */ - override onSendSignallingMessage() { - // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); - } -} - -class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; - - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { - - } - - get id(): string { return this.callEvent.state_key; } - - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); - for (const [,participant] of this.participants) { - participant.setMedia(this.localMedia.clone()); - } - // send m.call.member state event - - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.sendInvite(); - } - } - } - - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - } - - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); - if (participant) { - participant.updateSource(source); - } else { - participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); - } - } - - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall - } - } - - get id(): string { - return this.callEvent.state_key; - } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a02de99f..5040c805 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,26 +17,41 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, Disposable} from "../../utils/Disposables"; +import {Disposables, IDisposable} from "../../utils/Disposables"; import type {Room} from "../room/Room"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {TimeoutCreator, Timeout} from "../../platform/types/types"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import type {LocalMedia} from "./LocalMedia"; +import { + SDPStreamMetadataKey, + SDPStreamMetadataPurpose +} from "./callEventTypes"; +import type { + MCallBase, + MCallInvite, + MCallAnswer, + MCallSDPStreamMetadataChanged, + MCallCandidates, + MCallHangupReject, + SDPStreamMetadata, +} from "./callEventTypes"; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** * Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform * */ /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ -class PeerCall { +export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private state = CallState.Fledgling; private direction: CallDirection; + private localMedia?: LocalMedia; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible @@ -54,9 +69,13 @@ class PeerCall { private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); + // perfect negotiation flags + private makingOffer: boolean = false; + private ignoreOffer: boolean = false; + constructor( + private callId: string, // generated or from invite private readonly handler: PeerCallHandler, - private localMedia: LocalMedia, private readonly createTimeout: TimeoutCreator, webRTC: WebRTC ) { @@ -83,29 +102,8 @@ class PeerCall { } } - handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { - switch (message.type) { - case EventType.Invite: - // determining whether or not an incoming invite glares - // with an instance of PeerCall is different for group calls - // and 1:1 calls, so done outside of this class. - // If you pass an event for another call id in here it will assume it glares. - - //const newCallId = message.content.call_id; - //if (this.id && newCallId !== this.id) { - // this.handleInviteGlare(message.content); - //} else { - this.handleInvite(message.content, partyId); - //} - break; - case EventType.Answer: - this.handleAnswer(message.content, partyId); - break; - case EventType.Candidates: - this.handleRemoteIceCandidates(message.content, partyId); - break; - case EventType.Hangup: - } + get remoteTracks(): Track[] { + return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { @@ -125,7 +123,9 @@ class PeerCall { for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - await this.waitForState(CallState.InviteSent); + // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet + // but we would go straight to CreateAnswer, so also need to wait for that state + await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } async answer(localMediaPromise: Promise): Promise { @@ -166,15 +166,11 @@ class PeerCall { this.sendAnswer(); } - async hangup() { - - } - async setMedia(localMediaPromise: Promise) { const oldMedia = this.localMedia; this.localMedia = await localMediaPromise; - const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => { + const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => { const oldTrack = selectTrack(oldMedia); const newTrack = selectTrack(this.localMedia); if (oldTrack && newTrack) { @@ -187,50 +183,101 @@ class PeerCall { }; // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called - applyTrack(m => m.microphoneTrack); - applyTrack(m => m.cameraTrack); - applyTrack(m => m.screenShareTrack); + applyTrack(m => m?.microphoneTrack); + applyTrack(m => m?.cameraTrack); + applyTrack(m => m?.screenShareTrack); + } + + async reject() { + + } + + async hangup(errorCode: CallErrorCode) { + } + + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + switch (message.type) { + case EventType.Invite: + if (this.callId !== message.content.call_id) { + await this.handleInviteGlare(message.content, partyId); + } else { + await this.handleFirstInvite(message.content, partyId); + } + break; + case EventType.Answer: + await this.handleAnswer(message.content, partyId); + break; + //case EventType.Candidates: + // await this.handleRemoteIceCandidates(message.content, partyId); + // break; + case EventType.Hangup: + default: + throw new Error(`Unknown event type for call: ${message.type}`); + } + } + + private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise { + const content = { + call_id: callId, + version: 1, + }; + if (reason) { + content["reason"] = reason; + } + return this.handler.sendSignallingMessage({ + type: EventType.Hangup, + content + }); } // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { - // TODO: does this make sense to have this state if we're already connected? - this.setState(CallState.MakingOffer) + this.makingOffer = true; try { - await this.peerConnection.setLocalDescription(); - } catch (err) { - this.logger.debug(`Call ${this.callId} Error setting local description!`, err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; + try { + await this.peerConnection.setLocalDescription(); + } catch (err) { + this.logger.debug(`Call ${this.callId} Error setting local description!`, err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + + if (this.peerConnection.iceGatheringState === 'gathering') { + // Allow a short time for initial candidates to be gathered + await this.delay(200); + } + + if (this.state === CallState.Ended) { + return; + } + + const offer = this.peerConnection.localDescription!; + // Get rid of any candidates waiting to be sent: they'll be included in the local + // description we just got and will send in the offer. + this.logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); + this.candidateSendQueue = []; + + // need to queue this + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), + version: 1, + lifetime: CALL_TIMEOUT_MS + }; + if (this.state === CallState.CreateOffer) { + await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + this.setState(CallState.InviteSent); + } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + // send Negotiate message + //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //this.setState(CallState.InviteSent); + } + } finally { + this.makingOffer = false; } - if (this.peerConnection.iceGatheringState === 'gathering') { - // Allow a short time for initial candidates to be gathered - await this.delay(200); - } - - if (this.state === CallState.Ended) { - return; - } - - const offer = this.peerConnection.localDescription!; - // Get rid of any candidates waiting to be sent: they'll be included in the local - // description we just got and will send in the offer. - this.logger.info(`Call ${this.callId} Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; - - // need to queue this - const content = { - offer, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - version: 1, - lifetime: CALL_TIMEOUT_MS - }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); - this.setState(CallState.InviteSent); - } this.sendCandidateQueue(); if (this.state === CallState.InviteSent) { @@ -242,11 +289,47 @@ class PeerCall { } }; - private async handleInvite(content: InviteContent, partyId: PartyId): Promise { + private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise { + // this is only called when the ids are different + const newCallId = content.call_id; + if (this.callId! > newCallId) { + this.logger.log( + "Glare detected: answering incoming call " + newCallId + + " and canceling outgoing call " + this.callId, + ); + + /* + first, we should set CallDirection + we should anser the call + */ + + // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? + // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? + if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + + } else { + await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); + } + await this.handleInvite(content, partyId); + await this.answer(Promise.resolve(this.localMedia!)); + } else { + this.logger.log( + "Glare detected: rejecting incoming call " + newCallId + + " and keeping outgoing call " + this.callId, + ); + await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced); + } + } + + private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } + await this.handleInvite(content, partyId); + } + + private async handleInvite(content: MCallInvite, partyId: PartyId): Promise { // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if @@ -296,8 +379,8 @@ class PeerCall { } } - private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise { - this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`); + private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { + this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); if (this.state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); @@ -307,7 +390,7 @@ class PeerCall { if (this.opponentPartyId !== undefined) { this.logger.info( `Call ${this.callId} ` + - `Ignoring answer from party ID ${content.party_id}: ` + + `Ignoring answer from party ID ${partyId}: ` + `we already have an answer/reject from ${this.opponentPartyId}`, ); return; @@ -334,16 +417,66 @@ class PeerCall { } } + // private async onNegotiateReceived(event: MatrixEvent): Promise { + // const content = event.getContent(); + // const description = content.description; + // if (!description || !description.sdp || !description.type) { + // this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); + // return; + // } + // // Politeness always follows the direction of the call: in a glare situation, + // // we pick either the inbound or outbound call, so one side will always be + // // inbound and one outbound + // const polite = this.direction === CallDirection.Inbound; + + // // Here we follow the perfect negotiation logic from + // // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + // const offerCollision = ( + // (description.type === 'offer') && + // (this.makingOffer || this.peerConnection.signalingState !== 'stable') + // ); + + // this.ignoreOffer = !polite && offerCollision; + // if (this.ignoreOffer) { + // this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); + // return; + // } + + // const sdpStreamMetadata = content[SDPStreamMetadataKey]; + // if (sdpStreamMetadata) { + // this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); + // } else { + // this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); + // } + + // try { + // await this.peerConnection.setRemoteDescription(description); + + // if (description.type === 'offer') { + // await this.peerConnection.setLocalDescription(); + // await this.handler.sendSignallingMessage({ + // type: EventType.CallNegotiate, + // content: { + // description: this.peerConnection.localDescription!, + // [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), + // } + // }); + // } + // } catch (err) { + // this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); + // } + // } + private async sendAnswer(): Promise { - const answerMessage: AnswerMessage = { - type: EventType.Answer, - content: { - answer: { - sdp: this.peerConnection.localDescription!.sdp, - type: this.peerConnection.localDescription!.type, - }, - [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(), - } + const localDescription = this.peerConnection.localDescription!; + const answerContent: MCallAnswer = { + call_id: this.callId, + version: 1, + answer: { + sdp: localDescription.sdp, + type: localDescription.type, + }, + [SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(), }; // We have just taken the local description from the peerConn which will @@ -354,7 +487,7 @@ class PeerCall { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage(answerMessage); + await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -387,7 +520,6 @@ class PeerCall { }); } - private async sendCandidateQueue(): Promise { if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { return; @@ -395,15 +527,16 @@ class PeerCall { const candidates = this.candidateSendQueue; this.candidateSendQueue = []; - const candidatesMessage: CandidatesMessage = { - type: EventType.Candidates, - content: { - candidates: candidates, - } - }; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage(candidatesMessage); + await this.handler.sendSignallingMessage({ + type: EventType.Candidates, + content: { + call_id: this.callId, + version: 1, + candidates + } + }); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -430,7 +563,6 @@ class PeerCall { } } - private async addBufferedIceCandidates(): Promise { if (this.remoteCandidateBuffer && this.opponentPartyId) { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); @@ -463,7 +595,6 @@ class PeerCall { } } - private setState(state: CallState): void { const oldState = this.state; this.state = state; @@ -475,17 +606,20 @@ class PeerCall { this.handler.emitUpdate(this, undefined); } - private waitForState(state: CallState): Promise { - let deferred = this.statePromiseMap.get(state); - if (!deferred) { - let resolve; - const promise = new Promise(r => { - resolve = r; - }); - deferred = {resolve, promise}; - this.statePromiseMap.set(state, deferred); - } - return deferred.promise; + private waitForState(states: CallState[]): Promise { + // TODO: rework this, do we need to clean up the promises? + return Promise.race(states.map(state => { + let deferred = this.statePromiseMap.get(state); + if (!deferred) { + let resolve; + const promise = new Promise(r => { + resolve = r; + }); + deferred = {resolve, promise}; + this.statePromiseMap.set(state, deferred); + } + return deferred.promise; + })); } private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { @@ -493,8 +627,10 @@ class PeerCall { } private stopAllMedia(): void { - for (const track of this.localMedia.tracks) { - track.stop(); + if (this.localMedia) { + for (const track of this.localMedia.tracks) { + track.stop(); + } } } @@ -514,21 +650,6 @@ class PeerCall { //import { randomString } from '../randomstring'; -import { - MCallReplacesEvent, - MCallAnswer, - MCallInviteNegotiate, - CallCapabilities, - SDPStreamMetadataPurpose, - SDPStreamMetadata, - SDPStreamMetadataKey, - MCallSDPStreamMetadataChanged, - MCallSelectAnswer, - MCAllAssertedIdentity, - MCallCandidates, - MCallBase, - MCallHangupReject, -} from './callEventTypes'; // null is used as a special value meaning that the we're in a legacy 1:1 call // without MSC2746 that doesn't provide an id which device sent the message. @@ -681,46 +802,18 @@ export class CallError extends Error { } } -type InviteContent = { - offer: RTCSessionDescriptionInit, - [SDPStreamMetadataKey]: SDPStreamMetadata, - version?: number, - lifetime?: number -} - -export type InviteMessage = { - type: EventType.Invite, - content: InviteContent -} - -type AnwserContent = { - answer: { - sdp: string, - // type is now deprecated as of Matrix VoIP v1, but - // required to still be sent for backwards compat - type: RTCSdpType, - }, - [SDPStreamMetadataKey]: SDPStreamMetadata, -} - -export type AnswerMessage = { - type: EventType.Answer, - content: AnwserContent -} - -type CandidatesContent = { - candidates: RTCIceCandidate[] -} - -export type CandidatesMessage = { - type: EventType.Candidates, - content: CandidatesContent -} - - -export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage; +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); + sendSignallingMessage(message: SignallingMessage); +} + +export function tests() { + } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index da2b1ad6..a8fbeafd 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -130,57 +130,34 @@ write view I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... - - ## Thursday 3-3 notes we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? -List state transitions +## Peer call state transitions FROM CALLER FROM CALLEE Fledgling Fledgling - V calling `call()` V handleInvite + V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates WaitLocalMedia Ringing - V media promise resolves V answer() -CreateOffer WaitLocalMedia - V add tracks V media promise resolves - V wait for negotionneeded events CreateAnswer - V setLocalDescription() V - V send invite events -InviteSent - V receive anwser, setRemoteDescription() | - \__________________________________________________/ + V media promise resolves V `answer()` + V add local tracks WaitLocalMedia +CreateOffer V media promise resolves + V wait for negotionneeded events V add local tracks + V setLocalDescription() CreateAnswer + V send invite event V setLocalDescription(createAnswer()) +InviteSent | + V receive anwser, setRemoteDescription() | + \___________________________________________________/ V Connecting - V receive ice candidates and - iceConnectionState becomes 'connected' + V receive ice candidates and iceConnectionState becomes 'connected' Connected - V hangup for some reason + V `hangup()` or some terminate condition Ended -## From callee - -Fledgling -Ringing -WaitLocalMedia -CreateAnswer -Connecting -Connected -Ended - -Fledgling -WaitLocalMedia -CreateOffer -InviteSent -CreateAnswer -Connecting -Connected -Ringing -Ended - so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index cd0fbb9f..aa1bc079 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,11 +1,14 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ -import { CallErrorCode } from "./Call"; - // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; +export interface SessionDescription { + sdp?: string; + type: RTCSdpType +} + export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", @@ -32,40 +35,36 @@ export interface CallReplacesTarget { avatar_url: string; } -export interface MCallBase { +export type MCallBase = { call_id: string; version: string | number; - party_id?: string; - sender_session_id?: string; - dest_session_id?: string; } -export interface MCallAnswer extends MCallBase { - answer: RTCSessionDescription; +export type MGroupCallBase = MCallBase & { + conf_id: string; +} + +export type MCallAnswer = Base & { + answer: SessionDescription; capabilities?: CallCapabilities; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSelectAnswer extends MCallBase { +export type MCallSelectAnswer = Base & { selected_party_id: string; } -export interface MCallInviteNegotiate extends MCallBase { - offer: RTCSessionDescription; - description: RTCSessionDescription; +export type MCallInvite = Base & { + offer: SessionDescription; lifetime: number; - capabilities?: CallCapabilities; - invitee?: string; - sender_session_id?: string; - dest_session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallSDPStreamMetadataChanged extends MCallBase { +export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallReplacesEvent extends MCallBase { +export type MCallReplacesEvent = Base & { replacement_id: string; target_user: CallReplacesTarget; create_call: string; @@ -73,7 +72,7 @@ export interface MCallReplacesEvent extends MCallBase { target_room: string; } -export interface MCAllAssertedIdentity extends MCallBase { +export type MCAllAssertedIdentity = Base & { asserted_identity: { id: string; display_name: string; @@ -81,11 +80,11 @@ export interface MCAllAssertedIdentity extends MCallBase { }; } -export interface MCallCandidates extends MCallBase { +export type MCallCandidates = Base & { candidates: RTCIceCandidate[]; } -export interface MCallHangupReject extends MCallBase { +export type MCallHangupReject = Base & { reason?: CallErrorCode; } diff --git a/src/matrix/calls/group/Call.ts b/src/matrix/calls/group/GroupCall.ts similarity index 83% rename from src/matrix/calls/group/Call.ts rename to src/matrix/calls/group/GroupCall.ts index 9abef197..e05f572d 100644 --- a/src/matrix/calls/group/Call.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,14 +15,17 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; +import {Participant} from "./Participant"; +import {LocalMedia} from "../LocalMedia"; +import type {Track} from "../../../platform/types/MediaDevices"; -function participantId(senderUserId: string, senderDeviceId: string | null) { +function getParticipantId(senderUserId: string, senderDeviceId: string | null) { return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); } -class Call { +export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); - private localMedia?: LocalMedia; + private localMedia?: Promise; constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { @@ -30,17 +33,17 @@ class Call { get id(): string { return this.callEvent.state_key; } - async participate(tracks: Track[]) { - this.localMedia = LocalMedia.fromTracks(tracks); + async participate(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); } // send m.call.member state event // send invite to all participants that are < my userId for (const [,participant] of this.participants) { if (participant.userId < this.ownUserId) { - participant.sendInvite(); + participant.call(); } } } @@ -78,10 +81,6 @@ class Call { } } - get id(): string { - return this.callEvent.state_key; - } - get isTerminated(): boolean { return !!this.callEvent.content[CALL_TERMINATED]; } diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts index 26747e56..2b873aa0 100644 --- a/src/matrix/calls/group/Participant.ts +++ b/src/matrix/calls/group/Participant.ts @@ -14,35 +14,54 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {EventType} from "../PeerCall"; +import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; +import {makeTxnId} from "../../common"; + import type {PeerCallHandler} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; -class Participant implements PeerCallHandler { - private peerCall?: PeerCall; - +export class Participant implements PeerCallHandler { constructor( - private readonly userId: string, - private readonly deviceId: string, - private localMedia: LocalMedia | undefined, - private readonly webRTC: WebRTC, - private readonly hsApi: HomeServerApi + public readonly member: RoomMember, + private readonly deviceId: string | undefined, + private readonly peerCall: PeerCall, + private readonly hsApi: HomeServerApi, + private readonly groupCall: GroupCall ) {} - sendInvite() { - this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.call(this.localMedia); + /* @internal */ + call(localMedia: Promise) { + this.peerCall.call(localMedia); + } + + get remoteTracks(): Track[] { + return this.peerCall.remoteTracks; } /** From PeerCallHandler * @internal */ emitUpdate(params: any) { - + this.groupCall.emitParticipantUpdate(this, params); } /** From PeerCallHandler * @internal */ - onSendSignallingMessage(type: EventType, content: Record) { + async sendSignallingMessage(message: SignallingMessage) { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.groupCall.id; // TODO: this needs to be encrypted with olm first - this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); + + const request = this.hsApi.sendToDevice( + groupMessage.type, + {[this.member.userId]: { + [this.deviceId ?? "*"]: groupMessage.content + } + }, makeTxnId()); + await request.response(); } } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index e1991a1c..445ff22d 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -144,7 +144,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.VolumeChanged, -Infinity); + // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -186,7 +186,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); - this.emit(CallFeedEvent.VolumeChanged, maxVolume); + // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -201,7 +201,7 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); + // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 08c0d96d..0025dfec 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -114,7 +114,7 @@ class DOMPeerConnection implements PeerConnection { } createDataChannel(): DataChannel { - return new DataChannel(this.peerConnection.createDataChannel()); + return undefined as any;// new DataChannel(this.peerConnection.createDataChannel()); } private registerHandler() { diff --git a/yarn.lock b/yarn.lock index 7bcefdd4..0eb74b87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,9 +1486,9 @@ type-fest@^0.20.2: integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== typescript@^4.3.5: - version "4.3.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4" - integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA== + version "4.6.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4" + integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39"