diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 05245cd3..e6a6914f 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -57,9 +57,7 @@ export class CallViewModel extends ViewModel { } async toggleVideo() { - const localMedia = this.call.localMedia!; - const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); - await this.call.setMedia(toggledMedia); + //this.call.setMuted(); } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 868ca189..b88b17f0 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -367,9 +367,8 @@ export class RoomViewModel extends ViewModel { const session = this.getOption("session"); const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); // this will set the callViewModel above as a call will be added to callHandler.calls - const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); await call.join(localMedia); } catch (err) { console.error(err.stack); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index b49ca3c5..06e83d11 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,8 +15,8 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; +import {WebRTC, PeerConnection} from "../../platform/types/WebRTC"; +import {MediaDevices, Track} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; @@ -107,7 +107,7 @@ export class CallHandler { }); } - async createCall(roomId: string, localMedia: LocalMedia, name: string, intent: CallIntent = CallIntent.Ring): Promise { + async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(makeId("conf-"), true, { "m.name": name, @@ -116,7 +116,7 @@ export class CallHandler { this._calls.set(call.id, call); try { - await call.create(localMedia); + await call.create(type); // store call info so it will ring again when reopening the app const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); txn.calls.add({ diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 933ae89f..b79fe098 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -17,32 +17,28 @@ limitations under the License. import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; +import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { constructor( public readonly userMedia?: Stream, - public readonly microphoneMuted: boolean = false, - public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} - withMuted(microphone: boolean, camera: boolean) { - return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); - } - withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); + return new LocalMedia(this.userMedia, this.screenShare, options); } + /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { let userMedia; let screenShare; @@ -52,21 +48,21 @@ export class LocalMedia { stream = oldCloneStream; } else { stream = newStream?.clone(); - oldCloneStream?.audioTrack?.stop(); - oldCloneStream?.videoTrack?.stop(); + getStreamAudioTrack(oldCloneStream)?.stop(); + getStreamVideoTrack(oldCloneStream)?.stop(); } return stream; } return new LocalMedia( cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), - this.microphoneMuted, this.cameraMuted, cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare), this.dataChannelOptions ); } + /** @internal */ clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(),this.screenShare?.clone(), this.dataChannelOptions); } dispose() { @@ -75,11 +71,11 @@ export class LocalMedia { stopExcept(newMedia: LocalMedia | undefined) { if(newMedia?.userMedia?.id !== this.userMedia?.id) { - this.userMedia?.audioTrack?.stop(); - this.userMedia?.videoTrack?.stop(); + getStreamAudioTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMedia)?.stop(); } if(newMedia?.screenShare?.id !== this.screenShare?.id) { - this.screenShare?.videoTrack?.stop(); + getStreamVideoTrack(this.screenShare)?.stop(); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 79fb04fb..799fd9b8 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -16,10 +16,10 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; -import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, IDisposable} from "../../utils/Disposables"; -import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; +import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; +import {WebRTC, PeerConnection, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, @@ -69,6 +69,7 @@ export class PeerCall implements IDisposable { private direction: CallDirection; // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where @@ -85,7 +86,8 @@ export class PeerCall implements IDisposable { private hangupParty: CallParty; private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); - + private _remoteTrackToStreamId = new Map(); + private _remoteStreams = new Map(); // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; @@ -96,55 +98,62 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; private _remoteMedia: RemoteMedia; + private remoteMuteSettings?: MuteSettings; constructor( private callId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { - const outer = this; this._remoteMedia = new RemoteMedia(); - this.peerConnection = options.webRTC.createPeerConnection({ - onIceConnectionStateChange(state: RTCIceConnectionState) { - outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { - outer.onIceConnectionStateChange(state, log); + this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); + + const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { + this.peerConnection.addEventListener(type, listener); + const dispose = () => { + this.peerConnection.removeEventListener(type, listener); + }; + this.disposables.track(dispose); + }; + + listen("iceconnectionstatechange", () => { + const state = this.peerConnection.iceConnectionState; + this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { + this.onIceConnectionStateChange(state, log); + }); + }); + listen("icecandidate", event => { + this.logItem.wrap("onLocalIceCandidate", log => { + if (event.candidate) { + this.handleLocalIceCandidate(event.candidate, log); + } + }); + }); + listen("icegatheringstatechange", () => { + const state = this.peerConnection.iceGatheringState; + this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { + this.handleIceGatheringState(state, log); + }); + }); + listen("track", event => { + this.logItem.wrap("onRemoteTrack", log => { + this.onRemoteTrack(event.track, event.streams, log); + }); + }); + listen("datachannel", event => { + this.logItem.wrap("onRemoteDataChannel", log => { + this._dataChannel = event.channel; + this.options.emitUpdate(this, undefined); + }); + }); + listen("negotiationneeded", () => { + const promiseCreator = () => { + return this.logItem.wrap("onNegotiationNeeded", log => { + return this.handleNegotiation(log); }); - }, - onLocalIceCandidate(candidate: RTCIceCandidate) { - outer.logItem.wrap("onLocalIceCandidate", log => { - outer.handleLocalIceCandidate(candidate, log); - }); - }, - onIceGatheringStateChange(state: RTCIceGatheringState) { - outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { - outer.handleIceGatheringState(state, log); - }); - }, - onRemoteStreamRemoved(stream: Stream) { - outer.logItem.wrap("onRemoteStreamRemoved", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteTracksAdded(trackReceiver: TrackReceiver) { - outer.logItem.wrap("onRemoteTracksAdded", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteDataChannel(dataChannel: any | undefined) { - outer.logItem.wrap("onRemoteDataChannel", log => { - outer._dataChannel = dataChannel; - outer.options.emitUpdate(outer, undefined); - }); - }, - onNegotiationNeeded() { - const promiseCreator = () => { - return outer.logItem.wrap("onNegotiationNeeded", log => { - return outer.handleNegotiation(log); - }); - }; - outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); - } - }, this.options.forceTURN, this.options.turnServers, 0); + }; + this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + }); } get dataChannel(): any | undefined { return this._dataChannel; } @@ -166,7 +175,7 @@ export class PeerCall implements IDisposable { this.setState(CallState.CreateOffer, log); await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { - this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead @@ -211,11 +220,9 @@ export class PeerCall implements IDisposable { setMedia(localMedia: LocalMedia): Promise { return this.logItem.wrap("setMedia", async log => { - log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); - log.set("userMedia_audio_muted", localMedia.microphoneMuted); - log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); - log.set("userMedia_video_muted", localMedia.cameraMuted); - log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia)); + log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia)); + log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare)); log.set("datachannel", !!localMedia.dataChannelOptions); await this.updateLocalMedia(localMedia, log); const content: MCallSDPStreamMetadataChanged = { @@ -322,21 +329,26 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; // need to queue this - const content = { - call_id: this.callId, - offer, - [SDPStreamMetadataKey]: this.getSDPMetadata(), - version: 1, - seq: this.seq++, - lifetime: CALL_TIMEOUT_MS - }; if (this._state === CallState.CreateOffer) { + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { - // send Negotiate message - content.description = content.offer; - delete content.offer; + const content = { + call_id: this.callId, + description: offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { @@ -432,7 +444,7 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (this.peerConnection.remoteStreams.size === 0) { + if (this.peerConnection.getReceivers().length === 0) { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); @@ -843,11 +855,10 @@ export class PeerCall implements IDisposable { const metadata = {}; if (this.localMedia?.userMedia) { const streamId = this.localMedia.userMedia.id; - const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack, - video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack, + audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), + video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), }; } if (this.localMedia?.screenShare) { @@ -859,19 +870,67 @@ export class PeerCall implements IDisposable { return metadata; } - private updateRemoteMedia(log: ILogItem) { + private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { + return this.peerConnection.getReceivers().find(r => { + return r.track.kind === "audio" && this._remoteTrackToStreamId.get(r.track.id) === streamId; + }); + } + + private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { + if (streams.length === 0) { + log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); + return; + } + const stream = streams[0]; + this._remoteTrackToStreamId.set(track.id, stream.id); + if (!this._remoteStreams.has(stream.id)) { + const listener = (event: StreamTrackEvent): void => { + this.logItem.wrap({l: "removetrack", id: event.track.id}, log => { + const streamId = this._remoteTrackToStreamId.get(event.track.id); + if (streamId) { + this._remoteTrackToStreamId.delete(event.track.id); + const streamDetails = this._remoteStreams.get(streamId); + if (streamDetails && streamDetails.stream.getTracks().length === 0) { + this.disposables.disposeTracked(disposeListener); + this._remoteStreams.delete(stream.id); + this.updateRemoteMedia(log); + } + } + }) + }; + stream.addEventListener("removetrack", listener); + const disposeListener = () => { + stream.removeEventListener("removetrack", listener); + }; + this.disposables.track(disposeListener); + this._remoteStreams.set(stream.id, { + disposeListener, + stream + }); + this.updateRemoteMedia(log); + } + } + + private updateRemoteMedia(log: ILogItem): void { this._remoteMedia.userMedia = undefined; this._remoteMedia.screenShare = undefined; if (this.remoteSDPStreamMetadata) { - for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { - const metaData = this.remoteSDPStreamMetadata[streamId]; + for (const streamDetails of this._remoteStreams.values()) { + const {stream} = streamDetails; + const metaData = this.remoteSDPStreamMetadata[stream.id]; if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { - this._remoteMedia.userMedia = streamReceiver.stream; - streamReceiver.audioReceiver?.enable(!metaData.audio_muted); - streamReceiver.videoReceiver?.enable(!metaData.video_muted); + this._remoteMedia.userMedia = stream; + const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); + if (audioReceiver) { + audioReceiver.track.enabled = !metaData.audio_muted; + } + const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); + if (videoReceiver) { + videoReceiver.track.enabled = !metaData.audio_muted; + } } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { - this._remoteMedia.screenShare = streamReceiver.stream; + this._remoteMedia.screenShare = stream; } } } @@ -883,71 +942,46 @@ export class PeerCall implements IDisposable { return logItem.wrap("updateLocalMedia", async log => { const oldMedia = this.localMedia; this.localMedia = localMedia; - const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { - let streamSender; - if (oldStream) { - streamSender = this.peerConnection.localStreams.get(oldStream.id); - if (stream && stream.id !== oldStream.id) { - this.peerConnection.localStreams.set(stream.id, streamSender); - this.peerConnection.localStreams.delete(oldStream.id); - } - } - - const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { - const changed = (!track && oldTrack) || - (track && !oldTrack) || - (track && oldTrack && !track.equals(oldTrack)); - if (changed) { - if (track) { - if (oldTrack && sender && !track.equals(oldTrack)) { + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { + const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { + if (!oldTrack && newTrack) { + log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { + const sender = this.peerConnection.addTrack(newTrack, stream!); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); + }); + } else if (oldTrack) { + const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); + if (sender) { + if (newTrack && oldTrack.id !== newTrack.id) { try { - await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - return sender.replaceTrack(track); + await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { + return sender.replaceTrack(newTrack); }); } catch (err) { - // can't replace the track without renegotiating - log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + // can't replace the track without renegotiating{ + log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { this.peerConnection.removeTrack(sender); - this.peerConnection.addTrack(track); + this.peerConnection.addTrack(newTrack); }); } - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - // this will be used for muting, do we really want to trigger renegotiation here? - // we want to disable the sender, but also remove the track as we don't want to keep - // using the webcam if we don't need to - log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { - sender.track.enabled = false; + } else if (!newTrack) { + log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { this.peerConnection.removeTrack(sender); }); + } else { + log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); } } - } else if (track) { - log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id}); - if (sender && muted !== wasMuted) { - log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { - // sender.track.enabled = !muted; - // This doesn't always seem to trigger renegotiation?? - // We should probably always send the new metadata first ... - sender.enable(!muted); - }); - } else { - log.log(`${logLabel} ${track.kind} track hasn't changed`); - } + // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia } } - await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); - await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); - } + await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); + await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); + }; - await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); - await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); + await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -967,7 +1001,7 @@ export class PeerCall implements IDisposable { public dispose(): void { this.disposables.dispose(); - this.peerConnection.dispose(); + this.peerConnection.close(); } public close(reason: CallErrorCode | undefined, log: ILogItem): void { @@ -1038,6 +1072,7 @@ export function handlesEventType(eventType: string): boolean { eventType === EventType.Negotiate; } + export function tests() { } diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts new file mode 100644 index 00000000..c5970d4b --- /dev/null +++ b/src/matrix/calls/common.ts @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {Track, Stream} from "../../platform/types/MediaDevices"; + +export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { + return stream?.getAudioTracks()[0]; +} + +export function getStreamVideoTrack(stream: Stream | undefined): Track | undefined { + return stream?.getVideoTracks()[0]; +} + +export class MuteSettings { + constructor (public readonly microphone: boolean, public readonly camera: boolean) {} +} diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cb962b86..4992f6b6 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -168,7 +168,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(localMedia: LocalMedia): Promise { + create(type: "m.video" | "m.voice"): Promise { return this.logItem.wrap("create", async log => { if (this._state !== GroupCallState.Fledgling) { return; @@ -176,7 +176,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Creating; this.emitChange(); this.callContent = Object.assign({ - "m.type": localMedia.userMedia?.videoTrack ? "m.video" : "m.voice", + "m.type": type, }, this.callContent); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 93418a66..1b5f7afd 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -14,15 +14,39 @@ See the License for the specific language governing permissions and limitations under the License. */ +export interface Event {} + export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; - createVolumeMeasurer(stream: Stream): VolumeMeasurer; + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer; } +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface StreamTrackEvent extends Event { + readonly track: Track; +} + +export interface StreamEventMap { + "addtrack": StreamTrackEvent; + "removetrack": StreamTrackEvent; +} export interface Stream { getTracks(): ReadonlyArray; @@ -30,6 +54,8 @@ export interface Stream { getVideoTracks(): ReadonlyArray; readonly id: string; clone(): Stream; + addEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void; } export enum TrackKind { @@ -47,5 +73,7 @@ export interface Track { } export interface VolumeMeasurer { - + get isSpeaking(): boolean; + setSpeakingThreshold(threshold: number): void; + stop(); } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index edb26c0a..ca2ca646 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,63 +14,156 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, Stream} from "./MediaDevices"; +import {Track, Stream, Event} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void; } -export interface StreamSender { - get stream(): Stream; - get audioSender(): TrackSender | undefined; - get videoSender(): TrackSender | undefined; +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface DataChannelEventMap { + "bufferedamountlow": Event; + "close": Event; + "error": Event; + "message": MessageEvent; + "open": Event; } -export interface StreamReceiver { - get stream(): Stream; - get audioReceiver(): TrackReceiver | undefined; - get videoReceiver(): TrackReceiver | undefined; -} - -export interface TrackReceiver { - get track(): Track; - get enabled(): boolean; - enable(enabled: boolean); // this modifies the transceiver direction -} - -export interface TrackSender extends TrackReceiver { - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise; - /** make any needed adjustments to the sender or transceiver settings - * depending on the purpose, after adding the track to the connection */ - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; -} - -export interface PeerConnectionHandler { - onIceConnectionStateChange(state: RTCIceConnectionState); - onLocalIceCandidate(candidate: RTCIceCandidate); - onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteStreamRemoved(stream: Stream); - onRemoteTracksAdded(receiver: TrackReceiver); - onRemoteDataChannel(dataChannel: any | undefined); - onNegotiationNeeded(); -} - -export interface PeerConnection { - get iceGatheringState(): RTCIceGatheringState; - get signalingState(): RTCSignalingState; - get localDescription(): RTCSessionDescription | undefined; - get localStreams(): ReadonlyMap; - get remoteStreams(): ReadonlyMap; - createOffer(): Promise; - createAnswer(): Promise; - setLocalDescription(description?: RTCSessionDescriptionInit): Promise; - setRemoteDescription(description: RTCSessionDescriptionInit): Promise; - addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): TrackSender | undefined; - removeTrack(track: TrackSender): void; - createDataChannel(options: RTCDataChannelInit): any; - dispose(): void; +export interface DataChannel { + binaryType: BinaryType; + readonly id: number | null; + readonly label: string; + readonly negotiated: boolean; + readonly readyState: DataChannelState; close(): void; + send(data: string): void; + send(data: Blob): void; + send(data: ArrayBuffer): void; + send(data: ArrayBufferView): void; + addEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface DataChannelInit { + id?: number; + maxPacketLifeTime?: number; + maxRetransmits?: number; + negotiated?: boolean; + ordered?: boolean; + protocol?: string; +} + +export interface DataChannelEvent extends Event { + readonly channel: DataChannel; +} + +export interface PeerConnectionIceEvent extends Event { + readonly candidate: RTCIceCandidate | null; +} + +export interface TrackEvent extends Event { + readonly receiver: Receiver; + readonly streams: ReadonlyArray; + readonly track: Track; + readonly transceiver: Transceiver; +} + +export interface PeerConnectionEventMap { + "connectionstatechange": Event; + "datachannel": DataChannelEvent; + "icecandidate": PeerConnectionIceEvent; + "iceconnectionstatechange": Event; + "icegatheringstatechange": Event; + "negotiationneeded": Event; + "signalingstatechange": Event; + "track": TrackEvent; +} + +export type DataChannelState = "closed" | "closing" | "connecting" | "open"; +export type IceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new"; +export type PeerConnectionState = "closed" | "connected" | "connecting" | "disconnected" | "failed" | "new"; +export type SignalingState = "closed" | "have-local-offer" | "have-local-pranswer" | "have-remote-offer" | "have-remote-pranswer" | "stable"; +export type IceGatheringState = "complete" | "gathering" | "new"; +export type SdpType = "answer" | "offer" | "pranswer" | "rollback"; +export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendrecv" | "stopped"; +export interface SessionDescription { + readonly sdp: string; + readonly type: SdpType; + toJSON(): any; +} + +export interface AnswerOptions {} + +export interface OfferOptions { + iceRestart?: boolean; + offerToReceiveAudio?: boolean; + offerToReceiveVideo?: boolean; +} + +export interface SessionDescriptionInit { + sdp?: string; + type: SdpType; +} + +export interface LocalSessionDescriptionInit { + sdp?: string; + type?: SdpType; +} + +/** A WebRTC connection between the local computer and a remote peer. It provides methods to connect to a remote peer, maintain and monitor the connection, and close the connection once it's no longer needed. */ +export interface PeerConnection { + readonly connectionState: PeerConnectionState; + readonly iceConnectionState: IceConnectionState; + readonly iceGatheringState: IceGatheringState; + readonly localDescription: SessionDescription | null; + readonly remoteDescription: SessionDescription | null; + readonly signalingState: SignalingState; + addIceCandidate(candidate?: RTCIceCandidateInit): Promise; + addTrack(track: Track, ...streams: Stream[]): Sender; + close(): void; + createAnswer(options?: AnswerOptions): Promise; + createDataChannel(label: string, dataChannelDict?: DataChannelInit): DataChannel; + createOffer(options?: OfferOptions): Promise; + getReceivers(): Receiver[]; + getSenders(): Sender[]; + getTransceivers(): Transceiver[]; + removeTrack(sender: Sender): void; + restartIce(): void; + setLocalDescription(description?: LocalSessionDescriptionInit): Promise; + setRemoteDescription(description: SessionDescriptionInit): Promise; + addEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface Receiver { + readonly track: Track; +} + +export interface Sender { + readonly track: Track | null; + replaceTrack(withTrack: Track | null): Promise; +} + +export interface Transceiver { + readonly currentDirection: TransceiverDirection | null; + direction: TransceiverDirection; + readonly mid: string | null; + readonly receiver: Receiver; + readonly sender: Sender; + stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 4eb92259..c34ab85b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -30,12 +30,12 @@ export class MediaDevicesWrapper implements IMediaDevices { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - return new StreamWrapper(stream); + return stream as Stream; } async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - return new StreamWrapper(stream); + return stream as Stream; } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -68,70 +68,13 @@ export class MediaDevicesWrapper implements IMediaDevices { video: true, }; } -} -export class StreamWrapper implements Stream { - - public audioTrack: AudioTrackWrapper | undefined = undefined; - public videoTrack: TrackWrapper | undefined = undefined; - - constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { - if (clonedTracks) { - this.audioTrack = clonedTracks.audioTrack; - this.videoTrack = clonedTracks.videoTrack; - } else { - for (const track of stream.getTracks()) { - this.update(track); - } - } - } - - get id(): string { return this.stream.id; } - - clone(): Stream { - const clonedStream = this.stream.clone(); - const clonedTracks = { - audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, - videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, - }; - return new StreamWrapper(clonedStream, clonedTracks); - } - - update(track: MediaStreamTrack): TrackWrapper | undefined { - //console.trace("Stream.update " + JSON.stringify({id: track.id, vid: this.videoTrack?.id, aid: this.audioTrack?.id})); - if (track.kind === "video") { - if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream, track.id); - } - return this.videoTrack; - } else if (track.kind === "audio") { - if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); - } - return this.audioTrack; - } + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer { + return new WebAudioVolumeMeasurer(stream as MediaStream, callback); } } -export class TrackWrapper implements Track { - constructor( - public readonly track: MediaStreamTrack, - public readonly stream: MediaStream, - public readonly originalId: string, - ) {} - - get kind(): TrackKind { return this.track.kind as TrackKind; } - get label(): string { return this.track.label; } - get id(): string { return this.track.id; } - get settings(): MediaTrackSettings { return this.track.getSettings(); } - get enabled(): boolean { return this.track.enabled; } - set enabled(enabled: boolean) { this.track.enabled = enabled; } - // test equality across clones - equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } - stop() { this.track.stop(); } -} - -export class AudioTrackWrapper extends TrackWrapper { +export class WebAudioVolumeMeasurer implements VolumeMeasurer { private measuringVolumeActivity = false; private audioContext?: AudioContext; private analyser: AnalyserNode; @@ -140,9 +83,12 @@ export class AudioTrackWrapper extends TrackWrapper { private speaking = false; private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; + private callback: () => void; + private stream: MediaStream; - constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { - super(track, stream, originalId); + constructor(stream: MediaStream, callback: () => void) { + this.stream = stream; + this.callback = callback; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); @@ -162,6 +108,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -182,7 +129,6 @@ export class AudioTrackWrapper extends TrackWrapper { this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); } - public setSpeakingThreshold(threshold: number) { this.speakingThreshold = threshold; } @@ -204,6 +150,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -219,267 +166,16 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; + this.callback(); // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; }; - public dispose(): void { + public stop(): void { clearTimeout(this.volumeLooperTimeout); + this.analyser.disconnect(); + this.audioContext?.close(); } } - -// export interface ICallFeedOpts { -// client: MatrixClient; -// roomId: string; -// userId: string; -// stream: MediaStream; -// purpose: SDPStreamMetadataPurpose; -// audioMuted: boolean; -// videoMuted: boolean; -// } - -// export enum CallFeedEvent { -// NewStream = "new_stream", -// MuteStateChanged = "mute_state_changed", -// VolumeChanged = "volume_changed", -// Speaking = "speaking", -// } - -// export class CallFeed extends EventEmitter { -// public stream: MediaStream; -// public sdpMetadataStreamId: string; -// public userId: string; -// public purpose: SDPStreamMetadataPurpose; -// public speakingVolumeSamples: number[]; - -// private client: MatrixClient; -// private roomId: string; -// private audioMuted: boolean; -// private videoMuted: boolean; -// private measuringVolumeActivity = false; -// private audioContext: AudioContext; -// private analyser: AnalyserNode; -// private frequencyBinCount: Float32Array; -// private speakingThreshold = SPEAKING_THRESHOLD; -// private speaking = false; -// private volumeLooperTimeout: number; - -// constructor(opts: ICallFeedOpts) { -// super(); - -// this.client = opts.client; -// this.roomId = opts.roomId; -// this.userId = opts.userId; -// this.purpose = opts.purpose; -// this.audioMuted = opts.audioMuted; -// this.videoMuted = opts.videoMuted; -// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); -// this.sdpMetadataStreamId = opts.stream.id; - -// this.updateStream(null, opts.stream); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } -// } - -// private get hasAudioTrack(): boolean { -// return this.stream.getAudioTracks().length > 0; -// } - -// private updateStream(oldStream: MediaStream, newStream: MediaStream): void { -// if (newStream === oldStream) return; - -// if (oldStream) { -// oldStream.removeEventListener("addtrack", this.onAddTrack); -// this.measureVolumeActivity(false); -// } -// if (newStream) { -// this.stream = newStream; -// newStream.addEventListener("addtrack", this.onAddTrack); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } else { -// this.measureVolumeActivity(false); -// } -// } - -// this.emit(CallFeedEvent.NewStream, this.stream); -// } - -// private initVolumeMeasuring(): void { -// const AudioContext = window.AudioContext || window.webkitAudioContext; -// if (!this.hasAudioTrack || !AudioContext) return; - -// this.audioContext = new AudioContext(); - -// this.analyser = this.audioContext.createAnalyser(); -// this.analyser.fftSize = 512; -// this.analyser.smoothingTimeConstant = 0.1; - -// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); -// mediaStreamAudioSourceNode.connect(this.analyser); - -// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); -// } - -// private onAddTrack = (): void => { -// this.emit(CallFeedEvent.NewStream, this.stream); -// }; - -// /** -// * Returns callRoom member -// * @returns member of the callRoom -// */ -// public getMember(): RoomMember { -// const callRoom = this.client.getRoom(this.roomId); -// return callRoom.getMember(this.userId); -// } - -// /** -// * Returns true if CallFeed is local, otherwise returns false -// * @returns {boolean} is local? -// */ -// public isLocal(): boolean { -// return this.userId === this.client.getUserId(); -// } - -// /** -// * Returns true if audio is muted or if there are no audio -// * tracks, otherwise returns false -// * @returns {boolean} is audio muted? -// */ -// public isAudioMuted(): boolean { -// return this.stream.getAudioTracks().length === 0 || this.audioMuted; -// } - -// * -// * Returns true video is muted or if there are no video -// * tracks, otherwise returns false -// * @returns {boolean} is video muted? - -// public isVideoMuted(): boolean { -// // We assume only one video track -// return this.stream.getVideoTracks().length === 0 || this.videoMuted; -// } - -// public isSpeaking(): boolean { -// return this.speaking; -// } - -// /** -// * Replaces the current MediaStream with a new one. -// * This method should be only used by MatrixCall. -// * @param newStream new stream with which to replace the current one -// */ -// public setNewStream(newStream: MediaStream): void { -// this.updateStream(this.stream, newStream); -// } - -// /** -// * Set feed's internal audio mute state -// * @param muted is the feed's audio muted? -// */ -// public setAudioMuted(muted: boolean): void { -// this.audioMuted = muted; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Set feed's internal video mute state -// * @param muted is the feed's video muted? -// */ -// public setVideoMuted(muted: boolean): void { -// this.videoMuted = muted; -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Starts emitting volume_changed events where the emitter value is in decibels -// * @param enabled emit volume changes -// */ -// public measureVolumeActivity(enabled: boolean): void { -// if (enabled) { -// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - -// this.measuringVolumeActivity = true; -// this.volumeLooper(); -// } else { -// this.measuringVolumeActivity = false; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.VolumeChanged, -Infinity); -// } -// } - -// public setSpeakingThreshold(threshold: number) { -// this.speakingThreshold = threshold; -// } - -// private volumeLooper = () => { -// if (!this.analyser) return; - -// if (!this.measuringVolumeActivity) return; - -// this.analyser.getFloatFrequencyData(this.frequencyBinCount); - -// let maxVolume = -Infinity; -// for (let i = 0; i < this.frequencyBinCount.length; i++) { -// if (this.frequencyBinCount[i] > maxVolume) { -// maxVolume = this.frequencyBinCount[i]; -// } -// } - -// this.speakingVolumeSamples.shift(); -// this.speakingVolumeSamples.push(maxVolume); - -// this.emit(CallFeedEvent.VolumeChanged, maxVolume); - -// let newSpeaking = false; - -// for (let i = 0; i < this.speakingVolumeSamples.length; i++) { -// const volume = this.speakingVolumeSamples[i]; - -// if (volume > this.speakingThreshold) { -// newSpeaking = true; -// break; -// } -// } - -// if (this.speaking !== newSpeaking) { -// this.speaking = newSpeaking; -// this.emit(CallFeedEvent.Speaking, this.speaking); -// } - -// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); -// }; - -// public clone(): CallFeed { -// const mediaHandler = this.client.getMediaHandler(); -// const stream = this.stream.clone(); - -// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { -// mediaHandler.userMediaStreams.push(stream); -// } else { -// mediaHandler.screensharingStreams.push(stream); -// } - -// return new CallFeed({ -// client: this.client, -// roomId: this.roomId, -// userId: this.userId, -// stream, -// purpose: this.purpose, -// audioMuted: this.audioMuted, -// videoMuted: this.videoMuted, -// }); -// } - -// public dispose(): void { -// clearTimeout(this.volumeLooperTimeout); -// this.measureVolumeActivity(false); -// } -// } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 22096699..61d481d4 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; -import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; +import {Stream, Track, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, Sender, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -24,159 +23,21 @@ export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { - return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); - } -} - -export class RemoteStreamWrapper extends StreamWrapper { - constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { - super(stream); - this.stream.addEventListener("removetrack", this.onTrackRemoved); + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { + return new RTCPeerConnection({ + iceTransportPolicy: forceTURN ? 'relay' : undefined, + iceServers: turnServers, + iceCandidatePoolSize: iceCandidatePoolSize, + }) as PeerConnection; } - onTrackRemoved = (evt: MediaStreamTrackEvent) => { - if (evt.track.id === this.audioTrack?.track.id) { - this.audioTrack = undefined; - } else if (evt.track.id === this.videoTrack?.track.id) { - this.videoTrack = undefined; - } - if (!this.audioTrack && !this.videoTrack) { - this.emptyCallback(this); - } - }; - - dispose() { - this.stream.removeEventListener("removetrack", this.onTrackRemoved); - } -} - -export class DOMStreamSender implements StreamSender { - public audioSender: DOMTrackSender | undefined; - public videoSender: DOMTrackSender | undefined; - - constructor(public readonly stream: StreamWrapper) {} - - update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { - const transceiver = transceivers.find(t => t.sender === sender); - if (transceiver && sender.track) { - const trackWrapper = this.stream.update(sender.track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) { - this.videoSender = new DOMTrackSender(trackWrapper, transceiver); - return this.videoSender; - } else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) { - this.audioSender = new DOMTrackSender(trackWrapper, transceiver); - return this.audioSender; - } - } - } - } -} - -export class DOMStreamReceiver implements StreamReceiver { - public audioReceiver: DOMTrackReceiver | undefined; - public videoReceiver: DOMTrackReceiver | undefined; - - constructor(public readonly stream: RemoteStreamWrapper) {} - - update(event: RTCTrackEvent): DOMTrackReceiver | undefined { - const {receiver} = event; - const {track} = receiver; - const trackWrapper = this.stream.update(track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video) { - this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.videoReceiver; - } else { - this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.audioReceiver; - } - } - } -} - -export class DOMTrackSenderOrReceiver implements TrackReceiver { - constructor( - public readonly track: TrackWrapper, - public readonly transceiver: RTCRtpTransceiver, - private readonly exclusiveValue: RTCRtpTransceiverDirection, - private readonly excludedValue: RTCRtpTransceiverDirection - ) {} - - get enabled(): boolean { - return this.transceiver.direction === "sendrecv" || - this.transceiver.direction === this.exclusiveValue; - } - - enableWithoutRenegotiation(enabled: boolean) { - this.track.track.enabled = enabled; - } - - enable(enabled: boolean) { - if (enabled !== this.enabled) { - // do this first, so we stop sending track data immediately. - // this will still consume bandwidth though, so also disable the transceiver, - // which will trigger a renegotiation though. - this.enableWithoutRenegotiation(enabled); - if (enabled) { - if (this.transceiver.direction === "inactive") { - this.transceiver.direction = this.exclusiveValue; - } else { - this.transceiver.direction = "sendrecv"; - } - } else { - if (this.transceiver.direction === "sendrecv") { - this.transceiver.direction = this.excludedValue; - } else { - this.transceiver.direction = "inactive"; - } - } - } - } -} - -export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "recvonly", "sendonly"); - } -} - -export class DOMTrackSender extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "sendonly", "recvonly"); - } - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise { - return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); - } - - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void { if (purpose === SDPStreamMetadataPurpose.Screenshare) { - this.getRidOfRTXCodecs(); + this.getRidOfRTXCodecs(peerConnection as RTCPeerConnection, sender as RTCRtpSender); } } - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { + private getRidOfRTXCodecs(peerConnection: RTCPeerConnection, sender: RTCRtpSender): void { // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; @@ -190,172 +51,14 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { codecs.splice(rtxCodecIndex, 1); } } - if (this.transceiver.sender.track?.kind === "video" || - this.transceiver.receiver.track?.kind === "video") { - this.transceiver.setCodecPreferences(codecs); - } - } -} - -class DOMPeerConnection implements PeerConnection { - private readonly peerConnection: RTCPeerConnection; - private readonly handler: PeerConnectionHandler; - public readonly localStreams: Map = new Map(); - public readonly remoteStreams: Map = new Map(); - - constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { - this.handler = handler; - this.peerConnection = new RTCPeerConnection({ - iceTransportPolicy: forceTURN ? 'relay' : undefined, - iceServers: turnServers, - iceCandidatePoolSize: iceCandidatePoolSize, - }); - this.registerHandler(); - } - - get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } - get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } - get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } - - createOffer(): Promise { - return this.peerConnection.createOffer(); - } - - createAnswer(): Promise { - return this.peerConnection.createAnswer(); - } - - setLocalDescription(description?: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setLocalDescription(description); - } - - setRemoteDescription(description: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setRemoteDescription(description); - } - - addIceCandidate(candidate: RTCIceCandidate): Promise { - return this.peerConnection.addIceCandidate(candidate); - } - - close(): void { - return this.peerConnection.close(); - } - - addTrack(track: Track): DOMTrackSender | undefined { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.addTrack(track.track, track.stream); - let streamSender = this.localStreams.get(track.stream.id); - if (!streamSender) { - // TODO: reuse existing stream wrapper here? - streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); - this.localStreams.set(track.stream.id, streamSender); - } - const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); - return trackSender; - } - - removeTrack(sender: TrackSender): void { - if (!(sender instanceof DOMTrackSender)) { - throw new Error("Not a DOMTrackSender"); - } - this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); - // TODO: update localStreams - } - - createDataChannel(options: RTCDataChannelInit): any { - return this.peerConnection.createDataChannel("channel", options); - } - - private registerHandler() { - const pc = this.peerConnection; - pc.addEventListener('negotiationneeded', this); - pc.addEventListener('icecandidate', this); - pc.addEventListener('iceconnectionstatechange', this); - pc.addEventListener('icegatheringstatechange', this); - pc.addEventListener('signalingstatechange', this); - pc.addEventListener('track', this); - pc.addEventListener('datachannel', this); - } - - private deregisterHandler() { - const pc = this.peerConnection; - pc.removeEventListener('negotiationneeded', this); - pc.removeEventListener('icecandidate', this); - pc.removeEventListener('iceconnectionstatechange', this); - pc.removeEventListener('icegatheringstatechange', this); - pc.removeEventListener('signalingstatechange', this); - pc.removeEventListener('track', this); - pc.removeEventListener('datachannel', this); - } - - /** @internal */ - handleEvent(evt: Event) { - switch (evt.type) { - case "iceconnectionstatechange": - this.handleIceConnectionStateChange(); - break; - case "icecandidate": - this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent); - break; - case "icegatheringstatechange": - this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState); - break; - case "track": - this.handleRemoteTrack(evt as RTCTrackEvent); - break; - case "negotiationneeded": - this.handler.onNegotiationNeeded(); - break; - case "datachannel": - this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel); - break; - } - } - - dispose(): void { - this.deregisterHandler(); - for (const r of this.remoteStreams.values()) { - r.stream.dispose(); - } - } - - private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { - if (event.candidate) { - this.handler.onLocalIceCandidate(event.candidate); - } - }; - - private handleIceConnectionStateChange() { - const {iceConnectionState} = this.peerConnection; - if (iceConnectionState === "failed" && this.peerConnection.restartIce) { - this.peerConnection.restartIce(); - } else { - this.handler.onIceConnectionStateChange(iceConnectionState); - } - } - - onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { - if (this.remoteStreams.delete(stream.id)) { - this.handler.onRemoteStreamRemoved(stream); - } - } - - private handleRemoteTrack(evt: RTCTrackEvent) { - if (evt.streams.length !== 1) { - throw new Error("track in multiple streams is not supported"); - } - const stream = evt.streams[0]; - const transceivers = this.peerConnection.getTransceivers(); - let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id); - if (!streamReceiver) { - streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); - this.remoteStreams.set(stream.id, streamReceiver); - } - const trackReceiver = streamReceiver.update(evt); - if (trackReceiver) { - this.handler.onRemoteTracksAdded(trackReceiver); + + const transceiver = peerConnection.getTransceivers().find(t => t.sender === sender); + if (transceiver && ( + transceiver.sender.track?.kind === "video" || + transceiver.receiver.track?.kind === "video" + ) + ) { + transceiver.setCodecPreferences(codecs); } } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index b318c324..1c847cbb 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -17,15 +17,15 @@ limitations under the License. import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; import {ListView} from "../../general/ListView"; import {Stream} from "../../../../types/MediaDevices"; -import type {StreamWrapper} from "../../../dom/MediaDevices"; +import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../../matrix/calls/common"; import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; function bindStream(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { - t.mapSideEffect(vm => propSelector(vm)?.videoTrack?.enabled, (_,__, vm) => { + t.mapSideEffect(vm => getStreamVideoTrack(propSelector(vm))?.enabled, (_,__, vm) => { const stream = propSelector(vm); if (stream) { - video.srcObject = (stream as StreamWrapper).stream; - if (stream.videoTrack?.enabled) { + video.srcObject = stream as MediaStream; + if (getStreamVideoTrack(stream)?.enabled) { video.classList.remove("hidden"); } else { video.classList.add("hidden"); diff --git a/src/utils/AsyncQueue.ts b/src/utils/AsyncQueue.ts deleted file mode 100644 index 0686314c..00000000 --- a/src/utils/AsyncQueue.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export class AsyncQueue { - private isRunning = false; - private queue: T[] = []; - private error?: Error; - - constructor( - private readonly reducer: (v: V, t: T) => Promise, - private value: V, - private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t) - ) {} - - push(t: T) { - if (this.contains(t, this.queue)) { - return; - } - this.queue.push(t); - this.runLoopIfNeeded(); - } - - private async runLoopIfNeeded() { - if (this.isRunning || this.error) { - return; - } - this.isRunning = true; - try { - let item: T | undefined; - while (item = this.queue.shift()) { - this.value = await this.reducer(this.value, item); - } - } catch (err) { - this.error = err; - } finally { - this.isRunning = false; - } - } -}