From 2d4301fe5a5c451100f68e275f7a6fe90b0f7ba2 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:20:24 +0200 Subject: [PATCH] WIP: expose streams, senders and receivers --- .../session/room/timeline/tiles/CallTile.js | 1 + src/matrix/calls/LocalMedia.ts | 44 +-- src/matrix/calls/PeerCall.ts | 4 +- src/platform/types/MediaDevices.ts | 31 +- src/platform/types/WebRTC.ts | 42 ++- src/platform/web/dom/MediaDevices.ts | 79 +++-- src/platform/web/dom/WebRTC.ts | 299 ++++++++++++------ 7 files changed, 308 insertions(+), 192 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 32807958..2d2d9dab 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -75,6 +75,7 @@ export class CallTile extends SimpleTile { async join() { if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + // const screenShareTrack = await this.platform.mediaDevices.getScreenShareTrack(); const localMedia = new LocalMedia().withTracks(mediaTracks); await this._call.join(localMedia); } diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index fd91c61c..6eb7a225 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -15,37 +15,26 @@ limitations under the License. */ import {SDPStreamMetadataPurpose} from "./callEventTypes"; -import {Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( - public readonly cameraTrack?: Track, - public readonly screenShareTrack?: Track, - public readonly microphoneTrack?: AudioTrack, + public readonly userMedia?: Stream, + public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} - withTracks(tracks: Track[]) { - const cameraTrack = tracks.find(t => t.type === TrackType.Camera) ?? this.cameraTrack; - const screenShareTrack = tracks.find(t => t.type === TrackType.ScreenShare) ?? this.screenShareTrack; - const microphoneTrack = tracks.find(t => t.type === TrackType.Microphone) ?? this.microphoneTrack; - if (cameraTrack && microphoneTrack && cameraTrack.streamId !== microphoneTrack.streamId) { - throw new Error("The camera and audio track should have the same stream id"); - } - return new LocalMedia(cameraTrack, screenShareTrack, microphoneTrack as AudioTrack, this.dataChannelOptions); + withUserMedia(stream: Stream) { + return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + } + + withScreenShare(stream: Stream) { + return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.cameraTrack, this.screenShareTrack, this.microphoneTrack as AudioTrack, options); - } - - get tracks(): Track[] { - const tracks: Track[] = []; - if (this.cameraTrack) { tracks.push(this.cameraTrack); } - if (this.screenShareTrack) { tracks.push(this.screenShareTrack); } - if (this.microphoneTrack) { tracks.push(this.microphoneTrack); } - return tracks; + return new LocalMedia(this.userMedia, this.screenShare, options); } getSDPMetadata(): SDPStreamMetadata { @@ -54,8 +43,8 @@ export class LocalMedia { if (userMediaTrack) { metadata[userMediaTrack.streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.microphoneTrack?.muted ?? false, - video_muted: this.cameraTrack?.muted ?? false, + audio_muted: this.microphoneTrack?.muted ?? true, + video_muted: this.cameraTrack?.muted ?? true, }; } if (this.screenShareTrack) { @@ -67,13 +56,12 @@ export class LocalMedia { } clone() { - // TODO: implement - return this; + return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.cameraTrack?.stop(); - this.microphoneTrack?.stop(); - this.screenShareTrack?.stop(); + this.userMedia?.audioTrack?.stop(); + this.userMedia?.videoTrack?.stop(); + this.screenShare?.videoTrack?.stop(); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 8559982c..644ed160 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -701,8 +701,6 @@ export class PeerCall implements IDisposable { private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); - // will rerequest stream purpose for all tracks and set track.type accordingly - this.peerConnection.notifyStreamPurposeChanged(); for (const track of this.peerConnection.remoteTracks) { const streamMetaData = this.remoteSDPStreamMetadata?.[track.streamId]; if (streamMetaData) { @@ -757,6 +755,8 @@ export class PeerCall implements IDisposable { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; this.setState(CallState.Connected, log); + const transceivers = this.peerConnection.peerConnection.getTransceivers(); + console.log(transceivers); } else if (state == 'failed') { this.iceDisconnectedTimeout?.abort(); this.iceDisconnectedTimeout = undefined; diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index db267871..d0edbbea 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -18,29 +18,32 @@ export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. - getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; - getScreenShareTrack(): Promise; + getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getScreenShareTrack(): Promise; } -export enum TrackType { - ScreenShare, - Camera, - Microphone, +export interface Stream { + readonly audioTrack: AudioTrack | undefined; + readonly videoTrack: Track | undefined; + readonly id: string; + clone(): Stream; +} + +export enum TrackKind { + Video = "video", + Audio = "audio" } export interface Track { - get type(): TrackType; - get label(): string; - get id(): string; - get streamId(): string; - get settings(): MediaTrackSettings; - get muted(): boolean; - setMuted(muted: boolean): void; + readonly kind: TrackKind; + readonly label: string; + readonly id: string; + readonly settings: MediaTrackSettings; stop(): void; - clone(): Track; } export interface AudioTrack extends Track { + // TODO: how to emit updates on this? get isSpeaking(): boolean; } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index a1a7b7cd..a0512163 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,38 +14,62 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, TrackType} from "./MediaDevices"; +import {Track, Stream} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection; } +export interface StreamSender { + get stream(): Stream; + get audioSender(): TrackSender | undefined; + get videoSender(): TrackSender | undefined; +} + +export interface StreamReceiver { + get stream(): Stream; + get audioReceiver(): TrackReceiver | undefined; + get videoReceiver(): TrackReceiver | undefined; +} + +export interface TrackReceiver { + get track(): Track; + get enabled(): boolean; + enable(enabled: boolean); // this modifies the transceiver direction +} + +export interface TrackSender extends TrackReceiver { + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise; + /** make any needed adjustments to the sender or transceiver settings + * depending on the purpose, after adding the track to the connection */ + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; +} + export interface PeerConnectionHandler { onIceConnectionStateChange(state: RTCIceConnectionState); onLocalIceCandidate(candidate: RTCIceCandidate); onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteTracksChanged(tracks: Track[]); + onRemoteStreamRemoved(stream: Stream); + onRemoteTracksAdded(receiver: TrackReceiver); onRemoteDataChannel(dataChannel: any | undefined); onNegotiationNeeded(); - // request the type of incoming stream - getPurposeForStreamId(streamId: string): SDPStreamMetadataPurpose; } export interface PeerConnection { - notifyStreamPurposeChanged(): void; - get remoteTracks(): Track[]; get iceGatheringState(): RTCIceGatheringState; get signalingState(): RTCSignalingState; get localDescription(): RTCSessionDescription | undefined; + get localStreams(): ReadonlyArray; + get remoteStreams(): ReadonlyArray; createOffer(): Promise; createAnswer(): Promise; setLocalDescription(description?: RTCSessionDescriptionInit): Promise; setRemoteDescription(description: RTCSessionDescriptionInit): Promise; addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): void; - removeTrack(track: Track): boolean; - replaceTrack(oldTrack: Track, newTrack: Track): Promise; + addTrack(track: Track): TrackSender | undefined; + removeTrack(track: TrackSender): void; createDataChannel(options: RTCDataChannelInit): any; dispose(): void; close(): void; diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 49fd7fa5..52ca132b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, TrackType, Track, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -28,22 +28,14 @@ export class MediaDevicesWrapper implements IMediaDevices { return this.mediaDevices.enumerateDevices(); } - async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - const tracks = stream.getTracks().map(t => { - const type = t.kind === "audio" ? TrackType.Microphone : TrackType.Camera; - return wrapTrack(t, stream, type); - }); - return tracks; + return new StreamWrapper(stream); } - async getScreenShareTrack(): Promise { + async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - const videoTrack = stream.getTracks().find(t => t.kind === "video"); - if (videoTrack) { - return wrapTrack(videoTrack, stream, TrackType.ScreenShare); - } - return; + return new StreamWrapper(stream); } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -78,43 +70,50 @@ export class MediaDevicesWrapper implements IMediaDevices { } } -export function wrapTrack(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - if (track.kind === "audio") { - return new AudioTrackWrapper(track, stream, type); - } else { - return new TrackWrapper(track, stream, type); +export class StreamWrapper implements Stream { + + public audioTrack: AudioTrackWrapper | undefined; + public videoTrack: TrackWrapper | undefined; + + constructor(public readonly stream: MediaStream) { + for (const track of stream.getTracks()) { + this.update(track); + } + } + + get id(): string { return this.stream.id; } + + clone(): Stream { + return new StreamWrapper(this.stream.clone()); + } + + update(track: MediaStreamTrack): TrackWrapper | undefined { + if (track.kind === "video") { + if (!this.videoTrack || track.id !== this.videoTrack.track.id) { + this.videoTrack = new TrackWrapper(track, this.stream); + return this.videoTrack; + } + } else if (track.kind === "audio") { + if (!this.audioTrack || track.id !== this.audioTrack.track.id) { + this.audioTrack = new AudioTrackWrapper(track, this.stream); + return this.audioTrack; + } + } } } export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream, - private _type: TrackType, + public readonly stream: MediaStream ) {} - get type(): TrackType { return this._type; } + get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } - get streamId(): string { return this.stream.id; } - get muted(): boolean { return this.track.muted; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - setMuted(muted: boolean): void { - this.track.enabled = !muted; - } - - setType(type: TrackType): void { - this._type = type; - } - - stop() { - this.track.stop(); - } - - clone() { - return this.track.clone(); - } + stop() { this.track.stop(); } } export class AudioTrackWrapper extends TrackWrapper { @@ -127,8 +126,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream, type: TrackType) { - super(track, stream, type); + constructor(track: MediaStreamTrack, stream: MediaStream) { + super(track, stream); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 861a91f1..6760182d 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TrackWrapper, wrapTrack} from "./MediaDevices"; -import {Track, TrackType} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, DataChannel, PeerConnection} from "../../types/WebRTC"; +import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; +import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -29,11 +29,171 @@ export class DOMWebRTC implements WebRTC { } } +export class RemoteStreamWrapper extends StreamWrapper { + constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { + super(stream); + this.stream.addEventListener("removetrack", this.onTrackRemoved); + } + + onTrackRemoved = (evt: MediaStreamTrackEvent) => { + if (evt.track.id === this.audioTrack?.track.id) { + this.audioTrack = undefined; + } else if (evt.track.id === this.videoTrack?.track.id) { + this.videoTrack = undefined; + } + if (!this.audioTrack && !this.videoTrack) { + this.emptyCallback(this); + } + }; + + dispose() { + this.stream.removeEventListener("removetrack", this.onTrackRemoved); + } +} + +export class DOMStreamSender implements StreamSender { + public audioSender: DOMTrackSender | undefined; + public videoSender: DOMTrackSender | undefined; + + constructor(public readonly stream: StreamWrapper) {} + + update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { + const transceiver = transceivers.find(t => t.sender === sender); + if (transceiver && sender.track) { + const trackWrapper = this.stream.update(sender.track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoSender = new DOMTrackSender(trackWrapper, transceiver); + return this.videoSender; + } else { + this.audioSender = new DOMTrackSender(trackWrapper, transceiver); + return this.audioSender; + } + } + } + } +} + +export class DOMStreamReceiver implements StreamReceiver { + public audioReceiver: DOMTrackReceiver | undefined; + public videoReceiver: DOMTrackReceiver | undefined; + + constructor(public readonly stream: RemoteStreamWrapper) {} + + update(event: RTCTrackEvent): DOMTrackReceiver | undefined { + const {receiver} = event; + const {track} = receiver; + const trackWrapper = this.stream.update(track); + if (trackWrapper) { + if (trackWrapper.kind === TrackKind.Video) { + this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.videoReceiver; + } else { + this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); + return this.audioReceiver; + } + } + } +} + +export class DOMTrackSenderOrReceiver implements TrackReceiver { + constructor( + public readonly track: TrackWrapper, + public readonly transceiver: RTCRtpTransceiver, + private readonly exclusiveValue: RTCRtpTransceiverDirection, + private readonly excludedValue: RTCRtpTransceiverDirection + ) {} + + get enabled(): boolean { + return this.transceiver.currentDirection === "sendrecv" || + this.transceiver.currentDirection === this.exclusiveValue; + } + + enable(enabled: boolean) { + if (enabled !== this.enabled) { + if (enabled) { + if (this.transceiver.currentDirection === "inactive") { + this.transceiver.direction = this.exclusiveValue; + } else { + this.transceiver.direction = "sendrecv"; + } + } else { + if (this.transceiver.currentDirection === "sendrecv") { + this.transceiver.direction = this.excludedValue; + } else { + this.transceiver.direction = "inactive"; + } + } + } + } +} + +export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "recvonly", "sendonly"); + } +} + +export class DOMTrackSender extends DOMTrackSenderOrReceiver { + constructor( + track: TrackWrapper, + transceiver: RTCRtpTransceiver, + ) { + super(track, transceiver, "sendonly", "recvonly"); + } + /** replaces the track if possible without renegotiation. Can throw. */ + replaceTrack(track: Track): Promise { + return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); + } + + prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + if (purpose === SDPStreamMetadataPurpose.Screenshare) { + this.getRidOfRTXCodecs(); + } + } + + /** + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs(): void { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; + const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + if (this.transceiver.sender.track?.kind === "video" || + this.transceiver.receiver.track?.kind === "video") { + this.transceiver.setCodecPreferences(codecs); + } + } +} + class DOMPeerConnection implements PeerConnection { private readonly peerConnection: RTCPeerConnection; private readonly handler: PeerConnectionHandler; - //private dataChannelWrapper?: DOMDataChannel; - private _remoteTracks: TrackWrapper[] = []; + public readonly localStreams: DOMStreamSender[]; + public readonly remoteStreams: DOMStreamReceiver[]; constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { this.handler = handler; @@ -45,7 +205,6 @@ class DOMPeerConnection implements PeerConnection { this.registerHandler(); } - get remoteTracks(): Track[] { return this._remoteTracks; } get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } @@ -74,48 +233,26 @@ class DOMPeerConnection implements PeerConnection { return this.peerConnection.close(); } - addTrack(track: Track): void { + addTrack(track: Track): DOMTrackSender | undefined { if (!(track instanceof TrackWrapper)) { throw new Error("Not a TrackWrapper"); } - this.peerConnection.addTrack(track.track, track.stream); - if (track.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(track); + const sender = this.peerConnection.addTrack(track.track, track.stream); + let streamSender: DOMStreamSender | undefined = this.localStreams.find(s => s.stream.id === track.stream.id); + if (!streamSender) { + streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); + this.localStreams.push(streamSender); } + const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); + return trackSender; } - removeTrack(track: Track): boolean { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === track.track); - if (sender) { - this.peerConnection.removeTrack(sender); - return true; - } - return false; - } - - async replaceTrack(oldTrack: Track, newTrack: Track): Promise { - if (!(oldTrack instanceof TrackWrapper) || !(newTrack instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); - if (sender) { - await sender.replaceTrack(newTrack.track); - if (newTrack.type === TrackType.ScreenShare) { - this.getRidOfRTXCodecs(newTrack); - } - return true; - } - return false; - } - - notifyStreamPurposeChanged(): void { - for (const track of this.remoteTracks) { - const wrapper = track as TrackWrapper; - wrapper.setType(this.getRemoteTrackType(wrapper.track, wrapper.streamId)); + removeTrack(sender: TrackSender): void { + if (!(sender instanceof DOMTrackSender)) { + throw new Error("Not a DOMTrackSender"); } + this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); + // TODO: update localStreams } createDataChannel(options: RTCDataChannelInit): any { @@ -170,6 +307,9 @@ class DOMPeerConnection implements PeerConnection { dispose(): void { this.deregisterHandler(); + for (const r of this.remoteStreams) { + r.stream.dispose(); + } } private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { @@ -187,67 +327,28 @@ class DOMPeerConnection implements PeerConnection { } } + onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { + const idx = this.remoteStreams.findIndex(r => r.stream === stream); + if (idx !== -1) { + this.remoteStreams.splice(idx, 1); + this.handler.onRemoteStreamRemoved(stream); + } + } + private handleRemoteTrack(evt: RTCTrackEvent) { - // TODO: unit test this code somehow - // the tracks on the new stream (with their stream) - const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); - // of the tracks we already know about, filter the ones that aren't in the new stream - const withoutRemovedTracks = this._remoteTracks.filter(t => updatedTracks.some(ut => t.track.id === ut.track.id)); - // of the new tracks, filter the ones that we didn't already knew about - const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id)); - // wrap them - const wrappedAddedTracks = addedTracks.map(t => wrapTrack(t.track, t.stream, this.getRemoteTrackType(t.track, t.stream.id))); - // and concat the tracks for other streams with the added tracks - this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); - this.handler.onRemoteTracksChanged(this.remoteTracks); - } - - private getRemoteTrackType(track: MediaStreamTrack, streamId: string): TrackType { - if (track.kind === "video") { - const purpose = this.handler.getPurposeForStreamId(streamId); - return purpose === SDPStreamMetadataPurpose.Usermedia ? TrackType.Camera : TrackType.ScreenShare; - } else { - return TrackType.Microphone; + if (evt.streams.length !== 0) { + throw new Error("track in multiple streams is not supported"); } - } - - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void { - // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; - - const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? []; - const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? []; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } + const stream = evt.streams[0]; + const transceivers = this.peerConnection.getTransceivers(); + let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.find(r => r.stream.id === stream.id); + if (!streamReceiver) { + streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); + this.remoteStreams.push(streamReceiver); } - - for (const trans of this.peerConnection.getTransceivers()) { - if (trans.sender.track === screensharingTrack.track && - ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" - ) - ) { - trans.setCodecPreferences(codecs); - } + const trackReceiver = streamReceiver.update(evt); + if (trackReceiver) { + this.handler.onRemoteTracksAdded(trackReceiver); } } }