forked from mystiq/hydrogen-web
remove thick abstraction layer
instead just copy the DOM typing and make it part of the platform layer
This commit is contained in:
Bruno Windels
parent
baa884e9d0
commit
ac60d1b61d
13 changed files with 424 additions and 899 deletions
|
|
@ -57,9 +57,7 @@ export class CallViewModel extends ViewModel<Options> {
|
|||
}
|
||||
|
||||
async toggleVideo() {
|
||||
const localMedia = this.call.localMedia!;
|
||||
const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted);
|
||||
await this.call.setMedia(toggledMedia);
|
||||
//this.call.setMuted();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -367,9 +367,8 @@ export class RoomViewModel extends ViewModel {
|
|||
const session = this.getOption("session");
|
||||
const stream = await this.platform.mediaDevices.getMediaTracks(false, true);
|
||||
const localMedia = new LocalMedia().withUserMedia(stream);
|
||||
await this._call.join(localMedia);
|
||||
// this will set the callViewModel above as a call will be added to callHandler.calls
|
||||
const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100));
|
||||
const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100));
|
||||
await call.join(localMedia);
|
||||
} catch (err) {
|
||||
console.error(err.stack);
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ limitations under the License.
|
|||
*/
|
||||
|
||||
import {ObservableMap} from "../../observable/map/ObservableMap";
|
||||
import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC";
|
||||
import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices";
|
||||
import {WebRTC, PeerConnection} from "../../platform/types/WebRTC";
|
||||
import {MediaDevices, Track} from "../../platform/types/MediaDevices";
|
||||
import {handlesEventType} from "./PeerCall";
|
||||
import {EventType, CallIntent} from "./callEventTypes";
|
||||
import {GroupCall} from "./group/GroupCall";
|
||||
|
|
@ -107,7 +107,7 @@ export class CallHandler {
|
|||
});
|
||||
}
|
||||
|
||||
async createCall(roomId: string, localMedia: LocalMedia, name: string, intent: CallIntent = CallIntent.Ring): Promise<GroupCall> {
|
||||
async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise<GroupCall> {
|
||||
const logItem = this.options.logger.child({l: "call", incoming: false});
|
||||
const call = new GroupCall(makeId("conf-"), true, {
|
||||
"m.name": name,
|
||||
|
|
@ -116,7 +116,7 @@ export class CallHandler {
|
|||
this._calls.set(call.id, call);
|
||||
|
||||
try {
|
||||
await call.create(localMedia);
|
||||
await call.create(type);
|
||||
// store call info so it will ring again when reopening the app
|
||||
const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]);
|
||||
txn.calls.add({
|
||||
|
|
|
|||
|
|
@ -17,32 +17,28 @@ limitations under the License.
|
|||
import {SDPStreamMetadataPurpose} from "./callEventTypes";
|
||||
import {Stream} from "../../platform/types/MediaDevices";
|
||||
import {SDPStreamMetadata} from "./callEventTypes";
|
||||
import {getStreamVideoTrack, getStreamAudioTrack} from "./common";
|
||||
|
||||
export class LocalMedia {
|
||||
constructor(
|
||||
public readonly userMedia?: Stream,
|
||||
public readonly microphoneMuted: boolean = false,
|
||||
public readonly cameraMuted: boolean = false,
|
||||
public readonly screenShare?: Stream,
|
||||
public readonly dataChannelOptions?: RTCDataChannelInit,
|
||||
) {}
|
||||
|
||||
withMuted(microphone: boolean, camera: boolean) {
|
||||
return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions);
|
||||
}
|
||||
|
||||
withUserMedia(stream: Stream) {
|
||||
return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions);
|
||||
return new LocalMedia(stream, this.screenShare, this.dataChannelOptions);
|
||||
}
|
||||
|
||||
withScreenShare(stream: Stream) {
|
||||
return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions);
|
||||
return new LocalMedia(this.userMedia, stream, this.dataChannelOptions);
|
||||
}
|
||||
|
||||
withDataChannel(options: RTCDataChannelInit): LocalMedia {
|
||||
return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options);
|
||||
return new LocalMedia(this.userMedia, this.screenShare, options);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia {
|
||||
let userMedia;
|
||||
let screenShare;
|
||||
|
|
@ -52,21 +48,21 @@ export class LocalMedia {
|
|||
stream = oldCloneStream;
|
||||
} else {
|
||||
stream = newStream?.clone();
|
||||
oldCloneStream?.audioTrack?.stop();
|
||||
oldCloneStream?.videoTrack?.stop();
|
||||
getStreamAudioTrack(oldCloneStream)?.stop();
|
||||
getStreamVideoTrack(oldCloneStream)?.stop();
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
return new LocalMedia(
|
||||
cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia),
|
||||
this.microphoneMuted, this.cameraMuted,
|
||||
cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare),
|
||||
this.dataChannelOptions
|
||||
);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
clone(): LocalMedia {
|
||||
return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions);
|
||||
return new LocalMedia(this.userMedia?.clone(),this.screenShare?.clone(), this.dataChannelOptions);
|
||||
}
|
||||
|
||||
dispose() {
|
||||
|
|
@ -75,11 +71,11 @@ export class LocalMedia {
|
|||
|
||||
stopExcept(newMedia: LocalMedia | undefined) {
|
||||
if(newMedia?.userMedia?.id !== this.userMedia?.id) {
|
||||
this.userMedia?.audioTrack?.stop();
|
||||
this.userMedia?.videoTrack?.stop();
|
||||
getStreamAudioTrack(this.userMedia)?.stop();
|
||||
getStreamVideoTrack(this.userMedia)?.stop();
|
||||
}
|
||||
if(newMedia?.screenShare?.id !== this.screenShare?.id) {
|
||||
this.screenShare?.videoTrack?.stop();
|
||||
getStreamVideoTrack(this.screenShare)?.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,10 +16,10 @@ limitations under the License.
|
|||
|
||||
import {ObservableMap} from "../../observable/map/ObservableMap";
|
||||
import {recursivelyAssign} from "../../utils/recursivelyAssign";
|
||||
import {AsyncQueue} from "../../utils/AsyncQueue";
|
||||
import {Disposables, IDisposable} from "../../utils/Disposables";
|
||||
import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC";
|
||||
import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices";
|
||||
import {Disposables, Disposable, IDisposable} from "../../utils/Disposables";
|
||||
import {WebRTC, PeerConnection, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC";
|
||||
import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices";
|
||||
import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common";
|
||||
import {
|
||||
SDPStreamMetadataKey,
|
||||
SDPStreamMetadataPurpose,
|
||||
|
|
@ -69,6 +69,7 @@ export class PeerCall implements IDisposable {
|
|||
private direction: CallDirection;
|
||||
// we don't own localMedia and should hence not call dispose on it from here
|
||||
private localMedia?: LocalMedia;
|
||||
private localMuteSettings?: MuteSettings;
|
||||
private seq: number = 0;
|
||||
// A queue for candidates waiting to go out.
|
||||
// We try to amalgamate candidates into a single candidate message where
|
||||
|
|
@ -85,7 +86,8 @@ export class PeerCall implements IDisposable {
|
|||
private hangupParty: CallParty;
|
||||
private disposables = new Disposables();
|
||||
private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>();
|
||||
|
||||
private _remoteTrackToStreamId = new Map<string, string>();
|
||||
private _remoteStreams = new Map<string, {stream: Stream, disposeListener: Disposable}>();
|
||||
// perfect negotiation flags
|
||||
private makingOffer: boolean = false;
|
||||
private ignoreOffer: boolean = false;
|
||||
|
|
@ -96,55 +98,62 @@ export class PeerCall implements IDisposable {
|
|||
private _dataChannel?: any;
|
||||
private _hangupReason?: CallErrorCode;
|
||||
private _remoteMedia: RemoteMedia;
|
||||
private remoteMuteSettings?: MuteSettings;
|
||||
|
||||
constructor(
|
||||
private callId: string,
|
||||
private readonly options: Options,
|
||||
private readonly logItem: ILogItem,
|
||||
) {
|
||||
const outer = this;
|
||||
this._remoteMedia = new RemoteMedia();
|
||||
this.peerConnection = options.webRTC.createPeerConnection({
|
||||
onIceConnectionStateChange(state: RTCIceConnectionState) {
|
||||
outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => {
|
||||
outer.onIceConnectionStateChange(state, log);
|
||||
this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0);
|
||||
|
||||
const listen = <K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => {
|
||||
this.peerConnection.addEventListener(type, listener);
|
||||
const dispose = () => {
|
||||
this.peerConnection.removeEventListener(type, listener);
|
||||
};
|
||||
this.disposables.track(dispose);
|
||||
};
|
||||
|
||||
listen("iceconnectionstatechange", () => {
|
||||
const state = this.peerConnection.iceConnectionState;
|
||||
this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => {
|
||||
this.onIceConnectionStateChange(state, log);
|
||||
});
|
||||
},
|
||||
onLocalIceCandidate(candidate: RTCIceCandidate) {
|
||||
outer.logItem.wrap("onLocalIceCandidate", log => {
|
||||
outer.handleLocalIceCandidate(candidate, log);
|
||||
});
|
||||
},
|
||||
onIceGatheringStateChange(state: RTCIceGatheringState) {
|
||||
outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => {
|
||||
outer.handleIceGatheringState(state, log);
|
||||
listen("icecandidate", event => {
|
||||
this.logItem.wrap("onLocalIceCandidate", log => {
|
||||
if (event.candidate) {
|
||||
this.handleLocalIceCandidate(event.candidate, log);
|
||||
}
|
||||
});
|
||||
},
|
||||
onRemoteStreamRemoved(stream: Stream) {
|
||||
outer.logItem.wrap("onRemoteStreamRemoved", log => {
|
||||
outer.updateRemoteMedia(log);
|
||||
});
|
||||
},
|
||||
onRemoteTracksAdded(trackReceiver: TrackReceiver) {
|
||||
outer.logItem.wrap("onRemoteTracksAdded", log => {
|
||||
outer.updateRemoteMedia(log);
|
||||
listen("icegatheringstatechange", () => {
|
||||
const state = this.peerConnection.iceGatheringState;
|
||||
this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => {
|
||||
this.handleIceGatheringState(state, log);
|
||||
});
|
||||
},
|
||||
onRemoteDataChannel(dataChannel: any | undefined) {
|
||||
outer.logItem.wrap("onRemoteDataChannel", log => {
|
||||
outer._dataChannel = dataChannel;
|
||||
outer.options.emitUpdate(outer, undefined);
|
||||
});
|
||||
},
|
||||
onNegotiationNeeded() {
|
||||
listen("track", event => {
|
||||
this.logItem.wrap("onRemoteTrack", log => {
|
||||
this.onRemoteTrack(event.track, event.streams, log);
|
||||
});
|
||||
});
|
||||
listen("datachannel", event => {
|
||||
this.logItem.wrap("onRemoteDataChannel", log => {
|
||||
this._dataChannel = event.channel;
|
||||
this.options.emitUpdate(this, undefined);
|
||||
});
|
||||
});
|
||||
listen("negotiationneeded", () => {
|
||||
const promiseCreator = () => {
|
||||
return outer.logItem.wrap("onNegotiationNeeded", log => {
|
||||
return outer.handleNegotiation(log);
|
||||
return this.logItem.wrap("onNegotiationNeeded", log => {
|
||||
return this.handleNegotiation(log);
|
||||
});
|
||||
};
|
||||
outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator();
|
||||
}
|
||||
}, this.options.forceTURN, this.options.turnServers, 0);
|
||||
this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator();
|
||||
});
|
||||
}
|
||||
|
||||
get dataChannel(): any | undefined { return this._dataChannel; }
|
||||
|
|
@ -166,7 +175,7 @@ export class PeerCall implements IDisposable {
|
|||
this.setState(CallState.CreateOffer, log);
|
||||
await this.updateLocalMedia(localMedia, log);
|
||||
if (this.localMedia?.dataChannelOptions) {
|
||||
this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions);
|
||||
this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions);
|
||||
}
|
||||
// after adding the local tracks, and wait for handleNegotiation to be called,
|
||||
// or invite glare where we give up our invite and answer instead
|
||||
|
|
@ -211,11 +220,9 @@ export class PeerCall implements IDisposable {
|
|||
|
||||
setMedia(localMedia: LocalMedia): Promise<void> {
|
||||
return this.logItem.wrap("setMedia", async log => {
|
||||
log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack);
|
||||
log.set("userMedia_audio_muted", localMedia.microphoneMuted);
|
||||
log.set("userMedia_video", !!localMedia.userMedia?.videoTrack);
|
||||
log.set("userMedia_video_muted", localMedia.cameraMuted);
|
||||
log.set("screenShare_video", !!localMedia.screenShare?.videoTrack);
|
||||
log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia));
|
||||
log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia));
|
||||
log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare));
|
||||
log.set("datachannel", !!localMedia.dataChannelOptions);
|
||||
await this.updateLocalMedia(localMedia, log);
|
||||
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
|
||||
|
|
@ -322,6 +329,7 @@ export class PeerCall implements IDisposable {
|
|||
this.candidateSendQueue = [];
|
||||
|
||||
// need to queue this
|
||||
if (this._state === CallState.CreateOffer) {
|
||||
const content = {
|
||||
call_id: this.callId,
|
||||
offer,
|
||||
|
|
@ -330,13 +338,17 @@ export class PeerCall implements IDisposable {
|
|||
seq: this.seq++,
|
||||
lifetime: CALL_TIMEOUT_MS
|
||||
};
|
||||
if (this._state === CallState.CreateOffer) {
|
||||
await this.sendSignallingMessage({type: EventType.Invite, content}, log);
|
||||
this.setState(CallState.InviteSent, log);
|
||||
} else if (this._state === CallState.Connected || this._state === CallState.Connecting) {
|
||||
// send Negotiate message
|
||||
content.description = content.offer;
|
||||
delete content.offer;
|
||||
const content = {
|
||||
call_id: this.callId,
|
||||
description: offer,
|
||||
[SDPStreamMetadataKey]: this.getSDPMetadata(),
|
||||
version: 1,
|
||||
seq: this.seq++,
|
||||
lifetime: CALL_TIMEOUT_MS
|
||||
};
|
||||
await this.sendSignallingMessage({type: EventType.Negotiate, content}, log);
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -432,7 +444,7 @@ export class PeerCall implements IDisposable {
|
|||
// According to previous comments in this file, firefox at some point did not
|
||||
// add streams until media started arriving on them. Testing latest firefox
|
||||
// (81 at time of writing), this is no longer a problem, so let's do it the correct way.
|
||||
if (this.peerConnection.remoteStreams.size === 0) {
|
||||
if (this.peerConnection.getReceivers().length === 0) {
|
||||
await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => {
|
||||
return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
|
||||
});
|
||||
|
|
@ -843,11 +855,10 @@ export class PeerCall implements IDisposable {
|
|||
const metadata = {};
|
||||
if (this.localMedia?.userMedia) {
|
||||
const streamId = this.localMedia.userMedia.id;
|
||||
const streamSender = this.peerConnection.localStreams.get(streamId);
|
||||
metadata[streamId] = {
|
||||
purpose: SDPStreamMetadataPurpose.Usermedia,
|
||||
audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack,
|
||||
video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack,
|
||||
audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia),
|
||||
video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia),
|
||||
};
|
||||
}
|
||||
if (this.localMedia?.screenShare) {
|
||||
|
|
@ -859,19 +870,67 @@ export class PeerCall implements IDisposable {
|
|||
return metadata;
|
||||
}
|
||||
|
||||
private updateRemoteMedia(log: ILogItem) {
|
||||
private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined {
|
||||
return this.peerConnection.getReceivers().find(r => {
|
||||
return r.track.kind === "audio" && this._remoteTrackToStreamId.get(r.track.id) === streamId;
|
||||
});
|
||||
}
|
||||
|
||||
private onRemoteTrack(track: Track, streams: ReadonlyArray<Stream>, log: ILogItem) {
|
||||
if (streams.length === 0) {
|
||||
log.log({l: `ignoring ${track.kind} streamless track`, id: track.id});
|
||||
return;
|
||||
}
|
||||
const stream = streams[0];
|
||||
this._remoteTrackToStreamId.set(track.id, stream.id);
|
||||
if (!this._remoteStreams.has(stream.id)) {
|
||||
const listener = (event: StreamTrackEvent): void => {
|
||||
this.logItem.wrap({l: "removetrack", id: event.track.id}, log => {
|
||||
const streamId = this._remoteTrackToStreamId.get(event.track.id);
|
||||
if (streamId) {
|
||||
this._remoteTrackToStreamId.delete(event.track.id);
|
||||
const streamDetails = this._remoteStreams.get(streamId);
|
||||
if (streamDetails && streamDetails.stream.getTracks().length === 0) {
|
||||
this.disposables.disposeTracked(disposeListener);
|
||||
this._remoteStreams.delete(stream.id);
|
||||
this.updateRemoteMedia(log);
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
stream.addEventListener("removetrack", listener);
|
||||
const disposeListener = () => {
|
||||
stream.removeEventListener("removetrack", listener);
|
||||
};
|
||||
this.disposables.track(disposeListener);
|
||||
this._remoteStreams.set(stream.id, {
|
||||
disposeListener,
|
||||
stream
|
||||
});
|
||||
this.updateRemoteMedia(log);
|
||||
}
|
||||
}
|
||||
|
||||
private updateRemoteMedia(log: ILogItem): void {
|
||||
this._remoteMedia.userMedia = undefined;
|
||||
this._remoteMedia.screenShare = undefined;
|
||||
if (this.remoteSDPStreamMetadata) {
|
||||
for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) {
|
||||
const metaData = this.remoteSDPStreamMetadata[streamId];
|
||||
for (const streamDetails of this._remoteStreams.values()) {
|
||||
const {stream} = streamDetails;
|
||||
const metaData = this.remoteSDPStreamMetadata[stream.id];
|
||||
if (metaData) {
|
||||
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
|
||||
this._remoteMedia.userMedia = streamReceiver.stream;
|
||||
streamReceiver.audioReceiver?.enable(!metaData.audio_muted);
|
||||
streamReceiver.videoReceiver?.enable(!metaData.video_muted);
|
||||
this._remoteMedia.userMedia = stream;
|
||||
const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
|
||||
if (audioReceiver) {
|
||||
audioReceiver.track.enabled = !metaData.audio_muted;
|
||||
}
|
||||
const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
|
||||
if (videoReceiver) {
|
||||
videoReceiver.track.enabled = !metaData.audio_muted;
|
||||
}
|
||||
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
|
||||
this._remoteMedia.screenShare = streamReceiver.stream;
|
||||
this._remoteMedia.screenShare = stream;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -883,71 +942,46 @@ export class PeerCall implements IDisposable {
|
|||
return logItem.wrap("updateLocalMedia", async log => {
|
||||
const oldMedia = this.localMedia;
|
||||
this.localMedia = localMedia;
|
||||
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => {
|
||||
let streamSender;
|
||||
if (oldStream) {
|
||||
streamSender = this.peerConnection.localStreams.get(oldStream.id);
|
||||
if (stream && stream.id !== oldStream.id) {
|
||||
this.peerConnection.localStreams.set(stream.id, streamSender);
|
||||
this.peerConnection.localStreams.delete(oldStream.id);
|
||||
}
|
||||
}
|
||||
|
||||
const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => {
|
||||
const changed = (!track && oldTrack) ||
|
||||
(track && !oldTrack) ||
|
||||
(track && oldTrack && !track.equals(oldTrack));
|
||||
if (changed) {
|
||||
if (track) {
|
||||
if (oldTrack && sender && !track.equals(oldTrack)) {
|
||||
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => {
|
||||
const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => {
|
||||
if (!oldTrack && newTrack) {
|
||||
log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => {
|
||||
const sender = this.peerConnection.addTrack(newTrack, stream!);
|
||||
this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose);
|
||||
});
|
||||
} else if (oldTrack) {
|
||||
const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id);
|
||||
if (sender) {
|
||||
if (newTrack && oldTrack.id !== newTrack.id) {
|
||||
try {
|
||||
await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => {
|
||||
return sender.replaceTrack(track);
|
||||
await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => {
|
||||
return sender.replaceTrack(newTrack);
|
||||
});
|
||||
} catch (err) {
|
||||
// can't replace the track without renegotiating
|
||||
log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => {
|
||||
// can't replace the track without renegotiating{
|
||||
log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => {
|
||||
this.peerConnection.removeTrack(sender);
|
||||
this.peerConnection.addTrack(track);
|
||||
this.peerConnection.addTrack(newTrack);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
|
||||
this.peerConnection.addTrack(track);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (sender) {
|
||||
// this will be used for muting, do we really want to trigger renegotiation here?
|
||||
// we want to disable the sender, but also remove the track as we don't want to keep
|
||||
// using the webcam if we don't need to
|
||||
log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => {
|
||||
sender.track.enabled = false;
|
||||
} else if (!newTrack) {
|
||||
log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => {
|
||||
this.peerConnection.removeTrack(sender);
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (track) {
|
||||
log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id});
|
||||
if (sender && muted !== wasMuted) {
|
||||
log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => {
|
||||
// sender.track.enabled = !muted;
|
||||
// This doesn't always seem to trigger renegotiation??
|
||||
// We should probably always send the new metadata first ...
|
||||
sender.enable(!muted);
|
||||
});
|
||||
} else {
|
||||
log.log(`${logLabel} ${track.kind} track hasn't changed`);
|
||||
log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`);
|
||||
}
|
||||
}
|
||||
// TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia
|
||||
}
|
||||
}
|
||||
|
||||
await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted);
|
||||
await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted);
|
||||
}
|
||||
await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream));
|
||||
await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream));
|
||||
};
|
||||
|
||||
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia");
|
||||
await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare");
|
||||
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia);
|
||||
await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare);
|
||||
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
|
||||
});
|
||||
}
|
||||
|
|
@ -967,7 +1001,7 @@ export class PeerCall implements IDisposable {
|
|||
|
||||
public dispose(): void {
|
||||
this.disposables.dispose();
|
||||
this.peerConnection.dispose();
|
||||
this.peerConnection.close();
|
||||
}
|
||||
|
||||
public close(reason: CallErrorCode | undefined, log: ILogItem): void {
|
||||
|
|
@ -1038,6 +1072,7 @@ export function handlesEventType(eventType: string): boolean {
|
|||
eventType === EventType.Negotiate;
|
||||
}
|
||||
|
||||
|
||||
export function tests() {
|
||||
|
||||
}
|
||||
|
|
|
|||
29
src/matrix/calls/common.ts
Normal file
29
src/matrix/calls/common.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
Copyright 2022 The Matrix.org Foundation C.I.C.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
import type {Track, Stream} from "../../platform/types/MediaDevices";
|
||||
|
||||
export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined {
|
||||
return stream?.getAudioTracks()[0];
|
||||
}
|
||||
|
||||
export function getStreamVideoTrack(stream: Stream | undefined): Track | undefined {
|
||||
return stream?.getVideoTracks()[0];
|
||||
}
|
||||
|
||||
export class MuteSettings {
|
||||
constructor (public readonly microphone: boolean, public readonly camera: boolean) {}
|
||||
}
|
||||
|
|
@ -168,7 +168,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
|
|||
}
|
||||
|
||||
/** @internal */
|
||||
create(localMedia: LocalMedia): Promise<void> {
|
||||
create(type: "m.video" | "m.voice"): Promise<void> {
|
||||
return this.logItem.wrap("create", async log => {
|
||||
if (this._state !== GroupCallState.Fledgling) {
|
||||
return;
|
||||
|
|
@ -176,7 +176,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
|
|||
this._state = GroupCallState.Creating;
|
||||
this.emitChange();
|
||||
this.callContent = Object.assign({
|
||||
"m.type": localMedia.userMedia?.videoTrack ? "m.video" : "m.voice",
|
||||
"m.type": type,
|
||||
}, this.callContent);
|
||||
const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log});
|
||||
await request.response();
|
||||
|
|
|
|||
|
|
@ -14,15 +14,39 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
export interface Event {}
|
||||
|
||||
export interface MediaDevices {
|
||||
// filter out audiooutput
|
||||
enumerate(): Promise<MediaDeviceInfo[]>;
|
||||
// to assign to a video element, we downcast to WrappedTrack and use the stream property.
|
||||
getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream>;
|
||||
getScreenShareTrack(): Promise<Stream | undefined>;
|
||||
createVolumeMeasurer(stream: Stream): VolumeMeasurer;
|
||||
createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer;
|
||||
}
|
||||
|
||||
// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts
|
||||
/*! *****************************************************************************
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy of the
|
||||
License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
See the Apache Version 2.0 License for specific language governing permissions
|
||||
and limitations under the License.
|
||||
***************************************************************************** */
|
||||
|
||||
export interface StreamTrackEvent extends Event {
|
||||
readonly track: Track;
|
||||
}
|
||||
|
||||
export interface StreamEventMap {
|
||||
"addtrack": StreamTrackEvent;
|
||||
"removetrack": StreamTrackEvent;
|
||||
}
|
||||
|
||||
export interface Stream {
|
||||
getTracks(): ReadonlyArray<Track>;
|
||||
|
|
@ -30,6 +54,8 @@ export interface Stream {
|
|||
getVideoTracks(): ReadonlyArray<Track>;
|
||||
readonly id: string;
|
||||
clone(): Stream;
|
||||
addEventListener<K extends keyof StreamEventMap>(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
|
||||
removeEventListener<K extends keyof StreamEventMap>(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
|
||||
}
|
||||
|
||||
export enum TrackKind {
|
||||
|
|
@ -47,5 +73,7 @@ export interface Track {
|
|||
}
|
||||
|
||||
export interface VolumeMeasurer {
|
||||
|
||||
get isSpeaking(): boolean;
|
||||
setSpeakingThreshold(threshold: number): void;
|
||||
stop();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,63 +14,156 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
import {Track, Stream} from "./MediaDevices";
|
||||
import {Track, Stream, Event} from "./MediaDevices";
|
||||
import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes";
|
||||
|
||||
export interface WebRTC {
|
||||
createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection;
|
||||
createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection;
|
||||
prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void;
|
||||
}
|
||||
|
||||
export interface StreamSender {
|
||||
get stream(): Stream;
|
||||
get audioSender(): TrackSender | undefined;
|
||||
get videoSender(): TrackSender | undefined;
|
||||
// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts
|
||||
/*! *****************************************************************************
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
||||
this file except in compliance with the License. You may obtain a copy of the
|
||||
License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
||||
MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
See the Apache Version 2.0 License for specific language governing permissions
|
||||
and limitations under the License.
|
||||
***************************************************************************** */
|
||||
|
||||
export interface DataChannelEventMap {
|
||||
"bufferedamountlow": Event;
|
||||
"close": Event;
|
||||
"error": Event;
|
||||
"message": MessageEvent;
|
||||
"open": Event;
|
||||
}
|
||||
|
||||
export interface StreamReceiver {
|
||||
get stream(): Stream;
|
||||
get audioReceiver(): TrackReceiver | undefined;
|
||||
get videoReceiver(): TrackReceiver | undefined;
|
||||
}
|
||||
|
||||
export interface TrackReceiver {
|
||||
get track(): Track;
|
||||
get enabled(): boolean;
|
||||
enable(enabled: boolean); // this modifies the transceiver direction
|
||||
}
|
||||
|
||||
export interface TrackSender extends TrackReceiver {
|
||||
/** replaces the track if possible without renegotiation. Can throw. */
|
||||
replaceTrack(track: Track | undefined): Promise<void>;
|
||||
/** make any needed adjustments to the sender or transceiver settings
|
||||
* depending on the purpose, after adding the track to the connection */
|
||||
prepareForPurpose(purpose: SDPStreamMetadataPurpose): void;
|
||||
}
|
||||
|
||||
export interface PeerConnectionHandler {
|
||||
onIceConnectionStateChange(state: RTCIceConnectionState);
|
||||
onLocalIceCandidate(candidate: RTCIceCandidate);
|
||||
onIceGatheringStateChange(state: RTCIceGatheringState);
|
||||
onRemoteStreamRemoved(stream: Stream);
|
||||
onRemoteTracksAdded(receiver: TrackReceiver);
|
||||
onRemoteDataChannel(dataChannel: any | undefined);
|
||||
onNegotiationNeeded();
|
||||
}
|
||||
|
||||
export interface PeerConnection {
|
||||
get iceGatheringState(): RTCIceGatheringState;
|
||||
get signalingState(): RTCSignalingState;
|
||||
get localDescription(): RTCSessionDescription | undefined;
|
||||
get localStreams(): ReadonlyMap<string, StreamSender>;
|
||||
get remoteStreams(): ReadonlyMap<string, StreamReceiver>;
|
||||
createOffer(): Promise<RTCSessionDescriptionInit>;
|
||||
createAnswer(): Promise<RTCSessionDescriptionInit>;
|
||||
setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void>;
|
||||
setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>;
|
||||
addIceCandidate(candidate: RTCIceCandidate): Promise<void>;
|
||||
addTrack(track: Track): TrackSender | undefined;
|
||||
removeTrack(track: TrackSender): void;
|
||||
createDataChannel(options: RTCDataChannelInit): any;
|
||||
dispose(): void;
|
||||
export interface DataChannel {
|
||||
binaryType: BinaryType;
|
||||
readonly id: number | null;
|
||||
readonly label: string;
|
||||
readonly negotiated: boolean;
|
||||
readonly readyState: DataChannelState;
|
||||
close(): void;
|
||||
send(data: string): void;
|
||||
send(data: Blob): void;
|
||||
send(data: ArrayBuffer): void;
|
||||
send(data: ArrayBufferView): void;
|
||||
addEventListener<K extends keyof DataChannelEventMap>(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
|
||||
removeEventListener<K extends keyof DataChannelEventMap>(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
|
||||
}
|
||||
|
||||
export interface DataChannelInit {
|
||||
id?: number;
|
||||
maxPacketLifeTime?: number;
|
||||
maxRetransmits?: number;
|
||||
negotiated?: boolean;
|
||||
ordered?: boolean;
|
||||
protocol?: string;
|
||||
}
|
||||
|
||||
export interface DataChannelEvent extends Event {
|
||||
readonly channel: DataChannel;
|
||||
}
|
||||
|
||||
export interface PeerConnectionIceEvent extends Event {
|
||||
readonly candidate: RTCIceCandidate | null;
|
||||
}
|
||||
|
||||
export interface TrackEvent extends Event {
|
||||
readonly receiver: Receiver;
|
||||
readonly streams: ReadonlyArray<Stream>;
|
||||
readonly track: Track;
|
||||
readonly transceiver: Transceiver;
|
||||
}
|
||||
|
||||
export interface PeerConnectionEventMap {
|
||||
"connectionstatechange": Event;
|
||||
"datachannel": DataChannelEvent;
|
||||
"icecandidate": PeerConnectionIceEvent;
|
||||
"iceconnectionstatechange": Event;
|
||||
"icegatheringstatechange": Event;
|
||||
"negotiationneeded": Event;
|
||||
"signalingstatechange": Event;
|
||||
"track": TrackEvent;
|
||||
}
|
||||
|
||||
export type DataChannelState = "closed" | "closing" | "connecting" | "open";
|
||||
export type IceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new";
|
||||
export type PeerConnectionState = "closed" | "connected" | "connecting" | "disconnected" | "failed" | "new";
|
||||
export type SignalingState = "closed" | "have-local-offer" | "have-local-pranswer" | "have-remote-offer" | "have-remote-pranswer" | "stable";
|
||||
export type IceGatheringState = "complete" | "gathering" | "new";
|
||||
export type SdpType = "answer" | "offer" | "pranswer" | "rollback";
|
||||
export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendrecv" | "stopped";
|
||||
export interface SessionDescription {
|
||||
readonly sdp: string;
|
||||
readonly type: SdpType;
|
||||
toJSON(): any;
|
||||
}
|
||||
|
||||
export interface AnswerOptions {}
|
||||
|
||||
export interface OfferOptions {
|
||||
iceRestart?: boolean;
|
||||
offerToReceiveAudio?: boolean;
|
||||
offerToReceiveVideo?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionDescriptionInit {
|
||||
sdp?: string;
|
||||
type: SdpType;
|
||||
}
|
||||
|
||||
export interface LocalSessionDescriptionInit {
|
||||
sdp?: string;
|
||||
type?: SdpType;
|
||||
}
|
||||
|
||||
/** A WebRTC connection between the local computer and a remote peer. It provides methods to connect to a remote peer, maintain and monitor the connection, and close the connection once it's no longer needed. */
|
||||
export interface PeerConnection {
|
||||
readonly connectionState: PeerConnectionState;
|
||||
readonly iceConnectionState: IceConnectionState;
|
||||
readonly iceGatheringState: IceGatheringState;
|
||||
readonly localDescription: SessionDescription | null;
|
||||
readonly remoteDescription: SessionDescription | null;
|
||||
readonly signalingState: SignalingState;
|
||||
addIceCandidate(candidate?: RTCIceCandidateInit): Promise<void>;
|
||||
addTrack(track: Track, ...streams: Stream[]): Sender;
|
||||
close(): void;
|
||||
createAnswer(options?: AnswerOptions): Promise<SessionDescriptionInit>;
|
||||
createDataChannel(label: string, dataChannelDict?: DataChannelInit): DataChannel;
|
||||
createOffer(options?: OfferOptions): Promise<SessionDescriptionInit>;
|
||||
getReceivers(): Receiver[];
|
||||
getSenders(): Sender[];
|
||||
getTransceivers(): Transceiver[];
|
||||
removeTrack(sender: Sender): void;
|
||||
restartIce(): void;
|
||||
setLocalDescription(description?: LocalSessionDescriptionInit): Promise<void>;
|
||||
setRemoteDescription(description: SessionDescriptionInit): Promise<void>;
|
||||
addEventListener<K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
|
||||
removeEventListener<K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
|
||||
}
|
||||
|
||||
export interface Receiver {
|
||||
readonly track: Track;
|
||||
}
|
||||
|
||||
export interface Sender {
|
||||
readonly track: Track | null;
|
||||
replaceTrack(withTrack: Track | null): Promise<void>;
|
||||
}
|
||||
|
||||
export interface Transceiver {
|
||||
readonly currentDirection: TransceiverDirection | null;
|
||||
direction: TransceiverDirection;
|
||||
readonly mid: string | null;
|
||||
readonly receiver: Receiver;
|
||||
readonly sender: Sender;
|
||||
stop(): void;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices";
|
||||
import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices";
|
||||
|
||||
const POLLING_INTERVAL = 200; // ms
|
||||
export const SPEAKING_THRESHOLD = -60; // dB
|
||||
|
|
@ -30,12 +30,12 @@ export class MediaDevicesWrapper implements IMediaDevices {
|
|||
|
||||
async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream> {
|
||||
const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video));
|
||||
return new StreamWrapper(stream);
|
||||
return stream as Stream;
|
||||
}
|
||||
|
||||
async getScreenShareTrack(): Promise<Stream | undefined> {
|
||||
const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints());
|
||||
return new StreamWrapper(stream);
|
||||
return stream as Stream;
|
||||
}
|
||||
|
||||
private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints {
|
||||
|
|
@ -68,70 +68,13 @@ export class MediaDevicesWrapper implements IMediaDevices {
|
|||
video: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export class StreamWrapper implements Stream {
|
||||
|
||||
public audioTrack: AudioTrackWrapper | undefined = undefined;
|
||||
public videoTrack: TrackWrapper | undefined = undefined;
|
||||
|
||||
constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) {
|
||||
if (clonedTracks) {
|
||||
this.audioTrack = clonedTracks.audioTrack;
|
||||
this.videoTrack = clonedTracks.videoTrack;
|
||||
} else {
|
||||
for (const track of stream.getTracks()) {
|
||||
this.update(track);
|
||||
}
|
||||
createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer {
|
||||
return new WebAudioVolumeMeasurer(stream as MediaStream, callback);
|
||||
}
|
||||
}
|
||||
|
||||
get id(): string { return this.stream.id; }
|
||||
|
||||
clone(): Stream {
|
||||
const clonedStream = this.stream.clone();
|
||||
const clonedTracks = {
|
||||
audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined,
|
||||
videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined,
|
||||
};
|
||||
return new StreamWrapper(clonedStream, clonedTracks);
|
||||
}
|
||||
|
||||
update(track: MediaStreamTrack): TrackWrapper | undefined {
|
||||
//console.trace("Stream.update " + JSON.stringify({id: track.id, vid: this.videoTrack?.id, aid: this.audioTrack?.id}));
|
||||
if (track.kind === "video") {
|
||||
if (!this.videoTrack || track.id !== this.videoTrack.track.id) {
|
||||
this.videoTrack = new TrackWrapper(track, this.stream, track.id);
|
||||
}
|
||||
return this.videoTrack;
|
||||
} else if (track.kind === "audio") {
|
||||
if (!this.audioTrack || track.id !== this.audioTrack.track.id) {
|
||||
this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id);
|
||||
}
|
||||
return this.audioTrack;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class TrackWrapper implements Track {
|
||||
constructor(
|
||||
public readonly track: MediaStreamTrack,
|
||||
public readonly stream: MediaStream,
|
||||
public readonly originalId: string,
|
||||
) {}
|
||||
|
||||
get kind(): TrackKind { return this.track.kind as TrackKind; }
|
||||
get label(): string { return this.track.label; }
|
||||
get id(): string { return this.track.id; }
|
||||
get settings(): MediaTrackSettings { return this.track.getSettings(); }
|
||||
get enabled(): boolean { return this.track.enabled; }
|
||||
set enabled(enabled: boolean) { this.track.enabled = enabled; }
|
||||
// test equality across clones
|
||||
equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; }
|
||||
stop() { this.track.stop(); }
|
||||
}
|
||||
|
||||
export class AudioTrackWrapper extends TrackWrapper {
|
||||
export class WebAudioVolumeMeasurer implements VolumeMeasurer {
|
||||
private measuringVolumeActivity = false;
|
||||
private audioContext?: AudioContext;
|
||||
private analyser: AnalyserNode;
|
||||
|
|
@ -140,9 +83,12 @@ export class AudioTrackWrapper extends TrackWrapper {
|
|||
private speaking = false;
|
||||
private volumeLooperTimeout: number;
|
||||
private speakingVolumeSamples: number[];
|
||||
private callback: () => void;
|
||||
private stream: MediaStream;
|
||||
|
||||
constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) {
|
||||
super(track, stream, originalId);
|
||||
constructor(stream: MediaStream, callback: () => void) {
|
||||
this.stream = stream;
|
||||
this.callback = callback;
|
||||
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
|
||||
this.initVolumeMeasuring();
|
||||
this.measureVolumeActivity(true);
|
||||
|
|
@ -162,6 +108,7 @@ export class AudioTrackWrapper extends TrackWrapper {
|
|||
} else {
|
||||
this.measuringVolumeActivity = false;
|
||||
this.speakingVolumeSamples.fill(-Infinity);
|
||||
this.callback();
|
||||
// this.emit(CallFeedEvent.VolumeChanged, -Infinity);
|
||||
}
|
||||
}
|
||||
|
|
@ -182,7 +129,6 @@ export class AudioTrackWrapper extends TrackWrapper {
|
|||
this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
|
||||
}
|
||||
|
||||
|
||||
public setSpeakingThreshold(threshold: number) {
|
||||
this.speakingThreshold = threshold;
|
||||
}
|
||||
|
|
@ -204,6 +150,7 @@ export class AudioTrackWrapper extends TrackWrapper {
|
|||
this.speakingVolumeSamples.shift();
|
||||
this.speakingVolumeSamples.push(maxVolume);
|
||||
|
||||
this.callback();
|
||||
// this.emit(CallFeedEvent.VolumeChanged, maxVolume);
|
||||
|
||||
let newSpeaking = false;
|
||||
|
|
@ -219,267 +166,16 @@ export class AudioTrackWrapper extends TrackWrapper {
|
|||
|
||||
if (this.speaking !== newSpeaking) {
|
||||
this.speaking = newSpeaking;
|
||||
this.callback();
|
||||
// this.emit(CallFeedEvent.Speaking, this.speaking);
|
||||
}
|
||||
|
||||
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number;
|
||||
};
|
||||
|
||||
public dispose(): void {
|
||||
public stop(): void {
|
||||
clearTimeout(this.volumeLooperTimeout);
|
||||
this.analyser.disconnect();
|
||||
this.audioContext?.close();
|
||||
}
|
||||
}
|
||||
|
||||
// export interface ICallFeedOpts {
|
||||
// client: MatrixClient;
|
||||
// roomId: string;
|
||||
// userId: string;
|
||||
// stream: MediaStream;
|
||||
// purpose: SDPStreamMetadataPurpose;
|
||||
// audioMuted: boolean;
|
||||
// videoMuted: boolean;
|
||||
// }
|
||||
|
||||
// export enum CallFeedEvent {
|
||||
// NewStream = "new_stream",
|
||||
// MuteStateChanged = "mute_state_changed",
|
||||
// VolumeChanged = "volume_changed",
|
||||
// Speaking = "speaking",
|
||||
// }
|
||||
|
||||
// export class CallFeed extends EventEmitter {
|
||||
// public stream: MediaStream;
|
||||
// public sdpMetadataStreamId: string;
|
||||
// public userId: string;
|
||||
// public purpose: SDPStreamMetadataPurpose;
|
||||
// public speakingVolumeSamples: number[];
|
||||
|
||||
// private client: MatrixClient;
|
||||
// private roomId: string;
|
||||
// private audioMuted: boolean;
|
||||
// private videoMuted: boolean;
|
||||
// private measuringVolumeActivity = false;
|
||||
// private audioContext: AudioContext;
|
||||
// private analyser: AnalyserNode;
|
||||
// private frequencyBinCount: Float32Array;
|
||||
// private speakingThreshold = SPEAKING_THRESHOLD;
|
||||
// private speaking = false;
|
||||
// private volumeLooperTimeout: number;
|
||||
|
||||
// constructor(opts: ICallFeedOpts) {
|
||||
// super();
|
||||
|
||||
// this.client = opts.client;
|
||||
// this.roomId = opts.roomId;
|
||||
// this.userId = opts.userId;
|
||||
// this.purpose = opts.purpose;
|
||||
// this.audioMuted = opts.audioMuted;
|
||||
// this.videoMuted = opts.videoMuted;
|
||||
// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
|
||||
// this.sdpMetadataStreamId = opts.stream.id;
|
||||
|
||||
// this.updateStream(null, opts.stream);
|
||||
|
||||
// if (this.hasAudioTrack) {
|
||||
// this.initVolumeMeasuring();
|
||||
// }
|
||||
// }
|
||||
|
||||
// private get hasAudioTrack(): boolean {
|
||||
// return this.stream.getAudioTracks().length > 0;
|
||||
// }
|
||||
|
||||
// private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
|
||||
// if (newStream === oldStream) return;
|
||||
|
||||
// if (oldStream) {
|
||||
// oldStream.removeEventListener("addtrack", this.onAddTrack);
|
||||
// this.measureVolumeActivity(false);
|
||||
// }
|
||||
// if (newStream) {
|
||||
// this.stream = newStream;
|
||||
// newStream.addEventListener("addtrack", this.onAddTrack);
|
||||
|
||||
// if (this.hasAudioTrack) {
|
||||
// this.initVolumeMeasuring();
|
||||
// } else {
|
||||
// this.measureVolumeActivity(false);
|
||||
// }
|
||||
// }
|
||||
|
||||
// this.emit(CallFeedEvent.NewStream, this.stream);
|
||||
// }
|
||||
|
||||
// private initVolumeMeasuring(): void {
|
||||
// const AudioContext = window.AudioContext || window.webkitAudioContext;
|
||||
// if (!this.hasAudioTrack || !AudioContext) return;
|
||||
|
||||
// this.audioContext = new AudioContext();
|
||||
|
||||
// this.analyser = this.audioContext.createAnalyser();
|
||||
// this.analyser.fftSize = 512;
|
||||
// this.analyser.smoothingTimeConstant = 0.1;
|
||||
|
||||
// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
|
||||
// mediaStreamAudioSourceNode.connect(this.analyser);
|
||||
|
||||
// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
|
||||
// }
|
||||
|
||||
// private onAddTrack = (): void => {
|
||||
// this.emit(CallFeedEvent.NewStream, this.stream);
|
||||
// };
|
||||
|
||||
// /**
|
||||
// * Returns callRoom member
|
||||
// * @returns member of the callRoom
|
||||
// */
|
||||
// public getMember(): RoomMember {
|
||||
// const callRoom = this.client.getRoom(this.roomId);
|
||||
// return callRoom.getMember(this.userId);
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Returns true if CallFeed is local, otherwise returns false
|
||||
// * @returns {boolean} is local?
|
||||
// */
|
||||
// public isLocal(): boolean {
|
||||
// return this.userId === this.client.getUserId();
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Returns true if audio is muted or if there are no audio
|
||||
// * tracks, otherwise returns false
|
||||
// * @returns {boolean} is audio muted?
|
||||
// */
|
||||
// public isAudioMuted(): boolean {
|
||||
// return this.stream.getAudioTracks().length === 0 || this.audioMuted;
|
||||
// }
|
||||
|
||||
// *
|
||||
// * Returns true video is muted or if there are no video
|
||||
// * tracks, otherwise returns false
|
||||
// * @returns {boolean} is video muted?
|
||||
|
||||
// public isVideoMuted(): boolean {
|
||||
// // We assume only one video track
|
||||
// return this.stream.getVideoTracks().length === 0 || this.videoMuted;
|
||||
// }
|
||||
|
||||
// public isSpeaking(): boolean {
|
||||
// return this.speaking;
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Replaces the current MediaStream with a new one.
|
||||
// * This method should be only used by MatrixCall.
|
||||
// * @param newStream new stream with which to replace the current one
|
||||
// */
|
||||
// public setNewStream(newStream: MediaStream): void {
|
||||
// this.updateStream(this.stream, newStream);
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Set feed's internal audio mute state
|
||||
// * @param muted is the feed's audio muted?
|
||||
// */
|
||||
// public setAudioMuted(muted: boolean): void {
|
||||
// this.audioMuted = muted;
|
||||
// this.speakingVolumeSamples.fill(-Infinity);
|
||||
// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Set feed's internal video mute state
|
||||
// * @param muted is the feed's video muted?
|
||||
// */
|
||||
// public setVideoMuted(muted: boolean): void {
|
||||
// this.videoMuted = muted;
|
||||
// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
|
||||
// }
|
||||
|
||||
// /**
|
||||
// * Starts emitting volume_changed events where the emitter value is in decibels
|
||||
// * @param enabled emit volume changes
|
||||
// */
|
||||
// public measureVolumeActivity(enabled: boolean): void {
|
||||
// if (enabled) {
|
||||
// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
|
||||
|
||||
// this.measuringVolumeActivity = true;
|
||||
// this.volumeLooper();
|
||||
// } else {
|
||||
// this.measuringVolumeActivity = false;
|
||||
// this.speakingVolumeSamples.fill(-Infinity);
|
||||
// this.emit(CallFeedEvent.VolumeChanged, -Infinity);
|
||||
// }
|
||||
// }
|
||||
|
||||
// public setSpeakingThreshold(threshold: number) {
|
||||
// this.speakingThreshold = threshold;
|
||||
// }
|
||||
|
||||
// private volumeLooper = () => {
|
||||
// if (!this.analyser) return;
|
||||
|
||||
// if (!this.measuringVolumeActivity) return;
|
||||
|
||||
// this.analyser.getFloatFrequencyData(this.frequencyBinCount);
|
||||
|
||||
// let maxVolume = -Infinity;
|
||||
// for (let i = 0; i < this.frequencyBinCount.length; i++) {
|
||||
// if (this.frequencyBinCount[i] > maxVolume) {
|
||||
// maxVolume = this.frequencyBinCount[i];
|
||||
// }
|
||||
// }
|
||||
|
||||
// this.speakingVolumeSamples.shift();
|
||||
// this.speakingVolumeSamples.push(maxVolume);
|
||||
|
||||
// this.emit(CallFeedEvent.VolumeChanged, maxVolume);
|
||||
|
||||
// let newSpeaking = false;
|
||||
|
||||
// for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
|
||||
// const volume = this.speakingVolumeSamples[i];
|
||||
|
||||
// if (volume > this.speakingThreshold) {
|
||||
// newSpeaking = true;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
|
||||
// if (this.speaking !== newSpeaking) {
|
||||
// this.speaking = newSpeaking;
|
||||
// this.emit(CallFeedEvent.Speaking, this.speaking);
|
||||
// }
|
||||
|
||||
// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
|
||||
// };
|
||||
|
||||
// public clone(): CallFeed {
|
||||
// const mediaHandler = this.client.getMediaHandler();
|
||||
// const stream = this.stream.clone();
|
||||
|
||||
// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) {
|
||||
// mediaHandler.userMediaStreams.push(stream);
|
||||
// } else {
|
||||
// mediaHandler.screensharingStreams.push(stream);
|
||||
// }
|
||||
|
||||
// return new CallFeed({
|
||||
// client: this.client,
|
||||
// roomId: this.roomId,
|
||||
// userId: this.userId,
|
||||
// stream,
|
||||
// purpose: this.purpose,
|
||||
// audioMuted: this.audioMuted,
|
||||
// videoMuted: this.videoMuted,
|
||||
// });
|
||||
// }
|
||||
|
||||
// public dispose(): void {
|
||||
// clearTimeout(this.volumeLooperTimeout);
|
||||
// this.measureVolumeActivity(false);
|
||||
// }
|
||||
// }
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices";
|
||||
import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices";
|
||||
import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC";
|
||||
import {Stream, Track, TrackKind} from "../../types/MediaDevices";
|
||||
import {WebRTC, Sender, PeerConnection} from "../../types/WebRTC";
|
||||
import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes";
|
||||
|
||||
const POLLING_INTERVAL = 200; // ms
|
||||
|
|
@ -24,159 +23,21 @@ export const SPEAKING_THRESHOLD = -60; // dB
|
|||
const SPEAKING_SAMPLE_COUNT = 8; // samples
|
||||
|
||||
export class DOMWebRTC implements WebRTC {
|
||||
createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection {
|
||||
return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize);
|
||||
}
|
||||
createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection {
|
||||
return new RTCPeerConnection({
|
||||
iceTransportPolicy: forceTURN ? 'relay' : undefined,
|
||||
iceServers: turnServers,
|
||||
iceCandidatePoolSize: iceCandidatePoolSize,
|
||||
}) as PeerConnection;
|
||||
}
|
||||
|
||||
export class RemoteStreamWrapper extends StreamWrapper {
|
||||
constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) {
|
||||
super(stream);
|
||||
this.stream.addEventListener("removetrack", this.onTrackRemoved);
|
||||
}
|
||||
|
||||
onTrackRemoved = (evt: MediaStreamTrackEvent) => {
|
||||
if (evt.track.id === this.audioTrack?.track.id) {
|
||||
this.audioTrack = undefined;
|
||||
} else if (evt.track.id === this.videoTrack?.track.id) {
|
||||
this.videoTrack = undefined;
|
||||
}
|
||||
if (!this.audioTrack && !this.videoTrack) {
|
||||
this.emptyCallback(this);
|
||||
}
|
||||
};
|
||||
|
||||
dispose() {
|
||||
this.stream.removeEventListener("removetrack", this.onTrackRemoved);
|
||||
}
|
||||
}
|
||||
|
||||
export class DOMStreamSender implements StreamSender {
|
||||
public audioSender: DOMTrackSender | undefined;
|
||||
public videoSender: DOMTrackSender | undefined;
|
||||
|
||||
constructor(public readonly stream: StreamWrapper) {}
|
||||
|
||||
update(transceivers: ReadonlyArray<RTCRtpTransceiver>, sender: RTCRtpSender): DOMTrackSender | undefined {
|
||||
const transceiver = transceivers.find(t => t.sender === sender);
|
||||
if (transceiver && sender.track) {
|
||||
const trackWrapper = this.stream.update(sender.track);
|
||||
if (trackWrapper) {
|
||||
if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) {
|
||||
this.videoSender = new DOMTrackSender(trackWrapper, transceiver);
|
||||
return this.videoSender;
|
||||
} else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) {
|
||||
this.audioSender = new DOMTrackSender(trackWrapper, transceiver);
|
||||
return this.audioSender;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class DOMStreamReceiver implements StreamReceiver {
|
||||
public audioReceiver: DOMTrackReceiver | undefined;
|
||||
public videoReceiver: DOMTrackReceiver | undefined;
|
||||
|
||||
constructor(public readonly stream: RemoteStreamWrapper) {}
|
||||
|
||||
update(event: RTCTrackEvent): DOMTrackReceiver | undefined {
|
||||
const {receiver} = event;
|
||||
const {track} = receiver;
|
||||
const trackWrapper = this.stream.update(track);
|
||||
if (trackWrapper) {
|
||||
if (trackWrapper.kind === TrackKind.Video) {
|
||||
this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver);
|
||||
return this.videoReceiver;
|
||||
} else {
|
||||
this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver);
|
||||
return this.audioReceiver;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class DOMTrackSenderOrReceiver implements TrackReceiver {
|
||||
constructor(
|
||||
public readonly track: TrackWrapper,
|
||||
public readonly transceiver: RTCRtpTransceiver,
|
||||
private readonly exclusiveValue: RTCRtpTransceiverDirection,
|
||||
private readonly excludedValue: RTCRtpTransceiverDirection
|
||||
) {}
|
||||
|
||||
get enabled(): boolean {
|
||||
return this.transceiver.direction === "sendrecv" ||
|
||||
this.transceiver.direction === this.exclusiveValue;
|
||||
}
|
||||
|
||||
enableWithoutRenegotiation(enabled: boolean) {
|
||||
this.track.track.enabled = enabled;
|
||||
}
|
||||
|
||||
enable(enabled: boolean) {
|
||||
if (enabled !== this.enabled) {
|
||||
// do this first, so we stop sending track data immediately.
|
||||
// this will still consume bandwidth though, so also disable the transceiver,
|
||||
// which will trigger a renegotiation though.
|
||||
this.enableWithoutRenegotiation(enabled);
|
||||
if (enabled) {
|
||||
if (this.transceiver.direction === "inactive") {
|
||||
this.transceiver.direction = this.exclusiveValue;
|
||||
} else {
|
||||
this.transceiver.direction = "sendrecv";
|
||||
}
|
||||
} else {
|
||||
if (this.transceiver.direction === "sendrecv") {
|
||||
this.transceiver.direction = this.excludedValue;
|
||||
} else {
|
||||
this.transceiver.direction = "inactive";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class DOMTrackReceiver extends DOMTrackSenderOrReceiver {
|
||||
constructor(
|
||||
track: TrackWrapper,
|
||||
transceiver: RTCRtpTransceiver,
|
||||
) {
|
||||
super(track, transceiver, "recvonly", "sendonly");
|
||||
}
|
||||
}
|
||||
|
||||
export class DOMTrackSender extends DOMTrackSenderOrReceiver {
|
||||
constructor(
|
||||
track: TrackWrapper,
|
||||
transceiver: RTCRtpTransceiver,
|
||||
) {
|
||||
super(track, transceiver, "sendonly", "recvonly");
|
||||
}
|
||||
/** replaces the track if possible without renegotiation. Can throw. */
|
||||
replaceTrack(track: Track | undefined): Promise<void> {
|
||||
return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null);
|
||||
}
|
||||
|
||||
prepareForPurpose(purpose: SDPStreamMetadataPurpose): void {
|
||||
prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void {
|
||||
if (purpose === SDPStreamMetadataPurpose.Screenshare) {
|
||||
this.getRidOfRTXCodecs();
|
||||
this.getRidOfRTXCodecs(peerConnection as RTCPeerConnection, sender as RTCRtpSender);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method removes all video/rtx codecs from screensharing video
|
||||
* transceivers. This is necessary since they can cause problems. Without
|
||||
* this the following steps should produce an error:
|
||||
* Chromium calls Firefox
|
||||
* Firefox answers
|
||||
* Firefox starts screen-sharing
|
||||
* Chromium starts screen-sharing
|
||||
* Call crashes for Chromium with:
|
||||
* [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list.
|
||||
* [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs.
|
||||
* [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER)
|
||||
*/
|
||||
private getRidOfRTXCodecs(): void {
|
||||
private getRidOfRTXCodecs(peerConnection: RTCPeerConnection, sender: RTCRtpSender): void {
|
||||
// RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF
|
||||
if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return;
|
||||
|
||||
|
|
@ -190,172 +51,14 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver {
|
|||
codecs.splice(rtxCodecIndex, 1);
|
||||
}
|
||||
}
|
||||
if (this.transceiver.sender.track?.kind === "video" ||
|
||||
this.transceiver.receiver.track?.kind === "video") {
|
||||
this.transceiver.setCodecPreferences(codecs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class DOMPeerConnection implements PeerConnection {
|
||||
private readonly peerConnection: RTCPeerConnection;
|
||||
private readonly handler: PeerConnectionHandler;
|
||||
public readonly localStreams: Map<string, DOMStreamSender> = new Map();
|
||||
public readonly remoteStreams: Map<string, DOMStreamReceiver> = new Map();
|
||||
|
||||
constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) {
|
||||
this.handler = handler;
|
||||
this.peerConnection = new RTCPeerConnection({
|
||||
iceTransportPolicy: forceTURN ? 'relay' : undefined,
|
||||
iceServers: turnServers,
|
||||
iceCandidatePoolSize: iceCandidatePoolSize,
|
||||
});
|
||||
this.registerHandler();
|
||||
}
|
||||
|
||||
get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; }
|
||||
get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; }
|
||||
get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; }
|
||||
|
||||
createOffer(): Promise<RTCSessionDescriptionInit> {
|
||||
return this.peerConnection.createOffer();
|
||||
}
|
||||
|
||||
createAnswer(): Promise<RTCSessionDescriptionInit> {
|
||||
return this.peerConnection.createAnswer();
|
||||
}
|
||||
|
||||
setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void> {
|
||||
return this.peerConnection.setLocalDescription(description);
|
||||
}
|
||||
|
||||
setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void> {
|
||||
return this.peerConnection.setRemoteDescription(description);
|
||||
}
|
||||
|
||||
addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
|
||||
return this.peerConnection.addIceCandidate(candidate);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
return this.peerConnection.close();
|
||||
}
|
||||
|
||||
addTrack(track: Track): DOMTrackSender | undefined {
|
||||
if (!(track instanceof TrackWrapper)) {
|
||||
throw new Error("Not a TrackWrapper");
|
||||
}
|
||||
const sender = this.peerConnection.addTrack(track.track, track.stream);
|
||||
let streamSender = this.localStreams.get(track.stream.id);
|
||||
if (!streamSender) {
|
||||
// TODO: reuse existing stream wrapper here?
|
||||
streamSender = new DOMStreamSender(new StreamWrapper(track.stream));
|
||||
this.localStreams.set(track.stream.id, streamSender);
|
||||
}
|
||||
const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender);
|
||||
return trackSender;
|
||||
}
|
||||
|
||||
removeTrack(sender: TrackSender): void {
|
||||
if (!(sender instanceof DOMTrackSender)) {
|
||||
throw new Error("Not a DOMTrackSender");
|
||||
}
|
||||
this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender);
|
||||
// TODO: update localStreams
|
||||
}
|
||||
|
||||
createDataChannel(options: RTCDataChannelInit): any {
|
||||
return this.peerConnection.createDataChannel("channel", options);
|
||||
}
|
||||
|
||||
private registerHandler() {
|
||||
const pc = this.peerConnection;
|
||||
pc.addEventListener('negotiationneeded', this);
|
||||
pc.addEventListener('icecandidate', this);
|
||||
pc.addEventListener('iceconnectionstatechange', this);
|
||||
pc.addEventListener('icegatheringstatechange', this);
|
||||
pc.addEventListener('signalingstatechange', this);
|
||||
pc.addEventListener('track', this);
|
||||
pc.addEventListener('datachannel', this);
|
||||
}
|
||||
|
||||
private deregisterHandler() {
|
||||
const pc = this.peerConnection;
|
||||
pc.removeEventListener('negotiationneeded', this);
|
||||
pc.removeEventListener('icecandidate', this);
|
||||
pc.removeEventListener('iceconnectionstatechange', this);
|
||||
pc.removeEventListener('icegatheringstatechange', this);
|
||||
pc.removeEventListener('signalingstatechange', this);
|
||||
pc.removeEventListener('track', this);
|
||||
pc.removeEventListener('datachannel', this);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
handleEvent(evt: Event) {
|
||||
switch (evt.type) {
|
||||
case "iceconnectionstatechange":
|
||||
this.handleIceConnectionStateChange();
|
||||
break;
|
||||
case "icecandidate":
|
||||
this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent);
|
||||
break;
|
||||
case "icegatheringstatechange":
|
||||
this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState);
|
||||
break;
|
||||
case "track":
|
||||
this.handleRemoteTrack(evt as RTCTrackEvent);
|
||||
break;
|
||||
case "negotiationneeded":
|
||||
this.handler.onNegotiationNeeded();
|
||||
break;
|
||||
case "datachannel":
|
||||
this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.deregisterHandler();
|
||||
for (const r of this.remoteStreams.values()) {
|
||||
r.stream.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) {
|
||||
if (event.candidate) {
|
||||
this.handler.onLocalIceCandidate(event.candidate);
|
||||
}
|
||||
};
|
||||
|
||||
private handleIceConnectionStateChange() {
|
||||
const {iceConnectionState} = this.peerConnection;
|
||||
if (iceConnectionState === "failed" && this.peerConnection.restartIce) {
|
||||
this.peerConnection.restartIce();
|
||||
} else {
|
||||
this.handler.onIceConnectionStateChange(iceConnectionState);
|
||||
}
|
||||
}
|
||||
|
||||
onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => {
|
||||
if (this.remoteStreams.delete(stream.id)) {
|
||||
this.handler.onRemoteStreamRemoved(stream);
|
||||
}
|
||||
}
|
||||
|
||||
private handleRemoteTrack(evt: RTCTrackEvent) {
|
||||
if (evt.streams.length !== 1) {
|
||||
throw new Error("track in multiple streams is not supported");
|
||||
}
|
||||
const stream = evt.streams[0];
|
||||
const transceivers = this.peerConnection.getTransceivers();
|
||||
let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id);
|
||||
if (!streamReceiver) {
|
||||
streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty));
|
||||
this.remoteStreams.set(stream.id, streamReceiver);
|
||||
}
|
||||
const trackReceiver = streamReceiver.update(evt);
|
||||
if (trackReceiver) {
|
||||
this.handler.onRemoteTracksAdded(trackReceiver);
|
||||
const transceiver = peerConnection.getTransceivers().find(t => t.sender === sender);
|
||||
if (transceiver && (
|
||||
transceiver.sender.track?.kind === "video" ||
|
||||
transceiver.receiver.track?.kind === "video"
|
||||
)
|
||||
) {
|
||||
transceiver.setCodecPreferences(codecs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,15 +17,15 @@ limitations under the License.
|
|||
import {TemplateView, TemplateBuilder} from "../../general/TemplateView";
|
||||
import {ListView} from "../../general/ListView";
|
||||
import {Stream} from "../../../../types/MediaDevices";
|
||||
import type {StreamWrapper} from "../../../dom/MediaDevices";
|
||||
import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../../matrix/calls/common";
|
||||
import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel";
|
||||
|
||||
function bindStream<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) {
|
||||
t.mapSideEffect(vm => propSelector(vm)?.videoTrack?.enabled, (_,__, vm) => {
|
||||
t.mapSideEffect(vm => getStreamVideoTrack(propSelector(vm))?.enabled, (_,__, vm) => {
|
||||
const stream = propSelector(vm);
|
||||
if (stream) {
|
||||
video.srcObject = (stream as StreamWrapper).stream;
|
||||
if (stream.videoTrack?.enabled) {
|
||||
video.srcObject = stream as MediaStream;
|
||||
if (getStreamVideoTrack(stream)?.enabled) {
|
||||
video.classList.remove("hidden");
|
||||
} else {
|
||||
video.classList.add("hidden");
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
/*
|
||||
Copyright 2022 The Matrix.org Foundation C.I.C.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
export class AsyncQueue<T, V> {
|
||||
private isRunning = false;
|
||||
private queue: T[] = [];
|
||||
private error?: Error;
|
||||
|
||||
constructor(
|
||||
private readonly reducer: (v: V, t: T) => Promise<V>,
|
||||
private value: V,
|
||||
private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t)
|
||||
) {}
|
||||
|
||||
push(t: T) {
|
||||
if (this.contains(t, this.queue)) {
|
||||
return;
|
||||
}
|
||||
this.queue.push(t);
|
||||
this.runLoopIfNeeded();
|
||||
}
|
||||
|
||||
private async runLoopIfNeeded() {
|
||||
if (this.isRunning || this.error) {
|
||||
return;
|
||||
}
|
||||
this.isRunning = true;
|
||||
try {
|
||||
let item: T | undefined;
|
||||
while (item = this.queue.shift()) {
|
||||
this.value = await this.reducer(this.value, item);
|
||||
}
|
||||
} catch (err) {
|
||||
this.error = err;
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue