This commit is contained in:
Bruno Windels 2022-02-18 16:38:10 +01:00 committed by Bruno Windels
parent e5f44aecfb
commit 98e1dcf799
6 changed files with 404 additions and 2287 deletions

File diff suppressed because it is too large Load diff

View file

@ -1,3 +1,26 @@
## TODO
- PeerCall
- send invite
- find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether
we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer.
- handle receiving offer and send anwser
- handle sending ice candidates
- handle ice candidates finished (iceGatheringState === 'complete')
- handle receiving ice candidates
- handle sending renegotiation
- handle receiving renegotiation
- reject call
- hangup call
- handle muting tracks
- handle remote track being muted
- handle adding/removing tracks to an ongoing call
- handle sdp metadata
- Participant
- handle glare
- encrypt to_device message with olm
- batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute)
- find out if we should start muted or not?
## Store ongoing calls ## Store ongoing calls
Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing. Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing.

View file

@ -14,6 +14,9 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {EventType} from "../PeerCall";
import type {PeerCallHandler} from "../PeerCall";
class Participant implements PeerCallHandler { class Participant implements PeerCallHandler {
private peerCall?: PeerCall; private peerCall?: PeerCall;
@ -27,19 +30,18 @@ class Participant implements PeerCallHandler {
sendInvite() { sendInvite() {
this.peerCall = new PeerCall(this, this.webRTC); this.peerCall = new PeerCall(this, this.webRTC);
this.peerCall.setLocalMedia(this.localMedia); this.peerCall.call(this.localMedia);
this.peerCall.sendOffer();
} }
/** From PeerCallHandler /** From PeerCallHandler
* @internal */ * @internal */
override emitUpdate() { emitUpdate(params: any) {
} }
/** From PeerCallHandler /** From PeerCallHandler
* @internal */ * @internal */
override onSendSignallingMessage() { onSendSignallingMessage(type: EventType, content: Record<string, any>) {
// TODO: this needs to be encrypted with olm first // TODO: this needs to be encrypted with olm first
this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}}); this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
} }

View file

@ -33,7 +33,7 @@ export interface PeerConnectionHandler {
onDataChannelChanged(dataChannel: DataChannel | undefined); onDataChannelChanged(dataChannel: DataChannel | undefined);
onNegotiationNeeded(); onNegotiationNeeded();
// request the type of incoming stream // request the type of incoming stream
getPurposeForStreamId(trackId: string): StreamPurpose; getPurposeForStreamId(streamId: string): StreamPurpose;
} }
// does it make sense to wrap this? // does it make sense to wrap this?
export interface DataChannel { export interface DataChannel {
@ -46,11 +46,12 @@ export interface PeerConnection {
get dataChannel(): DataChannel | undefined; get dataChannel(): DataChannel | undefined;
createOffer(): Promise<RTCSessionDescriptionInit>; createOffer(): Promise<RTCSessionDescriptionInit>;
createAnswer(): Promise<RTCSessionDescriptionInit>; createAnswer(): Promise<RTCSessionDescriptionInit>;
setLocalDescription(description: RTCSessionDescriptionInit); setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void>;
setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>; setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>;
addIceCandidate(candidate: RTCIceCandidate): Promise<void>; addIceCandidate(candidate: RTCIceCandidate): Promise<void>;
addTrack(track: Track): void; addTrack(track: Track): void;
removeTrack(track: Track): boolean; removeTrack(track: Track): boolean;
replaceTrack(oldTrack: Track, newTrack: Track): Promise<boolean>; replaceTrack(oldTrack: Track, newTrack: Track): Promise<boolean>;
createDataChannel(): DataChannel; createDataChannel(): DataChannel;
dispose(): void;
} }

View file

@ -49,12 +49,12 @@ class DOMPeerConnection implements PeerConnection {
return this.peerConnection.createAnswer(); return this.peerConnection.createAnswer();
} }
setLocalDescription(description: RTCSessionDescriptionInit) { setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void> {
this.peerConnection.setLocalDescription(description); return this.peerConnection.setLocalDescription(description);
} }
setRemoteDescription(description: RTCSessionDescriptionInit) { setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void> {
this.peerConnection.setRemoteDescription(description); return this.peerConnection.setRemoteDescription(description);
} }
addIceCandidate(candidate: RTCIceCandidate): Promise<void> { addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
@ -66,6 +66,9 @@ class DOMPeerConnection implements PeerConnection {
throw new Error("Not a TrackWrapper"); throw new Error("Not a TrackWrapper");
} }
this.peerConnection.addTrack(track.track, track.stream); this.peerConnection.addTrack(track.track, track.stream);
if (track.type === TrackType.ScreenShare) {
this.getRidOfRTXCodecs(track);
}
} }
removeTrack(track: Track): boolean { removeTrack(track: Track): boolean {
@ -87,6 +90,9 @@ class DOMPeerConnection implements PeerConnection {
const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track); const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track);
if (sender) { if (sender) {
await sender.replaceTrack(newTrack.track); await sender.replaceTrack(newTrack.track);
if (newTrack.type === TrackType.ScreenShare) {
this.getRidOfRTXCodecs(newTrack);
}
return true; return true;
} }
return false; return false;
@ -106,6 +112,17 @@ class DOMPeerConnection implements PeerConnection {
pc.addEventListener('datachannel', this); pc.addEventListener('datachannel', this);
} }
private deregisterHandler() {
const pc = this.peerConnection;
pc.removeEventListener('negotiationneeded', this);
pc.removeEventListener('icecandidate', this);
pc.removeEventListener('iceconnectionstatechange', this);
pc.removeEventListener('icegatheringstatechange', this);
pc.removeEventListener('signalingstatechange', this);
pc.removeEventListener('track', this);
pc.removeEventListener('datachannel', this);
}
/** @internal */ /** @internal */
handleEvent(evt: Event) { handleEvent(evt: Event) {
switch (evt.type) { switch (evt.type) {
@ -129,6 +146,10 @@ class DOMPeerConnection implements PeerConnection {
} }
} }
dispose(): void {
this.deregisterHandler();
}
private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) {
if (event.candidate) { if (event.candidate) {
this.handler.onLocalIceCandidate(event.candidate); this.handler.onLocalIceCandidate(event.candidate);
@ -145,10 +166,15 @@ class DOMPeerConnection implements PeerConnection {
} }
private handleRemoteTrack(evt: RTCTrackEvent) { private handleRemoteTrack(evt: RTCTrackEvent) {
// the tracks on the new stream (with their stream)
const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};})); const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};}));
const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track)); // of the tracks we already know about, filter the ones that aren't in the new stream
const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track)); const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id));
// of the new tracks, filter the ones that we didn't already knew about
const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id));
// wrap them
const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream)); const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream));
// and concat the tracks for other streams with the added tracks
this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks); this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks);
this.handler.onRemoteTracksChanged(this.remoteTracks); this.handler.onRemoteTracksChanged(this.remoteTracks);
} }
@ -162,4 +188,44 @@ class DOMPeerConnection implements PeerConnection {
} }
return wrapTrack(track, stream, type); return wrapTrack(track, stream, type);
} }
/**
* This method removes all video/rtx codecs from screensharing video
* transceivers. This is necessary since they can cause problems. Without
* this the following steps should produce an error:
* Chromium calls Firefox
* Firefox answers
* Firefox starts screen-sharing
* Chromium starts screen-sharing
* Call crashes for Chromium with:
* [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list.
* [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs.
* [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER)
*/
private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void {
// RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF
if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return;
const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? [];
const codecs = [...sendCodecs, ...recvCodecs];
for (const codec of codecs) {
if (codec.mimeType === "video/rtx") {
const rtxCodecIndex = codecs.indexOf(codec);
codecs.splice(rtxCodecIndex, 1);
}
}
for (const trans of this.peerConnection.getTransceivers()) {
if (trans.sender.track === screensharingTrack.track &&
(
trans.sender.track?.kind === "video" ||
trans.receiver.track?.kind === "video"
)
) {
trans.setCodecPreferences(codecs);
}
}
}
} }

52
src/utils/AsyncQueue.ts Normal file
View file

@ -0,0 +1,52 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
export class AsyncQueue<T, V> {
private isRunning = false;
private queue: T[] = [];
private error?: Error;
constructor(
private readonly reducer: (v: V, t: T) => Promise<V>,
private value: V,
private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t)
) {}
push(t: T) {
if (this.contains(t, this.queue)) {
return;
}
this.queue.push(t);
this.runLoopIfNeeded();
}
private async runLoopIfNeeded() {
if (this.isRunning || this.error) {
return;
}
this.isRunning = true;
try {
let item: T | undefined;
while (item = this.queue.shift()) {
this.value = await this.reducer(this.value, item);
}
} catch (err) {
this.error = err;
} finally {
this.isRunning = false;
}
}
}