WIP5

src/matrix/calls/TODO.md

@@ -1,3 +1,26 @@
+## TODO
+ - PeerCall
+    - send invite
+    - find out if we need to do something different when renegotation is triggered (a subsequent onnegotiationneeded event) whether
+      we sent the invite/offer or answer. e.g. do we always do createOffer/setLocalDescription and then send it over a matrix negotiation event? even if we before called createAnswer.
+    - handle receiving offer and send anwser
+    - handle sending ice candidates
+        - handle ice candidates finished (iceGatheringState === 'complete')
+    - handle receiving ice candidates
+    - handle sending renegotiation
+    - handle receiving renegotiation
+    - reject call
+    - hangup call
+    - handle muting tracks
+    - handle remote track being muted
+    - handle adding/removing tracks to an ongoing call
+    - handle sdp metadata
+ - Participant
+    - handle glare
+    - encrypt to_device message with olm
+    - batch outgoing to_device messages in one request to homeserver for operations that will send out an event to all participants (e.g. mute)
+    - find out if we should start muted or not?
+
 ## Store ongoing calls
 
 Add store with all ongoing calls so when we quit and start again, we don't have to go through all the past calls to know which ones might still be ongoing.

src/matrix/calls/group/Participant.ts

@@ -14,6 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import {EventType} from "../PeerCall";
+import type {PeerCallHandler} from "../PeerCall";
+
 class Participant implements PeerCallHandler {
     private peerCall?: PeerCall;
 
@@ -27,19 +30,18 @@ class Participant implements PeerCallHandler {
 
     sendInvite() {
         this.peerCall = new PeerCall(this, this.webRTC);
-        this.peerCall.setLocalMedia(this.localMedia);
+        this.peerCall.call(this.localMedia);
-        this.peerCall.sendOffer();
     }
 
     /** From PeerCallHandler
      * @internal */
-    override emitUpdate() {
+    emitUpdate(params: any) {
 
     }
 
     /** From PeerCallHandler
      * @internal */
-    override onSendSignallingMessage() {
+    onSendSignallingMessage(type: EventType, content: Record<string, any>) {
         // TODO: this needs to be encrypted with olm first
         this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
     }

src/platform/types/WebRTC.ts

@@ -33,7 +33,7 @@ export interface PeerConnectionHandler {
     onDataChannelChanged(dataChannel: DataChannel | undefined);
     onNegotiationNeeded();
     // request the type of incoming stream
-    getPurposeForStreamId(trackId: string): StreamPurpose;
+    getPurposeForStreamId(streamId: string): StreamPurpose;
 }
 // does it make sense to wrap this?
 export interface DataChannel {
@@ -46,11 +46,12 @@ export interface PeerConnection {
     get dataChannel(): DataChannel | undefined;
     createOffer(): Promise<RTCSessionDescriptionInit>;
     createAnswer(): Promise<RTCSessionDescriptionInit>;
-    setLocalDescription(description: RTCSessionDescriptionInit);
+    setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void>;
     setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>;
     addIceCandidate(candidate: RTCIceCandidate): Promise<void>;
     addTrack(track: Track): void;
     removeTrack(track: Track): boolean;
     replaceTrack(oldTrack: Track, newTrack: Track): Promise<boolean>;
     createDataChannel(): DataChannel;
+    dispose(): void;
 }

src/platform/web/dom/WebRTC.ts

@@ -49,12 +49,12 @@ class DOMPeerConnection implements PeerConnection {
         return this.peerConnection.createAnswer();
     }
 
-    setLocalDescription(description: RTCSessionDescriptionInit) {
+    setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void> {
-        this.peerConnection.setLocalDescription(description);
+        return this.peerConnection.setLocalDescription(description);
     }
 
-    setRemoteDescription(description: RTCSessionDescriptionInit) {
+    setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void> {
-        this.peerConnection.setRemoteDescription(description);
+        return this.peerConnection.setRemoteDescription(description);
     }
 
     addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
@@ -66,6 +66,9 @@ class DOMPeerConnection implements PeerConnection {
             throw new Error("Not a TrackWrapper");
         }
         this.peerConnection.addTrack(track.track, track.stream);
+        if (track.type === TrackType.ScreenShare) {
+            this.getRidOfRTXCodecs(track);
+        }
     }
 
     removeTrack(track: Track): boolean {
@@ -87,6 +90,9 @@ class DOMPeerConnection implements PeerConnection {
         const sender = this.peerConnection.getSenders().find(s => s.track === oldTrack.track);
         if (sender) {
             await sender.replaceTrack(newTrack.track);
+            if (newTrack.type === TrackType.ScreenShare) {
+                this.getRidOfRTXCodecs(newTrack);
+            }
             return true;
         }
         return false;
@@ -106,6 +112,17 @@ class DOMPeerConnection implements PeerConnection {
         pc.addEventListener('datachannel', this);
     }
 
+    private deregisterHandler() {
+        const pc = this.peerConnection;
+        pc.removeEventListener('negotiationneeded', this);
+        pc.removeEventListener('icecandidate', this);
+        pc.removeEventListener('iceconnectionstatechange', this);
+        pc.removeEventListener('icegatheringstatechange', this);
+        pc.removeEventListener('signalingstatechange', this);
+        pc.removeEventListener('track', this);
+        pc.removeEventListener('datachannel', this);
+    }
+
     /** @internal */
     handleEvent(evt: Event) {
         switch (evt.type) {
@@ -129,6 +146,10 @@ class DOMPeerConnection implements PeerConnection {
         }
     }
 
+    dispose(): void {
+        this.deregisterHandler();
+    }
+
     private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) {
         if (event.candidate) {
             this.handler.onLocalIceCandidate(event.candidate);
@@ -145,10 +166,15 @@ class DOMPeerConnection implements PeerConnection {
     }
 
     private handleRemoteTrack(evt: RTCTrackEvent) {
+        // the tracks on the new stream (with their stream)
         const updatedTracks = evt.streams.flatMap(stream => stream.getTracks().map(track => {return {stream, track};}));
-        const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track == ut.track));
+        // of the tracks we already know about, filter the ones that aren't in the new stream
-        const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track === ut.track));
+        const withoutRemovedTracks = this._remoteTracks.filter(t => !updatedTracks.some(ut => t.track.id === ut.track.id));
+        // of the new tracks, filter the ones that we didn't already knew about
+        const addedTracks = updatedTracks.filter(ut => !this._remoteTracks.some(t => t.track.id === ut.track.id));
+        // wrap them
         const wrappedAddedTracks = addedTracks.map(t => this.wrapRemoteTrack(t.track, t.stream));
+        // and concat the tracks for other streams with the added tracks
         this._remoteTracks = withoutRemovedTracks.concat(...wrappedAddedTracks);
         this.handler.onRemoteTracksChanged(this.remoteTracks);
     }
@@ -162,4 +188,44 @@ class DOMPeerConnection implements PeerConnection {
         }
         return wrapTrack(track, stream, type);
     }
+
+    /**
+     * This method removes all video/rtx codecs from screensharing video
+     * transceivers. This is necessary since they can cause problems. Without
+     * this the following steps should produce an error:
+     *   Chromium calls Firefox
+     *   Firefox answers
+     *   Firefox starts screen-sharing
+     *   Chromium starts screen-sharing
+     *   Call crashes for Chromium with:
+     *       [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list.
+     *       [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs.
+     *       [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER)
+     */
+    private getRidOfRTXCodecs(screensharingTrack: TrackWrapper): void {
+        // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF
+        if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return;
+
+        const recvCodecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
+        const sendCodecs = RTCRtpSender.getCapabilities("video")?.codecs ?? [];
+        const codecs = [...sendCodecs, ...recvCodecs];
+
+        for (const codec of codecs) {
+            if (codec.mimeType === "video/rtx") {
+                const rtxCodecIndex = codecs.indexOf(codec);
+                codecs.splice(rtxCodecIndex, 1);
+            }
+        }
+
+        for (const trans of this.peerConnection.getTransceivers()) {
+            if (trans.sender.track === screensharingTrack.track &&
+                (
+                    trans.sender.track?.kind === "video" ||
+                    trans.receiver.track?.kind === "video"
+                )
+            ) {
+                trans.setCodecPreferences(codecs);
+            }
+        }
+    }
 }

src/utils/AsyncQueue.ts

@@ -0,0 +1,52 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export class AsyncQueue<T, V> {
+    private isRunning = false;
+    private queue: T[] = [];
+    private error?: Error;
+
+    constructor(
+        private readonly reducer: (v: V, t: T) => Promise<V>,
+        private value: V,
+        private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t)
+    ) {}
+
+    push(t: T) {
+        if (this.contains(t, this.queue)) {
+            return;
+        }
+        this.queue.push(t);
+        this.runLoopIfNeeded();
+    }
+
+    private async runLoopIfNeeded() {
+        if (this.isRunning || this.error) {
+            return;
+        }
+        this.isRunning = true;
+        try {
+            let item: T | undefined;
+            while (item = this.queue.shift()) {
+                this.value = await this.reducer(this.value, item);
+            }
+        } catch (err) {
+            this.error = err;
+        } finally {
+            this.isRunning = false;
+        }
+    }
+}