Merge branch 'bwindels/calls-wip' into bwindels/calls-thinner-abstraction

src/matrix/calls/LocalMedia.ts

@@ -43,13 +43,43 @@ export class LocalMedia {
         return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options);
     }
 
+    replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia {
+        let userMedia;
+        let screenShare;
+        const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => {
+            let stream;
+            if (oldOriginalStream?.id === newStream?.id) {
+                stream = oldCloneStream;
+            } else {
+                stream = newStream?.clone();
+                oldCloneStream?.audioTrack?.stop();
+                oldCloneStream?.videoTrack?.stop();
+            }
+            return stream;
+        }
+        return new LocalMedia(
+            cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia),
+            this.microphoneMuted, this.cameraMuted,
+            cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare),
+            this.dataChannelOptions
+        );
+    }
+
     clone(): LocalMedia {
         return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions);
     }
     
     dispose() {
-        this.userMedia?.audioTrack?.stop();
+        this.stopExcept(undefined);
-        this.userMedia?.videoTrack?.stop();
+    }
-        this.screenShare?.videoTrack?.stop();
+
+    stopExcept(newMedia: LocalMedia | undefined) {
+        if(newMedia?.userMedia?.id !== this.userMedia?.id) {
+            this.userMedia?.audioTrack?.stop();
+            this.userMedia?.videoTrack?.stop();
+        }
+        if(newMedia?.screenShare?.id !== this.screenShare?.id) {
+            this.screenShare?.videoTrack?.stop();
+        }
     }
 }

src/matrix/calls/PeerCall.ts

@@ -18,24 +18,24 @@ import {ObservableMap} from "../../observable/map/ObservableMap";
 import {recursivelyAssign} from "../../utils/recursivelyAssign";
 import {AsyncQueue} from "../../utils/AsyncQueue";
 import {Disposables, IDisposable} from "../../utils/Disposables";
-import type {Room} from "../room/Room";
-import type {StateEvent} from "../storage/types";
-import type {ILogItem} from "../../logging/types";
-
-import type {TimeoutCreator, Timeout} from "../../platform/types/types";
 import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC";
 import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices";
-import type {LocalMedia} from "./LocalMedia";
-
 import {
     SDPStreamMetadataKey,
     SDPStreamMetadataPurpose,
     EventType,
     CallErrorCode,
 } from "./callEventTypes";
+
+import type {Room} from "../room/Room";
+import type {StateEvent} from "../storage/types";
+import type {ILogItem} from "../../logging/types";
+import type {TimeoutCreator, Timeout} from "../../platform/types/types";
+import type {LocalMedia} from "./LocalMedia";
 import type {
     MCallBase,
     MCallInvite,
+    MCallNegotiate,
     MCallAnswer,
     MCallSDPStreamMetadataChanged,
     MCallCandidates,
@@ -217,24 +217,14 @@ export class PeerCall implements IDisposable {
             log.set("userMedia_video_muted", localMedia.cameraMuted);
             log.set("screenShare_video", !!localMedia.screenShare?.videoTrack);
             log.set("datachannel", !!localMedia.dataChannelOptions);
-
+            await this.updateLocalMedia(localMedia, log);
-            const oldMetaData = this.getSDPMetadata();
+            const content: MCallSDPStreamMetadataChanged<MCallBase> = {
-            const willRenegotiate = await this.updateLocalMedia(localMedia, log);
+                call_id: this.callId,
-            // TODO: if we will renegotiate, we don't bother sending the metadata changed event
+                version: 1,
-            // because the renegotiate event will send new metadata anyway, but is that the right
+                seq: this.seq++,
-            // call?
+                [SDPStreamMetadataKey]: this.getSDPMetadata()
-            if (!willRenegotiate) {
+            };
-                const newMetaData = this.getSDPMetadata();
+            await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
-                if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) {
-                    const content: MCallSDPStreamMetadataChanged<MCallBase> = {
-                        call_id: this.callId,
-                        version: 1,
-                        seq: this.seq++,
-                        [SDPStreamMetadataKey]: newMetaData
-                    };
-                    await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
-                }
-            }
         });
     }
 
@@ -265,6 +255,9 @@ export class PeerCall implements IDisposable {
                 case EventType.Answer:
                     await this.handleAnswer(message.content, partyId, log);
                     break;
+                case EventType.Negotiate:
+                    await this.handleRemoteNegotiate(message.content, partyId, log);
+                    break;
                 case EventType.Candidates:
                     await this.handleRemoteIceCandidates(message.content, partyId, log);
                     break;
@@ -341,10 +334,10 @@ export class PeerCall implements IDisposable {
                 await this.sendSignallingMessage({type: EventType.Invite, content}, log);
                 this.setState(CallState.InviteSent, log);
             } else if (this._state === CallState.Connected || this._state === CallState.Connecting) {
-                log.log("would send renegotiation now but not implemented");
                 // send Negotiate message
-                //await this.sendSignallingMessage({type: EventType.Invite, content});
+                content.description = content.offer;
-                //this.setState(CallState.InviteSent);
+                delete content.offer;
+                await this.sendSignallingMessage({type: EventType.Negotiate, content}, log);
             }
         } finally {
             this.makingOffer = false;
@@ -497,6 +490,36 @@ export class PeerCall implements IDisposable {
         }
     }
 
+
+    private async handleRemoteNegotiate(content: MCallNegotiate<MCallBase>, partyId: PartyId, log: ILogItem): Promise<void> {
+        if (this._state !== CallState.Connected) {
+            log.log({l: `Ignoring renegotiate because not connected`, status: this._state});
+            return;
+        }
+
+        if (this.opponentPartyId !== partyId) {
+            log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`);
+            return;
+        }
+
+        const sdpStreamMetadata = content[SDPStreamMetadataKey];
+        if (sdpStreamMetadata) {
+            this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log);
+        } else {
+            log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`);
+        }
+
+        try {
+            await this.peerConnection.setRemoteDescription(content.description);
+        } catch (e) {
+            await log.wrap(`Failed to set remote description`, log => {
+                log.catch(e);
+                this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
+            });
+            return;
+        }
+    }
+
     private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) {
         if (state === 'complete' && !this.sentEndOfCandidates) {
             // If we didn't get an empty-string candidate to signal the end of candidates,
@@ -823,8 +846,8 @@ export class PeerCall implements IDisposable {
             const streamSender = this.peerConnection.localStreams.get(streamId);
             metadata[streamId] = {
                 purpose: SDPStreamMetadataPurpose.Usermedia,
-                audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled),
+                audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack,
-                video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled),
+                video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack,
             };
         }
         if (this.localMedia?.screenShare) {
@@ -856,9 +879,8 @@ export class PeerCall implements IDisposable {
         this.options.emitUpdate(this, undefined);
     }
 
-    private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<boolean> {
+    private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<void> {
         return logItem.wrap("updateLocalMedia", async log => {
-            let willRenegotiate = false;
             const oldMedia = this.localMedia;
             this.localMedia = localMedia;
             const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => {
@@ -887,13 +909,11 @@ export class PeerCall implements IDisposable {
                                     log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => {
                                         this.peerConnection.removeTrack(sender);
                                         this.peerConnection.addTrack(track);
-                                        willRenegotiate = true;
                                     });
                                 }
                             } else {
                                 log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
                                     this.peerConnection.addTrack(track);
-                                    willRenegotiate = true;
                                 });
                             }
                         } else {
@@ -904,20 +924,17 @@ export class PeerCall implements IDisposable {
                                 log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => {
                                     sender.track.enabled = false;
                                     this.peerConnection.removeTrack(sender);
-                                    willRenegotiate = true;
                                 });
                             }
                         }
                     } else if (track) {
                         log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id});
                         if (sender && muted !== wasMuted) {
-                            // TODO: why does unmuting not work? wasMuted is false
                             log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => {
                                 // sender.track.enabled = !muted;
                                 // This doesn't always seem to trigger renegotiation??
                                 // We should probably always send the new metadata first ...
                                 sender.enable(!muted);
-                                willRenegotiate = true;
                             });
                         } else {
                             log.log(`${logLabel} ${track.kind} track hasn't changed`);
@@ -931,7 +948,6 @@ export class PeerCall implements IDisposable {
 
             await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia");
             await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare");
-            return willRenegotiate;
             // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
         });
     }

src/matrix/calls/TODO.md

@@ -9,14 +9,14 @@
 
 ## TODO
  - DONE: implement receiving hangup
- - making logging better
+ - DONE: implement cloning the localMedia so it works in safari?
+ - DONE: implement 3 retries per peer
+ - implement muting tracks with m.call.sdp_stream_metadata_changed
  - implement renegotiation
+ - making logging better
  - finish session id support
     - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
  - implement to_device messages arriving before m.call(.member) state event
- - implement muting tracks with m.call.sdp_stream_metadata_changed
- - DONE: implement cloning the localMedia so it works in safari?
- - DONE: implement 3 retries per peer
  - reeable crypto & implement fetching olm keys before sending encrypted signalling message
  - local echo for join/leave buttons?
  - make UI pretsy

src/matrix/calls/callEventTypes.ts

@@ -97,6 +97,12 @@ export type MCallInvite<Base extends MCallBase> = Base & {
     [SDPStreamMetadataKey]: SDPStreamMetadata;
 }
 
+export type MCallNegotiate<Base extends MCallBase> = Base & {
+    description: SessionDescription;
+    lifetime: number;
+    [SDPStreamMetadataKey]: SDPStreamMetadata;
+}
+
 export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & {
     [SDPStreamMetadataKey]: SDPStreamMetadata;
 }
@@ -213,6 +219,7 @@ export enum CallErrorCode {
 
 export type SignallingMessage<Base extends MCallBase> =
     {type: EventType.Invite, content: MCallInvite<Base>} |
+    {type: EventType.Negotiate, content: MCallNegotiate<Base>} |
     {type: EventType.Answer, content: MCallAnswer<Base>} |
     {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
     {type: EventType.Candidates, content: MCallCandidates<Base>} |

src/matrix/calls/group/GroupCall.ts

@@ -124,13 +124,13 @@ export class GroupCall extends EventEmitter<{change: never}> {
     }
 
     async setMedia(localMedia: LocalMedia): Promise<void> {
-        if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
+        if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) {
-            const oldMedia = this._localMedia;
+            const oldMedia = this._localMedia!;
             this._localMedia = localMedia;
             await Promise.all(Array.from(this._members.values()).map(m => {
-                return m.setMedia(localMedia!.clone());
+                return m.setMedia(localMedia, oldMedia);
             }));
-            oldMedia?.dispose();
+            oldMedia?.stopExcept(localMedia);
         }
     }
 

src/matrix/calls/group/Member.ts

@@ -185,11 +185,9 @@ export class Member {
     }
 
     /** @internal */
-    async setMedia(localMedia: LocalMedia): Promise<void> {
+    async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise<void> {
-        const oldMedia = this.localMedia;
+        this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia);
-        this.localMedia = localMedia;
+        await this.peerCall?.setMedia(this.localMedia);
-        await this.peerCall?.setMedia(localMedia);
-        oldMedia?.dispose();
     }
 
     private _createPeerCall(callId: string): PeerCall {