From 6fe90e60db68007359b88ed5673ba085b455feb4 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 7 Mar 2022 10:15:54 +0100 Subject: [PATCH] WIP9 --- src/matrix/DeviceMessageHandler.js | 6 ++- src/matrix/calls/CallHandler.ts | 5 +-- src/matrix/calls/PeerCall.ts | 22 ++++++++--- src/matrix/calls/TODO.md | 61 ++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index c6bce31e..20f4abd3 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -53,10 +53,14 @@ export class DeviceMessageHandler { } const newRoomKeys = this._megolmDecryption.roomKeysFromDeviceMessages(olmDecryptChanges.results, log); const callMessages = olmDecryptChanges.results.filter(dr => this._callHandler.handlesDeviceMessageEventType(dr.event?.type)); + // load devices by sender key await Promise.all(callMessages.map(async dr => { dr.setDevice(await this._getDevice(dr.senderCurve25519Key, txn)); - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); })); + // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? + for (const dr of callMessages) { + this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log); + } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? // well, we don't really need to update the room other then when a call starts or stops diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 4a2027f3..1dad9ce8 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -109,8 +109,7 @@ class GroupParticipant implements PeerCallHandler { sendInvite() { this.peerCall = new PeerCall(this, this.webRTC); - this.peerCall.setLocalMedia(this.localMedia); - this.peerCall.sendOffer(); + this.peerCall.call(this.localMedia); } /** From PeerCallHandler @@ -140,7 +139,7 @@ class GroupCall { async participate(tracks: Track[]) { this.localMedia = LocalMedia.fromTracks(tracks); for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.clone()); + participant.setMedia(this.localMedia.clone()); } // send m.call.member state event diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index a08263f6..a02de99f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -86,13 +86,23 @@ class PeerCall { handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) { switch (message.type) { case EventType.Invite: - this.handleInvite(message.content, partyId); + // determining whether or not an incoming invite glares + // with an instance of PeerCall is different for group calls + // and 1:1 calls, so done outside of this class. + // If you pass an event for another call id in here it will assume it glares. + + //const newCallId = message.content.call_id; + //if (this.id && newCallId !== this.id) { + // this.handleInviteGlare(message.content); + //} else { + this.handleInvite(message.content, partyId); + //} break; case EventType.Answer: this.handleAnswer(message.content, partyId); break; case EventType.Candidates: - this.handleRemoteIceCandidates(message.content); + this.handleRemoteIceCandidates(message.content, partyId); break; case EventType.Hangup: } @@ -184,6 +194,8 @@ class PeerCall { // calls are serialized and deduplicated by responsePromiseChain private handleNegotiation = async (): Promise => { + // TODO: does this make sense to have this state if we're already connected? + this.setState(CallState.MakingOffer) try { await this.peerConnection.setLocalDescription(); } catch (err) { @@ -221,7 +233,7 @@ class PeerCall { } this.sendCandidateQueue(); - if (this.state === CallState.CreateOffer) { + if (this.state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between if (this.state === CallState.InviteSent) { @@ -495,7 +507,7 @@ class PeerCall { public dispose(): void { this.disposables.dispose(); - // TODO: dispose peerConnection? + this.peerConnection.dispose(); } } @@ -529,9 +541,9 @@ export enum CallParty { export enum CallState { Fledgling = 'fledgling', - InviteSent = 'invite_sent', WaitLocalMedia = 'wait_local_media', CreateOffer = 'create_offer', + InviteSent = 'invite_sent', CreateAnswer = 'create_answer', Connecting = 'connecting', Connected = 'connected', diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 69268b1d..da2b1ad6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -124,3 +124,64 @@ write view - you send m.call.negotiate - SOLVED: wrt to MSC2746, is the screen share track and the audio track (and video track) part of the same stream? or do screen share tracks need to go in a different stream? it sounds incompatible with the MSC2746 requirement. - SOLVED: how does muting work? MediaStreamTrack.enabled + - SOLVED: so, what's the difference between the call_id and the conf_id in group call events? + - call_id is the specific 1:1 call, conf_id is the thing in the m.call state event key + - so a group call has a conf_id with MxN peer calls, each having their call_id. + +I think we need to synchronize the negotiation needed because we don't use a CallState to guard it... + + + +## Thursday 3-3 notes + +we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags? + + +List state transitions + +FROM CALLER FROM CALLEE + +Fledgling Fledgling + V calling `call()` V handleInvite +WaitLocalMedia Ringing + V media promise resolves V answer() +CreateOffer WaitLocalMedia + V add tracks V media promise resolves + V wait for negotionneeded events CreateAnswer + V setLocalDescription() V + V send invite events +InviteSent + V receive anwser, setRemoteDescription() | + \__________________________________________________/ + V + Connecting + V receive ice candidates and + iceConnectionState becomes 'connected' + Connected + V hangup for some reason + Ended + +## From callee + +Fledgling +Ringing +WaitLocalMedia +CreateAnswer +Connecting +Connected +Ended + +Fledgling +WaitLocalMedia +CreateOffer +InviteSent +CreateAnswer +Connecting +Connected +Ringing +Ended + +so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection. + + +when glare, won't we drop both calls? No: https://github.com/matrix-org/matrix-spec-proposals/pull/2746#discussion_r819388754