WIP11

src/matrix/calls/CallHandler.ts

@@ -17,33 +17,34 @@ limitations under the License.
 import {ObservableMap} from "../../observable/map/ObservableMap";
 
 import type {Room} from "../room/Room";
+import type {MemberChange} from "../room/members/RoomMember";
 import type {StateEvent} from "../storage/types";
 import type {ILogItem} from "../../logging/types";
+import type {Platform} from "../../platform/web/Platform";
 
 import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC";
 import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
-import type {SignallingMessage} from "./PeerCall";
-import type {MGroupCallBase} from "./callEventTypes";
+import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall";
+import {EventType} from "./callEventTypes";
+import type {SignallingMessage, MGroupCallBase} from "./callEventTypes";
+import type {GroupCall} from "./group/GroupCall";
 
 const GROUP_CALL_TYPE = "m.call";
 const GROUP_CALL_MEMBER_TYPE = "m.call.member";
-
-enum CallSetupMessageType {
-    Invite = "m.call.invite",
-    Answer = "m.call.answer",
-    Candidates = "m.call.candidates",
-    Hangup = "m.call.hangup",
-}
-
-const CONF_ID = "conf_id";
 const CALL_TERMINATED = "m.terminated";
 
 export class GroupCallHandler {
+
+    private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall;
     // group calls by call id
     public readonly calls: ObservableMap<string, GroupCall> = new ObservableMap<string, GroupCall>();
+    // map of userId to set of conf_id's they are in
+    private memberToCallIds: Map<string, Set<string>> = new Map();
 
-    constructor() {
-
+    constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) {
+        this.createPeerCall = (callId: string, handler: PeerCallHandler) => {
+            return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC);
+        }
     }
 
     // TODO: check and poll turn server credentials here
@@ -51,7 +52,23 @@ export class GroupCallHandler {
     handleRoomState(room: Room, events: StateEvent[], log: ILogItem) {
         // first update call events
         for (const event of events) {
-            if (event.type === GROUP_CALL_TYPE) {
+            if (event.type === EventType.GroupCall) {
+                this.handleCallEvent(event);
+            }
+        }
+        // then update participants
+        for (const event of events) {
+            if (event.type === EventType.GroupCallMember) {
+                this.handleCallMemberEvent(event);
+            }
+        }
+    }
+
+    updateRoomMembers(room: Room, memberChanges: Map<string, MemberChange>) {
+
+    }
+
+    private handleCallEvent(event: StateEvent) {
         const callId = event.state_key;
         let call = this.calls.get(callId);
         if (call) {
@@ -60,34 +77,44 @@ export class GroupCallHandler {
                 this.calls.remove(call.id);
             }
         } else {
-                    call = new GroupCall(event, room);
+            call = new GroupCall(event, room, this.createPeerCall);
             this.calls.set(call.id, call);
         }
     }
-        }
-        // then update participants
-        for (const event of events) {
-            if (event.type === GROUP_CALL_MEMBER_TYPE) {
+
+    private handleCallMemberEvent(event: StateEvent) {
         const participant = event.state_key;
-                const sources = event.content["m.sources"];
-                for (const source of sources) {
-                    const call = this.calls.get(source[CONF_ID]);
-                    if (call && !call.isTerminated) {
-                        call.addParticipant(participant, source);
+        const calls = event.content["m.calls"] ?? [];
+        const newCallIdsMemberOf = new Set<string>(calls.map(call => {
+            const callId = call["m.call_id"];
+            const groupCall = this.calls.get(callId);
+            // TODO: also check the participant when receiving the m.call event
+            groupCall?.addParticipant(participant, call);
+            return callId;
+        }));
+        let previousCallIdsMemberOf = this.memberToCallIds.get(participant);
+        // remove user as participant of any calls not present anymore
+        if (previousCallIdsMemberOf) {
+            for (const previousCallId of previousCallIdsMemberOf) {
+                if (!newCallIdsMemberOf.has(previousCallId)) {
+                    const groupCall = this.calls.get(previousCallId);
+                    groupCall?.removeParticipant(participant);
                 }
             }
         }
+        if (newCallIdsMemberOf.size === 0) {
+            this.memberToCallIds.delete(participant);
+        } else {
+            this.memberToCallIds.set(participant, newCallIdsMemberOf);
         }
     }
 
-    handlesDeviceMessageEventType(eventType: string | undefined): boolean {
-        return  eventType === CallSetupMessageType.Invite ||
-                eventType === CallSetupMessageType.Candidates ||
-                eventType === CallSetupMessageType.Answer ||
-                eventType === CallSetupMessageType.Hangup;
+    handlesDeviceMessageEventType(eventType: string): boolean {
+        return handlesEventType(eventType);
     }
 
     handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage<MGroupCallBase>, log: ILogItem) {
+        // TODO: buffer messages for calls we haven't received the state event for yet?
         const call = this.calls.get(event.content.conf_id);
         call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log);
     }

src/matrix/calls/PeerCall.ts

@@ -29,7 +29,8 @@ import type {LocalMedia} from "./LocalMedia";
 
 import {
     SDPStreamMetadataKey,
-    SDPStreamMetadataPurpose
+    SDPStreamMetadataPurpose,
+    EventType,
 } from "./callEventTypes";
 import type {
     MCallBase,
@@ -39,6 +40,7 @@ import type {
     MCallCandidates,
     MCallHangupReject,
     SDPStreamMetadata,
+    SignallingMessage
 } from "./callEventTypes";
 
 // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already
@@ -677,21 +679,6 @@ export enum CallDirection {
     Outbound = 'outbound',
 }
 
-export enum EventType {
-    Invite = "m.call.invite",
-    Candidates = "m.call.candidates",
-    Answer = "m.call.answer",
-    Hangup = "m.call.hangup",
-    Reject = "m.call.reject",
-    SelectAnswer = "m.call.select_answer",
-    Negotiate = "m.call.negotiate",
-    SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed",
-    SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed",
-    Replaces = "m.call.replaces",
-    AssertedIdentity = "m.call.asserted_identity",
-    AssertedIdentityPrefix = "org.matrix.call.asserted_identity",
-}
-
 export enum CallErrorCode {
     /** The user chose to end the call */
     UserHangup = 'user_hangup',
@@ -802,18 +789,18 @@ export class CallError extends Error {
     }
 }
 
-export type SignallingMessage<Base extends MCallBase> =
-    {type: EventType.Invite, content: MCallInvite<Base>} |
-    {type: EventType.Answer, content: MCallAnswer<Base>} |
-    {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
-    {type: EventType.Candidates, content: MCallCandidates<Base>} |
-    {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};
-
 export interface PeerCallHandler {
     emitUpdate(peerCall: PeerCall, params: any);
     sendSignallingMessage(message: SignallingMessage<MCallBase>);
 }
 
+export function handlesEventType(eventType: string): boolean {
+    return  eventType === EventType.Invite ||
+            eventType === EventType.Candidates ||
+            eventType === EventType.Answer ||
+            eventType === EventType.Hangup;
+}
+
 export function tests() {
 
 }

src/matrix/calls/callEventTypes.ts

@@ -1,6 +1,24 @@
 // allow non-camelcase as these are events type that go onto the wire
 /* eslint-disable camelcase */
 
+
+export enum EventType {
+    GroupCall = "m.call",
+    GroupCallMember = "m.call.member",
+    Invite = "m.call.invite",
+    Candidates = "m.call.candidates",
+    Answer = "m.call.answer",
+    Hangup = "m.call.hangup",
+    Reject = "m.call.reject",
+    SelectAnswer = "m.call.select_answer",
+    Negotiate = "m.call.negotiate",
+    SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed",
+    SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed",
+    Replaces = "m.call.replaces",
+    AssertedIdentity = "m.call.asserted_identity",
+    AssertedIdentityPrefix = "org.matrix.call.asserted_identity",
+}
+
 // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
 export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
 
@@ -88,4 +106,95 @@ export type MCallHangupReject<Base extends MCallBase> = Base & {
     reason?: CallErrorCode;
 }
 
-/* eslint-enable camelcase */
+export enum CallErrorCode {
+    /** The user chose to end the call */
+    UserHangup = 'user_hangup',
+
+    /** An error code when the local client failed to create an offer. */
+    LocalOfferFailed = 'local_offer_failed',
+    /**
+     * An error code when there is no local mic/camera to use. This may be because
+     * the hardware isn't plugged in, or the user has explicitly denied access.
+     */
+    NoUserMedia = 'no_user_media',
+
+    /**
+     * Error code used when a call event failed to send
+     * because unknown devices were present in the room
+     */
+    UnknownDevices = 'unknown_devices',
+
+    /**
+     * Error code used when we fail to send the invite
+     * for some reason other than there being unknown devices
+     */
+    SendInvite = 'send_invite',
+
+    /**
+     * An answer could not be created
+     */
+    CreateAnswer = 'create_answer',
+
+    /**
+     * Error code used when we fail to send the answer
+     * for some reason other than there being unknown devices
+     */
+    SendAnswer = 'send_answer',
+
+    /**
+     * The session description from the other side could not be set
+     */
+    SetRemoteDescription = 'set_remote_description',
+
+    /**
+     * The session description from this side could not be set
+     */
+    SetLocalDescription = 'set_local_description',
+
+    /**
+     * A different device answered the call
+     */
+    AnsweredElsewhere = 'answered_elsewhere',
+
+    /**
+     * No media connection could be established to the other party
+     */
+    IceFailed = 'ice_failed',
+
+    /**
+     * The invite timed out whilst waiting for an answer
+     */
+    InviteTimeout = 'invite_timeout',
+
+    /**
+     * The call was replaced by another call
+     */
+    Replaced = 'replaced',
+
+    /**
+     * Signalling for the call could not be sent (other than the initial invite)
+     */
+    SignallingFailed = 'signalling_timeout',
+
+    /**
+     * The remote party is busy
+     */
+    UserBusy = 'user_busy',
+
+    /**
+     * We transferred the call off to somewhere else
+     */
+    Transfered = 'transferred',
+
+    /**
+     * A call from the same user was found with a new session id
+     */
+    NewSession = 'new_session',
+}
+
+export type SignallingMessage<Base extends MCallBase> =
+    {type: EventType.Invite, content: MCallInvite<Base>} |
+    {type: EventType.Answer, content: MCallAnswer<Base>} |
+    {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
+    {type: EventType.Candidates, content: MCallCandidates<Base>} |
+    {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};

src/matrix/calls/group/GroupCall.ts

@@ -18,16 +18,21 @@ import {ObservableMap} from "../../../observable/map/ObservableMap";
 import {Participant} from "./Participant";
 import {LocalMedia} from "../LocalMedia";
 import type {Track} from "../../../platform/types/MediaDevices";
-
-function getParticipantId(senderUserId: string, senderDeviceId: string | null) {
-    return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
-}
+import type {SignallingMessage, MGroupCallBase} from "../callEventTypes";
+import type {Room} from "../../room/Room";
+import type {StateEvent} from "../../storage/types";
+import type {Platform} from "../../../platform/web/Platform";
 
 export class GroupCall {
     private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
     private localMedia?: Promise<LocalMedia>;
 
-    constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
+    constructor(
+        private readonly ownUserId: string,
+        private callEvent: StateEvent,
+        private readonly room: Room,
+        private readonly platform: Platform
+    ) {
 
     }
 
@@ -52,32 +57,25 @@ export class GroupCall {
         this.callEvent = callEvent;
     }
 
-    addParticipant(userId, source) {
-        const participantId = getParticipantId(userId, source.device_id);
-        const participant = this.participants.get(participantId);
+    addParticipant(userId, memberCallInfo) {
+        let participant = this.participants.get(userId);
         if (participant) {
-            participant.updateSource(source);
+            participant.updateCallInfo(memberCallInfo);
         } else {
-            participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC));
+            participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC);
+            participant.updateCallInfo(memberCallInfo);
+            this.participants.add(userId, participant);
         }
     }
 
-    handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) {
-        const participantId = getParticipantId(senderUserId, senderDeviceId);
-        let peerCall = this.participants.get(participantId);
-        let hasDeviceInKey = true;
-        if (!peerCall) {
-            hasDeviceInKey = false;
-            peerCall = this.participants.get(getParticipantId(senderUserId, null))
+    removeParticipant(userId) {
+
     }
-        if (peerCall) {
-            peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId);
-            if (!hasDeviceInKey && peerCall.opponentPartyId) {
-                this.participants.delete(getParticipantId(senderUserId, null));
-                this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId));
-            }
-        } else {
-            // create peerCall
+
+    handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) {
+        let participant = this.participants.get(userId);
+        if (participant) {
+            participant.handleIncomingSignallingMessage(message, senderDeviceId);
         }
     }
 

src/matrix/room/Room.js

@@ -218,6 +218,9 @@ export class Room extends BaseRoom {
             if (this._memberList) {
                 this._memberList.afterSync(memberChanges);
             }
+            if (this._callHandler) {
+                this._callHandler.updateRoomMembers(this, memberChanges);
+            }
             if (this._observedMembers) {
                 this._updateObservedMembers(memberChanges);
             }