From 4bedd4737b59e4455642e83c21f553c828933cc5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 9 Mar 2022 18:53:51 +0100 Subject: [PATCH] WIP11 --- src/matrix/calls/CallHandler.ts | 105 ++++++++++++++++---------- src/matrix/calls/PeerCall.ts | 33 +++------ src/matrix/calls/callEventTypes.ts | 111 +++++++++++++++++++++++++++- src/matrix/calls/group/GroupCall.ts | 50 ++++++------- src/matrix/room/Room.js | 3 + 5 files changed, 213 insertions(+), 89 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d7d673ed..d84be9e3 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -17,33 +17,34 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import type {Room} from "../room/Room"; +import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; +import type {Platform} from "../../platform/web/Platform"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import type {SignallingMessage} from "./PeerCall"; -import type {MGroupCallBase} from "./callEventTypes"; +import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; +import type {GroupCall} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; - -enum CallSetupMessageType { - Invite = "m.call.invite", - Answer = "m.call.answer", - Candidates = "m.call.candidates", - Hangup = "m.call.hangup", -} - -const CONF_ID = "conf_id"; const CALL_TERMINATED = "m.terminated"; export class GroupCallHandler { + + private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; // group calls by call id public readonly calls: ObservableMap = new ObservableMap(); + // map of userId to set of conf_id's they are in + private memberToCallIds: Map> = new Map(); - constructor() { - + constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { + this.createPeerCall = (callId: string, handler: PeerCallHandler) => { + return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); + } } // TODO: check and poll turn server credentials here @@ -51,43 +52,69 @@ export class GroupCallHandler { handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { - if (event.type === GROUP_CALL_TYPE) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room); - this.calls.set(call.id, call); - } + if (event.type === EventType.GroupCall) { + this.handleCallEvent(event); } } // then update participants for (const event of events) { - if (event.type === GROUP_CALL_MEMBER_TYPE) { - const participant = event.state_key; - const sources = event.content["m.sources"]; - for (const source of sources) { - const call = this.calls.get(source[CONF_ID]); - if (call && !call.isTerminated) { - call.addParticipant(participant, source); - } - } + if (event.type === EventType.GroupCallMember) { + this.handleCallMemberEvent(event); } } } - handlesDeviceMessageEventType(eventType: string | undefined): boolean { - return eventType === CallSetupMessageType.Invite || - eventType === CallSetupMessageType.Candidates || - eventType === CallSetupMessageType.Answer || - eventType === CallSetupMessageType.Hangup; + updateRoomMembers(room: Room, memberChanges: Map) { + + } + + private handleCallEvent(event: StateEvent) { + const callId = event.state_key; + let call = this.calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this.calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.createPeerCall); + this.calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const participant = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this.calls.get(callId); + // TODO: also check the participant when receiving the m.call event + groupCall?.addParticipant(participant, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(participant); + // remove user as participant of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this.calls.get(previousCallId); + groupCall?.removeParticipant(participant); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(participant); + } else { + this.memberToCallIds.set(participant, newCallIdsMemberOf); + } + } + + handlesDeviceMessageEventType(eventType: string): boolean { + return handlesEventType(eventType); } handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + // TODO: buffer messages for calls we haven't received the state event for yet? const call = this.calls.get(event.content.conf_id); call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5040c805..f06b291d 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -29,7 +29,8 @@ import type {LocalMedia} from "./LocalMedia"; import { SDPStreamMetadataKey, - SDPStreamMetadataPurpose + SDPStreamMetadataPurpose, + EventType, } from "./callEventTypes"; import type { MCallBase, @@ -39,6 +40,7 @@ import type { MCallCandidates, MCallHangupReject, SDPStreamMetadata, + SignallingMessage } from "./callEventTypes"; // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already @@ -677,21 +679,6 @@ export enum CallDirection { Outbound = 'outbound', } -export enum EventType { - Invite = "m.call.invite", - Candidates = "m.call.candidates", - Answer = "m.call.answer", - Hangup = "m.call.hangup", - Reject = "m.call.reject", - SelectAnswer = "m.call.select_answer", - Negotiate = "m.call.negotiate", - SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", - SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", - Replaces = "m.call.replaces", - AssertedIdentity = "m.call.asserted_identity", - AssertedIdentityPrefix = "org.matrix.call.asserted_identity", -} - export enum CallErrorCode { /** The user chose to end the call */ UserHangup = 'user_hangup', @@ -802,18 +789,18 @@ export class CallError extends Error { } } -export type SignallingMessage = - {type: EventType.Invite, content: MCallInvite} | - {type: EventType.Answer, content: MCallAnswer} | - {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | - {type: EventType.Candidates, content: MCallCandidates} | - {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; - export interface PeerCallHandler { emitUpdate(peerCall: PeerCall, params: any); sendSignallingMessage(message: SignallingMessage); } +export function handlesEventType(eventType: string): boolean { + return eventType === EventType.Invite || + eventType === EventType.Candidates || + eventType === EventType.Answer || + eventType === EventType.Hangup; +} + export function tests() { } diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index aa1bc079..0e9eb8f8 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -1,6 +1,24 @@ // allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ + +export enum EventType { + GroupCall = "m.call", + GroupCallMember = "m.call.member", + Invite = "m.call.invite", + Candidates = "m.call.candidates", + Answer = "m.call.answer", + Hangup = "m.call.hangup", + Reject = "m.call.reject", + SelectAnswer = "m.call.select_answer", + Negotiate = "m.call.negotiate", + SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed", + SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed", + Replaces = "m.call.replaces", + AssertedIdentity = "m.call.asserted_identity", + AssertedIdentityPrefix = "org.matrix.call.asserted_identity", +} + // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; @@ -88,4 +106,95 @@ export type MCallHangupReject = Base & { reason?: CallErrorCode; } -/* eslint-enable camelcase */ +export enum CallErrorCode { + /** The user chose to end the call */ + UserHangup = 'user_hangup', + + /** An error code when the local client failed to create an offer. */ + LocalOfferFailed = 'local_offer_failed', + /** + * An error code when there is no local mic/camera to use. This may be because + * the hardware isn't plugged in, or the user has explicitly denied access. + */ + NoUserMedia = 'no_user_media', + + /** + * Error code used when a call event failed to send + * because unknown devices were present in the room + */ + UnknownDevices = 'unknown_devices', + + /** + * Error code used when we fail to send the invite + * for some reason other than there being unknown devices + */ + SendInvite = 'send_invite', + + /** + * An answer could not be created + */ + CreateAnswer = 'create_answer', + + /** + * Error code used when we fail to send the answer + * for some reason other than there being unknown devices + */ + SendAnswer = 'send_answer', + + /** + * The session description from the other side could not be set + */ + SetRemoteDescription = 'set_remote_description', + + /** + * The session description from this side could not be set + */ + SetLocalDescription = 'set_local_description', + + /** + * A different device answered the call + */ + AnsweredElsewhere = 'answered_elsewhere', + + /** + * No media connection could be established to the other party + */ + IceFailed = 'ice_failed', + + /** + * The invite timed out whilst waiting for an answer + */ + InviteTimeout = 'invite_timeout', + + /** + * The call was replaced by another call + */ + Replaced = 'replaced', + + /** + * Signalling for the call could not be sent (other than the initial invite) + */ + SignallingFailed = 'signalling_timeout', + + /** + * The remote party is busy + */ + UserBusy = 'user_busy', + + /** + * We transferred the call off to somewhere else + */ + Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', +} + +export type SignallingMessage = + {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Answer, content: MCallAnswer} | + {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | + {type: EventType.Candidates, content: MCallCandidates} | + {type: EventType.Hangup | EventType.Reject, content: MCallHangupReject}; diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index e05f572d..dce50846 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,16 +18,21 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Participant} from "./Participant"; import {LocalMedia} from "../LocalMedia"; import type {Track} from "../../../platform/types/MediaDevices"; - -function getParticipantId(senderUserId: string, senderDeviceId: string | null) { - return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); -} +import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; +import type {Room} from "../../room/Room"; +import type {StateEvent} from "../../storage/types"; +import type {Platform} from "../../../platform/web/Platform"; export class GroupCall { private readonly participants: ObservableMap = new ObservableMap(); private localMedia?: Promise; - constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { + constructor( + private readonly ownUserId: string, + private callEvent: StateEvent, + private readonly room: Room, + private readonly platform: Platform + ) { } @@ -52,32 +57,25 @@ export class GroupCall { this.callEvent = callEvent; } - addParticipant(userId, source) { - const participantId = getParticipantId(userId, source.device_id); - const participant = this.participants.get(participantId); + addParticipant(userId, memberCallInfo) { + let participant = this.participants.get(userId); if (participant) { - participant.updateSource(source); + participant.updateCallInfo(memberCallInfo); } else { - participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); + participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); + participant.updateCallInfo(memberCallInfo); + this.participants.add(userId, participant); } } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record, log: ILogItem) { - const participantId = getParticipantId(senderUserId, senderDeviceId); - let peerCall = this.participants.get(participantId); - let hasDeviceInKey = true; - if (!peerCall) { - hasDeviceInKey = false; - peerCall = this.participants.get(getParticipantId(senderUserId, null)) - } - if (peerCall) { - peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); - if (!hasDeviceInKey && peerCall.opponentPartyId) { - this.participants.delete(getParticipantId(senderUserId, null)); - this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); - } - } else { - // create peerCall + removeParticipant(userId) { + + } + + handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { + let participant = this.participants.get(userId); + if (participant) { + participant.handleIncomingSignallingMessage(message, senderDeviceId); } } diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index b9ec82a3..d0fcbc84 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -218,6 +218,9 @@ export class Room extends BaseRoom { if (this._memberList) { this._memberList.afterSync(memberChanges); } + if (this._callHandler) { + this._callHandler.updateRoomMembers(this, memberChanges); + } if (this._observedMembers) { this._updateObservedMembers(memberChanges); }