From 6da4a4209c95598da1daf451a82f6aa0649d8f62 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 10 Mar 2022 14:53:31 +0100 Subject: [PATCH] WIP: work on group calling code --- src/matrix/DeviceMessageHandler.js | 2 +- src/matrix/Session.js | 14 ++- src/matrix/calls/CallHandler.ts | 126 ++++++++++++++------------ src/matrix/calls/PeerCall.ts | 79 ++++++++-------- src/matrix/calls/group/GroupCall.ts | 90 +++++++++++------- src/matrix/calls/group/Member.ts | 112 +++++++++++++++++++++++ src/matrix/calls/group/Participant.ts | 67 -------------- src/matrix/common.js | 6 +- src/matrix/e2ee/olm/Encryption.ts | 2 +- src/matrix/net/HomeServerApi.ts | 4 + src/platform/types/MediaDevices.ts | 1 + 11 files changed, 301 insertions(+), 202 deletions(-) create mode 100644 src/matrix/calls/group/Member.ts delete mode 100644 src/matrix/calls/group/Participant.ts diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 470559a9..91ef82f6 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -59,7 +59,7 @@ export class DeviceMessageHandler { })); // TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)? for (const dr of callMessages) { - this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log); + this._callHandler.handleDeviceMessage(dr.event, dr.device.userId, dr.device.deviceId, log); } // TODO: somehow include rooms that received a call to_device message in the sync state? // or have updates flow through event emitter? diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 3d9b13c8..94fb5dee 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -73,6 +73,19 @@ export class Session { }; this._roomsBeingCreated = new ObservableMap(); this._user = new User(sessionInfo.userId); + this._callHandler = new CallHandler({ + createTimeout: this._platform.clock.createTimeout, + hsApi: this._hsApi, + encryptDeviceMessage: async (roomId, message, log) => { + if (!this._deviceTracker || !this._olmEncryption) { + throw new Error("encryption is not enabled"); + } + await this._deviceTracker.trackRoom(roomId, log); + const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); + return encryptedMessage; + } + }); this._deviceMessageHandler = new DeviceMessageHandler({storage, callHandler: this._callHandler}); this._olm = olm; this._olmUtil = null; @@ -100,7 +113,6 @@ export class Session { this._createRoomEncryption = this._createRoomEncryption.bind(this); this._forgetArchivedRoom = this._forgetArchivedRoom.bind(this); this.needsKeyBackup = new ObservableValue(false); - this._callHandler = new CallHandler(this._platform, this._hsApi); } get fingerprintKey() { diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index d84be9e3..0e70fb5a 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,48 +15,53 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; +import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; +import {handlesEventType} from "./PeerCall"; +import {EventType} from "./callEventTypes"; +import {GroupCall} from "./group/GroupCall"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; import type {ILogItem} from "../../logging/types"; import type {Platform} from "../../platform/web/Platform"; - -import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; -import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall"; -import {EventType} from "./callEventTypes"; +import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; import type {SignallingMessage, MGroupCallBase} from "./callEventTypes"; -import type {GroupCall} from "./group/GroupCall"; +import type {Options as GroupCallOptions} from "./group/GroupCall"; const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const CALL_TERMINATED = "m.terminated"; -export class GroupCallHandler { +export type Options = Omit; - private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall; +export class GroupCallHandler { // group calls by call id - public readonly calls: ObservableMap = new ObservableMap(); + private readonly _calls: ObservableMap = new ObservableMap(); // map of userId to set of conf_id's they are in private memberToCallIds: Map> = new Map(); + private groupCallOptions: GroupCallOptions; - constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) { - this.createPeerCall = (callId: string, handler: PeerCallHandler) => { - return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC); - } + constructor(private readonly options: Options) { + this.groupCallOptions = Object.assign({}, this.options, { + emitUpdate: (groupCall, params) => this._calls.update(groupCall.id, params) + }); } + get calls(): BaseObservableMap { return this._calls; } + // TODO: check and poll turn server credentials here + /** @internal */ handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event); + this.handleCallEvent(event, room); } } - // then update participants + // then update members for (const event of events) { if (event.type === EventType.GroupCallMember) { this.handleCallMemberEvent(event); @@ -64,59 +69,62 @@ export class GroupCallHandler { } } + /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { } - private handleCallEvent(event: StateEvent) { - const callId = event.state_key; - let call = this.calls.get(callId); - if (call) { - call.updateCallEvent(event); - if (call.isTerminated) { - this.calls.remove(call.id); - } - } else { - call = new GroupCall(event, room, this.createPeerCall); - this.calls.set(call.id, call); - } - } - - private handleCallMemberEvent(event: StateEvent) { - const participant = event.state_key; - const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { - const callId = call["m.call_id"]; - const groupCall = this.calls.get(callId); - // TODO: also check the participant when receiving the m.call event - groupCall?.addParticipant(participant, call); - return callId; - })); - let previousCallIdsMemberOf = this.memberToCallIds.get(participant); - // remove user as participant of any calls not present anymore - if (previousCallIdsMemberOf) { - for (const previousCallId of previousCallIdsMemberOf) { - if (!newCallIdsMemberOf.has(previousCallId)) { - const groupCall = this.calls.get(previousCallId); - groupCall?.removeParticipant(participant); - } - } - } - if (newCallIdsMemberOf.size === 0) { - this.memberToCallIds.delete(participant); - } else { - this.memberToCallIds.set(participant, newCallIdsMemberOf); - } - } - + /** @internal */ handlesDeviceMessageEventType(eventType: string): boolean { return handlesEventType(eventType); } - handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage, log: ILogItem) { + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { // TODO: buffer messages for calls we haven't received the state event for yet? - const call = this.calls.get(event.content.conf_id); - call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); + const call = this._calls.get(message.content.conf_id); + call?.handleDeviceMessage(message, userId, deviceId, log); + } + + private handleCallEvent(event: StateEvent, room: Room) { + const callId = event.state_key; + let call = this._calls.get(callId); + if (call) { + call.updateCallEvent(event); + if (call.isTerminated) { + this._calls.remove(call.id); + } + } else { + call = new GroupCall(event, room, this.groupCallOptions); + this._calls.set(call.id, call); + } + } + + private handleCallMemberEvent(event: StateEvent) { + const userId = event.state_key; + const calls = event.content["m.calls"] ?? []; + const newCallIdsMemberOf = new Set(calls.map(call => { + const callId = call["m.call_id"]; + const groupCall = this._calls.get(callId); + // TODO: also check the member when receiving the m.call event + groupCall?.addMember(userId, call); + return callId; + })); + let previousCallIdsMemberOf = this.memberToCallIds.get(userId); + // remove user as member of any calls not present anymore + if (previousCallIdsMemberOf) { + for (const previousCallId of previousCallIdsMemberOf) { + if (!newCallIdsMemberOf.has(previousCallId)) { + const groupCall = this._calls.get(previousCallId); + groupCall?.removeMember(userId); + } + } + } + if (newCallIdsMemberOf.size === 0) { + this.memberToCallIds.delete(userId); + } else { + this.memberToCallIds.set(userId, newCallIdsMemberOf); + } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index f06b291d..ed8351ac 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -43,6 +43,13 @@ import type { SignallingMessage } from "./callEventTypes"; +export type Options = { + webRTC: WebRTC, + createTimeout: TimeoutCreator, + emitUpdate: (peerCall: PeerCall, params: any) => void; + sendSignallingMessage: (message: SignallingMessage, log: ILogItem) => Promise; +}; + // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // do for sharing keys will be best as that already deals with room tracking. /** @@ -51,7 +58,7 @@ import type { /** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; - private state = CallState.Fledgling; + private _state = CallState.Fledgling; private direction: CallDirection; private localMedia?: LocalMedia; // A queue for candidates waiting to go out. @@ -74,15 +81,12 @@ export class PeerCall implements IDisposable { // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; - constructor( private callId: string, // generated or from invite - private readonly handler: PeerCallHandler, - private readonly createTimeout: TimeoutCreator, - webRTC: WebRTC + private readonly options: Options ) { const outer = this; - this.peerConnection = webRTC.createPeerConnection({ + this.peerConnection = options.webRTC.createPeerConnection({ onIceConnectionStateChange(state: RTCIceConnectionState) {}, onLocalIceCandidate(candidate: RTCIceCandidate) {}, onIceGatheringStateChange(state: RTCIceGatheringState) {}, @@ -104,12 +108,14 @@ export class PeerCall implements IDisposable { } } + get state(): CallState { return this._state; } + get remoteTracks(): Track[] { return this.peerConnection.remoteTracks; } async call(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Fledgling) { + if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; @@ -131,7 +137,7 @@ export class PeerCall implements IDisposable { } async answer(localMediaPromise: Promise): Promise { - if (this.state !== CallState.Ringing) { + if (this._state !== CallState.Ringing) { return; } this.setState(CallState.WaitLocalMedia); @@ -197,7 +203,7 @@ export class PeerCall implements IDisposable { async hangup(errorCode: CallErrorCode) { } - async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId): Promise { + async handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId, log: ILogItem): Promise { switch (message.type) { case EventType.Invite: if (this.callId !== message.content.call_id) { @@ -226,10 +232,10 @@ export class PeerCall implements IDisposable { if (reason) { content["reason"] = reason; } - return this.handler.sendSignallingMessage({ + return this.options.sendSignallingMessage({ type: EventType.Hangup, content - }); + }, undefined); } // calls are serialized and deduplicated by responsePromiseChain @@ -249,7 +255,7 @@ export class PeerCall implements IDisposable { await this.delay(200); } - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { return; } @@ -268,12 +274,12 @@ export class PeerCall implements IDisposable { version: 1, lifetime: CALL_TIMEOUT_MS }; - if (this.state === CallState.CreateOffer) { - await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + if (this._state === CallState.CreateOffer) { + await this.options.sendSignallingMessage({type: EventType.Invite, content}); this.setState(CallState.InviteSent); - } else if (this.state === CallState.Connected || this.state === CallState.Connecting) { + } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { // send Negotiate message - //await this.handler.sendSignallingMessage({type: EventType.Invite, content}); + //await this.options.sendSignallingMessage({type: EventType.Invite, content}); //this.setState(CallState.InviteSent); } } finally { @@ -282,10 +288,10 @@ export class PeerCall implements IDisposable { this.sendCandidateQueue(); - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { await this.delay(CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.InviteSent) { + if (this._state === CallState.InviteSent) { this.hangup(CallErrorCode.InviteTimeout); } } @@ -307,7 +313,7 @@ export class PeerCall implements IDisposable { // TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer? // How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer? - if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) { + if (this._state === CallState.Fledgling || this._state === CallState.CreateOffer || this._state === CallState.WaitLocalMedia) { } else { await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced); @@ -324,7 +330,7 @@ export class PeerCall implements IDisposable { } private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise { - if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) { + if (this._state !== CallState.Fledgling || this.opponentPartyId !== undefined) { // TODO: hangup or ignore? return; } @@ -370,7 +376,7 @@ export class PeerCall implements IDisposable { await this.delay(content.lifetime ?? CALL_TIMEOUT_MS); // @ts-ignore TS doesn't take the await above into account to know that the state could have changed in between - if (this.state === CallState.Ringing) { + if (this._state === CallState.Ringing) { this.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); @@ -384,7 +390,7 @@ export class PeerCall implements IDisposable { private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise { this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`); - if (this.state === CallState.Ended) { + if (this._state === CallState.Ended) { this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`); return; } @@ -456,7 +462,7 @@ export class PeerCall implements IDisposable { // if (description.type === 'offer') { // await this.peerConnection.setLocalDescription(); - // await this.handler.sendSignallingMessage({ + // await this.options.sendSignallingMessage({ // type: EventType.CallNegotiate, // content: { // description: this.peerConnection.localDescription!, @@ -471,7 +477,7 @@ export class PeerCall implements IDisposable { private async sendAnswer(): Promise { const localDescription = this.peerConnection.localDescription!; - const answerContent: MCallAnswer = { + const answerContent: MCallAnswer = { call_id: this.callId, version: 1, answer: { @@ -489,7 +495,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; try { - await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent}); + await this.options.sendSignallingMessage({type: EventType.Answer, content: answerContent}, undefined); } catch (error) { this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false); throw error; @@ -513,7 +519,7 @@ export class PeerCall implements IDisposable { this.candidateSendQueue.push(content); // Don't send the ICE candidates yet if the call is in the ringing state - if (this.state === CallState.Ringing) return; + if (this._state === CallState.Ringing) return; // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) @@ -523,7 +529,7 @@ export class PeerCall implements IDisposable { } private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) { + if (this.candidateSendQueue.length === 0 || this._state === CallState.Ended) { return; } @@ -531,14 +537,14 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { - await this.handler.sendSignallingMessage({ + await this.options.sendSignallingMessage({ type: EventType.Candidates, content: { call_id: this.callId, version: 1, candidates - } - }); + }, + }, undefined); // Try to send candidates again just in case we received more candidates while sending. this.sendCandidateQueue(); } catch (error) { @@ -598,14 +604,14 @@ export class PeerCall implements IDisposable { } private setState(state: CallState): void { - const oldState = this.state; - this.state = state; + const oldState = this._state; + this._state = state; let deferred = this.statePromiseMap.get(state); if (deferred) { deferred.resolve(); this.statePromiseMap.delete(state); } - this.handler.emitUpdate(this, undefined); + this.options.emitUpdate(this, undefined); } private waitForState(states: CallState[]): Promise { @@ -638,7 +644,7 @@ export class PeerCall implements IDisposable { private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered - const timeout = this.disposables.track(this.createTimeout(timeoutMs)); + const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); await timeout.elapsed(); this.disposables.untrack(timeout); } @@ -789,11 +795,6 @@ export class CallError extends Error { } } -export interface PeerCallHandler { - emitUpdate(peerCall: PeerCall, params: any); - sendSignallingMessage(message: SignallingMessage); -} - export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index dce50846..7266645d 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -15,71 +15,95 @@ limitations under the License. */ import {ObservableMap} from "../../../observable/map/ObservableMap"; -import {Participant} from "./Participant"; +import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {RoomMember} from "../../room/members/RoomMember"; +import type {Options as MemberOptions} from "./Member"; +import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; import type {SignallingMessage, MGroupCallBase} from "../callEventTypes"; import type {Room} from "../../room/Room"; import type {StateEvent} from "../../storage/types"; import type {Platform} from "../../../platform/web/Platform"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + emitUpdate: (call: GroupCall, params?: any) => void; + encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, +}; export class GroupCall { - private readonly participants: ObservableMap = new ObservableMap(); + private readonly _members: ObservableMap = new ObservableMap(); private localMedia?: Promise; + private _memberOptions: MemberOptions; constructor( - private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, - private readonly platform: Platform + private readonly options: Options ) { - + this._memberOptions = Object.assign({ + confId: this.id, + emitUpdate: member => this._members.update(member.member.userId, member), + encryptDeviceMessage: (message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.room.id, message, log); + } + }, options); } + get members(): BaseObservableMap { return this._members; } + get id(): string { return this.callEvent.state_key; } - async participate(tracks: Promise) { - this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); - for (const [,participant] of this.participants) { - participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone())); - } - // send m.call.member state event + get isTerminated(): boolean { + return this.callEvent.content["m.terminated"] === true; + } - // send invite to all participants that are < my userId - for (const [,participant] of this.participants) { - if (participant.userId < this.ownUserId) { - participant.call(); - } + async join(tracks: Promise) { + this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks)); + // send m.call.member state event + const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, { + + }); + await request.response(); + // send invite to all members that are < my userId + for (const [,member] of this._members) { + member.connect(this.localMedia); } } + /** @internal */ updateCallEvent(callEvent: StateEvent) { this.callEvent = callEvent; + // TODO: emit update } - addParticipant(userId, memberCallInfo) { - let participant = this.participants.get(userId); - if (participant) { - participant.updateCallInfo(memberCallInfo); + /** @internal */ + addMember(userId, memberCallInfo) { + let member = this._members.get(userId); + if (member) { + member.updateCallInfo(memberCallInfo); } else { - participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC); - participant.updateCallInfo(memberCallInfo); - this.participants.add(userId, participant); + member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); + member.updateCallInfo(memberCallInfo); + this._members.add(userId, member); } } - removeParticipant(userId) { - + /** @internal */ + removeMember(userId) { + this._members.remove(userId); } - handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage, log: ILogItem) { - let participant = this.participants.get(userId); - if (participant) { - participant.handleIncomingSignallingMessage(message, senderDeviceId); + /** @internal */ + handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + // TODO: return if we are not membering to the call + let member = this._members.get(userId); + if (member) { + member.handleDeviceMessage(message, deviceId, log); + } else { + // we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway? } } - - get isTerminated(): boolean { - return !!this.callEvent.content[CALL_TERMINATED]; - } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts new file mode 100644 index 00000000..c6791568 --- /dev/null +++ b/src/matrix/calls/group/Member.ts @@ -0,0 +1,112 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {PeerCall, CallState} from "../PeerCall"; +import {makeTxnId, makeId} from "../../common"; +import {EventType} from "../callEventTypes"; + +import type {Options as PeerCallOptions} from "../PeerCall"; +import type {LocalMedia} from "../LocalMedia"; +import type {HomeServerApi} from "../../net/HomeServerApi"; +import type {Track} from "../../../platform/types/MediaDevices"; +import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes"; +import type {GroupCall} from "./GroupCall"; +import type {RoomMember} from "../../room/members/RoomMember"; +import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; +import type {ILogItem} from "../../../logging/types"; + +export type Options = Omit & { + confId: string, + ownUserId: string, + hsApi: HomeServerApi, + encryptDeviceMessage: (message: SignallingMessage, log: ILogItem) => Promise, + emitUpdate: (participant: Member, params?: any) => void, +} + +export class Member { + private peerCall?: PeerCall; + private localMedia?: Promise; + + constructor( + public readonly member: RoomMember, + private readonly options: Options + ) {} + + get remoteTracks(): Track[] { + return this.peerCall?.remoteTracks ?? []; + } + + get isConnected(): boolean { + return this.peerCall?.state === CallState.Connected; + } + + /* @internal */ + connect(localMedia: Promise) { + this.localMedia = localMedia; + // otherwise wait for it to connect + if (this.member.userId < this.options.ownUserId) { + this.peerCall = this._createPeerCall(makeId("c")); + this.peerCall.call(localMedia); + } + } + + /** @internal */ + updateCallInfo(memberCallInfo) { + + } + + /** @internal */ + emitUpdate = (peerCall: PeerCall, params: any) => { + if (peerCall.state === CallState.Ringing) { + peerCall.answer(this.localMedia!); + } + this.options.emitUpdate(this, params); + } + + /** From PeerCallHandler + * @internal */ + sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { + const groupMessage = message as SignallingMessage; + groupMessage.content.conf_id = this.options.confId; + const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const request = this.options.hsApi.sendToDevice( + "m.room.encrypted", + {[this.member.userId]: { + ["*"]: encryptedMessage.content + } + }, makeTxnId(), {log}); + await request.response(); + } + + /** @internal */ + handleDeviceMessage(message: SignallingMessage, deviceId: string, log: ILogItem) { + if (message.type === EventType.Invite && !this.peerCall) { + this.peerCall = this._createPeerCall(message.content.call_id); + } + if (this.peerCall) { + this.peerCall.handleIncomingSignallingMessage(message, deviceId, log); + } else { + // TODO: need to buffer events until invite comes? + } + } + + private _createPeerCall(callId: string): PeerCall { + return new PeerCall(callId, Object.assign({}, this.options, { + emitUpdate: this.emitUpdate, + sendSignallingMessage: this.sendSignallingMessage + })); + } +} diff --git a/src/matrix/calls/group/Participant.ts b/src/matrix/calls/group/Participant.ts deleted file mode 100644 index 2b873aa0..00000000 --- a/src/matrix/calls/group/Participant.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import {EventType, PeerCall, SignallingMessage} from "../PeerCall"; -import {makeTxnId} from "../../common"; - -import type {PeerCallHandler} from "../PeerCall"; -import type {LocalMedia} from "../LocalMedia"; -import type {HomeServerApi} from "../../net/HomeServerApi"; -import type {Track} from "../../../platform/types/MediaDevices"; -import type {MCallBase, MGroupCallBase} from "../callEventTypes"; -import type {GroupCall} from "./GroupCall"; -import type {RoomMember} from "../../room/members/RoomMember"; - -export class Participant implements PeerCallHandler { - constructor( - public readonly member: RoomMember, - private readonly deviceId: string | undefined, - private readonly peerCall: PeerCall, - private readonly hsApi: HomeServerApi, - private readonly groupCall: GroupCall - ) {} - - /* @internal */ - call(localMedia: Promise) { - this.peerCall.call(localMedia); - } - - get remoteTracks(): Track[] { - return this.peerCall.remoteTracks; - } - - /** From PeerCallHandler - * @internal */ - emitUpdate(params: any) { - this.groupCall.emitParticipantUpdate(this, params); - } - - /** From PeerCallHandler - * @internal */ - async sendSignallingMessage(message: SignallingMessage) { - const groupMessage = message as SignallingMessage; - groupMessage.content.conf_id = this.groupCall.id; - // TODO: this needs to be encrypted with olm first - - const request = this.hsApi.sendToDevice( - groupMessage.type, - {[this.member.userId]: { - [this.deviceId ?? "*"]: groupMessage.content - } - }, makeTxnId()); - await request.response(); - } -} diff --git a/src/matrix/common.js b/src/matrix/common.js index ba7876ed..5919ad9c 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -16,9 +16,13 @@ limitations under the License. */ export function makeTxnId() { + return makeId("t"); +} + +export function makeId(prefix) { const n = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); const str = n.toString(16); - return "t" + "0".repeat(14 - str.length) + str; + return prefix + "0".repeat(14 - str.length) + str; } export function isTxnId(txnId) { diff --git a/src/matrix/e2ee/olm/Encryption.ts b/src/matrix/e2ee/olm/Encryption.ts index 9b754272..dcc9f0b1 100644 --- a/src/matrix/e2ee/olm/Encryption.ts +++ b/src/matrix/e2ee/olm/Encryption.ts @@ -311,7 +311,7 @@ class EncryptionTarget { } } -class EncryptedMessage { +export class EncryptedMessage { constructor( public readonly content: OlmEncryptedMessageContent, public readonly device: DeviceIdentity diff --git a/src/matrix/net/HomeServerApi.ts b/src/matrix/net/HomeServerApi.ts index e9902ef8..30406c34 100644 --- a/src/matrix/net/HomeServerApi.ts +++ b/src/matrix/net/HomeServerApi.ts @@ -159,6 +159,10 @@ export class HomeServerApi { state(roomId: string, eventType: string, stateKey: string, options?: BaseRequestOptions): IHomeServerRequest { return this._get(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, undefined, options); } + + sendState(roomId: string, eventType: string, stateKey: string, content: Record, options?: BaseRequestOptions): IHomeServerRequest { + return this._put(`/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent(eventType)}/${encodeURIComponent(stateKey)}`, {}, content, options); + } getLoginFlows(): IHomeServerRequest { return this._unauthedRequest("GET", this._url("/login")); diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 8bf608ce..ed9015bf 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -17,6 +17,7 @@ limitations under the License. export interface MediaDevices { // filter out audiooutput enumerate(): Promise; + // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; }