From b213a45c5c759ad90d3fc7adde489dcbf2ae5c09 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 11 Mar 2022 14:40:37 +0100 Subject: [PATCH] WIP: work on group call state transitions --- src/matrix/calls/CallHandler.ts | 26 ++++++- src/matrix/calls/PeerCall.ts | 5 +- src/matrix/calls/TODO.md | 8 +- src/matrix/calls/group/GroupCall.ts | 116 ++++++++++++++++++++-------- src/matrix/calls/group/Member.ts | 14 ++-- 5 files changed, 119 insertions(+), 50 deletions(-) diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 0e70fb5a..a0cd8473 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -21,6 +21,7 @@ import {handlesEventType} from "./PeerCall"; import {EventType} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; +import type {LocalMedia} from "./LocalMedia"; import type {Room} from "../room/Room"; import type {MemberChange} from "../room/members/RoomMember"; import type {StateEvent} from "../storage/types"; @@ -49,6 +50,22 @@ export class GroupCallHandler { }); } + async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise { + const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions); + this._calls.set(call.id, call); + try { + await call.create(localMedia, name); + } catch (err) { + if (err.name === "ConnectionError") { + // if we're offline, give up and remove the call again + this._calls.remove(call.id); + } + throw err; + } + await call.join(localMedia); + return call; + } + get calls(): BaseObservableMap { return this._calls; } // TODO: check and poll turn server credentials here @@ -58,7 +75,7 @@ export class GroupCallHandler { // first update call events for (const event of events) { if (event.type === EventType.GroupCall) { - this.handleCallEvent(event, room); + this.handleCallEvent(event, room.id); } } // then update members @@ -71,7 +88,8 @@ export class GroupCallHandler { /** @internal */ updateRoomMembers(room: Room, memberChanges: Map) { - + // TODO: also have map for roomId to calls, so we can easily update members + // we will also need this to get the call for a room } /** @internal */ @@ -86,7 +104,7 @@ export class GroupCallHandler { call?.handleDeviceMessage(message, userId, deviceId, log); } - private handleCallEvent(event: StateEvent, room: Room) { + private handleCallEvent(event: StateEvent, roomId: string) { const callId = event.state_key; let call = this._calls.get(callId); if (call) { @@ -95,7 +113,7 @@ export class GroupCallHandler { this._calls.remove(call.id); } } else { - call = new GroupCall(event, room, this.groupCallOptions); + call = new GroupCall(event.state_key, event.content, roomId, this.groupCallOptions); this._calls.set(call.id, call); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ed8351ac..270e0fa4 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -127,12 +127,11 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateOffer); - // add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called for (const t of this.localMedia.tracks) { this.peerConnection.addTrack(t); } - // TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet - // but we would go straight to CreateAnswer, so also need to wait for that state + // after adding the local tracks, and wait for handleNegotiation to be called, + // or invite glare where we give up our invite and answer instead await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]); } diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index ad8fe185..83d706f6 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -97,12 +97,12 @@ party identification Build basic version of PeerCall - add candidates code -Build basic version of GroupCall - - add state, block invalid actions +DONE: Build basic version of GroupCall + - DONE: add state, block invalid actions DONE: Make it possible to olm encrypt the messages Do work needed for state events - - receiving (almost done?) - - sending + - DONEish: receiving (almost done?) + - DONEish: sending Expose call objects expose volume events from audiotrack to group call Write view model diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9e80b0ff..6c26c995 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -18,6 +18,8 @@ import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; +import {makeId} from "../../common"; + import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; import type {Track} from "../../../platform/types/MediaDevices"; @@ -30,12 +32,11 @@ import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; export enum GroupCallState { - LocalCallFeedUninitialized = "local_call_feed_uninitialized", - InitializingLocalCallFeed = "initializing_local_call_feed", - LocalCallFeedInitialized = "local_call_feed_initialized", - Joining = "entering", - Joined = "entered", - Ended = "ended", + Fledgling = "fledgling", + Creating = "creating", + Created = "created", + Joining = "joining", + Joined = "joined", } export type Options = Omit & { @@ -46,70 +47,112 @@ export type Options = Omit = new ObservableMap(); - private localMedia?: Promise; + private _localMedia?: LocalMedia; private _memberOptions: MemberOptions; - private _state: GroupCallState = GroupCallState.LocalCallFeedInitialized; - - // TODO: keep connected state and deal + private _state: GroupCallState; + constructor( - private callEvent: StateEvent, - private readonly room: Room, + id: string | undefined, + private callContent: Record | undefined, + private readonly roomId: string, private readonly options: Options ) { + this.id = id ?? makeId("conf-"); + this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({ confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.room.id, message, log); + return this.options.encryptDeviceMessage(this.roomId, message, log); } }, options); } - static async create(roomId: string, options: Options): Promise { - - } - + get localMedia(): LocalMedia | undefined { return this._localMedia; } get members(): BaseObservableMap { return this._members; } - get id(): string { return this.callEvent.state_key; } - get isTerminated(): boolean { - return this.callEvent.content["m.terminated"] === true; + return this.callContent?.["m.terminated"] === true; } - async join(localMedia: Promise) { - this.localMedia = localMedia; - const memberContent = await this._createOrUpdateOwnMemberStateContent(); + async join(localMedia: LocalMedia) { + if (this._state !== GroupCallState.Created) { + return; + } + this._state = GroupCallState.Joining; + this._localMedia = localMedia; + const memberContent = await this._joinCallMemberContent(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.room.id, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); await request.response(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this.localMedia); + member.connect(this._localMedia); + } + } + + async leave() { + const memberContent = await this._leaveCallMemberContent(); + // send m.call.member state event + if (memberContent) { + const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + await request.response(); } } /** @internal */ - updateCallEvent(callEvent: StateEvent) { - this.callEvent = callEvent; - // TODO: emit update + async create(localMedia: LocalMedia, name: string) { + if (this._state !== GroupCallState.Fledgling) { + return; + } + this._state = GroupCallState.Creating; + this.callContent = { + "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", + "m.name": name, + "m.intent": "m.ring" + }; + const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + await request.response(); + } + + /** @internal */ + updateCallEvent(callContent: Record) { + this.callContent = callContent; + if (this._state === GroupCallState.Creating) { + this._state = GroupCallState.Created; + } } /** @internal */ addMember(userId, memberCallInfo) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joining) { + this._state = GroupCallState.Joined; + } + return; + } let member = this._members.get(userId); if (member) { member.updateCallInfo(memberCallInfo); } else { - member = new Member(RoomMember.fromUserId(this.room.id, userId, "join"), this._memberOptions); - member.updateCallInfo(memberCallInfo); + member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions); this._members.add(userId, member); + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + member.connect(this._localMedia!); + } } } /** @internal */ removeMember(userId) { + if (userId === this.options.ownUserId) { + if (this._state === GroupCallState.Joined) { + this._state = GroupCallState.Created; + } + return; + } this._members.remove(userId); } @@ -124,10 +167,10 @@ export class GroupCall { } } - private async _createOrUpdateOwnMemberStateContent() { + private async _joinCallMemberContent() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.room.id, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -150,4 +193,13 @@ export class GroupCall { } return stateContent; } + + private async _leaveCallMemberContent(): Promise | undefined> { + const {storage} = this.options; + const txn = await storage.readTxn([storage.storeNames.roomState]); + const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const callsInfo = stateEvent?.event?.content?.["m.calls"]; + callsInfo?.filter(c => c["m.call_id"] === this.id); + return stateEvent?.event.content; + } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index cf020f37..0c574bb0 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -38,10 +38,11 @@ export type Options = Omit; + private localMedia?: LocalMedia; constructor( public readonly member: RoomMember, + private memberCallInfo: Record, private readonly options: Options ) {} @@ -53,13 +54,13 @@ export class Member { return this.peerCall?.state === CallState.Connected; } - /* @internal */ - connect(localMedia: Promise) { + /** @internal */ + connect(localMedia: LocalMedia) { this.localMedia = localMedia; // otherwise wait for it to connect if (this.member.userId < this.options.ownUserId) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(Promise.resolve(localMedia.clone())); } } @@ -71,13 +72,12 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(Promise.resolve(this.localMedia!)); } this.options.emitUpdate(this, params); } - /** From PeerCallHandler - * @internal */ + /** @internal */ sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId;