From a0a07355d47c336883eab7b00b06a9033701259b Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:52:19 +0100 Subject: [PATCH] more improvements, make hangup work --- src/domain/session/room/CallViewModel.ts | 26 +++++-- src/domain/session/room/RoomViewModel.js | 9 ++- .../room/timeline/tiles/BaseMessageTile.js | 8 --- .../session/room/timeline/tiles/CallTile.js | 59 +++++++++++++-- .../session/room/timeline/tiles/SimpleTile.js | 8 +++ .../session/room/timeline/tilesCreator.js | 4 +- src/matrix/Session.js | 1 - src/matrix/calls/CallHandler.ts | 8 +-- src/matrix/calls/LocalMedia.ts | 6 ++ src/matrix/calls/PeerCall.ts | 4 ++ src/matrix/calls/group/GroupCall.ts | 72 +++++++++++++++---- src/matrix/calls/group/Member.ts | 8 +++ src/matrix/room/Room.js | 2 +- .../value/PickMapObservableValue.ts | 8 +-- src/platform/web/ui/session/room/CallView.ts | 7 +- .../ui/session/room/timeline/CallTileView.ts | 8 ++- 16 files changed, 182 insertions(+), 56 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 7fcabf02..30b18bc1 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -33,16 +33,26 @@ export class CallViewModel extends ViewModel { .sortValues((a, b) => a.compare(b)); } + private get call(): GroupCall { + return this.getOption("call"); + } + get name(): string { - return this.getOption("call").name; + return this.call.name; } get id(): string { - return this.getOption("call").id; + return this.call.id; } get localTracks(): Track[] { - return this.getOption("call").localMedia?.tracks ?? []; + return this.call.localMedia?.tracks ?? []; + } + + leave() { + if (this.call.hasJoined) { + this.call.leave(); + } } } @@ -50,12 +60,16 @@ type MemberOptions = BaseOptions & {member: Member}; export class CallMemberViewModel extends ViewModel { get tracks(): Track[] { - return this.getOption("member").remoteTracks; + return this.member.remoteTracks; + } + + private get member(): Member { + return this.getOption("member"); } compare(other: CallMemberViewModel): number { - const myUserId = this.getOption("member").member.userId; - const otherUserId = other.getOption("member").member.userId; + const myUserId = this.member.member.userId; + const otherUserId = other.member.member.userId; if(myUserId === otherUserId) { return 0; } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 2b47673f..6dd8c8bd 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -49,12 +49,17 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key const calls = this.getOption("session").callHandler.calls; - this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); + this._callObservable = new PickMapObservableValue(calls.filterValues(c => { + return c.roomId === this._room.id && c.hasJoined; + })); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { + if (call && this._callViewModel && call.id === this._callViewModel.id) { + return; + } this._callViewModel = this.disposeTracked(this._callViewModel); if (call) { - this._callViewModel = new CallViewModel(this.childOptions({call})); + this._callViewModel = this.track(new CallViewModel(this.childOptions({call}))); } this.emitChange("callViewModel"); })); diff --git a/src/domain/session/room/timeline/tiles/BaseMessageTile.js b/src/domain/session/room/timeline/tiles/BaseMessageTile.js index 3385a587..c2041a2c 100644 --- a/src/domain/session/room/timeline/tiles/BaseMessageTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMessageTile.js @@ -49,14 +49,6 @@ export class BaseMessageTile extends SimpleTile { return `https://matrix.to/#/${encodeURIComponent(this.sender)}`; } - get displayName() { - return this._entry.displayName || this.sender; - } - - get sender() { - return this._entry.sender; - } - get memberPanelLink() { return `${this.urlCreator.urlUntilSegment("room")}/member/${this.sender}`; } diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js index 7129bb75..3e3918b6 100644 --- a/src/domain/session/room/timeline/tiles/CallTile.js +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -23,6 +23,27 @@ import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; // alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates export class CallTile extends SimpleTile { + + constructor(options) { + super(options); + const calls = this.getOption("session").callHandler.calls; + this._call = calls.get(this._entry.stateKey); + this._callSubscription = undefined; + if (this._call) { + this._callSubscription = this._call.disposableOn("change", () => { + // unsubscribe when terminated + if (this._call.isTerminated) { + this._callSubscription = this._callSubscription(); + this._call = undefined; + } + this.emitChange(); + }); + } + } + + get confId() { + return this._entry.stateKey; + } get shape() { return "call"; @@ -32,17 +53,43 @@ export class CallTile extends SimpleTile { return this._entry.content["m.name"]; } - get _call() { - const calls = this.getOption("session").callHandler.calls; - return calls.get(this._entry.stateKey); + get canJoin() { + return this._call && !this._call.hasJoined; + } + + get canLeave() { + return this._call && this._call.hasJoined; + } + + get label() { + if (this._call) { + if (this._call.hasJoined) { + return `Ongoing call (${this.name}, ${this.confId})`; + } else { + return `${this.displayName} started a call (${this.name}, ${this.confId})`; + } + } else { + return `Call finished, started by ${this.displayName} (${this.name}, ${this.confId})`; + } } async join() { - const call = this._call; - if (call) { + if (this.canJoin) { const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - await call.join(localMedia); + await this._call.join(localMedia); + } + } + + async leave() { + if (this.canLeave) { + this._call.leave(); + } + } + + dispose() { + if (this._callSubscription) { + this._callSubscription = this._callSubscription(); } } } diff --git a/src/domain/session/room/timeline/tiles/SimpleTile.js b/src/domain/session/room/timeline/tiles/SimpleTile.js index af2b0e12..d70a0a37 100644 --- a/src/domain/session/room/timeline/tiles/SimpleTile.js +++ b/src/domain/session/room/timeline/tiles/SimpleTile.js @@ -154,4 +154,12 @@ export class SimpleTile extends ViewModel { get _ownMember() { return this._options.timeline.me; } + + get displayName() { + return this._entry.displayName || this.sender; + } + + get sender() { + return this._entry.sender; + } } diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index 659a5e76..f35f6536 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -73,7 +73,9 @@ export function tilesCreator(baseOptions) { case "m.room.encryption": return new EncryptionEnabledTile(options); case "m.call": - return entry.stateKey ? new CallTile(options) : null; + // if prevContent is present, it's an update to a call event, which we don't render + // as the original event is updated through the call object which receive state event updates + return entry.stateKey && !entry.prevContent ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index 27689013..56822c70 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -86,7 +86,6 @@ export class Session { // although we probably already fetched all devices to send messages in the likely e2ee room await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); - console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index 269f6020..7b8a7f16 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -112,7 +112,7 @@ export class CallHandler { const callId = event.state_key; let call = this._calls.get(callId); if (call) { - call.updateCallEvent(event); + call.updateCallEvent(event.content); if (call.isTerminated) { this._calls.remove(call.id); } @@ -125,13 +125,13 @@ export class CallHandler { private handleCallMemberEvent(event: StateEvent) { const userId = event.state_key; const calls = event.content["m.calls"] ?? []; - const newCallIdsMemberOf = new Set(calls.map(call => { + for (const call of calls) { const callId = call["m.call_id"]; const groupCall = this._calls.get(callId); // TODO: also check the member when receiving the m.call event groupCall?.addMember(userId, call); - return callId; - })); + }; + const newCallIdsMemberOf = new Set(calls.map(call => call["m.call_id"])); let previousCallIdsMemberOf = this.memberToCallIds.get(userId); // remove user as member of any calls not present anymore if (previousCallIdsMemberOf) { diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 1fd4aad6..b148dd92 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -60,4 +60,10 @@ export class LocalMedia { } return metadata; } + + dispose() { + this.cameraTrack?.stop(); + this.microphoneTrack?.stop(); + this.screenShareTrack?.stop(); + } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 319fde3a..c6c6fca3 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -748,6 +748,10 @@ export class PeerCall implements IDisposable { this.disposables.dispose(); this.peerConnection.dispose(); } + + public close(): void { + this.peerConnection.close(); + } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 901eb3b3..a3148518 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -19,6 +19,7 @@ import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; import {RoomMember} from "../../room/members/RoomMember"; import {makeId} from "../../common"; +import {EventEmitter} from "../../../utils/EventEmitter"; import type {Options as MemberOptions} from "./Member"; import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; @@ -31,6 +32,9 @@ import type {EncryptedMessage} from "../../e2ee/olm/Encryption"; import type {ILogItem} from "../../../logging/types"; import type {Storage} from "../../storage/idb/Storage"; +const CALL_TYPE = "m.call"; +const CALL_MEMBER_TYPE = "m.call.member"; + export enum GroupCallState { Fledgling = "fledgling", Creating = "creating", @@ -46,7 +50,7 @@ export type Options = Omit { public readonly id: string; private readonly _members: ObservableMap = new ObservableMap(); private _localMedia?: LocalMedia = undefined; @@ -59,6 +63,7 @@ export class GroupCall { public readonly roomId: string, private readonly options: Options ) { + super(); this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; this._memberOptions = Object.assign({}, options, { @@ -87,12 +92,12 @@ export class GroupCall { } this._state = GroupCallState.Joining; this._localMedia = localMedia; - this.options.emitUpdate(this); - const memberContent = await this._joinCallMemberContent(); + this.emitChange(); + const memberContent = await this._createJoinPayload(); // send m.call.member state event - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); - this.options.emitUpdate(this); + this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { member.connect(this._localMedia); @@ -107,25 +112,41 @@ export class GroupCall { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event if (memberContent) { - const request = this.options.hsApi.sendState(this.roomId, "m.call.member", this.options.ownUserId, memberContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId, memberContent); await request.response(); + // our own user isn't included in members, so not in the count + if (this._members.size === 0) { + this.terminate(); + } } } + async terminate() { + if (this._state === GroupCallState.Fledgling) { + return; + } + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, Object.assign({}, this.callContent, { + "m.terminated": true + })); + await request.response(); + } + /** @internal */ async create(localMedia: LocalMedia, name: string) { if (this._state !== GroupCallState.Fledgling) { return; } this._state = GroupCallState.Creating; + this.emitChange(); this.callContent = { "m.type": localMedia.cameraTrack ? "m.video" : "m.voice", "m.name": name, "m.intent": "m.ring" }; - const request = this.options.hsApi.sendState(this.roomId, "m.call", this.id, this.callContent); + const request = this.options.hsApi.sendState(this.roomId, CALL_TYPE, this.id, this.callContent); await request.response(); this._state = GroupCallState.Created; + this.emitChange(); } /** @internal */ @@ -134,6 +155,7 @@ export class GroupCall { if (this._state === GroupCallState.Creating) { this._state = GroupCallState.Created; } + this.emitChange(); } /** @internal */ @@ -141,6 +163,7 @@ export class GroupCall { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joining) { this._state = GroupCallState.Joined; + this.emitChange(); } return; } @@ -160,11 +183,21 @@ export class GroupCall { removeMember(userId) { if (userId === this.options.ownUserId) { if (this._state === GroupCallState.Joined) { + this._localMedia?.dispose(); + this._localMedia = undefined; + for (const [,member] of this._members) { + member.disconnect(); + } this._state = GroupCallState.Created; } - return; + } else { + const member = this._members.get(userId); + if (member) { + this._members.remove(userId); + member.disconnect(); + } } - this._members.remove(userId); + this.emitChange(); } /** @internal */ @@ -179,10 +212,10 @@ export class GroupCall { } } - private async _joinCallMemberContent() { + private async _createJoinPayload() { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); const stateContent = stateEvent?.event?.content ?? { ["m.calls"]: [] }; @@ -209,9 +242,18 @@ export class GroupCall { private async _leaveCallMemberContent(): Promise | undefined> { const {storage} = this.options; const txn = await storage.readTxn([storage.storeNames.roomState]); - const stateEvent = await txn.roomState.get(this.roomId, "m.call.member", this.options.ownUserId); - const callsInfo = stateEvent?.event?.content?.["m.calls"]; - callsInfo?.filter(c => c["m.call_id"] === this.id); - return stateEvent?.event.content; + const stateEvent = await txn.roomState.get(this.roomId, CALL_MEMBER_TYPE, this.options.ownUserId); + if (stateEvent) { + const content = stateEvent.event.content; + const callsInfo = content["m.calls"]; + content["m.calls"] = callsInfo?.filter(c => c["m.call_id"] !== this.id); + return content; + + } + } + + protected emitChange() { + this.emit("change"); + this.options.emitUpdate(this); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 0a80bbef..dedde429 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -65,6 +65,14 @@ export class Member { } } + /** @internal */ + disconnect() { + this.peerCall?.close(); + this.peerCall?.dispose(); + this.peerCall = undefined; + this.localMedia = undefined; + } + /** @internal */ updateCallInfo(memberCallInfo) { // m.calls object from the m.call.member event diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js index ca11cdd8..ff1926b4 100644 --- a/src/matrix/room/Room.js +++ b/src/matrix/room/Room.js @@ -451,7 +451,7 @@ export class Room extends BaseRoom { _updateCallHandler(roomResponse, log) { if (this._callHandler) { const stateEvents = roomResponse.state?.events; - if (stateEvents) { + if (stateEvents?.length) { this._callHandler.handleRoomState(this, stateEvents, log); } let timelineEvents = roomResponse.timeline?.events; diff --git a/src/observable/value/PickMapObservableValue.ts b/src/observable/value/PickMapObservableValue.ts index 67f9d562..b493d841 100644 --- a/src/observable/value/PickMapObservableValue.ts +++ b/src/observable/value/PickMapObservableValue.ts @@ -60,13 +60,11 @@ export class PickMapObservableValue extends BaseObservableValue(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Track[]) { t.mapSideEffect(propSelector, tracks => { - console.log("tracks", tracks); if (tracks.length) { video.srcObject = (tracks[0] as TrackWrapper).stream; } @@ -33,8 +32,8 @@ function bindVideoTracks(t: TemplateBuilder, video: HTMLVideoElement, prop export class CallView extends TemplateView { render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { return t.div({class: "CallView"}, [ - t.p(["Call ", vm => vm.name, vm => ` (${vm.id})`]), - t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true}), vm => vm.localTracks)), + t.p(vm => `Call ${vm.name} (${vm.id})`), + t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localTracks)), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))) ]); } @@ -42,6 +41,6 @@ export class CallView extends TemplateView { class MemberView extends TemplateView { render(t: TemplateBuilder, vm: CallMemberViewModel) { - return bindVideoTracks(t, t.video({autoplay: true}), vm => vm.tracks); + return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.tracks); } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts index dfb04228..e0ca00bd 100644 --- a/src/platform/web/ui/session/room/timeline/CallTileView.ts +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -22,9 +22,9 @@ export class CallTileView extends TemplateView { return t.li( {className: "AnnouncementView"}, t.div([ - "Call ", - vm => vm.name, - t.button({className: "CallTileView_join"}, "Join") + vm => vm.label, + t.button({className: "CallTileView_join", hidden: vm => !vm.canJoin}, "Join"), + t.button({className: "CallTileView_leave", hidden: vm => !vm.canLeave}, "Leave") ]) ); } @@ -33,6 +33,8 @@ export class CallTileView extends TemplateView { onClick(evt) { if (evt.target.className === "CallTileView_join") { this.value.join(); + } else if (evt.target.className === "CallTileView_leave") { + this.value.leave(); } } }