From 0a37fd561e3c713faf4639d3413918d5bd111d06 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 23 Mar 2022 12:23:10 +0100 Subject: [PATCH] just enough view code to join a call --- src/domain/session/room/RoomViewModel.js | 7 +-- .../session/room/timeline/tiles/CallTile.js | 48 +++++++++++++++++++ .../session/room/timeline/tilesCreator.js | 3 ++ src/matrix/DeviceMessageHandler.js | 1 + src/matrix/Session.js | 7 +-- src/matrix/Sync.js | 1 + src/matrix/calls/PeerCall.ts | 1 + src/matrix/calls/group/GroupCall.ts | 15 ++++-- src/matrix/calls/group/Member.ts | 14 +++--- src/matrix/common.js | 17 +++++++ src/matrix/e2ee/RoomEncryption.js | 14 ++---- src/platform/web/ui/session/room/common.ts | 3 ++ .../ui/session/room/timeline/CallTileView.ts | 38 +++++++++++++++ 13 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/CallTile.js create mode 100644 src/platform/web/ui/session/room/timeline/CallTileView.ts diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 81ca58d2..2b47673f 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -48,7 +48,8 @@ export class RoomViewModel extends ViewModel { _setupCallViewModel() { // pick call for this room with lowest key - this._callObservable = new PickMapObservableValue(this.getOption("session").callHandler.calls.filterValues(c => c.roomId === this._room.id)); + const calls = this.getOption("session").callHandler.calls; + this._callObservable = new PickMapObservableValue(calls.filterValues(c => c.roomId === this._room.id && c.hasJoined)); this._callViewModel = undefined; this.track(this._callObservable.subscribe(call => { this._callViewModel = this.disposeTracked(this._callViewModel); @@ -68,6 +69,7 @@ export class RoomViewModel extends ViewModel { try { const timeline = await this._room.openTimeline(); this._tilesCreator = tilesCreator(this.childOptions({ + session: this.getOption("session"), roomVM: this, timeline, })); @@ -349,9 +351,8 @@ export class RoomViewModel extends ViewModel { async startCall() { try { const session = this.getOption("session"); - const mediaTracks = await this.platform.mediaDevices.getMediaTracks(true, true); + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withTracks(mediaTracks); - console.log("localMedia", localMedia.tracks); // this will set the callViewModel above as a call will be added to callHandler.calls await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); } catch (err) { diff --git a/src/domain/session/room/timeline/tiles/CallTile.js b/src/domain/session/room/timeline/tiles/CallTile.js new file mode 100644 index 00000000..7129bb75 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/CallTile.js @@ -0,0 +1,48 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {SimpleTile} from "./SimpleTile.js"; +import {LocalMedia} from "../../../../../matrix/calls/LocalMedia"; + +// TODO: timeline entries for state events with the same state key and type +// should also update previous entries in the timeline, so we can update the name of the call, whether it is terminated, etc ... + +// alternatively, we could just subscribe to the GroupCall and spontanously emit an update when it updates + +export class CallTile extends SimpleTile { + + get shape() { + return "call"; + } + + get name() { + return this._entry.content["m.name"]; + } + + get _call() { + const calls = this.getOption("session").callHandler.calls; + return calls.get(this._entry.stateKey); + } + + async join() { + const call = this._call; + if (call) { + const mediaTracks = await this.platform.mediaDevices.getMediaTracks(false, true); + const localMedia = new LocalMedia().withTracks(mediaTracks); + await call.join(localMedia); + } + } +} diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index dc9a850e..659a5e76 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -26,6 +26,7 @@ import {RoomMemberTile} from "./tiles/RoomMemberTile.js"; import {EncryptedEventTile} from "./tiles/EncryptedEventTile.js"; import {EncryptionEnabledTile} from "./tiles/EncryptionEnabledTile.js"; import {MissingAttachmentTile} from "./tiles/MissingAttachmentTile.js"; +import {CallTile} from "./tiles/CallTile.js"; export function tilesCreator(baseOptions) { const tilesCreator = function tilesCreator(entry, emitUpdate) { @@ -71,6 +72,8 @@ export function tilesCreator(baseOptions) { return new EncryptedEventTile(options); case "m.room.encryption": return new EncryptionEnabledTile(options); + case "m.call": + return entry.stateKey ? new CallTile(options) : null; default: // unknown type not rendered return null; diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js index 80fd1592..11c10750 100644 --- a/src/matrix/DeviceMessageHandler.js +++ b/src/matrix/DeviceMessageHandler.js @@ -86,6 +86,7 @@ export class DeviceMessageHandler { this._senderDeviceCache.set(device); } } + console.log("incoming device message", senderKey, device, this._senderDeviceCache); return device; } } diff --git a/src/matrix/Session.js b/src/matrix/Session.js index e604e068..27689013 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -78,14 +78,15 @@ export class Session { this._callHandler = new CallHandler({ createTimeout: this._platform.clock.createTimeout, hsApi: this._hsApi, - encryptDeviceMessage: async (roomId, message, log) => { + encryptDeviceMessage: async (roomId, userId, message, log) => { if (!this._deviceTracker || !this._olmEncryption) { throw new Error("encryption is not enabled"); } // TODO: just get the devices we're sending the message to, not all the room devices // although we probably already fetched all devices to send messages in the likely e2ee room - await this._deviceTracker.trackRoom(roomId, log); - const devices = await this._deviceTracker.devicesForTrackedRoom(roomId, this._hsApi, log); + await this._deviceTracker.trackRoom(this.rooms.get(roomId), log); + const devices = await this._deviceTracker.devicesForRoomMembers(roomId, [userId], this._hsApi, log); + console.log("devices", devices); const encryptedMessage = await this._olmEncryption.encrypt(message.type, message.content, devices, this._hsApi, log); return encryptedMessage; }, diff --git a/src/matrix/Sync.js b/src/matrix/Sync.js index 8e880def..b4ea702f 100644 --- a/src/matrix/Sync.js +++ b/src/matrix/Sync.js @@ -224,6 +224,7 @@ export class Sync { _openPrepareSyncTxn() { const storeNames = this._storage.storeNames; return this._storage.readTxn([ + storeNames.deviceIdentities, // to read device from olm messages storeNames.olmSessions, storeNames.inboundGroupSessions, // to read fragments when loading sync writer when rejoining archived room diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 9a702ebd..319fde3a 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -114,6 +114,7 @@ export class PeerCall implements IDisposable { } }); this.logger = { + info(...args) { console.info.apply(console, ["WebRTC debug:", ...args])}, debug(...args) { console.log.apply(console, ["WebRTC debug:", ...args])}, log(...args) { console.log.apply(console, ["WebRTC log:", ...args])}, warn(...args) { console.log.apply(console, ["WebRTC warn:", ...args])}, diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index d22135d1..901eb3b3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -41,7 +41,7 @@ export enum GroupCallState { export type Options = Omit & { emitUpdate: (call: GroupCall, params?: any) => void; - encryptDeviceMessage: (roomId: string, message: SignallingMessage, log: ILogItem) => Promise, + encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage, log: ILogItem) => Promise, storage: Storage, ownDeviceId: string }; @@ -61,13 +61,13 @@ export class GroupCall { ) { this.id = id ?? makeId("conf-"); this._state = id ? GroupCallState.Created : GroupCallState.Fledgling; - this._memberOptions = Object.assign({ + this._memberOptions = Object.assign({}, options, { confId: this.id, emitUpdate: member => this._members.update(member.member.userId, member), - encryptDeviceMessage: (message: SignallingMessage, log) => { - return this.options.encryptDeviceMessage(this.roomId, message, log); + encryptDeviceMessage: (userId: string, message: SignallingMessage, log) => { + return this.options.encryptDeviceMessage(this.roomId, userId, message, log); } - }, options); + }); } get localMedia(): LocalMedia | undefined { return this._localMedia; } @@ -99,6 +99,10 @@ export class GroupCall { } } + get hasJoined() { + return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; + } + async leave() { const memberContent = await this._leaveCallMemberContent(); // send m.call.member state event @@ -165,6 +169,7 @@ export class GroupCall { /** @internal */ handleDeviceMessage(message: SignallingMessage, userId: string, deviceId: string, log: ILogItem) { + console.log("incoming to_device call signalling message from", userId, deviceId, message); // TODO: return if we are not membering to the call let member = this._members.get(userId); if (member) { diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index bd1613cd..0a80bbef 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -17,6 +17,7 @@ limitations under the License. import {PeerCall, CallState} from "../PeerCall"; import {makeTxnId, makeId} from "../../common"; import {EventType} from "../callEventTypes"; +import {formatToDeviceMessagesPayload} from "../../common"; import type {Options as PeerCallOptions} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; @@ -32,7 +33,7 @@ export type Options = Omit, log: ILogItem) => Promise, + encryptDeviceMessage: (userId: string, message: SignallingMessage, log: ILogItem) => Promise, emitUpdate: (participant: Member, params?: any) => void, } @@ -81,13 +82,14 @@ export class Member { sendSignallingMessage = async (message: SignallingMessage, log: ILogItem) => { const groupMessage = message as SignallingMessage; groupMessage.content.conf_id = this.options.confId; - const encryptedMessage = await this.options.encryptDeviceMessage(groupMessage, log); + const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log); + const payload = formatToDeviceMessagesPayload(encryptedMessages); const request = this.options.hsApi.sendToDevice( "m.room.encrypted", - {[this.member.userId]: { - ["*"]: encryptedMessage.content - } - }, makeTxnId(), {log}); + payload, + makeTxnId(), + {log} + ); await request.response(); } diff --git a/src/matrix/common.js b/src/matrix/common.js index 5919ad9c..7cd72ae1 100644 --- a/src/matrix/common.js +++ b/src/matrix/common.js @@ -15,6 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +import {groupBy} from "../utils/groupBy"; + + export function makeTxnId() { return makeId("t"); } @@ -29,6 +32,20 @@ export function isTxnId(txnId) { return txnId.startsWith("t") && txnId.length === 15; } +export function formatToDeviceMessagesPayload(messages) { + const messagesByUser = groupBy(messages, message => message.device.userId); + const payload = { + messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { + userMap[userId] = messages.reduce((deviceMap, message) => { + deviceMap[message.device.deviceId] = message.content; + return deviceMap; + }, {}); + return userMap; + }, {}) + }; + return payload; +} + export function tests() { return { "isTxnId succeeds on result of makeTxnId": assert => { diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js index 80f57507..cb0dd333 100644 --- a/src/matrix/e2ee/RoomEncryption.js +++ b/src/matrix/e2ee/RoomEncryption.js @@ -18,7 +18,7 @@ import {MEGOLM_ALGORITHM, DecryptionSource} from "./common.js"; import {groupEventsBySession} from "./megolm/decryption/utils"; import {mergeMap} from "../../utils/mergeMap"; import {groupBy} from "../../utils/groupBy"; -import {makeTxnId} from "../common.js"; +import {makeTxnId, formatToDeviceMessagesPayload} from "../common.js"; const ENCRYPTED_TYPE = "m.room.encrypted"; // how often ensureMessageKeyIsShared can check if it needs to @@ -386,6 +386,7 @@ export class RoomEncryption { await writeTxn.complete(); } + // TODO: make this use _sendMessagesToDevices async _sendSharedMessageToDevices(type, message, devices, hsApi, log) { const devicesByUser = groupBy(devices, device => device.userId); const payload = { @@ -403,16 +404,7 @@ export class RoomEncryption { async _sendMessagesToDevices(type, messages, hsApi, log) { log.set("messages", messages.length); - const messagesByUser = groupBy(messages, message => message.device.userId); - const payload = { - messages: Array.from(messagesByUser.entries()).reduce((userMap, [userId, messages]) => { - userMap[userId] = messages.reduce((deviceMap, message) => { - deviceMap[message.device.deviceId] = message.content; - return deviceMap; - }, {}); - return userMap; - }, {}) - }; + const payload = formatToDeviceMessagesPayload(messages); const txnId = makeTxnId(); await hsApi.sendToDevice(type, payload, txnId, {log}).response(); } diff --git a/src/platform/web/ui/session/room/common.ts b/src/platform/web/ui/session/room/common.ts index 5048211a..a2732ff4 100644 --- a/src/platform/web/ui/session/room/common.ts +++ b/src/platform/web/ui/session/room/common.ts @@ -24,6 +24,7 @@ import {AnnouncementView} from "./timeline/AnnouncementView.js"; import {RedactedView} from "./timeline/RedactedView.js"; import {SimpleTile} from "../../../../../domain/session/room/timeline/tiles/SimpleTile.js"; import {GapView} from "./timeline/GapView.js"; +import {CallTileView} from "./timeline/CallTileView"; export type TileView = GapView | AnnouncementView | TextMessageView | ImageView | VideoView | FileView | LocationView | MissingAttachmentView | RedactedView; @@ -51,5 +52,7 @@ export function viewClassForEntry(entry: SimpleTile): TileViewConstructor | unde return MissingAttachmentView; case "redacted": return RedactedView; + case "call": + return CallTileView; } } diff --git a/src/platform/web/ui/session/room/timeline/CallTileView.ts b/src/platform/web/ui/session/room/timeline/CallTileView.ts new file mode 100644 index 00000000..dfb04228 --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/CallTileView.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView} from "../../../general/TemplateView"; +import type {CallTile} from "../../../../../../domain/session/room/timeline/tiles/CallTile"; + +export class CallTileView extends TemplateView { + render(t, vm) { + return t.li( + {className: "AnnouncementView"}, + t.div([ + "Call ", + vm => vm.name, + t.button({className: "CallTileView_join"}, "Join") + ]) + ); + } + + /* This is called by the parent ListView, which just has 1 listener for the whole list */ + onClick(evt) { + if (evt.target.className === "CallTileView_join") { + this.value.join(); + } + } +}