diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index c60ce649..464a39b0 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -16,8 +16,8 @@ limitations under the License. */ import {TimelineViewModel} from "./timeline/TimelineViewModel.js"; -import {ComposerViewModel} from "./ComposerViewModel.js" -import {CallViewModel} from "./CallViewModel" +import {ComposerViewModel} from "./ComposerViewModel.js"; +import {CallViewModel} from "./call/CallViewModel"; import {PickMapObservableValue} from "../../../observable/value/PickMapObservableValue"; import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; import {ViewModel} from "../../ViewModel"; diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/call/CallViewModel.ts similarity index 86% rename from src/domain/session/room/CallViewModel.ts rename to src/domain/session/room/call/CallViewModel.ts index 08bc3691..c41e1fa1 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/call/CallViewModel.ts @@ -14,19 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {AvatarSource} from "../../AvatarSource"; -import {ViewModel, Options as BaseOptions} from "../../ViewModel"; -import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/common"; -import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar"; -import {EventObservableValue} from "../../../observable/value/EventObservableValue"; -import {ObservableValueMap} from "../../../observable/map/ObservableValueMap"; -import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; -import type {Member} from "../../../matrix/calls/group/Member"; -import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; -import type {Stream} from "../../../platform/types/MediaDevices"; -import type {MediaRepository} from "../../../matrix/net/MediaRepository"; +import {AvatarSource} from "../../../AvatarSource"; +import {ViewModel, Options as BaseOptions} from "../../../ViewModel"; +import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../matrix/calls/common"; +import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../../avatar"; +import {EventObservableValue} from "../../../../observable/value/EventObservableValue"; +import {ObservableValueMap} from "../../../../observable/map/ObservableValueMap"; +import type {GroupCall} from "../../../../matrix/calls/group/GroupCall"; +import type {Member} from "../../../../matrix/calls/group/Member"; +import type {BaseObservableList} from "../../../../observable/list/BaseObservableList"; +import type {Stream} from "../../../../platform/types/MediaDevices"; +import type {MediaRepository} from "../../../../matrix/net/MediaRepository"; -type Options = BaseOptions & { +export type Options = BaseOptions & { call: GroupCall, mediaRepository: MediaRepository }; @@ -146,8 +146,7 @@ export class CallMemberViewModel extends ViewModel implements ISt avatarUrl(size: number): string | undefined { const {avatarUrl} = this.member.member; - const mediaRepository = this.getOption("mediaRepository"); - return getAvatarHttpUrl(avatarUrl, size, this.platform, mediaRepository); + return getAvatarHttpUrl(avatarUrl, size, this.platform, this.options.mediaRepository); } get avatarTitle(): string { diff --git a/src/domain/session/room/call/PrepareCallViewModel.ts b/src/domain/session/room/call/PrepareCallViewModel.ts new file mode 100644 index 00000000..6cfc4303 --- /dev/null +++ b/src/domain/session/room/call/PrepareCallViewModel.ts @@ -0,0 +1,128 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {AvatarSource} from "../../../AvatarSource"; +import {ViewModel, Options as BaseOptions} from "../../../ViewModel"; +import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../matrix/calls/common"; +import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../../avatar"; +import {EventObservableValue} from "../../../../observable/value/EventObservableValue"; +import {ObservableValueMap} from "../../../../observable/map/ObservableValueMap"; +import type {GroupCall} from "../../../../matrix/calls/group/GroupCall"; +import type {Member} from "../../../../matrix/calls/group/Member"; +import type {BaseObservableList} from "../../../../observable/list/BaseObservableList"; +import type {BaseObservableMap} from "../../../../observable/map/BaseObservableMap"; +import type {Stream, VolumeMeasurer, MediaDevices} from "../../../../platform/types/MediaDevices"; +import type {MediaRepository} from "../../../../matrix/net/MediaRepository"; +import type {Options as CallViewModelOptions} from "./CallViewModel"; + +type Options = Omit & { + call?: GroupCall +}; + +export class PrepareCallViewModel extends ViewModel { + private _availableVideoDevices?: BaseObservableList; + private _availableAudioDevices?: BaseObservableList; + private _audioDevice?: AudioDeviceViewModel; + private _videoDevice?: VideoDeviceViewModel; + + async init() { + const devices = await (this.platform.mediaDevices as MediaDevices).observeDevices(); + const sortByNameAndId = (a: BaseDeviceViewModel, b: BaseDeviceViewModel) => { + const labelCmp = a.label.localeCompare(b.label); + if (labelCmp === 0) { + return a.deviceId.localeCompare(b.deviceId); + } else { + return labelCmp; + } + }; + this._availableAudioDevices = devices + .filterValues(v => v.kind === "videoinput") + .mapValues((device, emitChange) => { + const vm = new VideoDeviceViewModel(this.childOptions({device, emitChange})); + vm.init(); + return vm; + }).sortValues(sortByNameAndId); + this._availableAudioDevices = devices + .filterValues(v => v.kind === "audioinput") + .mapValues((device, emitChange) => { + const vm = new AudioDeviceViewModel(this.childOptions({device, emitChange})); + vm.init(); + return vm; + }).sortValues(sortByNameAndId); + } + + get availableVideoDevices(): BaseObservableList | undefined { + return this._availableVideoDevices; + } + + get availableAudioDevices(): BaseObservableList | undefined { + return this._availableAudioDevices; + } + + get videoDevice() : DeviceViewModel | undefined { + + } + + setVideoDeviceId(deviceId: string) { + this.videoDevice = this._availableVideoDevices.get(deviceId); + } + + get audioDevice() : DeviceViewModel | undefined { + + } +} + +type DeviceOptions = BaseOptions & { + device: MediaDeviceInfo, +} + +class BaseDeviceViewModel extends ViewModel { + get label(): string { return this.options.device.label; } + get deviceId(): string { return this.options.device.deviceId; } +} + +class VideoDeviceViewModel extends BaseDeviceViewModel { + private _stream: Stream; + + async init() { + try { + this._stream = await (this.platform.mediaDevices as MediaDevices).getMediaTracks(false, {deviceId: this.deviceId}); + this.track(() => getStreamVideoTrack(this._stream)?.stop()); + } catch (err) {} + } + + get stream(): Stream { + return this._stream; + } +} + +class AudioDeviceViewModel extends BaseDeviceViewModel { + private volumeMeasurer?: VolumeMeasurer; + + async init() { + try { + const stream = await (this.platform.mediaDevices as MediaDevices).getMediaTracks({deviceId: this.deviceId}, false); + this.track(() => getStreamAudioTrack(stream)?.stop()); + this.volumeMeasurer = this.track((this.platform.mediaDevices as MediaDevices).createVolumeMeasurer(stream, () => { + this.emitChange("volume"); + })); + } catch (err) {} + } + + get volume(): number { + return this.volumeMeasurer?.volume ?? 0; + } +} diff --git a/src/observable/index.ts b/src/observable/index.ts index 040ec761..f513c85a 100644 --- a/src/observable/index.ts +++ b/src/observable/index.ts @@ -34,8 +34,8 @@ Object.assign(BaseObservableMap.prototype, { return new SortedMapList(this, comparator); }, - mapValues(mapper, updater) { - return new MappedMap(this, mapper, updater); + mapValues(mapper, updater, remover) { + return new MappedMap(this, mapper, updater, remover); }, filterValues(filter) { diff --git a/src/observable/map/MappedMap.js b/src/observable/map/MappedMap.js index a6b65c41..c8b57931 100644 --- a/src/observable/map/MappedMap.js +++ b/src/observable/map/MappedMap.js @@ -20,11 +20,12 @@ so a mapped value can emit updates on it's own with this._emitSpontaneousUpdate how should the mapped value be notified of an update though? and can it then decide to not propagate the update? */ export class MappedMap extends BaseObservableMap { - constructor(source, mapper, updater) { + constructor(source, mapper, updater, remover) { super(); this._source = source; this._mapper = mapper; this._updater = updater; + this._remover = remover; this._mappedValues = new Map(); } @@ -45,6 +46,9 @@ export class MappedMap extends BaseObservableMap { onRemove(key/*, _value*/) { const mappedValue = this._mappedValues.get(key); if (this._mappedValues.delete(key)) { + if (this._remover) { + this._remover(mappedValue); + } this.emitRemove(key, mappedValue); } } diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 1b5f7afd..b9596a57 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -14,13 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ +import type {BaseObservableMap} from "../../observable/map/BaseObservableMap"; + export interface Event {} +export type DeviceFilter = { + deviceId: string +}; + export interface MediaDevices { // filter out audiooutput - enumerate(): Promise; + observeDevices(): Promise>; // to assign to a video element, we downcast to WrappedTrack and use the stream property. - getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; + getMediaTracks(audio: boolean | DeviceFilter, video: boolean | DeviceFilter): Promise; getScreenShareTrack(): Promise; createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer; } @@ -74,6 +80,7 @@ export interface Track { export interface VolumeMeasurer { get isSpeaking(): boolean; + get volume(): number; setSpeakingThreshold(threshold: number): void; - stop(); + dispose(); } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index c34ab85b..d6857659 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer, DeviceFilter} from "../../types/MediaDevices"; +import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap"; +import {ObservableMap} from "../../../observable/map/ObservableMap"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -24,11 +26,12 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export class MediaDevicesWrapper implements IMediaDevices { constructor(private readonly mediaDevices: MediaDevices) {} - enumerate(): Promise { - return this.mediaDevices.enumerateDevices(); + async observeDevices(): Promise> { + const initialDevices = await this.mediaDevices.enumerateDevices(); + return new DeviceObservableMap(initialDevices, this.mediaDevices); } - async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { + async getMediaTracks(audio: boolean | DeviceFilter, video: boolean | DeviceFilter): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); return stream as Stream; } @@ -38,7 +41,7 @@ export class MediaDevicesWrapper implements IMediaDevices { return stream as Stream; } - private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { + private getUserMediaContraints(audio: boolean | DeviceFilter, video: boolean | DeviceFilter): MediaStreamConstraints { const isWebkit = !!navigator["webkitGetUserMedia"]; return { @@ -109,7 +112,6 @@ export class WebAudioVolumeMeasurer implements VolumeMeasurer { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); this.callback(); - // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -133,6 +135,16 @@ export class WebAudioVolumeMeasurer implements VolumeMeasurer { this.speakingThreshold = threshold; } + get volume(): number { + for (let i = SPEAKING_SAMPLE_COUNT -1; i <= 0; i -= 1) { + const sample = this.speakingVolumeSamples[i]; + if (Number.isSafeInteger(sample)) { + return sample; + } + } + return 0; + } + private volumeLooper = () => { if (!this.analyser) return; @@ -151,7 +163,6 @@ export class WebAudioVolumeMeasurer implements VolumeMeasurer { this.speakingVolumeSamples.push(maxVolume); this.callback(); - // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -167,15 +178,58 @@ export class WebAudioVolumeMeasurer implements VolumeMeasurer { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; this.callback(); - // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; }; - public stop(): void { + public dispose(): void { clearTimeout(this.volumeLooperTimeout); this.analyser.disconnect(); this.audioContext?.close(); } } + +class DeviceObservableMap extends ObservableMap { + private updatePromise?: Promise = undefined; + + constructor(initialDevices: ReadonlyArray, private readonly mediaDevices: MediaDevices) { + super(); + this.processDevices(initialDevices); + } + + handleEvent(evt) { + if (evt.type === "devicechange") { + // serialize updates + this.updatePromise = (this.updatePromise ?? Promise.resolve()).then(() => this.updateDevices()); + } + } + + private async updateDevices() { + const devices = await this.mediaDevices.enumerateDevices(); + this.processDevices(devices); + } + + private processDevices(devices: ReadonlyArray) { + const inputDevices = devices.filter(d => d.kind === "videoinput" || d.kind === "audioinput"); + for (const [,device] of this) { + const stillPresent = inputDevices.some(d => d.deviceId === device.deviceId); + if (!stillPresent) { + this.remove(device.deviceId); + } + } + for (const device of inputDevices) { + this.add(device.deviceId, device); + } + } + + onSubscribeFirst() { + super.onSubscribeFirst(); + this.mediaDevices.addEventListener("devicechange", this); + } + + onUnsubscribeLast() { + this.mediaDevices.removeEventListener("devicechange", this); + super.onUnsubscribeLast(); + } +}