From 41288683fc9547ef1cf80cfe56ac472bf6afd712 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 10 Jun 2022 17:10:23 +0200 Subject: [PATCH] allow unmuting when we don't yet have a mic/cam track --- scripts/postcss/css-url-to-variables.js | 8 ++- src/domain/session/room/CallViewModel.ts | 55 +++++++++---------- src/matrix/calls/PeerCall.ts | 12 ++-- src/matrix/calls/common.ts | 28 +++++++++- src/matrix/calls/group/GroupCall.ts | 25 +++++++-- .../web/ui/css/themes/element/call.css | 9 +++ src/platform/web/ui/session/room/CallView.ts | 14 ++++- 7 files changed, 105 insertions(+), 46 deletions(-) diff --git a/scripts/postcss/css-url-to-variables.js b/scripts/postcss/css-url-to-variables.js index 1d4666f4..9e2e0077 100644 --- a/scripts/postcss/css-url-to-variables.js +++ b/scripts/postcss/css-url-to-variables.js @@ -26,7 +26,13 @@ const idToPrepend = "icon-url"; function findAndReplaceUrl(decl) { const value = decl.value; - const parsed = valueParser(value); + let parsed; + try { + parsed = valueParser(value); + } catch (err) { + console.log(`Error trying to parse ${decl}`); + throw err; + } parsed.walk(node => { if (node.type !== "function" || node.value !== "url") { return; diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 3e7fd951..5b6f05b4 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -60,18 +60,17 @@ export class CallViewModel extends ViewModel { } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get memberCount(): number { return this.memberViewModels.length; } - get name(): string { return this.call.name; } @@ -84,22 +83,36 @@ export class CallViewModel extends ViewModel { return this.getOption("call"); } - hangup() { + async hangup() { if (this.call.hasJoined) { - this.call.leave(); + await this.call.leave(); } } async toggleCamera() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleCamera()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.camera && !getStreamVideoTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(!muteSettings.microphone, true); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleCamera()); + } this.emitChange(); } } async toggleMicrophone() { - if (this.call.muteSettings) { - this.call.setMuted(this.call.muteSettings.toggleMicrophone()); + const {localMedia, muteSettings} = this.call; + if (muteSettings && localMedia) { + // unmute but no track? + if (muteSettings.microphone && !getStreamAudioTrack(localMedia.userMedia)) { + const stream = await this.platform.mediaDevices.getMediaTracks(true, !muteSettings.camera); + await this.call.setMedia(localMedia.withUserMedia(stream)); + } else { + await this.call.setMuted(muteSettings.toggleMicrophone()); + } this.emitChange(); } } @@ -130,11 +143,11 @@ class OwnMemberViewModel extends ViewModel implements IStreamViewModel } get isCameraMuted(): boolean { - return isLocalCameraMuted(this.call); + return this.call.muteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isLocalMicrophoneMuted(this.call); + return this.call.muteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -186,11 +199,11 @@ export class CallMemberViewModel extends ViewModel implements ISt } get isCameraMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream)); + return this.member.remoteMuteSettings?.camera ?? true; } get isMicrophoneMuted(): boolean { - return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream)); + return this.member.remoteMuteSettings?.microphone ?? true; } get avatarLetter(): string { @@ -229,19 +242,3 @@ export interface IStreamViewModel extends AvatarSource, ViewModel { get isCameraMuted(): boolean; get isMicrophoneMuted(): boolean; } - -function isMuted(muted: boolean | undefined, hasTrack: boolean) { - if (muted) { - return true; - } else { - return !hasTrack; - } -} - -function isLocalCameraMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.camera, !!getStreamVideoTrack(call.localMedia?.userMedia)); -} - -function isLocalMicrophoneMuted(call: GroupCall): boolean { - return isMuted(call.muteSettings?.microphone, !!getStreamAudioTrack(call.localMedia?.userMedia)); -} diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index ea7e9d06..809decbf 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -887,8 +887,8 @@ export class PeerCall implements IDisposable { const streamId = this.localMedia.userMedia.id; metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), - video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), + audio_muted: this.localMuteSettings?.microphone ?? false, + video_muted: this.localMuteSettings?.camera ?? false, }; } if (this.localMedia?.screenShare) { @@ -936,7 +936,7 @@ export class PeerCall implements IDisposable { this.updateRemoteMedia(log); } } - }) + }); }; stream.addEventListener("removetrack", listener); const disposeListener = () => { @@ -971,8 +971,10 @@ export class PeerCall implements IDisposable { videoReceiver.track.enabled = !metaData.video_muted; } this._remoteMuteSettings = new MuteSettings( - metaData.audio_muted || !audioReceiver?.track, - metaData.video_muted || !videoReceiver?.track + metaData.audio_muted ?? false, + metaData.video_muted ?? false, + !!audioReceiver?.track ?? false, + !!videoReceiver?.track ?? false ); log.log({ l: "setting userMedia", diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index db49a168..66db6edc 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -25,14 +25,36 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin } export class MuteSettings { - constructor (public readonly microphone: boolean = false, public readonly camera: boolean = false) {} + constructor ( + private readonly isMicrophoneMuted: boolean = false, + private readonly isCameraMuted: boolean = false, + private hasMicrophoneTrack: boolean = false, + private hasCameraTrack: boolean = false, + ) {} + + updateTrackInfo(userMedia: Stream | undefined) { + this.hasMicrophoneTrack = !!getStreamAudioTrack(userMedia); + this.hasCameraTrack = !!getStreamVideoTrack(userMedia); + } + + get microphone(): boolean { + return !this.hasMicrophoneTrack || this.isMicrophoneMuted; + } + + get camera(): boolean { + return !this.hasCameraTrack || this.isCameraMuted; + } toggleCamera(): MuteSettings { - return new MuteSettings(this.microphone, !this.camera); + return new MuteSettings(this.microphone, !this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); } toggleMicrophone(): MuteSettings { - return new MuteSettings(!this.microphone, this.camera); + return new MuteSettings(!this.microphone, this.camera, this.hasMicrophoneTrack, this.hasCameraTrack); + } + + equals(other: MuteSettings) { + return this.microphone === other.microphone && this.camera === other.camera; } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index ebacf844..19e40dbf 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -137,11 +137,13 @@ export class GroupCall extends EventEmitter<{change: never}> { ownSessionId: this.options.sessionId }); const membersLogItem = logItem.child("member connections"); + const localMuteSettings = new MuteSettings(); + localMuteSettings.updateTrackInfo(localMedia.userMedia); const joinedData = new JoinedData( logItem, membersLogItem, localMedia, - new MuteSettings() + localMuteSettings ); this.joinedData = joinedData; await joinedData.logItem.wrap("join", async log => { @@ -163,9 +165,14 @@ export class GroupCall extends EventEmitter<{change: never}> { if ((this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) && this.joinedData) { const oldMedia = this.joinedData.localMedia; this.joinedData.localMedia = localMedia; + // reflect the fact we gained or lost local tracks in the local mute settings + // and update the track info so PeerCall can use it to send up to date metadata, + this.joinedData.localMuteSettings.updateTrackInfo(localMedia.userMedia); + this.emitChange(); //allow listeners to see new media/mute settings await Promise.all(Array.from(this._members.values()).map(m => { return m.setMedia(localMedia, oldMedia); })); + oldMedia?.stopExcept(localMedia); } } @@ -175,11 +182,19 @@ export class GroupCall extends EventEmitter<{change: never}> { if (!joinedData) { return; } + const prevMuteSettings = joinedData.localMuteSettings; + // we still update the mute settings if nothing changed because + // you might be muted because you don't have a track or because + // you actively chosen to mute + // (which we want to respect in the future when you add a track) joinedData.localMuteSettings = muteSettings; - await Promise.all(Array.from(this._members.values()).map(m => { - return m.setMuted(joinedData.localMuteSettings); - })); - this.emitChange(); + joinedData.localMuteSettings.updateTrackInfo(joinedData.localMedia.userMedia); + if (!prevMuteSettings.equals(muteSettings)) { + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMuted(joinedData.localMuteSettings); + })); + this.emitChange(); + } } get muteSettings(): MuteSettings | undefined { diff --git a/src/platform/web/ui/css/themes/element/call.css b/src/platform/web/ui/css/themes/element/call.css index 4398f9c6..9bdc4abb 100644 --- a/src/platform/web/ui/css/themes/element/call.css +++ b/src/platform/web/ui/css/themes/element/call.css @@ -40,6 +40,7 @@ limitations under the License. border-radius: 8px; overflow: hidden; background-color: black; + display: block; } .StreamView > * { @@ -98,11 +99,19 @@ limitations under the License. background-repeat: no-repeat; } +.CallView_buttons button:disabled { + background-color: var(--accent-color--lighter-10); +} + .CallView_buttons .CallView_hangup { background-color: var(--error-color); background-image: url("./icons/hangup.svg?primary=background-color-primary"); } +.CallView_buttons .CallView_hangup:disabled { + background-color: var(--error-color--lighter-10); +} + .CallView_buttons .CallView_mutedMicrophone { background-color: var(--background-color-primary); background-image: url("./icons/mic-muted.svg?primary=text-color"); diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 5ccdaa84..6961dc53 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -37,12 +37,12 @@ export class CallView extends TemplateView { t.button({className: { "CallView_mutedMicrophone": vm => vm.isMicrophoneMuted, "CallView_unmutedMicrophone": vm => !vm.isMicrophoneMuted, - }, onClick: () => vm.toggleMicrophone()}), + }, onClick: disableTargetCallback(() => vm.toggleMicrophone())}), t.button({className: { "CallView_mutedCamera": vm => vm.isCameraMuted, "CallView_unmutedCamera": vm => !vm.isCameraMuted, - }, onClick: () => vm.toggleCamera()}), - t.button({className: "CallView_hangup", onClick: () => vm.hangup()}), + }, onClick: disableTargetCallback(() => vm.toggleCamera())}), + t.button({className: "CallView_hangup", onClick: disableTargetCallback(() => vm.hangup())}), ]) ]); } @@ -122,3 +122,11 @@ class StreamView extends TemplateView { this.updateSubViews(value, props); } } + +function disableTargetCallback(callback: (evt: Event) => Promise): (evt: Event) => Promise { + return async (evt: Event) => { + (evt.target as HTMLElement)?.setAttribute("disabled", "disabled"); + await callback(evt); + (evt.target as HTMLElement)?.removeAttribute("disabled"); + } +}