From 480c5c1584c43c0f2d1ad82054e33b62a442a730 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:49:54 +0200 Subject: [PATCH 01/35] update SDK docs with new style location --- doc/SDK.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/SDK.md b/doc/SDK.md index 8ce0b304..54e37cca 100644 --- a/doc/SDK.md +++ b/doc/SDK.md @@ -47,7 +47,8 @@ const assetPaths = { wasmBundle: olmJsPath } }; -import "hydrogen-view-sdk/style.css"; +import "hydrogen-view-sdk/theme-element-light.css"; +// OR import "hydrogen-view-sdk/theme-element-dark.css"; async function main() { const app = document.querySelector('#app')! From 382fba88bdf757887ac978712fc8004788006bfb Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 14 Apr 2022 23:19:44 +0200 Subject: [PATCH 02/35] WIP for muting --- src/domain/session/room/CallViewModel.ts | 6 + src/matrix/calls/LocalMedia.ts | 14 +- src/matrix/calls/PeerCall.ts | 256 +++++++++---------- src/matrix/calls/TODO.md | 2 +- src/matrix/calls/group/GroupCall.ts | 11 + src/matrix/calls/group/Member.ts | 8 + src/platform/types/MediaDevices.ts | 3 + src/platform/web/dom/MediaDevices.ts | 34 ++- src/platform/web/dom/WebRTC.ts | 8 + src/platform/web/index.html | 2 +- src/platform/web/ui/general/TemplateView.ts | 6 +- src/platform/web/ui/session/room/CallView.ts | 23 +- 12 files changed, 210 insertions(+), 163 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index bc7b54ff..05245cd3 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel { this.call.leave(); } } + + async toggleVideo() { + const localMedia = this.call.localMedia!; + const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); + await this.call.setMedia(toggledMedia); + } } type MemberOptions = BaseOptions & {member: Member}; diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 6564adac..7dbc9e17 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes"; export class LocalMedia { constructor( public readonly userMedia?: Stream, + public readonly microphoneMuted: boolean = false, + public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} + withMuted(microphone: boolean, camera: boolean) { + return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); + } + withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.screenShare, options); + return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 5b6e2ca8..b450ac3f 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -31,6 +31,7 @@ import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, + CallErrorCode, } from "./callEventTypes"; import type { MCallBase, @@ -66,6 +67,7 @@ export class PeerCall implements IDisposable { private readonly peerConnection: PeerConnection; private _state = CallState.Fledgling; private direction: CallDirection; + // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; private seq: number = 0; // A queue for candidates waiting to go out. @@ -151,7 +153,6 @@ export class PeerCall implements IDisposable { get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } - // we should keep an object with streams by purpose ... e.g. RemoteMedia? get remoteMedia(): Readonly { return this._remoteMedia; } @@ -163,7 +164,7 @@ export class PeerCall implements IDisposable { } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.setMedia(localMedia); + this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } @@ -179,7 +180,7 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateAnswer, log); - this.setMedia(localMedia, log); + this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -208,48 +209,32 @@ export class PeerCall implements IDisposable { }); } - setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise { - return logItem.wrap("setMedia", async log => { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { - const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; + setMedia(localMedia: LocalMedia): Promise { + return this.logItem.wrap("setMedia", async log => { + log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); + log.set("userMedia_audio_muted", localMedia.microphoneMuted); + log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); + log.set("userMedia_video_muted", localMedia.cameraMuted); + log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("datachannel", !!localMedia.dataChannelOptions); - const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { - if (track) { - if (oldTrack && sender) { - log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - sender.replaceTrack(track); - }); - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => { - this.peerConnection.removeTrack(sender); - }); - } - } + const oldMetaData = this.getSDPMetadata(); + const willRenegotiate = await this.updateLocalMedia(localMedia, log); + if (!willRenegotiate) { + const newMetaData = this.getSDPMetadata(); + if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: newMetaData + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log); } - - applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack); - applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack); } - - applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare"); - // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } - /** group calls would handle reject at the group call level, not at the peer call level */ - async reject() { - - } - hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); @@ -280,7 +265,14 @@ export class PeerCall implements IDisposable { case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; + case EventType.SDPStreamMetadataChanged: + case EventType.SDPStreamMetadataChangedPrefix: + this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log); + break; case EventType.Hangup: + // TODO: this is a bit hacky, double check its what we need + this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log); + break; default: log.log(`Unknown event type for call: ${message.type}`); break; @@ -444,12 +436,12 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - // if (this.peerConnection.remoteTracks.length === 0) { - // await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { - // return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); - // }); - // return; - // } + if (this.peerConnection.remoteStreams.size === 0) { + await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { + return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } this.setState(CallState.Ringing, log); @@ -719,7 +711,7 @@ export class PeerCall implements IDisposable { // this will accumulate all updates into one object, so we still have the old stream info when we change stream id this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.updateRemoteMedia(log); - // TODO: apply muting + } private async addBufferedIceCandidates(log: ILogItem): Promise { @@ -828,10 +820,9 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled), - video_muted: !(streamSender?.videoSender?.enabled), + audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), + video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), }; - console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata)); } if (this.localMedia?.screenShare) { const streamId = this.localMedia.screenShare.id; @@ -851,6 +842,8 @@ export class PeerCall implements IDisposable { if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { this._remoteMedia.userMedia = streamReceiver.stream; + streamReceiver.audioReceiver?.enable(!metaData.audio_muted); + streamReceiver.videoReceiver?.enable(!metaData.video_muted); } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { this._remoteMedia.screenShare = streamReceiver.stream; } @@ -860,6 +853,77 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + return logItem.wrap("updateLocalMedia", async log => { + let willRenegotiate = false; + const oldMedia = this.localMedia; + this.localMedia = localMedia; + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { + const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined; + + const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { + const changed = (!track && oldTrack) || + (track && !oldTrack) || + (track && oldTrack && !track.equals(oldTrack)); + if (changed) { + if (track) { + if (oldTrack && sender && !track.equals(oldTrack)) { + try { + await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { + return sender.replaceTrack(track); + }); + } catch (err) { + // can't replace the track without renegotiating + log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + this.peerConnection.removeTrack(sender); + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + log.wrap(`adding ${logLabel} ${track.kind} track`, log => { + this.peerConnection.addTrack(track); + willRenegotiate = true; + }); + } + } else { + if (sender) { + // this will be used for muting, do we really want to trigger renegotiation here? + // we want to disable the sender, but also remove the track as we don't want to keep + // using the webcam if we don't need to + log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { + sender.track.enabled = false; + this.peerConnection.removeTrack(sender); + willRenegotiate = true; + }); + } + } + } else if (track) { + console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted}); + if (sender && muted !== wasMuted) { + // TODO: why does unmuting not work? wasMuted is false + log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { + sender.track.enabled = !muted; + sender.enable(!muted); + willRenegotiate = true; + }); + } else { + log.log(`${logLabel} ${track.kind} track hasn't changed`); + } + } + } + + await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); + await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); + } + + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); + applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + return willRenegotiate; + // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method + }); + } + private async delay(timeoutMs: number): Promise { // Allow a short time for initial candidates to be gathered const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); @@ -915,100 +979,11 @@ export enum CallDirection { Outbound = 'outbound', } -export enum CallErrorCode { - /** The user chose to end the call */ - UserHangup = 'user_hangup', - - /** An error code when the local client failed to create an offer. */ - LocalOfferFailed = 'local_offer_failed', - /** - * An error code when there is no local mic/camera to use. This may be because - * the hardware isn't plugged in, or the user has explicitly denied access. - */ - NoUserMedia = 'no_user_media', - - /** - * Error code used when a call event failed to send - * because unknown devices were present in the room - */ - UnknownDevices = 'unknown_devices', - - /** - * Error code used when we fail to send the invite - * for some reason other than there being unknown devices - */ - SendInvite = 'send_invite', - - /** - * An answer could not be created - */ - CreateAnswer = 'create_answer', - - /** - * Error code used when we fail to send the answer - * for some reason other than there being unknown devices - */ - SendAnswer = 'send_answer', - - /** - * The session description from the other side could not be set - */ - SetRemoteDescription = 'set_remote_description', - - /** - * The session description from this side could not be set - */ - SetLocalDescription = 'set_local_description', - - /** - * A different device answered the call - */ - AnsweredElsewhere = 'answered_elsewhere', - - /** - * No media connection could be established to the other party - */ - IceFailed = 'ice_failed', - - /** - * The invite timed out whilst waiting for an answer - */ - InviteTimeout = 'invite_timeout', - - /** - * The call was replaced by another call - */ - Replaced = 'replaced', - - /** - * Signalling for the call could not be sent (other than the initial invite) - */ - SignallingFailed = 'signalling_timeout', - - /** - * The remote party is busy - */ - UserBusy = 'user_busy', - - /** - * We transferred the call off to somewhere else - */ - Transfered = 'transferred', - - /** - * A call from the same user was found with a new session id - */ - NewSession = 'new_session', -} - /** * The version field that we set in m.call.* events */ const VOIP_PROTO_VERSION = 1; -/** The fallback ICE server to use for STUN or TURN protocols. */ -const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; - /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; @@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean { return eventType === EventType.Invite || eventType === EventType.Candidates || eventType === EventType.Answer || - eventType === EventType.Hangup; + eventType === EventType.Hangup || + eventType === EventType.SDPStreamMetadataChanged || + eventType === EventType.SDPStreamMetadataChangedPrefix || + eventType === EventType.Negotiate; } export function tests() { diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index 8c041fcc..a0a64752 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -15,7 +15,7 @@ - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - implement muting tracks with m.call.sdp_stream_metadata_changed - - implement cloning the localMedia so it works in safari? + - DONE: implement cloning the localMedia so it works in safari? - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 892ed67c..9fc8d1ee 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> { }); } + async setMedia(localMedia: LocalMedia): Promise { + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + const oldMedia = this._localMedia; + this._localMedia = localMedia; + await Promise.all(Array.from(this._members.values()).map(m => { + return m.setMedia(localMedia!.clone()); + })); + oldMedia?.dispose(); + } + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 06693030..e821e3da 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -182,6 +182,14 @@ export class Member { } } + /** @internal */ + async setMedia(localMedia: LocalMedia): Promise { + const oldMedia = this.localMedia; + this.localMedia = localMedia; + await this.peerCall?.setMedia(localMedia); + oldMedia?.dispose(); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index d0edbbea..85f64f95 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -39,6 +39,9 @@ export interface Track { readonly label: string; readonly id: string; readonly settings: MediaTrackSettings; + get enabled(): boolean; + set enabled(value: boolean); + equals(track: Track): boolean; stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 3723162a..a04fca91 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -75,27 +75,37 @@ export class StreamWrapper implements Stream { public audioTrack: AudioTrackWrapper | undefined = undefined; public videoTrack: TrackWrapper | undefined = undefined; - constructor(public readonly stream: MediaStream) { - for (const track of stream.getTracks()) { - this.update(track); + constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { + if (clonedTracks) { + this.audioTrack = clonedTracks.audioTrack; + this.videoTrack = clonedTracks.videoTrack; + } else { + for (const track of stream.getTracks()) { + this.update(track); + } } } get id(): string { return this.stream.id; } clone(): Stream { - return new StreamWrapper(this.stream.clone()); + const clonedStream = this.stream.clone(); + const clonedTracks = { + audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, + videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, + }; + return new StreamWrapper(clonedStream, clonedTracks); } update(track: MediaStreamTrack): TrackWrapper | undefined { if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream); + this.videoTrack = new TrackWrapper(track, this.stream, track.id); } return this.videoTrack; } else if (track.kind === "audio") { if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream); + this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); } return this.audioTrack; } @@ -105,14 +115,18 @@ export class StreamWrapper implements Stream { export class TrackWrapper implements Track { constructor( public readonly track: MediaStreamTrack, - public readonly stream: MediaStream + public readonly stream: MediaStream, + public readonly originalId: string, ) {} get kind(): TrackKind { return this.track.kind as TrackKind; } get label(): string { return this.track.label; } get id(): string { return this.track.id; } get settings(): MediaTrackSettings { return this.track.getSettings(); } - + get enabled(): boolean { return this.track.enabled; } + set enabled(enabled: boolean) { this.track.enabled = enabled; } + // test equality across clones + equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } stop() { this.track.stop(); } } @@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper { private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; - constructor(track: MediaStreamTrack, stream: MediaStream) { - super(track, stream); + constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { + super(track, stream, originalId); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 672c14d4..22096699 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver { this.transceiver.direction === this.exclusiveValue; } + enableWithoutRenegotiation(enabled: boolean) { + this.track.track.enabled = enabled; + } + enable(enabled: boolean) { if (enabled !== this.enabled) { + // do this first, so we stop sending track data immediately. + // this will still consume bandwidth though, so also disable the transceiver, + // which will trigger a renegotiation though. + this.enableWithoutRenegotiation(enabled); if (enabled) { if (this.transceiver.direction === "inactive") { this.transceiver.direction = this.exclusiveValue; diff --git a/src/platform/web/index.html b/src/platform/web/index.html index 5950d89f..064c61a1 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -11,7 +11,7 @@ - + From 6cd3c8ee2bfab6498f260eb5e91cacb16c11688a Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Wed, 20 Apr 2022 12:25:23 +0530 Subject: [PATCH 04/35] Read config from URL --- src/platform/web/Platform.js | 19 +++++++++++++++++-- src/platform/web/index.html | 7 +++---- src/platform/web/main.js | 1 + vite.common-config.js | 1 + 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 1c630aed..92608d75 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -126,10 +126,11 @@ function adaptUIOnVisualViewportResize(container) { } export class Platform { - constructor({ container, assetPaths, config, options = null, cryptoExtras = null }) { + constructor({ container, assetPaths, config, configURL, options = null, cryptoExtras = null }) { this._container = container; this._assetPaths = assetPaths; this._config = config; + this._configURL = configURL; this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_"); this.clock = new Clock(); this.encoding = new Encoding(); @@ -142,7 +143,7 @@ export class Platform { this._serviceWorkerHandler = new ServiceWorkerHandler(); this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker); } - this.notificationService = new NotificationService(this._serviceWorkerHandler, config.push); + this.notificationService = null; // Only try to use crypto when olm is provided if(this._assetPaths.olm) { this.crypto = new Crypto(cryptoExtras); @@ -165,6 +166,20 @@ export class Platform { this._workerPromise = undefined; } + async init() { + if (!this._config) { + if (!this._configURL) { + throw new Error("Neither config nor configURL was provided!"); + } + const {body}= await this.request(this._configURL, {method: "GET", format: "json"}).response(); + this._config = body; + } + this._notificationService = new NotificationService( + this._serviceWorkerHandler, + this._config.push + ); + } + _createLogger(isDevelopment) { // Make sure that loginToken does not end up in the logs const transformer = (item) => { diff --git a/src/platform/web/index.html b/src/platform/web/index.html index ed0e8eb0..16418699 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -17,7 +17,7 @@ diff --git a/src/platform/web/main.js b/src/platform/web/main.js index 1729c17c..edc2cf14 100644 --- a/src/platform/web/main.js +++ b/src/platform/web/main.js @@ -32,6 +32,7 @@ export async function main(platform) { // const recorder = new RecordRequester(createFetchRequest(clock.createTimeout)); // const request = recorder.request; // window.getBrawlFetchLog = () => recorder.log(); + await platform.init(); const navigation = createNavigation(); platform.setNavigation(navigation); const urlRouter = createRouter({navigation, history: platform.history}); diff --git a/vite.common-config.js b/vite.common-config.js index f5a90154..8a82a9da 100644 --- a/vite.common-config.js +++ b/vite.common-config.js @@ -31,6 +31,7 @@ const commonOptions = { assetsInlineLimit: 0, polyfillModulePreload: false, }, + assetsInclude: ['**/config.json'], define: { DEFINE_VERSION: JSON.stringify(version), DEFINE_GLOBAL_HASH: JSON.stringify(null), From c42292f1b062444ff144bc6426f07606709e40b5 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 20 Apr 2022 10:57:07 +0200 Subject: [PATCH 05/35] more WIP --- src/matrix/calls/PeerCall.ts | 26 +++++++++++++++++++------- src/matrix/calls/group/Member.ts | 2 ++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index b450ac3f..515e1b65 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -164,7 +164,7 @@ export class PeerCall implements IDisposable { } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); - this.updateLocalMedia(localMedia, log); + await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); } @@ -180,7 +180,7 @@ export class PeerCall implements IDisposable { return; } this.setState(CallState.CreateAnswer, log); - this.updateLocalMedia(localMedia, log); + await this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { myAnswer = await this.peerConnection.createAnswer(); @@ -220,6 +220,9 @@ export class PeerCall implements IDisposable { const oldMetaData = this.getSDPMetadata(); const willRenegotiate = await this.updateLocalMedia(localMedia, log); + // TODO: if we will renegotiate, we don't bother sending the metadata changed event + // because the renegotiate event will send new metadata anyway, but is that the right + // call? if (!willRenegotiate) { const newMetaData = this.getSDPMetadata(); if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { @@ -229,7 +232,7 @@ export class PeerCall implements IDisposable { seq: this.seq++, [SDPStreamMetadataKey]: newMetaData }; - await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log); + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); } } }); @@ -859,7 +862,14 @@ export class PeerCall implements IDisposable { const oldMedia = this.localMedia; this.localMedia = localMedia; const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { - const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined; + let streamSender; + if (oldStream) { + streamSender = this.peerConnection.localStreams.get(oldStream.id); + if (stream && stream.id !== oldStream.id) { + this.peerConnection.localStreams.set(stream.id, streamSender); + this.peerConnection.localStreams.delete(oldStream.id); + } + } const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { const changed = (!track && oldTrack) || @@ -899,11 +909,13 @@ export class PeerCall implements IDisposable { } } } else if (track) { - console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted}); + log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id}); if (sender && muted !== wasMuted) { // TODO: why does unmuting not work? wasMuted is false log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { - sender.track.enabled = !muted; + // sender.track.enabled = !muted; + // This doesn't always seem to trigger renegotiation?? + // We should probably always send the new metadata first ... sender.enable(!muted); willRenegotiate = true; }); @@ -918,7 +930,7 @@ export class PeerCall implements IDisposable { } await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); - applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); return willRenegotiate; // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index e821e3da..22a3d1fa 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -154,6 +154,8 @@ export class Member { } } }; + // TODO: remove this for release + log.set("payload", groupMessage.content); const request = this.options.hsApi.sendToDevice( message.type, //"m.room.encrypted", From 4a8af83c8fdc5fbf46686e030101bc0d111f6bc9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Wed, 20 Apr 2022 10:57:42 +0200 Subject: [PATCH 06/35] WIP --- src/platform/types/MediaDevices.ts | 19 +++++++++---------- src/platform/web/dom/MediaDevices.ts | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 85f64f95..93418a66 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -20,11 +20,14 @@ export interface MediaDevices { // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; + createVolumeMeasurer(stream: Stream): VolumeMeasurer; } + export interface Stream { - readonly audioTrack: AudioTrack | undefined; - readonly videoTrack: Track | undefined; + getTracks(): ReadonlyArray; + getAudioTracks(): ReadonlyArray; + getVideoTracks(): ReadonlyArray; readonly id: string; clone(): Stream; } @@ -38,15 +41,11 @@ export interface Track { readonly kind: TrackKind; readonly label: string; readonly id: string; - readonly settings: MediaTrackSettings; - get enabled(): boolean; - set enabled(value: boolean); - equals(track: Track): boolean; + enabled: boolean; + // getSettings(): MediaTrackSettings; stop(): void; } -export interface AudioTrack extends Track { - // TODO: how to emit updates on this? - get isSpeaking(): boolean; +export interface VolumeMeasurer { + } - diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index a04fca91..4eb92259 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -98,6 +98,7 @@ export class StreamWrapper implements Stream { } update(track: MediaStreamTrack): TrackWrapper | undefined { + //console.trace("Stream.update " + JSON.stringify({id: track.id, vid: this.videoTrack?.id, aid: this.audioTrack?.id})); if (track.kind === "video") { if (!this.videoTrack || track.id !== this.videoTrack.track.id) { this.videoTrack = new TrackWrapper(track, this.stream, track.id); From 5f8a171c2c706a884321c16f9798874fc1bf364b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 20 Apr 2022 11:55:48 -0500 Subject: [PATCH 07/35] Fix asset build throwing and swallowing errors (#721) - Fix `svg-colorizer` throwing errors with Windows file paths - Fix `css-url-parser` swallowing errors because it was `async` - Fail SDK build script (`yarn build:sdk`, `build.sh`) overall when some commands are failing --- scripts/postcss/css-url-processor.js | 2 +- scripts/postcss/svg-colorizer.js | 2 +- scripts/sdk/build.sh | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/postcss/css-url-processor.js b/scripts/postcss/css-url-processor.js index 3ae7c60d..f58818f1 100644 --- a/scripts/postcss/css-url-processor.js +++ b/scripts/postcss/css-url-processor.js @@ -39,7 +39,7 @@ function colorsFromURL(url, colorMap) { function processURL(decl, replacer, colorMap) { const value = decl.value; const parsed = valueParser(value); - parsed.walk(async node => { + parsed.walk(node => { if (node.type !== "function" || node.value !== "url") { return; } diff --git a/scripts/postcss/svg-colorizer.js b/scripts/postcss/svg-colorizer.js index 95355ea8..06b7b14b 100644 --- a/scripts/postcss/svg-colorizer.js +++ b/scripts/postcss/svg-colorizer.js @@ -37,7 +37,7 @@ module.exports.buildColorizedSVG = function (svgLocation, primaryColor, secondar if (svgCode === coloredSVGCode) { throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive)."); } - const fileName = svgLocation.match(/.+\/(.+\.svg)/)[1]; + const fileName = svgLocation.match(/.+[/\\](.+\.svg)/)[1]; const outputName = `${fileName.substring(0, fileName.length - 4)}-${createHash(coloredSVGCode)}.svg`; const outputPath = path.resolve(__dirname, "../../.tmp"); try { diff --git a/scripts/sdk/build.sh b/scripts/sdk/build.sh index 2ac4be3a..ae3a794e 100755 --- a/scripts/sdk/build.sh +++ b/scripts/sdk/build.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Exit whenever one of the commands fail with a non-zero exit code +set -e +set -o pipefail + rm -rf target yarn run vite build -c vite.sdk-assets-config.js yarn run vite build -c vite.sdk-lib-config.js From 468b7e15954df04bff84a99252fc6ced16aa5816 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 21 Apr 2022 12:52:42 +0530 Subject: [PATCH 08/35] Cache config.json --- src/platform/web/Platform.js | 2 +- src/platform/web/sw.js | 35 ++++++++++++++++++++++++++++++++++- vite.config.js | 5 +++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 92608d75..7d5126dc 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -171,7 +171,7 @@ export class Platform { if (!this._configURL) { throw new Error("Neither config nor configURL was provided!"); } - const {body}= await this.request(this._configURL, {method: "GET", format: "json"}).response(); + const {body}= await this.request(this._configURL, {method: "GET", format: "json", cache: true}).response(); this._config = body; } this._notificationService = new NotificationService( diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index c5f69438..e57634fc 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -75,7 +75,14 @@ self.addEventListener('fetch', (event) => { This has to do with xhr not being supported in service workers. */ if (event.request.method === "GET") { - event.respondWith(handleRequest(event.request)); + if (event.request.url.includes("config.json")) { + /** + * Use a different strategy for this file. + */ + event.respondWith(handleConfigRequest(event.request)); + } else { + event.respondWith(handleRequest(event.request)); + } } }); @@ -119,6 +126,32 @@ async function handleRequest(request) { } } +async function handleConfigRequest(request) { + const url = new URL(request.url); + // rewrite / to /index.html so it hits the cache + if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) { + request = new Request(new URL("index.html", baseURL.href)); + } + let response = await readCache(request); + if (response) { + fetchAndUpdateConfig(request); + return response; + } + response = await fetchAndUpdateConfig(request); + return response; +} + +async function fetchAndUpdateConfig(request) { + const response = await fetch(request, { + signal: pendingFetchAbortController.signal, + headers: { + "Cache-Control": "no-cache", + }, + }); + updateCache(request, response.clone()); + return response; +} + async function updateCache(request, response) { // don't write error responses to the cache if (response.status >= 400) { diff --git a/vite.config.js b/vite.config.js index 4dd35af2..87e3d063 100644 --- a/vite.config.js +++ b/vite.config.js @@ -14,6 +14,11 @@ export default defineConfig(({mode}) => { outDir: "../../../target", minify: true, sourcemap: true, + rollupOptions: { + output: { + assetFileNames: (asset) => asset.name.includes("config.json") ? "assets/[name][extname]": "assets/[name].[hash][extname]", + }, + }, }, plugins: [ themeBuilder({ From 82ffb557e5cfa1082b6bdd5c799bd5635346ebd0 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:09:31 +0200 Subject: [PATCH 09/35] update TODO --- src/matrix/calls/TODO.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/matrix/calls/TODO.md b/src/matrix/calls/TODO.md index a0a64752..a7f4e82c 100644 --- a/src/matrix/calls/TODO.md +++ b/src/matrix/calls/TODO.md @@ -9,14 +9,14 @@ ## TODO - DONE: implement receiving hangup - - making logging better + - DONE: implement cloning the localMedia so it works in safari? + - DONE: implement 3 retries per peer + - implement muting tracks with m.call.sdp_stream_metadata_changed - implement renegotiation + - making logging better - finish session id support - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - implement to_device messages arriving before m.call(.member) state event - - implement muting tracks with m.call.sdp_stream_metadata_changed - - DONE: implement cloning the localMedia so it works in safari? - - DONE: implement 3 retries per peer - reeable crypto & implement fetching olm keys before sending encrypted signalling message - local echo for join/leave buttons? - make UI pretsy From 99769eb84e0f7cd577fdee5a60def920ecb50a4f Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:10:49 +0200 Subject: [PATCH 10/35] implement basic renegotiation --- src/matrix/calls/PeerCall.ts | 53 ++++++++++++++++++++++++------ src/matrix/calls/callEventTypes.ts | 7 ++++ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 515e1b65..98747e40 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -18,24 +18,24 @@ import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {AsyncQueue} from "../../utils/AsyncQueue"; import {Disposables, IDisposable} from "../../utils/Disposables"; -import type {Room} from "../room/Room"; -import type {StateEvent} from "../storage/types"; -import type {ILogItem} from "../../logging/types"; - -import type {TimeoutCreator, Timeout} from "../../platform/types/types"; import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; -import type {LocalMedia} from "./LocalMedia"; - import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, EventType, CallErrorCode, } from "./callEventTypes"; + +import type {Room} from "../room/Room"; +import type {StateEvent} from "../storage/types"; +import type {ILogItem} from "../../logging/types"; +import type {TimeoutCreator, Timeout} from "../../platform/types/types"; +import type {LocalMedia} from "./LocalMedia"; import type { MCallBase, MCallInvite, + MCallNegotiate, MCallAnswer, MCallSDPStreamMetadataChanged, MCallCandidates, @@ -265,6 +265,9 @@ export class PeerCall implements IDisposable { case EventType.Answer: await this.handleAnswer(message.content, partyId, log); break; + case EventType.Negotiate: + await this.handleRemoteNegotiate(message.content, partyId, log); + break; case EventType.Candidates: await this.handleRemoteIceCandidates(message.content, partyId, log); break; @@ -341,10 +344,10 @@ export class PeerCall implements IDisposable { await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { - log.log("would send renegotiation now but not implemented"); // send Negotiate message - //await this.sendSignallingMessage({type: EventType.Invite, content}); - //this.setState(CallState.InviteSent); + content.description = content.offer; + delete content.offer; + await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { this.makingOffer = false; @@ -497,6 +500,36 @@ export class PeerCall implements IDisposable { } } + + private async handleRemoteNegotiate(content: MCallNegotiate, partyId: PartyId, log: ILogItem): Promise { + if (this._state !== CallState.Connected) { + log.log({l: `Ignoring renegotiate because not connected`, status: this._state}); + return; + } + + if (this.opponentPartyId !== partyId) { + log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`); + return; + } + + const sdpStreamMetadata = content[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log); + } else { + log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); + } + + try { + await this.peerConnection.setRemoteDescription(content.description); + } catch (e) { + await log.wrap(`Failed to set remote description`, log => { + log.catch(e); + this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); + }); + return; + } + } + private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) { if (state === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, diff --git a/src/matrix/calls/callEventTypes.ts b/src/matrix/calls/callEventTypes.ts index 4726175c..449bd469 100644 --- a/src/matrix/calls/callEventTypes.ts +++ b/src/matrix/calls/callEventTypes.ts @@ -97,6 +97,12 @@ export type MCallInvite = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } +export type MCallNegotiate = Base & { + description: SessionDescription; + lifetime: number; + [SDPStreamMetadataKey]: SDPStreamMetadata; +} + export type MCallSDPStreamMetadataChanged = Base & { [SDPStreamMetadataKey]: SDPStreamMetadata; } @@ -213,6 +219,7 @@ export enum CallErrorCode { export type SignallingMessage = {type: EventType.Invite, content: MCallInvite} | + {type: EventType.Negotiate, content: MCallNegotiate} | {type: EventType.Answer, content: MCallAnswer} | {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged} | {type: EventType.Candidates, content: MCallCandidates} | From 55c6dcf613ad8c3c4272bd09bf679b1acff74dba Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:11:24 +0200 Subject: [PATCH 11/35] don't re-clone streams when not needed --- src/matrix/calls/LocalMedia.ts | 36 ++++++++++++++++++++++++++--- src/matrix/calls/group/GroupCall.ts | 8 +++---- src/matrix/calls/group/Member.ts | 8 +++---- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 7dbc9e17..933ae89f 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -43,13 +43,43 @@ export class LocalMedia { return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); } + replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { + let userMedia; + let screenShare; + const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => { + let stream; + if (oldOriginalStream?.id === newStream?.id) { + stream = oldCloneStream; + } else { + stream = newStream?.clone(); + oldCloneStream?.audioTrack?.stop(); + oldCloneStream?.videoTrack?.stop(); + } + return stream; + } + return new LocalMedia( + cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), + this.microphoneMuted, this.cameraMuted, + cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare), + this.dataChannelOptions + ); + } + clone(): LocalMedia { return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); } dispose() { - this.userMedia?.audioTrack?.stop(); - this.userMedia?.videoTrack?.stop(); - this.screenShare?.videoTrack?.stop(); + this.stopExcept(undefined); + } + + stopExcept(newMedia: LocalMedia | undefined) { + if(newMedia?.userMedia?.id !== this.userMedia?.id) { + this.userMedia?.audioTrack?.stop(); + this.userMedia?.videoTrack?.stop(); + } + if(newMedia?.screenShare?.id !== this.screenShare?.id) { + this.screenShare?.videoTrack?.stop(); + } } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 9fc8d1ee..cb962b86 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -124,13 +124,13 @@ export class GroupCall extends EventEmitter<{change: never}> { } async setMedia(localMedia: LocalMedia): Promise { - if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { - const oldMedia = this._localMedia; + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) { + const oldMedia = this._localMedia!; this._localMedia = localMedia; await Promise.all(Array.from(this._members.values()).map(m => { - return m.setMedia(localMedia!.clone()); + return m.setMedia(localMedia, oldMedia); })); - oldMedia?.dispose(); + oldMedia?.stopExcept(localMedia); } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 22a3d1fa..714cd821 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -185,11 +185,9 @@ export class Member { } /** @internal */ - async setMedia(localMedia: LocalMedia): Promise { - const oldMedia = this.localMedia; - this.localMedia = localMedia; - await this.peerCall?.setMedia(localMedia); - oldMedia?.dispose(); + async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise { + this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia); + await this.peerCall?.setMedia(this.localMedia); } private _createPeerCall(callId: string): PeerCall { From 10a6269147df9b00ff322a793c07e31dad129cf9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:15:57 +0200 Subject: [PATCH 12/35] always send new metadata after calling setMedia --- src/matrix/calls/PeerCall.ts | 39 ++++++++++-------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 98747e40..79fb04fb 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -217,24 +217,14 @@ export class PeerCall implements IDisposable { log.set("userMedia_video_muted", localMedia.cameraMuted); log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); log.set("datachannel", !!localMedia.dataChannelOptions); - - const oldMetaData = this.getSDPMetadata(); - const willRenegotiate = await this.updateLocalMedia(localMedia, log); - // TODO: if we will renegotiate, we don't bother sending the metadata changed event - // because the renegotiate event will send new metadata anyway, but is that the right - // call? - if (!willRenegotiate) { - const newMetaData = this.getSDPMetadata(); - if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) { - const content: MCallSDPStreamMetadataChanged = { - call_id: this.callId, - version: 1, - seq: this.seq++, - [SDPStreamMetadataKey]: newMetaData - }; - await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); - } - } + await this.updateLocalMedia(localMedia, log); + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); }); } @@ -856,8 +846,8 @@ export class PeerCall implements IDisposable { const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled), - video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled), + audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack, + video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack, }; } if (this.localMedia?.screenShare) { @@ -889,9 +879,8 @@ export class PeerCall implements IDisposable { this.options.emitUpdate(this, undefined); } - private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { + private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise { return logItem.wrap("updateLocalMedia", async log => { - let willRenegotiate = false; const oldMedia = this.localMedia; this.localMedia = localMedia; const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { @@ -920,13 +909,11 @@ export class PeerCall implements IDisposable { log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { this.peerConnection.removeTrack(sender); this.peerConnection.addTrack(track); - willRenegotiate = true; }); } } else { log.wrap(`adding ${logLabel} ${track.kind} track`, log => { this.peerConnection.addTrack(track); - willRenegotiate = true; }); } } else { @@ -937,20 +924,17 @@ export class PeerCall implements IDisposable { log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { sender.track.enabled = false; this.peerConnection.removeTrack(sender); - willRenegotiate = true; }); } } } else if (track) { log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id}); if (sender && muted !== wasMuted) { - // TODO: why does unmuting not work? wasMuted is false log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { // sender.track.enabled = !muted; // This doesn't always seem to trigger renegotiation?? // We should probably always send the new metadata first ... sender.enable(!muted); - willRenegotiate = true; }); } else { log.log(`${logLabel} ${track.kind} track hasn't changed`); @@ -964,7 +948,6 @@ export class PeerCall implements IDisposable { await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); - return willRenegotiate; // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } From 1cdc76f5a464871a50ca909a1eb4d126e5632703 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 21 Apr 2022 14:14:38 +0530 Subject: [PATCH 13/35] Use undefine instead of null --- src/platform/web/Platform.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 7d5126dc..1c999598 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -143,7 +143,7 @@ export class Platform { this._serviceWorkerHandler = new ServiceWorkerHandler(); this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker); } - this.notificationService = null; + this.notificationService = undefined; // Only try to use crypto when olm is provided if(this._assetPaths.olm) { this.crypto = new Crypto(cryptoExtras); From 4f239445816df107fb446ba3b46345a20e77cbb4 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 21 Apr 2022 14:17:47 +0530 Subject: [PATCH 14/35] Use named param in Legacy Platform --- src/platform/web/LegacyPlatform.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/web/LegacyPlatform.js b/src/platform/web/LegacyPlatform.js index 85632bf2..b8a6d7e7 100644 --- a/src/platform/web/LegacyPlatform.js +++ b/src/platform/web/LegacyPlatform.js @@ -19,6 +19,6 @@ import {hkdf} from "../../utils/crypto/hkdf"; import {Platform as ModernPlatform} from "./Platform.js"; -export function Platform(container, assetPaths, config, options = null) { - return new ModernPlatform(container, assetPaths, config, options, {aesjs, hkdf}); +export function Platform({ container, assetPaths, config, configURL, options = null }) { + return new ModernPlatform({ container, assetPaths, config, configURL, options, cryptoExtras: { aesjs, hkdf }}); } From b6e55ef59c263901421f8e53db23f301a91f0633 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 21 Apr 2022 14:46:55 +0530 Subject: [PATCH 15/35] Remove comment --- src/platform/web/sw.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index e57634fc..aaf56c4b 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -76,9 +76,6 @@ self.addEventListener('fetch', (event) => { */ if (event.request.method === "GET") { if (event.request.url.includes("config.json")) { - /** - * Use a different strategy for this file. - */ event.respondWith(handleConfigRequest(event.request)); } else { event.respondWith(handleRequest(event.request)); From ac60d1b61dde0eee0e4e20907300c131c1d80435 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 21 Apr 2022 17:39:11 +0200 Subject: [PATCH 16/35] remove thick abstraction layer instead just copy the DOM typing and make it part of the platform layer --- src/domain/session/room/CallViewModel.ts | 4 +- src/domain/session/room/RoomViewModel.js | 3 +- src/matrix/calls/CallHandler.ts | 8 +- src/matrix/calls/LocalMedia.ts | 28 +- src/matrix/calls/PeerCall.ts | 287 +++++++++------- src/matrix/calls/common.ts | 29 ++ src/matrix/calls/group/GroupCall.ts | 4 +- src/platform/types/MediaDevices.ts | 32 +- src/platform/types/WebRTC.ts | 195 ++++++++--- src/platform/web/dom/MediaDevices.ts | 338 +------------------ src/platform/web/dom/WebRTC.ts | 335 ++---------------- src/platform/web/ui/session/room/CallView.ts | 8 +- src/utils/AsyncQueue.ts | 52 --- 13 files changed, 424 insertions(+), 899 deletions(-) create mode 100644 src/matrix/calls/common.ts delete mode 100644 src/utils/AsyncQueue.ts diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index 05245cd3..e6a6914f 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -57,9 +57,7 @@ export class CallViewModel extends ViewModel { } async toggleVideo() { - const localMedia = this.call.localMedia!; - const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted); - await this.call.setMedia(toggledMedia); + //this.call.setMuted(); } } diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 868ca189..b88b17f0 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -367,9 +367,8 @@ export class RoomViewModel extends ViewModel { const session = this.getOption("session"); const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const localMedia = new LocalMedia().withUserMedia(stream); - await this._call.join(localMedia); // this will set the callViewModel above as a call will be added to callHandler.calls - const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); + const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100)); await call.join(localMedia); } catch (err) { console.error(err.stack); diff --git a/src/matrix/calls/CallHandler.ts b/src/matrix/calls/CallHandler.ts index b49ca3c5..06e83d11 100644 --- a/src/matrix/calls/CallHandler.ts +++ b/src/matrix/calls/CallHandler.ts @@ -15,8 +15,8 @@ limitations under the License. */ import {ObservableMap} from "../../observable/map/ObservableMap"; -import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; +import {WebRTC, PeerConnection} from "../../platform/types/WebRTC"; +import {MediaDevices, Track} from "../../platform/types/MediaDevices"; import {handlesEventType} from "./PeerCall"; import {EventType, CallIntent} from "./callEventTypes"; import {GroupCall} from "./group/GroupCall"; @@ -107,7 +107,7 @@ export class CallHandler { }); } - async createCall(roomId: string, localMedia: LocalMedia, name: string, intent: CallIntent = CallIntent.Ring): Promise { + async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise { const logItem = this.options.logger.child({l: "call", incoming: false}); const call = new GroupCall(makeId("conf-"), true, { "m.name": name, @@ -116,7 +116,7 @@ export class CallHandler { this._calls.set(call.id, call); try { - await call.create(localMedia); + await call.create(type); // store call info so it will ring again when reopening the app const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); txn.calls.add({ diff --git a/src/matrix/calls/LocalMedia.ts b/src/matrix/calls/LocalMedia.ts index 933ae89f..b79fe098 100644 --- a/src/matrix/calls/LocalMedia.ts +++ b/src/matrix/calls/LocalMedia.ts @@ -17,32 +17,28 @@ limitations under the License. import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {Stream} from "../../platform/types/MediaDevices"; import {SDPStreamMetadata} from "./callEventTypes"; +import {getStreamVideoTrack, getStreamAudioTrack} from "./common"; export class LocalMedia { constructor( public readonly userMedia?: Stream, - public readonly microphoneMuted: boolean = false, - public readonly cameraMuted: boolean = false, public readonly screenShare?: Stream, public readonly dataChannelOptions?: RTCDataChannelInit, ) {} - withMuted(microphone: boolean, camera: boolean) { - return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions); - } - withUserMedia(stream: Stream) { - return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions); + return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); } withScreenShare(stream: Stream) { - return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions); + return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); } withDataChannel(options: RTCDataChannelInit): LocalMedia { - return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options); + return new LocalMedia(this.userMedia, this.screenShare, options); } + /** @internal */ replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia { let userMedia; let screenShare; @@ -52,21 +48,21 @@ export class LocalMedia { stream = oldCloneStream; } else { stream = newStream?.clone(); - oldCloneStream?.audioTrack?.stop(); - oldCloneStream?.videoTrack?.stop(); + getStreamAudioTrack(oldCloneStream)?.stop(); + getStreamVideoTrack(oldCloneStream)?.stop(); } return stream; } return new LocalMedia( cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia), - this.microphoneMuted, this.cameraMuted, cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare), this.dataChannelOptions ); } + /** @internal */ clone(): LocalMedia { - return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions); + return new LocalMedia(this.userMedia?.clone(),this.screenShare?.clone(), this.dataChannelOptions); } dispose() { @@ -75,11 +71,11 @@ export class LocalMedia { stopExcept(newMedia: LocalMedia | undefined) { if(newMedia?.userMedia?.id !== this.userMedia?.id) { - this.userMedia?.audioTrack?.stop(); - this.userMedia?.videoTrack?.stop(); + getStreamAudioTrack(this.userMedia)?.stop(); + getStreamVideoTrack(this.userMedia)?.stop(); } if(newMedia?.screenShare?.id !== this.screenShare?.id) { - this.screenShare?.videoTrack?.stop(); + getStreamVideoTrack(this.screenShare)?.stop(); } } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 79fb04fb..799fd9b8 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -16,10 +16,10 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; -import {AsyncQueue} from "../../utils/AsyncQueue"; -import {Disposables, IDisposable} from "../../utils/Disposables"; -import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC"; -import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices"; +import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; +import {WebRTC, PeerConnection, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; +import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; +import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose, @@ -69,6 +69,7 @@ export class PeerCall implements IDisposable { private direction: CallDirection; // we don't own localMedia and should hence not call dispose on it from here private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private seq: number = 0; // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where @@ -85,7 +86,8 @@ export class PeerCall implements IDisposable { private hangupParty: CallParty; private disposables = new Disposables(); private statePromiseMap = new Map void, promise: Promise}>(); - + private _remoteTrackToStreamId = new Map(); + private _remoteStreams = new Map(); // perfect negotiation flags private makingOffer: boolean = false; private ignoreOffer: boolean = false; @@ -96,55 +98,62 @@ export class PeerCall implements IDisposable { private _dataChannel?: any; private _hangupReason?: CallErrorCode; private _remoteMedia: RemoteMedia; + private remoteMuteSettings?: MuteSettings; constructor( private callId: string, private readonly options: Options, private readonly logItem: ILogItem, ) { - const outer = this; this._remoteMedia = new RemoteMedia(); - this.peerConnection = options.webRTC.createPeerConnection({ - onIceConnectionStateChange(state: RTCIceConnectionState) { - outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { - outer.onIceConnectionStateChange(state, log); + this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0); + + const listen = (type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => { + this.peerConnection.addEventListener(type, listener); + const dispose = () => { + this.peerConnection.removeEventListener(type, listener); + }; + this.disposables.track(dispose); + }; + + listen("iceconnectionstatechange", () => { + const state = this.peerConnection.iceConnectionState; + this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { + this.onIceConnectionStateChange(state, log); + }); + }); + listen("icecandidate", event => { + this.logItem.wrap("onLocalIceCandidate", log => { + if (event.candidate) { + this.handleLocalIceCandidate(event.candidate, log); + } + }); + }); + listen("icegatheringstatechange", () => { + const state = this.peerConnection.iceGatheringState; + this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { + this.handleIceGatheringState(state, log); + }); + }); + listen("track", event => { + this.logItem.wrap("onRemoteTrack", log => { + this.onRemoteTrack(event.track, event.streams, log); + }); + }); + listen("datachannel", event => { + this.logItem.wrap("onRemoteDataChannel", log => { + this._dataChannel = event.channel; + this.options.emitUpdate(this, undefined); + }); + }); + listen("negotiationneeded", () => { + const promiseCreator = () => { + return this.logItem.wrap("onNegotiationNeeded", log => { + return this.handleNegotiation(log); }); - }, - onLocalIceCandidate(candidate: RTCIceCandidate) { - outer.logItem.wrap("onLocalIceCandidate", log => { - outer.handleLocalIceCandidate(candidate, log); - }); - }, - onIceGatheringStateChange(state: RTCIceGatheringState) { - outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => { - outer.handleIceGatheringState(state, log); - }); - }, - onRemoteStreamRemoved(stream: Stream) { - outer.logItem.wrap("onRemoteStreamRemoved", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteTracksAdded(trackReceiver: TrackReceiver) { - outer.logItem.wrap("onRemoteTracksAdded", log => { - outer.updateRemoteMedia(log); - }); - }, - onRemoteDataChannel(dataChannel: any | undefined) { - outer.logItem.wrap("onRemoteDataChannel", log => { - outer._dataChannel = dataChannel; - outer.options.emitUpdate(outer, undefined); - }); - }, - onNegotiationNeeded() { - const promiseCreator = () => { - return outer.logItem.wrap("onNegotiationNeeded", log => { - return outer.handleNegotiation(log); - }); - }; - outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); - } - }, this.options.forceTURN, this.options.turnServers, 0); + }; + this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator(); + }); } get dataChannel(): any | undefined { return this._dataChannel; } @@ -166,7 +175,7 @@ export class PeerCall implements IDisposable { this.setState(CallState.CreateOffer, log); await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { - this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); + this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); } // after adding the local tracks, and wait for handleNegotiation to be called, // or invite glare where we give up our invite and answer instead @@ -211,11 +220,9 @@ export class PeerCall implements IDisposable { setMedia(localMedia: LocalMedia): Promise { return this.logItem.wrap("setMedia", async log => { - log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack); - log.set("userMedia_audio_muted", localMedia.microphoneMuted); - log.set("userMedia_video", !!localMedia.userMedia?.videoTrack); - log.set("userMedia_video_muted", localMedia.cameraMuted); - log.set("screenShare_video", !!localMedia.screenShare?.videoTrack); + log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia)); + log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia)); + log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare)); log.set("datachannel", !!localMedia.dataChannelOptions); await this.updateLocalMedia(localMedia, log); const content: MCallSDPStreamMetadataChanged = { @@ -322,21 +329,26 @@ export class PeerCall implements IDisposable { this.candidateSendQueue = []; // need to queue this - const content = { - call_id: this.callId, - offer, - [SDPStreamMetadataKey]: this.getSDPMetadata(), - version: 1, - seq: this.seq++, - lifetime: CALL_TIMEOUT_MS - }; if (this._state === CallState.CreateOffer) { + const content = { + call_id: this.callId, + offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; await this.sendSignallingMessage({type: EventType.Invite, content}, log); this.setState(CallState.InviteSent, log); } else if (this._state === CallState.Connected || this._state === CallState.Connecting) { - // send Negotiate message - content.description = content.offer; - delete content.offer; + const content = { + call_id: this.callId, + description: offer, + [SDPStreamMetadataKey]: this.getSDPMetadata(), + version: 1, + seq: this.seq++, + lifetime: CALL_TIMEOUT_MS + }; await this.sendSignallingMessage({type: EventType.Negotiate, content}, log); } } finally { @@ -432,7 +444,7 @@ export class PeerCall implements IDisposable { // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (this.peerConnection.remoteStreams.size === 0) { + if (this.peerConnection.getReceivers().length === 0) { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); }); @@ -843,11 +855,10 @@ export class PeerCall implements IDisposable { const metadata = {}; if (this.localMedia?.userMedia) { const streamId = this.localMedia.userMedia.id; - const streamSender = this.peerConnection.localStreams.get(streamId); metadata[streamId] = { purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack, - video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack, + audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia), + video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia), }; } if (this.localMedia?.screenShare) { @@ -859,19 +870,67 @@ export class PeerCall implements IDisposable { return metadata; } - private updateRemoteMedia(log: ILogItem) { + private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { + return this.peerConnection.getReceivers().find(r => { + return r.track.kind === "audio" && this._remoteTrackToStreamId.get(r.track.id) === streamId; + }); + } + + private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { + if (streams.length === 0) { + log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); + return; + } + const stream = streams[0]; + this._remoteTrackToStreamId.set(track.id, stream.id); + if (!this._remoteStreams.has(stream.id)) { + const listener = (event: StreamTrackEvent): void => { + this.logItem.wrap({l: "removetrack", id: event.track.id}, log => { + const streamId = this._remoteTrackToStreamId.get(event.track.id); + if (streamId) { + this._remoteTrackToStreamId.delete(event.track.id); + const streamDetails = this._remoteStreams.get(streamId); + if (streamDetails && streamDetails.stream.getTracks().length === 0) { + this.disposables.disposeTracked(disposeListener); + this._remoteStreams.delete(stream.id); + this.updateRemoteMedia(log); + } + } + }) + }; + stream.addEventListener("removetrack", listener); + const disposeListener = () => { + stream.removeEventListener("removetrack", listener); + }; + this.disposables.track(disposeListener); + this._remoteStreams.set(stream.id, { + disposeListener, + stream + }); + this.updateRemoteMedia(log); + } + } + + private updateRemoteMedia(log: ILogItem): void { this._remoteMedia.userMedia = undefined; this._remoteMedia.screenShare = undefined; if (this.remoteSDPStreamMetadata) { - for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { - const metaData = this.remoteSDPStreamMetadata[streamId]; + for (const streamDetails of this._remoteStreams.values()) { + const {stream} = streamDetails; + const metaData = this.remoteSDPStreamMetadata[stream.id]; if (metaData) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { - this._remoteMedia.userMedia = streamReceiver.stream; - streamReceiver.audioReceiver?.enable(!metaData.audio_muted); - streamReceiver.videoReceiver?.enable(!metaData.video_muted); + this._remoteMedia.userMedia = stream; + const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id); + if (audioReceiver) { + audioReceiver.track.enabled = !metaData.audio_muted; + } + const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id); + if (videoReceiver) { + videoReceiver.track.enabled = !metaData.audio_muted; + } } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { - this._remoteMedia.screenShare = streamReceiver.stream; + this._remoteMedia.screenShare = stream; } } } @@ -883,71 +942,46 @@ export class PeerCall implements IDisposable { return logItem.wrap("updateLocalMedia", async log => { const oldMedia = this.localMedia; this.localMedia = localMedia; - const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => { - let streamSender; - if (oldStream) { - streamSender = this.peerConnection.localStreams.get(oldStream.id); - if (stream && stream.id !== oldStream.id) { - this.peerConnection.localStreams.set(stream.id, streamSender); - this.peerConnection.localStreams.delete(oldStream.id); - } - } - - const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => { - const changed = (!track && oldTrack) || - (track && !oldTrack) || - (track && oldTrack && !track.equals(oldTrack)); - if (changed) { - if (track) { - if (oldTrack && sender && !track.equals(oldTrack)) { + const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => { + const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => { + if (!oldTrack && newTrack) { + log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => { + const sender = this.peerConnection.addTrack(newTrack, stream!); + this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose); + }); + } else if (oldTrack) { + const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id); + if (sender) { + if (newTrack && oldTrack.id !== newTrack.id) { try { - await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { - return sender.replaceTrack(track); + await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => { + return sender.replaceTrack(newTrack); }); } catch (err) { - // can't replace the track without renegotiating - log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => { + // can't replace the track without renegotiating{ + log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => { this.peerConnection.removeTrack(sender); - this.peerConnection.addTrack(track); + this.peerConnection.addTrack(newTrack); }); } - } else { - log.wrap(`adding ${logLabel} ${track.kind} track`, log => { - this.peerConnection.addTrack(track); - }); - } - } else { - if (sender) { - // this will be used for muting, do we really want to trigger renegotiation here? - // we want to disable the sender, but also remove the track as we don't want to keep - // using the webcam if we don't need to - log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => { - sender.track.enabled = false; + } else if (!newTrack) { + log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => { this.peerConnection.removeTrack(sender); }); + } else { + log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`); } } - } else if (track) { - log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id}); - if (sender && muted !== wasMuted) { - log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => { - // sender.track.enabled = !muted; - // This doesn't always seem to trigger renegotiation?? - // We should probably always send the new metadata first ... - sender.enable(!muted); - }); - } else { - log.log(`${logLabel} ${track.kind} track hasn't changed`); - } + // TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia } } - await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted); - await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted); - } + await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream)); + await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream)); + }; - await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia"); - await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare"); + await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia); + await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare); // TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method }); } @@ -967,7 +1001,7 @@ export class PeerCall implements IDisposable { public dispose(): void { this.disposables.dispose(); - this.peerConnection.dispose(); + this.peerConnection.close(); } public close(reason: CallErrorCode | undefined, log: ILogItem): void { @@ -1038,6 +1072,7 @@ export function handlesEventType(eventType: string): boolean { eventType === EventType.Negotiate; } + export function tests() { } diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts new file mode 100644 index 00000000..c5970d4b --- /dev/null +++ b/src/matrix/calls/common.ts @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type {Track, Stream} from "../../platform/types/MediaDevices"; + +export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined { + return stream?.getAudioTracks()[0]; +} + +export function getStreamVideoTrack(stream: Stream | undefined): Track | undefined { + return stream?.getVideoTracks()[0]; +} + +export class MuteSettings { + constructor (public readonly microphone: boolean, public readonly camera: boolean) {} +} diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index cb962b86..4992f6b6 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -168,7 +168,7 @@ export class GroupCall extends EventEmitter<{change: never}> { } /** @internal */ - create(localMedia: LocalMedia): Promise { + create(type: "m.video" | "m.voice"): Promise { return this.logItem.wrap("create", async log => { if (this._state !== GroupCallState.Fledgling) { return; @@ -176,7 +176,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._state = GroupCallState.Creating; this.emitChange(); this.callContent = Object.assign({ - "m.type": localMedia.userMedia?.videoTrack ? "m.video" : "m.voice", + "m.type": type, }, this.callContent); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); await request.response(); diff --git a/src/platform/types/MediaDevices.ts b/src/platform/types/MediaDevices.ts index 93418a66..1b5f7afd 100644 --- a/src/platform/types/MediaDevices.ts +++ b/src/platform/types/MediaDevices.ts @@ -14,15 +14,39 @@ See the License for the specific language governing permissions and limitations under the License. */ +export interface Event {} + export interface MediaDevices { // filter out audiooutput enumerate(): Promise; // to assign to a video element, we downcast to WrappedTrack and use the stream property. getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise; getScreenShareTrack(): Promise; - createVolumeMeasurer(stream: Stream): VolumeMeasurer; + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer; } +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface StreamTrackEvent extends Event { + readonly track: Track; +} + +export interface StreamEventMap { + "addtrack": StreamTrackEvent; + "removetrack": StreamTrackEvent; +} export interface Stream { getTracks(): ReadonlyArray; @@ -30,6 +54,8 @@ export interface Stream { getVideoTracks(): ReadonlyArray; readonly id: string; clone(): Stream; + addEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void; } export enum TrackKind { @@ -47,5 +73,7 @@ export interface Track { } export interface VolumeMeasurer { - + get isSpeaking(): boolean; + setSpeakingThreshold(threshold: number): void; + stop(); } diff --git a/src/platform/types/WebRTC.ts b/src/platform/types/WebRTC.ts index edb26c0a..ca2ca646 100644 --- a/src/platform/types/WebRTC.ts +++ b/src/platform/types/WebRTC.ts @@ -14,63 +14,156 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {Track, Stream} from "./MediaDevices"; +import {Track, Stream, Event} from "./MediaDevices"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; export interface WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void; } -export interface StreamSender { - get stream(): Stream; - get audioSender(): TrackSender | undefined; - get videoSender(): TrackSender | undefined; +// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts +/*! ***************************************************************************** +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions +and limitations under the License. +***************************************************************************** */ + +export interface DataChannelEventMap { + "bufferedamountlow": Event; + "close": Event; + "error": Event; + "message": MessageEvent; + "open": Event; } -export interface StreamReceiver { - get stream(): Stream; - get audioReceiver(): TrackReceiver | undefined; - get videoReceiver(): TrackReceiver | undefined; -} - -export interface TrackReceiver { - get track(): Track; - get enabled(): boolean; - enable(enabled: boolean); // this modifies the transceiver direction -} - -export interface TrackSender extends TrackReceiver { - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise; - /** make any needed adjustments to the sender or transceiver settings - * depending on the purpose, after adding the track to the connection */ - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void; -} - -export interface PeerConnectionHandler { - onIceConnectionStateChange(state: RTCIceConnectionState); - onLocalIceCandidate(candidate: RTCIceCandidate); - onIceGatheringStateChange(state: RTCIceGatheringState); - onRemoteStreamRemoved(stream: Stream); - onRemoteTracksAdded(receiver: TrackReceiver); - onRemoteDataChannel(dataChannel: any | undefined); - onNegotiationNeeded(); -} - -export interface PeerConnection { - get iceGatheringState(): RTCIceGatheringState; - get signalingState(): RTCSignalingState; - get localDescription(): RTCSessionDescription | undefined; - get localStreams(): ReadonlyMap; - get remoteStreams(): ReadonlyMap; - createOffer(): Promise; - createAnswer(): Promise; - setLocalDescription(description?: RTCSessionDescriptionInit): Promise; - setRemoteDescription(description: RTCSessionDescriptionInit): Promise; - addIceCandidate(candidate: RTCIceCandidate): Promise; - addTrack(track: Track): TrackSender | undefined; - removeTrack(track: TrackSender): void; - createDataChannel(options: RTCDataChannelInit): any; - dispose(): void; +export interface DataChannel { + binaryType: BinaryType; + readonly id: number | null; + readonly label: string; + readonly negotiated: boolean; + readonly readyState: DataChannelState; close(): void; + send(data: string): void; + send(data: Blob): void; + send(data: ArrayBuffer): void; + send(data: ArrayBufferView): void; + addEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface DataChannelInit { + id?: number; + maxPacketLifeTime?: number; + maxRetransmits?: number; + negotiated?: boolean; + ordered?: boolean; + protocol?: string; +} + +export interface DataChannelEvent extends Event { + readonly channel: DataChannel; +} + +export interface PeerConnectionIceEvent extends Event { + readonly candidate: RTCIceCandidate | null; +} + +export interface TrackEvent extends Event { + readonly receiver: Receiver; + readonly streams: ReadonlyArray; + readonly track: Track; + readonly transceiver: Transceiver; +} + +export interface PeerConnectionEventMap { + "connectionstatechange": Event; + "datachannel": DataChannelEvent; + "icecandidate": PeerConnectionIceEvent; + "iceconnectionstatechange": Event; + "icegatheringstatechange": Event; + "negotiationneeded": Event; + "signalingstatechange": Event; + "track": TrackEvent; +} + +export type DataChannelState = "closed" | "closing" | "connecting" | "open"; +export type IceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new"; +export type PeerConnectionState = "closed" | "connected" | "connecting" | "disconnected" | "failed" | "new"; +export type SignalingState = "closed" | "have-local-offer" | "have-local-pranswer" | "have-remote-offer" | "have-remote-pranswer" | "stable"; +export type IceGatheringState = "complete" | "gathering" | "new"; +export type SdpType = "answer" | "offer" | "pranswer" | "rollback"; +export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendrecv" | "stopped"; +export interface SessionDescription { + readonly sdp: string; + readonly type: SdpType; + toJSON(): any; +} + +export interface AnswerOptions {} + +export interface OfferOptions { + iceRestart?: boolean; + offerToReceiveAudio?: boolean; + offerToReceiveVideo?: boolean; +} + +export interface SessionDescriptionInit { + sdp?: string; + type: SdpType; +} + +export interface LocalSessionDescriptionInit { + sdp?: string; + type?: SdpType; +} + +/** A WebRTC connection between the local computer and a remote peer. It provides methods to connect to a remote peer, maintain and monitor the connection, and close the connection once it's no longer needed. */ +export interface PeerConnection { + readonly connectionState: PeerConnectionState; + readonly iceConnectionState: IceConnectionState; + readonly iceGatheringState: IceGatheringState; + readonly localDescription: SessionDescription | null; + readonly remoteDescription: SessionDescription | null; + readonly signalingState: SignalingState; + addIceCandidate(candidate?: RTCIceCandidateInit): Promise; + addTrack(track: Track, ...streams: Stream[]): Sender; + close(): void; + createAnswer(options?: AnswerOptions): Promise; + createDataChannel(label: string, dataChannelDict?: DataChannelInit): DataChannel; + createOffer(options?: OfferOptions): Promise; + getReceivers(): Receiver[]; + getSenders(): Sender[]; + getTransceivers(): Transceiver[]; + removeTrack(sender: Sender): void; + restartIce(): void; + setLocalDescription(description?: LocalSessionDescriptionInit): Promise; + setRemoteDescription(description: SessionDescriptionInit): Promise; + addEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void; + removeEventListener(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void; +} + +export interface Receiver { + readonly track: Track; +} + +export interface Sender { + readonly track: Track | null; + replaceTrack(withTrack: Track | null): Promise; +} + +export interface Transceiver { + readonly currentDirection: TransceiverDirection | null; + direction: TransceiverDirection; + readonly mid: string | null; + readonly receiver: Receiver; + readonly sender: Sender; + stop(): void; } diff --git a/src/platform/web/dom/MediaDevices.ts b/src/platform/web/dom/MediaDevices.ts index 4eb92259..c34ab85b 100644 --- a/src/platform/web/dom/MediaDevices.ts +++ b/src/platform/web/dom/MediaDevices.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; +import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -30,12 +30,12 @@ export class MediaDevicesWrapper implements IMediaDevices { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise { const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); - return new StreamWrapper(stream); + return stream as Stream; } async getScreenShareTrack(): Promise { const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); - return new StreamWrapper(stream); + return stream as Stream; } private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { @@ -68,70 +68,13 @@ export class MediaDevicesWrapper implements IMediaDevices { video: true, }; } -} -export class StreamWrapper implements Stream { - - public audioTrack: AudioTrackWrapper | undefined = undefined; - public videoTrack: TrackWrapper | undefined = undefined; - - constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) { - if (clonedTracks) { - this.audioTrack = clonedTracks.audioTrack; - this.videoTrack = clonedTracks.videoTrack; - } else { - for (const track of stream.getTracks()) { - this.update(track); - } - } - } - - get id(): string { return this.stream.id; } - - clone(): Stream { - const clonedStream = this.stream.clone(); - const clonedTracks = { - audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined, - videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined, - }; - return new StreamWrapper(clonedStream, clonedTracks); - } - - update(track: MediaStreamTrack): TrackWrapper | undefined { - //console.trace("Stream.update " + JSON.stringify({id: track.id, vid: this.videoTrack?.id, aid: this.audioTrack?.id})); - if (track.kind === "video") { - if (!this.videoTrack || track.id !== this.videoTrack.track.id) { - this.videoTrack = new TrackWrapper(track, this.stream, track.id); - } - return this.videoTrack; - } else if (track.kind === "audio") { - if (!this.audioTrack || track.id !== this.audioTrack.track.id) { - this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id); - } - return this.audioTrack; - } + createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer { + return new WebAudioVolumeMeasurer(stream as MediaStream, callback); } } -export class TrackWrapper implements Track { - constructor( - public readonly track: MediaStreamTrack, - public readonly stream: MediaStream, - public readonly originalId: string, - ) {} - - get kind(): TrackKind { return this.track.kind as TrackKind; } - get label(): string { return this.track.label; } - get id(): string { return this.track.id; } - get settings(): MediaTrackSettings { return this.track.getSettings(); } - get enabled(): boolean { return this.track.enabled; } - set enabled(enabled: boolean) { this.track.enabled = enabled; } - // test equality across clones - equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; } - stop() { this.track.stop(); } -} - -export class AudioTrackWrapper extends TrackWrapper { +export class WebAudioVolumeMeasurer implements VolumeMeasurer { private measuringVolumeActivity = false; private audioContext?: AudioContext; private analyser: AnalyserNode; @@ -140,9 +83,12 @@ export class AudioTrackWrapper extends TrackWrapper { private speaking = false; private volumeLooperTimeout: number; private speakingVolumeSamples: number[]; + private callback: () => void; + private stream: MediaStream; - constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) { - super(track, stream, originalId); + constructor(stream: MediaStream, callback: () => void) { + this.stream = stream; + this.callback = callback; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.initVolumeMeasuring(); this.measureVolumeActivity(true); @@ -162,6 +108,7 @@ export class AudioTrackWrapper extends TrackWrapper { } else { this.measuringVolumeActivity = false; this.speakingVolumeSamples.fill(-Infinity); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -182,7 +129,6 @@ export class AudioTrackWrapper extends TrackWrapper { this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); } - public setSpeakingThreshold(threshold: number) { this.speakingThreshold = threshold; } @@ -204,6 +150,7 @@ export class AudioTrackWrapper extends TrackWrapper { this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.push(maxVolume); + this.callback(); // this.emit(CallFeedEvent.VolumeChanged, maxVolume); let newSpeaking = false; @@ -219,267 +166,16 @@ export class AudioTrackWrapper extends TrackWrapper { if (this.speaking !== newSpeaking) { this.speaking = newSpeaking; + this.callback(); // this.emit(CallFeedEvent.Speaking, this.speaking); } this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; }; - public dispose(): void { + public stop(): void { clearTimeout(this.volumeLooperTimeout); + this.analyser.disconnect(); + this.audioContext?.close(); } } - -// export interface ICallFeedOpts { -// client: MatrixClient; -// roomId: string; -// userId: string; -// stream: MediaStream; -// purpose: SDPStreamMetadataPurpose; -// audioMuted: boolean; -// videoMuted: boolean; -// } - -// export enum CallFeedEvent { -// NewStream = "new_stream", -// MuteStateChanged = "mute_state_changed", -// VolumeChanged = "volume_changed", -// Speaking = "speaking", -// } - -// export class CallFeed extends EventEmitter { -// public stream: MediaStream; -// public sdpMetadataStreamId: string; -// public userId: string; -// public purpose: SDPStreamMetadataPurpose; -// public speakingVolumeSamples: number[]; - -// private client: MatrixClient; -// private roomId: string; -// private audioMuted: boolean; -// private videoMuted: boolean; -// private measuringVolumeActivity = false; -// private audioContext: AudioContext; -// private analyser: AnalyserNode; -// private frequencyBinCount: Float32Array; -// private speakingThreshold = SPEAKING_THRESHOLD; -// private speaking = false; -// private volumeLooperTimeout: number; - -// constructor(opts: ICallFeedOpts) { -// super(); - -// this.client = opts.client; -// this.roomId = opts.roomId; -// this.userId = opts.userId; -// this.purpose = opts.purpose; -// this.audioMuted = opts.audioMuted; -// this.videoMuted = opts.videoMuted; -// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); -// this.sdpMetadataStreamId = opts.stream.id; - -// this.updateStream(null, opts.stream); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } -// } - -// private get hasAudioTrack(): boolean { -// return this.stream.getAudioTracks().length > 0; -// } - -// private updateStream(oldStream: MediaStream, newStream: MediaStream): void { -// if (newStream === oldStream) return; - -// if (oldStream) { -// oldStream.removeEventListener("addtrack", this.onAddTrack); -// this.measureVolumeActivity(false); -// } -// if (newStream) { -// this.stream = newStream; -// newStream.addEventListener("addtrack", this.onAddTrack); - -// if (this.hasAudioTrack) { -// this.initVolumeMeasuring(); -// } else { -// this.measureVolumeActivity(false); -// } -// } - -// this.emit(CallFeedEvent.NewStream, this.stream); -// } - -// private initVolumeMeasuring(): void { -// const AudioContext = window.AudioContext || window.webkitAudioContext; -// if (!this.hasAudioTrack || !AudioContext) return; - -// this.audioContext = new AudioContext(); - -// this.analyser = this.audioContext.createAnalyser(); -// this.analyser.fftSize = 512; -// this.analyser.smoothingTimeConstant = 0.1; - -// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); -// mediaStreamAudioSourceNode.connect(this.analyser); - -// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); -// } - -// private onAddTrack = (): void => { -// this.emit(CallFeedEvent.NewStream, this.stream); -// }; - -// /** -// * Returns callRoom member -// * @returns member of the callRoom -// */ -// public getMember(): RoomMember { -// const callRoom = this.client.getRoom(this.roomId); -// return callRoom.getMember(this.userId); -// } - -// /** -// * Returns true if CallFeed is local, otherwise returns false -// * @returns {boolean} is local? -// */ -// public isLocal(): boolean { -// return this.userId === this.client.getUserId(); -// } - -// /** -// * Returns true if audio is muted or if there are no audio -// * tracks, otherwise returns false -// * @returns {boolean} is audio muted? -// */ -// public isAudioMuted(): boolean { -// return this.stream.getAudioTracks().length === 0 || this.audioMuted; -// } - -// * -// * Returns true video is muted or if there are no video -// * tracks, otherwise returns false -// * @returns {boolean} is video muted? - -// public isVideoMuted(): boolean { -// // We assume only one video track -// return this.stream.getVideoTracks().length === 0 || this.videoMuted; -// } - -// public isSpeaking(): boolean { -// return this.speaking; -// } - -// /** -// * Replaces the current MediaStream with a new one. -// * This method should be only used by MatrixCall. -// * @param newStream new stream with which to replace the current one -// */ -// public setNewStream(newStream: MediaStream): void { -// this.updateStream(this.stream, newStream); -// } - -// /** -// * Set feed's internal audio mute state -// * @param muted is the feed's audio muted? -// */ -// public setAudioMuted(muted: boolean): void { -// this.audioMuted = muted; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Set feed's internal video mute state -// * @param muted is the feed's video muted? -// */ -// public setVideoMuted(muted: boolean): void { -// this.videoMuted = muted; -// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); -// } - -// /** -// * Starts emitting volume_changed events where the emitter value is in decibels -// * @param enabled emit volume changes -// */ -// public measureVolumeActivity(enabled: boolean): void { -// if (enabled) { -// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; - -// this.measuringVolumeActivity = true; -// this.volumeLooper(); -// } else { -// this.measuringVolumeActivity = false; -// this.speakingVolumeSamples.fill(-Infinity); -// this.emit(CallFeedEvent.VolumeChanged, -Infinity); -// } -// } - -// public setSpeakingThreshold(threshold: number) { -// this.speakingThreshold = threshold; -// } - -// private volumeLooper = () => { -// if (!this.analyser) return; - -// if (!this.measuringVolumeActivity) return; - -// this.analyser.getFloatFrequencyData(this.frequencyBinCount); - -// let maxVolume = -Infinity; -// for (let i = 0; i < this.frequencyBinCount.length; i++) { -// if (this.frequencyBinCount[i] > maxVolume) { -// maxVolume = this.frequencyBinCount[i]; -// } -// } - -// this.speakingVolumeSamples.shift(); -// this.speakingVolumeSamples.push(maxVolume); - -// this.emit(CallFeedEvent.VolumeChanged, maxVolume); - -// let newSpeaking = false; - -// for (let i = 0; i < this.speakingVolumeSamples.length; i++) { -// const volume = this.speakingVolumeSamples[i]; - -// if (volume > this.speakingThreshold) { -// newSpeaking = true; -// break; -// } -// } - -// if (this.speaking !== newSpeaking) { -// this.speaking = newSpeaking; -// this.emit(CallFeedEvent.Speaking, this.speaking); -// } - -// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); -// }; - -// public clone(): CallFeed { -// const mediaHandler = this.client.getMediaHandler(); -// const stream = this.stream.clone(); - -// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { -// mediaHandler.userMediaStreams.push(stream); -// } else { -// mediaHandler.screensharingStreams.push(stream); -// } - -// return new CallFeed({ -// client: this.client, -// roomId: this.roomId, -// userId: this.userId, -// stream, -// purpose: this.purpose, -// audioMuted: this.audioMuted, -// videoMuted: this.videoMuted, -// }); -// } - -// public dispose(): void { -// clearTimeout(this.volumeLooperTimeout); -// this.measureVolumeActivity(false); -// } -// } diff --git a/src/platform/web/dom/WebRTC.ts b/src/platform/web/dom/WebRTC.ts index 22096699..61d481d4 100644 --- a/src/platform/web/dom/WebRTC.ts +++ b/src/platform/web/dom/WebRTC.ts @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; -import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; -import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC"; +import {Stream, Track, TrackKind} from "../../types/MediaDevices"; +import {WebRTC, Sender, PeerConnection} from "../../types/WebRTC"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; const POLLING_INTERVAL = 200; // ms @@ -24,159 +23,21 @@ export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples export class DOMWebRTC implements WebRTC { - createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { - return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); - } -} - -export class RemoteStreamWrapper extends StreamWrapper { - constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) { - super(stream); - this.stream.addEventListener("removetrack", this.onTrackRemoved); + createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { + return new RTCPeerConnection({ + iceTransportPolicy: forceTURN ? 'relay' : undefined, + iceServers: turnServers, + iceCandidatePoolSize: iceCandidatePoolSize, + }) as PeerConnection; } - onTrackRemoved = (evt: MediaStreamTrackEvent) => { - if (evt.track.id === this.audioTrack?.track.id) { - this.audioTrack = undefined; - } else if (evt.track.id === this.videoTrack?.track.id) { - this.videoTrack = undefined; - } - if (!this.audioTrack && !this.videoTrack) { - this.emptyCallback(this); - } - }; - - dispose() { - this.stream.removeEventListener("removetrack", this.onTrackRemoved); - } -} - -export class DOMStreamSender implements StreamSender { - public audioSender: DOMTrackSender | undefined; - public videoSender: DOMTrackSender | undefined; - - constructor(public readonly stream: StreamWrapper) {} - - update(transceivers: ReadonlyArray, sender: RTCRtpSender): DOMTrackSender | undefined { - const transceiver = transceivers.find(t => t.sender === sender); - if (transceiver && sender.track) { - const trackWrapper = this.stream.update(sender.track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) { - this.videoSender = new DOMTrackSender(trackWrapper, transceiver); - return this.videoSender; - } else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) { - this.audioSender = new DOMTrackSender(trackWrapper, transceiver); - return this.audioSender; - } - } - } - } -} - -export class DOMStreamReceiver implements StreamReceiver { - public audioReceiver: DOMTrackReceiver | undefined; - public videoReceiver: DOMTrackReceiver | undefined; - - constructor(public readonly stream: RemoteStreamWrapper) {} - - update(event: RTCTrackEvent): DOMTrackReceiver | undefined { - const {receiver} = event; - const {track} = receiver; - const trackWrapper = this.stream.update(track); - if (trackWrapper) { - if (trackWrapper.kind === TrackKind.Video) { - this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.videoReceiver; - } else { - this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver); - return this.audioReceiver; - } - } - } -} - -export class DOMTrackSenderOrReceiver implements TrackReceiver { - constructor( - public readonly track: TrackWrapper, - public readonly transceiver: RTCRtpTransceiver, - private readonly exclusiveValue: RTCRtpTransceiverDirection, - private readonly excludedValue: RTCRtpTransceiverDirection - ) {} - - get enabled(): boolean { - return this.transceiver.direction === "sendrecv" || - this.transceiver.direction === this.exclusiveValue; - } - - enableWithoutRenegotiation(enabled: boolean) { - this.track.track.enabled = enabled; - } - - enable(enabled: boolean) { - if (enabled !== this.enabled) { - // do this first, so we stop sending track data immediately. - // this will still consume bandwidth though, so also disable the transceiver, - // which will trigger a renegotiation though. - this.enableWithoutRenegotiation(enabled); - if (enabled) { - if (this.transceiver.direction === "inactive") { - this.transceiver.direction = this.exclusiveValue; - } else { - this.transceiver.direction = "sendrecv"; - } - } else { - if (this.transceiver.direction === "sendrecv") { - this.transceiver.direction = this.excludedValue; - } else { - this.transceiver.direction = "inactive"; - } - } - } - } -} - -export class DOMTrackReceiver extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "recvonly", "sendonly"); - } -} - -export class DOMTrackSender extends DOMTrackSenderOrReceiver { - constructor( - track: TrackWrapper, - transceiver: RTCRtpTransceiver, - ) { - super(track, transceiver, "sendonly", "recvonly"); - } - /** replaces the track if possible without renegotiation. Can throw. */ - replaceTrack(track: Track | undefined): Promise { - return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null); - } - - prepareForPurpose(purpose: SDPStreamMetadataPurpose): void { + prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void { if (purpose === SDPStreamMetadataPurpose.Screenshare) { - this.getRidOfRTXCodecs(); + this.getRidOfRTXCodecs(peerConnection as RTCPeerConnection, sender as RTCRtpSender); } } - /** - * This method removes all video/rtx codecs from screensharing video - * transceivers. This is necessary since they can cause problems. Without - * this the following steps should produce an error: - * Chromium calls Firefox - * Firefox answers - * Firefox starts screen-sharing - * Chromium starts screen-sharing - * Call crashes for Chromium with: - * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. - * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. - * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) - */ - private getRidOfRTXCodecs(): void { + private getRidOfRTXCodecs(peerConnection: RTCPeerConnection, sender: RTCRtpSender): void { // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; @@ -190,172 +51,14 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver { codecs.splice(rtxCodecIndex, 1); } } - if (this.transceiver.sender.track?.kind === "video" || - this.transceiver.receiver.track?.kind === "video") { - this.transceiver.setCodecPreferences(codecs); - } - } -} - -class DOMPeerConnection implements PeerConnection { - private readonly peerConnection: RTCPeerConnection; - private readonly handler: PeerConnectionHandler; - public readonly localStreams: Map = new Map(); - public readonly remoteStreams: Map = new Map(); - - constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) { - this.handler = handler; - this.peerConnection = new RTCPeerConnection({ - iceTransportPolicy: forceTURN ? 'relay' : undefined, - iceServers: turnServers, - iceCandidatePoolSize: iceCandidatePoolSize, - }); - this.registerHandler(); - } - - get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; } - get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; } - get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; } - - createOffer(): Promise { - return this.peerConnection.createOffer(); - } - - createAnswer(): Promise { - return this.peerConnection.createAnswer(); - } - - setLocalDescription(description?: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setLocalDescription(description); - } - - setRemoteDescription(description: RTCSessionDescriptionInit): Promise { - return this.peerConnection.setRemoteDescription(description); - } - - addIceCandidate(candidate: RTCIceCandidate): Promise { - return this.peerConnection.addIceCandidate(candidate); - } - - close(): void { - return this.peerConnection.close(); - } - - addTrack(track: Track): DOMTrackSender | undefined { - if (!(track instanceof TrackWrapper)) { - throw new Error("Not a TrackWrapper"); - } - const sender = this.peerConnection.addTrack(track.track, track.stream); - let streamSender = this.localStreams.get(track.stream.id); - if (!streamSender) { - // TODO: reuse existing stream wrapper here? - streamSender = new DOMStreamSender(new StreamWrapper(track.stream)); - this.localStreams.set(track.stream.id, streamSender); - } - const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender); - return trackSender; - } - - removeTrack(sender: TrackSender): void { - if (!(sender instanceof DOMTrackSender)) { - throw new Error("Not a DOMTrackSender"); - } - this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender); - // TODO: update localStreams - } - - createDataChannel(options: RTCDataChannelInit): any { - return this.peerConnection.createDataChannel("channel", options); - } - - private registerHandler() { - const pc = this.peerConnection; - pc.addEventListener('negotiationneeded', this); - pc.addEventListener('icecandidate', this); - pc.addEventListener('iceconnectionstatechange', this); - pc.addEventListener('icegatheringstatechange', this); - pc.addEventListener('signalingstatechange', this); - pc.addEventListener('track', this); - pc.addEventListener('datachannel', this); - } - - private deregisterHandler() { - const pc = this.peerConnection; - pc.removeEventListener('negotiationneeded', this); - pc.removeEventListener('icecandidate', this); - pc.removeEventListener('iceconnectionstatechange', this); - pc.removeEventListener('icegatheringstatechange', this); - pc.removeEventListener('signalingstatechange', this); - pc.removeEventListener('track', this); - pc.removeEventListener('datachannel', this); - } - - /** @internal */ - handleEvent(evt: Event) { - switch (evt.type) { - case "iceconnectionstatechange": - this.handleIceConnectionStateChange(); - break; - case "icecandidate": - this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent); - break; - case "icegatheringstatechange": - this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState); - break; - case "track": - this.handleRemoteTrack(evt as RTCTrackEvent); - break; - case "negotiationneeded": - this.handler.onNegotiationNeeded(); - break; - case "datachannel": - this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel); - break; - } - } - - dispose(): void { - this.deregisterHandler(); - for (const r of this.remoteStreams.values()) { - r.stream.dispose(); - } - } - - private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) { - if (event.candidate) { - this.handler.onLocalIceCandidate(event.candidate); - } - }; - - private handleIceConnectionStateChange() { - const {iceConnectionState} = this.peerConnection; - if (iceConnectionState === "failed" && this.peerConnection.restartIce) { - this.peerConnection.restartIce(); - } else { - this.handler.onIceConnectionStateChange(iceConnectionState); - } - } - - onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => { - if (this.remoteStreams.delete(stream.id)) { - this.handler.onRemoteStreamRemoved(stream); - } - } - - private handleRemoteTrack(evt: RTCTrackEvent) { - if (evt.streams.length !== 1) { - throw new Error("track in multiple streams is not supported"); - } - const stream = evt.streams[0]; - const transceivers = this.peerConnection.getTransceivers(); - let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id); - if (!streamReceiver) { - streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty)); - this.remoteStreams.set(stream.id, streamReceiver); - } - const trackReceiver = streamReceiver.update(evt); - if (trackReceiver) { - this.handler.onRemoteTracksAdded(trackReceiver); + + const transceiver = peerConnection.getTransceivers().find(t => t.sender === sender); + if (transceiver && ( + transceiver.sender.track?.kind === "video" || + transceiver.receiver.track?.kind === "video" + ) + ) { + transceiver.setCodecPreferences(codecs); } } } diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index b318c324..1c847cbb 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -17,15 +17,15 @@ limitations under the License. import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; import {ListView} from "../../general/ListView"; import {Stream} from "../../../../types/MediaDevices"; -import type {StreamWrapper} from "../../../dom/MediaDevices"; +import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../../matrix/calls/common"; import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; function bindStream(t: TemplateBuilder, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { - t.mapSideEffect(vm => propSelector(vm)?.videoTrack?.enabled, (_,__, vm) => { + t.mapSideEffect(vm => getStreamVideoTrack(propSelector(vm))?.enabled, (_,__, vm) => { const stream = propSelector(vm); if (stream) { - video.srcObject = (stream as StreamWrapper).stream; - if (stream.videoTrack?.enabled) { + video.srcObject = stream as MediaStream; + if (getStreamVideoTrack(stream)?.enabled) { video.classList.remove("hidden"); } else { video.classList.add("hidden"); diff --git a/src/utils/AsyncQueue.ts b/src/utils/AsyncQueue.ts deleted file mode 100644 index 0686314c..00000000 --- a/src/utils/AsyncQueue.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -export class AsyncQueue { - private isRunning = false; - private queue: T[] = []; - private error?: Error; - - constructor( - private readonly reducer: (v: V, t: T) => Promise, - private value: V, - private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t) - ) {} - - push(t: T) { - if (this.contains(t, this.queue)) { - return; - } - this.queue.push(t); - this.runLoopIfNeeded(); - } - - private async runLoopIfNeeded() { - if (this.isRunning || this.error) { - return; - } - this.isRunning = true; - try { - let item: T | undefined; - while (item = this.queue.shift()) { - this.value = await this.reducer(this.value, item); - } - } catch (err) { - this.error = err; - } finally { - this.isRunning = false; - } - } -} From 826835e518666d3fd7d09f4e944c98ee3650d369 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 22 Apr 2022 12:07:53 +0530 Subject: [PATCH 17/35] No need to rewrite to index.html --- src/platform/web/sw.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index aaf56c4b..91ae85c5 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -124,11 +124,6 @@ async function handleRequest(request) { } async function handleConfigRequest(request) { - const url = new URL(request.url); - // rewrite / to /index.html so it hits the cache - if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) { - request = new Request(new URL("index.html", baseURL.href)); - } let response = await readCache(request); if (response) { fetchAndUpdateConfig(request); From c6691cf1cbd6fe69b617f445d1ed6a3fe297c216 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 22 Apr 2022 12:10:25 +0530 Subject: [PATCH 18/35] Simplify code --- src/platform/web/sw.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index 91ae85c5..2d43d1ee 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -125,12 +125,12 @@ async function handleRequest(request) { async function handleConfigRequest(request) { let response = await readCache(request); + const networkResponsePromise = fetchAndUpdateConfig(request); if (response) { - fetchAndUpdateConfig(request); return response; + } else { + return await networkResponsePromise; } - response = await fetchAndUpdateConfig(request); - return response; } async function fetchAndUpdateConfig(request) { From 5a94a2feba344029699705dd15f33891266b7b6c Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 22 Apr 2022 12:22:30 +0530 Subject: [PATCH 19/35] Move handleConfigRequest inside handleRequest --- src/platform/web/sw.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index 2d43d1ee..a9a92979 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -75,11 +75,7 @@ self.addEventListener('fetch', (event) => { This has to do with xhr not being supported in service workers. */ if (event.request.method === "GET") { - if (event.request.url.includes("config.json")) { - event.respondWith(handleConfigRequest(event.request)); - } else { - event.respondWith(handleRequest(event.request)); - } + event.respondWith(handleRequest(event.request)); } }); @@ -96,8 +92,12 @@ function isCacheableThumbnail(url) { const baseURL = new URL(self.registration.scope); let pendingFetchAbortController = new AbortController(); + async function handleRequest(request) { try { + if (request.url.includes("config.json")) { + return handleConfigRequest(request); + } const url = new URL(request.url); // rewrite / to /index.html so it hits the cache if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) { From 7a33c2e00d69f09979a3f668bff6e130441f5623 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 22 Apr 2022 12:26:29 +0530 Subject: [PATCH 20/35] await --- src/platform/web/sw.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index a9a92979..49789804 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -96,7 +96,7 @@ let pendingFetchAbortController = new AbortController(); async function handleRequest(request) { try { if (request.url.includes("config.json")) { - return handleConfigRequest(request); + return await handleConfigRequest(request); } const url = new URL(request.url); // rewrite / to /index.html so it hits the cache From d8da1287804921bd785be8383b51827913b2c1d6 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Fri, 22 Apr 2022 14:34:16 +0530 Subject: [PATCH 21/35] remove await --- src/platform/web/sw.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/sw.js b/src/platform/web/sw.js index 49789804..a9a92979 100644 --- a/src/platform/web/sw.js +++ b/src/platform/web/sw.js @@ -96,7 +96,7 @@ let pendingFetchAbortController = new AbortController(); async function handleRequest(request) { try { if (request.url.includes("config.json")) { - return await handleConfigRequest(request); + return handleConfigRequest(request); } const url = new URL(request.url); // rewrite / to /index.html so it hits the cache From cdb2a79b62f27e7e1e25d36a200adcff3bfa96f3 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Fri, 22 Apr 2022 14:48:14 +0100 Subject: [PATCH 22/35] add muting again, separate from changing media --- src/domain/session/room/CallViewModel.ts | 2 +- src/matrix/calls/PeerCall.ts | 80 +++++++++++++++++++++--- src/matrix/calls/common.ts | 8 +++ src/matrix/calls/group/GroupCall.ts | 19 +++++- src/matrix/calls/group/Member.ts | 16 +++-- 5 files changed, 111 insertions(+), 14 deletions(-) diff --git a/src/domain/session/room/CallViewModel.ts b/src/domain/session/room/CallViewModel.ts index e6a6914f..df61d334 100644 --- a/src/domain/session/room/CallViewModel.ts +++ b/src/domain/session/room/CallViewModel.ts @@ -57,7 +57,7 @@ export class CallViewModel extends ViewModel { } async toggleVideo() { - //this.call.setMuted(); + this.call.setMuted(this.call.muteSettings.toggleCamera()); } } diff --git a/src/matrix/calls/PeerCall.ts b/src/matrix/calls/PeerCall.ts index 799fd9b8..81828c32 100644 --- a/src/matrix/calls/PeerCall.ts +++ b/src/matrix/calls/PeerCall.ts @@ -17,7 +17,7 @@ limitations under the License. import {ObservableMap} from "../../observable/map/ObservableMap"; import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables"; -import {WebRTC, PeerConnection, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; +import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices"; import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common"; import { @@ -166,13 +166,14 @@ export class PeerCall implements IDisposable { return this._remoteMedia; } - call(localMedia: LocalMedia): Promise { + call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("call", async log => { if (this._state !== CallState.Fledgling) { return; } this.direction = CallDirection.Outbound; this.setState(CallState.CreateOffer, log); + this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); if (this.localMedia?.dataChannelOptions) { this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions); @@ -183,12 +184,13 @@ export class PeerCall implements IDisposable { }); } - answer(localMedia: LocalMedia): Promise { + answer(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise { return this.logItem.wrap("answer", async log => { if (this._state !== CallState.Ringing) { return; } this.setState(CallState.CreateAnswer, log); + this.localMuteSettings = localMuteSettings; await this.updateLocalMedia(localMedia, log); let myAnswer: RTCSessionDescriptionInit; try { @@ -235,12 +237,54 @@ export class PeerCall implements IDisposable { }); } + setMuted(localMuteSettings: MuteSettings) { + return this.logItem.wrap("setMuted", async log => { + this.localMuteSettings = localMuteSettings; + log.set("cameraMuted", localMuteSettings.camera); + log.set("microphoneMuted", localMuteSettings.microphone); + + if (this.localMedia) { + const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia); + if (userMediaAudio) { + this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log); + } + const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia); + if (userMediaVideo) { + this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log); + } + const content: MCallSDPStreamMetadataChanged = { + call_id: this.callId, + version: 1, + seq: this.seq++, + [SDPStreamMetadataKey]: this.getSDPMetadata() + }; + await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log); + } + }); + } + hangup(errorCode: CallErrorCode): Promise { return this.logItem.wrap("hangup", log => { return this._hangup(errorCode, log); }); } + private muteTrack(track: Track, muted: boolean, log: ILogItem): void { + log.wrap({l: "track", kind: track.kind, id: track.id}, log => { + const enabled = !muted; + log.set("enabled", enabled); + const transceiver = this.findTransceiverForTrack(track); + if (transceiver) { + if (transceiver.sender.track) { + transceiver.sender.track.enabled = enabled; + } + log.set("fromDirection", transceiver.direction); + enableSenderOnTransceiver(transceiver, enabled); + log.set("toDirection", transceiver.direction); + } + }); + } + private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise { if (this._state === CallState.Ended) { return; @@ -872,10 +916,17 @@ export class PeerCall implements IDisposable { private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined { return this.peerConnection.getReceivers().find(r => { - return r.track.kind === "audio" && this._remoteTrackToStreamId.get(r.track.id) === streamId; + return r.track.kind === kind && this._remoteTrackToStreamId.get(r.track.id) === streamId; }); } + private findTransceiverForTrack(track: Track): Transceiver | undefined { + return this.peerConnection.getTransceivers().find(t => { + return t.sender.track?.id === track.id; + }); + } + + private onRemoteTrack(track: Track, streams: ReadonlyArray, log: ILogItem) { if (streams.length === 0) { log.log({l: `ignoring ${track.kind} streamless track`, id: track.id}); @@ -1072,7 +1123,22 @@ export function handlesEventType(eventType: string): boolean { eventType === EventType.Negotiate; } - -export function tests() { - +function enableSenderOnTransceiver(transceiver: Transceiver, enabled: boolean) { + return enableTransceiver(transceiver, enabled, "sendonly", "recvonly"); +} + +function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusiveValue: TransceiverDirection, excludedValue: TransceiverDirection) { + if (enabled) { + if (transceiver.direction === "inactive") { + transceiver.direction = exclusiveValue; + } else { + transceiver.direction = "sendrecv"; + } + } else { + if (transceiver.direction === "sendrecv") { + transceiver.direction = excludedValue; + } else { + transceiver.direction = "inactive"; + } + } } diff --git a/src/matrix/calls/common.ts b/src/matrix/calls/common.ts index c5970d4b..6770aec1 100644 --- a/src/matrix/calls/common.ts +++ b/src/matrix/calls/common.ts @@ -26,4 +26,12 @@ export function getStreamVideoTrack(stream: Stream | undefined): Track | undefin export class MuteSettings { constructor (public readonly microphone: boolean, public readonly camera: boolean) {} + + toggleCamera(): MuteSettings { + return new MuteSettings(this.microphone, !this.camera); + } + + toggleMicrophone(): MuteSettings { + return new MuteSettings(!this.microphone, this.camera); + } } diff --git a/src/matrix/calls/group/GroupCall.ts b/src/matrix/calls/group/GroupCall.ts index 4992f6b6..57d08cd3 100644 --- a/src/matrix/calls/group/GroupCall.ts +++ b/src/matrix/calls/group/GroupCall.ts @@ -17,6 +17,7 @@ limitations under the License. import {ObservableMap} from "../../../observable/map/ObservableMap"; import {Member} from "./Member"; import {LocalMedia} from "../LocalMedia"; +import {MuteSettings} from "../common"; import {RoomMember} from "../../room/members/RoomMember"; import {EventEmitter} from "../../../utils/EventEmitter"; import {EventType, CallIntent} from "../callEventTypes"; @@ -63,6 +64,7 @@ export class GroupCall extends EventEmitter<{change: never}> { private _localMedia?: LocalMedia = undefined; private _memberOptions: MemberOptions; private _state: GroupCallState; + private localMuteSettings: MuteSettings = new MuteSettings(false, false); constructor( public readonly id: string, @@ -118,7 +120,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this.emitChange(); // send invite to all members that are < my userId for (const [,member] of this._members) { - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } }); } @@ -134,6 +136,19 @@ export class GroupCall extends EventEmitter<{change: never}> { } } + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { + for (const [,member] of this._members) { + member.setMuted(this.localMuteSettings); + } + } + } + + get muteSettings(): MuteSettings { + return this.localMuteSettings; + } + get hasJoined() { return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; } @@ -230,7 +245,7 @@ export class GroupCall extends EventEmitter<{change: never}> { this._members.add(memberKey, member); if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { // Safari can't send a MediaStream to multiple sources, so clone it - member.connect(this._localMedia!.clone()); + member.connect(this._localMedia!.clone(), this.localMuteSettings); } } } diff --git a/src/matrix/calls/group/Member.ts b/src/matrix/calls/group/Member.ts index 714cd821..e7eefd4d 100644 --- a/src/matrix/calls/group/Member.ts +++ b/src/matrix/calls/group/Member.ts @@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common"; import {EventType, CallErrorCode} from "../callEventTypes"; import {formatToDeviceMessagesPayload} from "../../common"; +import type {MuteSettings} from "../common"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {LocalMedia} from "../LocalMedia"; import type {HomeServerApi} from "../../net/HomeServerApi"; @@ -50,6 +51,7 @@ const errorCodesWithoutRetry = [ export class Member { private peerCall?: PeerCall; private localMedia?: LocalMedia; + private localMuteSettings?: MuteSettings; private retryCount: number = 0; constructor( @@ -80,9 +82,10 @@ export class Member { } /** @internal */ - connect(localMedia: LocalMedia) { + connect(localMedia: LocalMedia, localMuteSettings: MuteSettings) { this.logItem.wrap("connect", () => { this.localMedia = localMedia; + this.localMuteSettings = localMuteSettings; // otherwise wait for it to connect let shouldInitiateCall; // the lexicographically lower side initiates the call @@ -93,7 +96,7 @@ export class Member { } if (shouldInitiateCall) { this.peerCall = this._createPeerCall(makeId("c")); - this.peerCall.call(localMedia); + this.peerCall.call(localMedia, localMuteSettings); } }); } @@ -121,7 +124,7 @@ export class Member { /** @internal */ emitUpdate = (peerCall: PeerCall, params: any) => { if (peerCall.state === CallState.Ringing) { - peerCall.answer(this.localMedia!); + peerCall.answer(this.localMedia!, this.localMuteSettings!); } else if (peerCall.state === CallState.Ended) { const hangupReason = peerCall.hangupReason; @@ -130,7 +133,7 @@ export class Member { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { this.retryCount += 1; if (this.retryCount <= 3) { - this.connect(this.localMedia!); + this.connect(this.localMedia!, this.localMuteSettings!); } } } @@ -190,6 +193,11 @@ export class Member { await this.peerCall?.setMedia(this.localMedia); } + setMuted(muteSettings: MuteSettings) { + this.localMuteSettings = muteSettings; + this.peerCall?.setMuted(muteSettings); + } + private _createPeerCall(callId: string): PeerCall { return new PeerCall(callId, Object.assign({}, this.options, { emitUpdate: this.emitUpdate, From 39ecc6cc6d60847ee201020ea2533fc003686a70 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:27:33 +0200 Subject: [PATCH 23/35] WIP typing errors --- src/platform/web/ui/session/room/CallView.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/web/ui/session/room/CallView.ts b/src/platform/web/ui/session/room/CallView.ts index 1c847cbb..95d9c027 100644 --- a/src/platform/web/ui/session/room/CallView.ts +++ b/src/platform/web/ui/session/room/CallView.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; +import {TemplateView, Builder} from "../../general/TemplateView"; import {ListView} from "../../general/ListView"; import {Stream} from "../../../../types/MediaDevices"; import {getStreamVideoTrack, getStreamAudioTrack} from "../../../../../matrix/calls/common"; @@ -38,7 +38,7 @@ function bindStream(t: TemplateBuilder, video: HTMLVideoElement, propSelec } export class CallView extends TemplateView { - render(t: TemplateBuilder, vm: CallViewModel): HTMLElement { + render(t: Builder, vm: CallViewModel): Element { return t.div({class: "CallView"}, [ t.p(vm => `Call ${vm.name} (${vm.id})`), t.div({class: "CallView_me"}, bindStream(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), From 6c57c96cb9a7a3b77f4fd1c92723bfbbad59b860 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:07:28 +0200 Subject: [PATCH 24/35] add typing for text bindings in template view --- src/platform/web/ui/general/TemplateView.ts | 24 +++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/platform/web/ui/general/TemplateView.ts b/src/platform/web/ui/general/TemplateView.ts index ce593f75..a0e2079a 100644 --- a/src/platform/web/ui/general/TemplateView.ts +++ b/src/platform/web/ui/general/TemplateView.ts @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child} from "./html"; +import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child as NonBoundChild} from "./html"; import {mountView} from "./utils"; import {BaseUpdateView, IObservableValue} from "./BaseUpdateView"; import {IMountArgs, ViewNode, IView} from "./types"; @@ -30,12 +30,15 @@ function objHasFns(obj: ClassNames): obj is { [className: string]: bool } export type RenderFn = (t: Builder, vm: T) => ViewNode; +type TextBinding = (T) => string | number | boolean | undefined | null; +type Child = NonBoundChild | TextBinding; +type Children = Child | Child[]; type EventHandler = ((event: Event) => void); type AttributeStaticValue = string | boolean; type AttributeBinding = (value: T) => AttributeStaticValue; export type AttrValue = AttributeStaticValue | AttributeBinding | EventHandler | ClassNames; export type Attributes = { [attribute: string]: AttrValue }; -type ElementFn = (attributes?: Attributes | Child | Child[], children?: Child | Child[]) => Element; +type ElementFn = (attributes?: Attributes | Children, children?: Children) => Element; export type Builder = TemplateBuilder & { [tagName in typeof TAG_NAMES[string][number]]: ElementFn }; /** @@ -195,15 +198,15 @@ export class TemplateBuilder { this._addAttributeBinding(node, "className", value => classNames(obj, value)); } - _addTextBinding(fn: (value: T) => string): Text { - const initialValue = fn(this._value); + _addTextBinding(fn: (value: T) => ReturnType>): Text { + const initialValue = fn(this._value)+""; const node = text(initialValue); let prevValue = initialValue; const binding = () => { - const newValue = fn(this._value); + const newValue = fn(this._value)+""; if (prevValue !== newValue) { prevValue = newValue; - node.textContent = newValue+""; + node.textContent = newValue; } }; @@ -242,7 +245,7 @@ export class TemplateBuilder { } } - _setNodeChildren(node: Element, children: Child | Child[]): void{ + _setNodeChildren(node: Element, children: Children): void{ if (!Array.isArray(children)) { children = [children]; } @@ -276,14 +279,17 @@ export class TemplateBuilder { return node; } - el(name: string, attributes?: Attributes | Child | Child[], children?: Child | Child[]): ViewNode { + el(name: string, attributes?: Attributes | Children, children?: Children): ViewNode { return this.elNS(HTML_NS, name, attributes, children); } - elNS(ns: string, name: string, attributes?: Attributes | Child | Child[], children?: Child | Child[]): ViewNode { + elNS(ns: string, name: string, attributesOrChildren?: Attributes | Children, children?: Children): ViewNode { + let attributes: Attributes | undefined; if (attributes !== undefined && isChildren(attributes)) { children = attributes; attributes = undefined; + } else { + attributes = attributesOrChildren as Attributes; } const node = document.createElementNS(ns, name); From ab893f63b5534cd40df096ef70f6a04456ec3f6d Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:40:25 +0200 Subject: [PATCH 25/35] remove unneeded assignment --- src/platform/web/ui/general/TemplateView.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/platform/web/ui/general/TemplateView.ts b/src/platform/web/ui/general/TemplateView.ts index a0e2079a..7f7f4c13 100644 --- a/src/platform/web/ui/general/TemplateView.ts +++ b/src/platform/web/ui/general/TemplateView.ts @@ -287,7 +287,6 @@ export class TemplateBuilder { let attributes: Attributes | undefined; if (attributes !== undefined && isChildren(attributes)) { children = attributes; - attributes = undefined; } else { attributes = attributesOrChildren as Attributes; } From 22df062bbb4723f8f028fdb89827f3e8d3e10d23 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:05:02 +0200 Subject: [PATCH 26/35] fix observable typescript errors --- src/domain/SessionPickerViewModel.js | 2 +- src/matrix/Session.js | 2 +- src/matrix/room/timeline/Timeline.js | 2 +- src/observable/{index.js => index.ts} | 9 +++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) rename src/observable/{index.js => index.ts} (78%) diff --git a/src/domain/SessionPickerViewModel.js b/src/domain/SessionPickerViewModel.js index e486c64f..f65fd2e6 100644 --- a/src/domain/SessionPickerViewModel.js +++ b/src/domain/SessionPickerViewModel.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SortedArray} from "../observable/index.js"; +import {SortedArray} from "../observable/index"; import {ViewModel} from "./ViewModel"; import {avatarInitials, getIdentifierColorNumber} from "./avatar"; diff --git a/src/matrix/Session.js b/src/matrix/Session.js index bd361434..9d63d335 100644 --- a/src/matrix/Session.js +++ b/src/matrix/Session.js @@ -21,7 +21,7 @@ import {RoomStatus} from "./room/common"; import {RoomBeingCreated} from "./room/RoomBeingCreated"; import {Invite} from "./room/Invite.js"; import {Pusher} from "./push/Pusher"; -import { ObservableMap } from "../observable/index.js"; +import { ObservableMap } from "../observable/index"; import {User} from "./User.js"; import {DeviceMessageHandler} from "./DeviceMessageHandler.js"; import {Account as E2EEAccount} from "./e2ee/Account.js"; diff --git a/src/matrix/room/timeline/Timeline.js b/src/matrix/room/timeline/Timeline.js index 3332a5b0..de17fce2 100644 --- a/src/matrix/room/timeline/Timeline.js +++ b/src/matrix/room/timeline/Timeline.js @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index.js"; +import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index"; import {Disposables} from "../../../utils/Disposables"; import {Direction} from "./Direction"; import {TimelineReader} from "./persistence/TimelineReader.js"; diff --git a/src/observable/index.js b/src/observable/index.ts similarity index 78% rename from src/observable/index.js rename to src/observable/index.ts index 6057174b..040ec761 100644 --- a/src/observable/index.js +++ b/src/observable/index.ts @@ -46,3 +46,12 @@ Object.assign(BaseObservableMap.prototype, { return new JoinedMap([this].concat(otherMaps)); } }); + +declare module "./map/BaseObservableMap" { + interface BaseObservableMap { + sortValues(comparator: (a: V, b: V) => number): SortedMapList; + mapValues(mapper: (V, emitSpontaneousUpdate: (params: any) => void) => M, updater: (mappedValue: M, params: any, value: V) => void): MappedMap; + filterValues(filter: (V, K) => boolean): FilteredMap; + join(...otherMaps: BaseObservableMap[]): JoinedMap; + } +} From a52423856d77cb60d202ba66b05a721b71129a57 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:05:31 +0200 Subject: [PATCH 27/35] template view: remove type duplication --- src/platform/web/ui/general/TemplateView.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/ui/general/TemplateView.ts b/src/platform/web/ui/general/TemplateView.ts index ef6320e7..059dfaf6 100644 --- a/src/platform/web/ui/general/TemplateView.ts +++ b/src/platform/web/ui/general/TemplateView.ts @@ -181,7 +181,7 @@ export class TemplateBuilder { this._templateView._addEventListener(node, name, fn, useCapture); } - _addAttributeBinding(node: Element, name: string, fn: (value: T) => boolean | string): void { + _addAttributeBinding(node: Element, name: string, fn: AttributeBinding): void { let prevValue: string | boolean | undefined = undefined; const binding = () => { const newValue = fn(this._value); From bec8cea583af5d6a2a2b57ae70831a27d87ea3ac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:17:07 +0200 Subject: [PATCH 28/35] fix for breaking in #725 --- src/platform/web/ui/general/TemplateView.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/platform/web/ui/general/TemplateView.ts b/src/platform/web/ui/general/TemplateView.ts index 7f7f4c13..d6e3dd3f 100644 --- a/src/platform/web/ui/general/TemplateView.ts +++ b/src/platform/web/ui/general/TemplateView.ts @@ -285,10 +285,12 @@ export class TemplateBuilder { elNS(ns: string, name: string, attributesOrChildren?: Attributes | Children, children?: Children): ViewNode { let attributes: Attributes | undefined; - if (attributes !== undefined && isChildren(attributes)) { - children = attributes; - } else { - attributes = attributesOrChildren as Attributes; + if (attributesOrChildren) { + if (isChildren(attributesOrChildren)) { + children = attributesOrChildren as Children; + } else { + attributes = attributesOrChildren as Attributes; + } } const node = document.createElementNS(ns, name); From fa34315210c4ca9c072c5f096058d8076feb1ba9 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 25 Apr 2022 16:44:31 +0200 Subject: [PATCH 29/35] undo refactoring typo from #723 --- src/platform/web/Platform.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 1c999598..7d66301d 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -174,7 +174,7 @@ export class Platform { const {body}= await this.request(this._configURL, {method: "GET", format: "json", cache: true}).response(); this._config = body; } - this._notificationService = new NotificationService( + this.notificationService = new NotificationService( this._serviceWorkerHandler, this._config.push ); From 3767f6a4206ccfdde0be00bf9f1f9ebd09968a8a Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Tue, 26 Apr 2022 14:19:13 +0200 Subject: [PATCH 30/35] put theme back to default --- src/platform/web/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/web/index.html b/src/platform/web/index.html index f73e65ed..16418699 100644 --- a/src/platform/web/index.html +++ b/src/platform/web/index.html @@ -11,7 +11,7 @@ - +