Merge branch 'bwindels/calls' into thirdroom/dev

This commit is contained in:
Robert Long 2022-04-26 11:53:50 -07:00
commit f6a0986b3c
38 changed files with 1135 additions and 1085 deletions

View file

@ -47,7 +47,8 @@ const assetPaths = {
wasmBundle: olmJsPath wasmBundle: olmJsPath
} }
}; };
import "hydrogen-view-sdk/style.css"; import "hydrogen-view-sdk/theme-element-light.css";
// OR import "hydrogen-view-sdk/theme-element-dark.css";
async function main() { async function main() {
const app = document.querySelector<HTMLDivElement>('#app')! const app = document.querySelector<HTMLDivElement>('#app')!

View file

@ -39,7 +39,7 @@ function colorsFromURL(url, colorMap) {
function processURL(decl, replacer, colorMap) { function processURL(decl, replacer, colorMap) {
const value = decl.value; const value = decl.value;
const parsed = valueParser(value); const parsed = valueParser(value);
parsed.walk(async node => { parsed.walk(node => {
if (node.type !== "function" || node.value !== "url") { if (node.type !== "function" || node.value !== "url") {
return; return;
} }

View file

@ -37,7 +37,7 @@ module.exports.buildColorizedSVG = function (svgLocation, primaryColor, secondar
if (svgCode === coloredSVGCode) { if (svgCode === coloredSVGCode) {
throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive)."); throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive).");
} }
const fileName = svgLocation.match(/.+\/(.+\.svg)/)[1]; const fileName = svgLocation.match(/.+[/\\](.+\.svg)/)[1];
const outputName = `${fileName.substring(0, fileName.length - 4)}-${createHash(coloredSVGCode)}.svg`; const outputName = `${fileName.substring(0, fileName.length - 4)}-${createHash(coloredSVGCode)}.svg`;
const outputPath = path.resolve(__dirname, "../../.tmp"); const outputPath = path.resolve(__dirname, "../../.tmp");
try { try {

View file

@ -1,4 +1,8 @@
#!/bin/bash #!/bin/bash
# Exit whenever one of the commands fail with a non-zero exit code
set -e
set -o pipefail
rm -rf target rm -rf target
yarn run vite build -c vite.sdk-assets-config.js yarn run vite build -c vite.sdk-assets-config.js
yarn run vite build -c vite.sdk-lib-config.js yarn run vite build -c vite.sdk-lib-config.js

View file

@ -0,0 +1,23 @@
/*
Copyright 2020 Bruno Windels <bruno@windels.cloud>
Copyright 2020, 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
export interface AvatarSource {
get avatarLetter(): string;
get avatarColorNumber(): number;
avatarUrl(size: number): string | undefined;
get avatarTitle(): string;
}

View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {SortedArray} from "../observable/index.js"; import {SortedArray} from "../observable/index";
import {ViewModel} from "./ViewModel"; import {ViewModel} from "./ViewModel";
import {avatarInitials, getIdentifierColorNumber} from "./avatar"; import {avatarInitials, getIdentifierColorNumber} from "./avatar";

View file

@ -51,10 +51,10 @@ export function getIdentifierColorNumber(id: string): number {
return (hashCode(id) % 8) + 1; return (hashCode(id) % 8) + 1;
} }
export function getAvatarHttpUrl(avatarUrl: string, cssSize: number, platform: Platform, mediaRepository: MediaRepository): string | null { export function getAvatarHttpUrl(avatarUrl: string | undefined, cssSize: number, platform: Platform, mediaRepository: MediaRepository): string | undefined {
if (avatarUrl) { if (avatarUrl) {
const imageSize = cssSize * platform.devicePixelRatio; const imageSize = cssSize * platform.devicePixelRatio;
return mediaRepository.mxcUrlThumbnail(avatarUrl, imageSize, imageSize, "crop"); return mediaRepository.mxcUrlThumbnail(avatarUrl, imageSize, imageSize, "crop");
} }
return null; return undefined;
} }

View file

@ -14,23 +14,34 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {AvatarSource} from "../../AvatarSource";
import {ViewModel, Options as BaseOptions} from "../../ViewModel"; import {ViewModel, Options as BaseOptions} from "../../ViewModel";
import {getStreamVideoTrack, getStreamAudioTrack} from "../../../matrix/calls/common";
import {avatarInitials, getIdentifierColorNumber, getAvatarHttpUrl} from "../../avatar";
import {EventObservableValue} from "../../../observable/value/EventObservableValue";
import {ObservableValueMap} from "../../../observable/map/ObservableValueMap";
import type {GroupCall} from "../../../matrix/calls/group/GroupCall"; import type {GroupCall} from "../../../matrix/calls/group/GroupCall";
import type {Member} from "../../../matrix/calls/group/Member"; import type {Member} from "../../../matrix/calls/group/Member";
import type {BaseObservableList} from "../../../observable/list/BaseObservableList"; import type {BaseObservableList} from "../../../observable/list/BaseObservableList";
import type {Stream} from "../../../platform/types/MediaDevices"; import type {Stream} from "../../../platform/types/MediaDevices";
import type {MediaRepository} from "../../../matrix/net/MediaRepository";
type Options = BaseOptions & {call: GroupCall}; type Options = BaseOptions & {
call: GroupCall,
mediaRepository: MediaRepository
};
export class CallViewModel extends ViewModel<Options> { export class CallViewModel extends ViewModel<Options> {
public readonly memberViewModels: BaseObservableList<IStreamViewModel>;
public readonly memberViewModels: BaseObservableList<CallMemberViewModel>;
constructor(options: Options) { constructor(options: Options) {
super(options); super(options);
this.memberViewModels = this.getOption("call").members const ownMemberViewModelMap = new ObservableValueMap("self", new EventObservableValue(this.call, "change"))
.mapValues(call => new OwnMemberViewModel(this.childOptions({call: this.call, mediaRepository: this.getOption("mediaRepository")})), () => {});
this.memberViewModels = this.call.members
.filterValues(member => member.isConnected) .filterValues(member => member.isConnected)
.mapValues(member => new CallMemberViewModel(this.childOptions({member}))) .mapValues(member => new CallMemberViewModel(this.childOptions({member, mediaRepository: this.getOption("mediaRepository")})))
.join(ownMemberViewModelMap)
.sortValues((a, b) => a.compare(b)); .sortValues((a, b) => a.compare(b));
} }
@ -46,7 +57,7 @@ export class CallViewModel extends ViewModel<Options> {
return this.call.id; return this.call.id;
} }
get localStream(): Stream | undefined { get stream(): Stream | undefined {
return this.call.localMedia?.userMedia; return this.call.localMedia?.userMedia;
} }
@ -55,11 +66,66 @@ export class CallViewModel extends ViewModel<Options> {
this.call.leave(); this.call.leave();
} }
} }
get isCameraMuted(): boolean {
return this.call.muteSettings.camera;
}
get isMicrophoneMuted(): boolean {
return this.call.muteSettings.microphone;
}
async toggleVideo() {
this.call.setMuted(this.call.muteSettings.toggleCamera());
}
} }
type MemberOptions = BaseOptions & {member: Member}; type OwnMemberOptions = BaseOptions & {
call: GroupCall,
mediaRepository: MediaRepository
}
export class CallMemberViewModel extends ViewModel<MemberOptions> { class OwnMemberViewModel extends ViewModel<OwnMemberOptions> implements IStreamViewModel {
get stream(): Stream | undefined {
return this.call.localMedia?.userMedia;
}
private get call(): GroupCall {
return this.getOption("call");
}
get isCameraMuted(): boolean {
return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream);
}
get isMicrophoneMuted(): boolean {
return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream);
}
get avatarLetter(): string {
return "I";
}
get avatarColorNumber(): number {
return 3;
}
avatarUrl(size: number): string | undefined {
return undefined;
}
get avatarTitle(): string {
return "Me";
}
compare(other: OwnMemberViewModel | CallMemberViewModel): number {
return -1;
}
}
type MemberOptions = BaseOptions & {member: Member, mediaRepository: MediaRepository};
export class CallMemberViewModel extends ViewModel<MemberOptions> implements IStreamViewModel {
get stream(): Stream | undefined { get stream(): Stream | undefined {
return this.member.remoteMedia?.userMedia; return this.member.remoteMedia?.userMedia;
} }
@ -68,7 +134,36 @@ export class CallMemberViewModel extends ViewModel<MemberOptions> {
return this.getOption("member"); return this.getOption("member");
} }
compare(other: CallMemberViewModel): number { get isCameraMuted(): boolean {
return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream);
}
get isMicrophoneMuted(): boolean {
return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream);
}
get avatarLetter(): string {
return avatarInitials(this.member.member.name);
}
get avatarColorNumber(): number {
return getIdentifierColorNumber(this.member.userId);
}
avatarUrl(size: number): string | undefined {
const {avatarUrl} = this.member.member;
const mediaRepository = this.getOption("mediaRepository");
return getAvatarHttpUrl(avatarUrl, size, this.platform, mediaRepository);
}
get avatarTitle(): string {
return this.member.member.name;
}
compare(other: OwnMemberViewModel | CallMemberViewModel): number {
if (other instanceof OwnMemberViewModel) {
return -other.compare(this);
}
const myUserId = this.member.member.userId; const myUserId = this.member.member.userId;
const otherUserId = other.member.member.userId; const otherUserId = other.member.member.userId;
if(myUserId === otherUserId) { if(myUserId === otherUserId) {
@ -77,3 +172,9 @@ export class CallMemberViewModel extends ViewModel<MemberOptions> {
return myUserId < otherUserId ? -1 : 1; return myUserId < otherUserId ? -1 : 1;
} }
} }
export interface IStreamViewModel extends AvatarSource, ViewModel {
get stream(): Stream | undefined;
get isCameraMuted(): boolean;
get isMicrophoneMuted(): boolean;
}

View file

@ -62,7 +62,7 @@ export class RoomViewModel extends ViewModel {
} }
this._callViewModel = this.disposeTracked(this._callViewModel); this._callViewModel = this.disposeTracked(this._callViewModel);
if (call) { if (call) {
this._callViewModel = this.track(new CallViewModel(this.childOptions({call}))); this._callViewModel = this.track(new CallViewModel(this.childOptions({call, mediaRepository: this._room.mediaRepository})));
} }
this.emitChange("callViewModel"); this.emitChange("callViewModel");
})); }));
@ -367,9 +367,8 @@ export class RoomViewModel extends ViewModel {
const session = this.getOption("session"); const session = this.getOption("session");
const stream = await this.platform.mediaDevices.getMediaTracks(false, true); const stream = await this.platform.mediaDevices.getMediaTracks(false, true);
const localMedia = new LocalMedia().withUserMedia(stream); const localMedia = new LocalMedia().withUserMedia(stream);
await this._call.join(localMedia);
// this will set the callViewModel above as a call will be added to callHandler.calls // this will set the callViewModel above as a call will be added to callHandler.calls
const call = await session.callHandler.createCall(this._room.id, localMedia, "A call " + Math.round(this.platform.random() * 100)); const call = await session.callHandler.createCall(this._room.id, "m.video", "A call " + Math.round(this.platform.random() * 100));
await call.join(localMedia); await call.join(localMedia);
} catch (err) { } catch (err) {
console.error(err.stack); console.error(err.stack);

View file

@ -21,7 +21,7 @@ import {RoomStatus} from "./room/common";
import {RoomBeingCreated} from "./room/RoomBeingCreated"; import {RoomBeingCreated} from "./room/RoomBeingCreated";
import {Invite} from "./room/Invite.js"; import {Invite} from "./room/Invite.js";
import {Pusher} from "./push/Pusher"; import {Pusher} from "./push/Pusher";
import { ObservableMap } from "../observable/index.js"; import { ObservableMap } from "../observable/index";
import {User} from "./User.js"; import {User} from "./User.js";
import {DeviceMessageHandler} from "./DeviceMessageHandler.js"; import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
import {Account as E2EEAccount} from "./e2ee/Account.js"; import {Account as E2EEAccount} from "./e2ee/Account.js";

View file

@ -15,8 +15,8 @@ limitations under the License.
*/ */
import {ObservableMap} from "../../observable/map/ObservableMap"; import {ObservableMap} from "../../observable/map/ObservableMap";
import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC"; import {WebRTC, PeerConnection} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack} from "../../platform/types/MediaDevices"; import {MediaDevices, Track} from "../../platform/types/MediaDevices";
import {handlesEventType} from "./PeerCall"; import {handlesEventType} from "./PeerCall";
import {EventType, CallIntent} from "./callEventTypes"; import {EventType, CallIntent} from "./callEventTypes";
import {GroupCall} from "./group/GroupCall"; import {GroupCall} from "./group/GroupCall";
@ -107,7 +107,7 @@ export class CallHandler {
}); });
} }
async createCall(roomId: string, callType: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise<GroupCall> { async createCall(roomId: string, type: "m.video" | "m.voice", name: string, intent: CallIntent = CallIntent.Ring): Promise<GroupCall> {
const logItem = this.options.logger.child({l: "call", incoming: false}); const logItem = this.options.logger.child({l: "call", incoming: false});
const call = new GroupCall(makeId("conf-"), true, { const call = new GroupCall(makeId("conf-"), true, {
"m.name": name, "m.name": name,
@ -116,7 +116,7 @@ export class CallHandler {
this._calls.set(call.id, call); this._calls.set(call.id, call);
try { try {
await call.create(callType); await call.create(type);
// store call info so it will ring again when reopening the app // store call info so it will ring again when reopening the app
const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]); const txn = await this.options.storage.readWriteTxn([this.options.storage.storeNames.calls]);
txn.calls.add({ txn.calls.add({

View file

@ -17,6 +17,7 @@ limitations under the License.
import {SDPStreamMetadataPurpose} from "./callEventTypes"; import {SDPStreamMetadataPurpose} from "./callEventTypes";
import {Stream} from "../../platform/types/MediaDevices"; import {Stream} from "../../platform/types/MediaDevices";
import {SDPStreamMetadata} from "./callEventTypes"; import {SDPStreamMetadata} from "./callEventTypes";
import {getStreamVideoTrack, getStreamAudioTrack} from "./common";
export class LocalMedia { export class LocalMedia {
constructor( constructor(
@ -37,13 +38,44 @@ export class LocalMedia {
return new LocalMedia(this.userMedia, this.screenShare, options); return new LocalMedia(this.userMedia, this.screenShare, options);
} }
/** @internal */
replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia {
let userMedia;
let screenShare;
const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => {
let stream;
if (oldOriginalStream?.id === newStream?.id) {
stream = oldCloneStream;
} else {
stream = newStream?.clone();
getStreamAudioTrack(oldCloneStream)?.stop();
getStreamVideoTrack(oldCloneStream)?.stop();
}
return stream;
}
return new LocalMedia(
cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia),
cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare),
this.dataChannelOptions
);
}
/** @internal */
clone(): LocalMedia { clone(): LocalMedia {
return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); return new LocalMedia(this.userMedia?.clone(),this.screenShare?.clone(), this.dataChannelOptions);
} }
dispose() { dispose() {
this.userMedia?.audioTrack?.stop(); this.stopExcept(undefined);
this.userMedia?.videoTrack?.stop(); }
this.screenShare?.videoTrack?.stop();
stopExcept(newMedia: LocalMedia | undefined) {
if(newMedia?.userMedia?.id !== this.userMedia?.id) {
getStreamAudioTrack(this.userMedia)?.stop();
getStreamVideoTrack(this.userMedia)?.stop();
}
if(newMedia?.screenShare?.id !== this.screenShare?.id) {
getStreamVideoTrack(this.screenShare)?.stop();
}
} }
} }

View file

@ -16,25 +16,26 @@ limitations under the License.
import {ObservableMap} from "../../observable/map/ObservableMap"; import {ObservableMap} from "../../observable/map/ObservableMap";
import {recursivelyAssign} from "../../utils/recursivelyAssign"; import {recursivelyAssign} from "../../utils/recursivelyAssign";
import {AsyncQueue} from "../../utils/AsyncQueue"; import {Disposables, Disposable, IDisposable} from "../../utils/Disposables";
import {Disposables, IDisposable} from "../../utils/Disposables"; import {WebRTC, PeerConnection, Transceiver, TransceiverDirection, Sender, Receiver, PeerConnectionEventMap} from "../../platform/types/WebRTC";
import type {Room} from "../room/Room"; import {MediaDevices, Track, TrackKind, Stream, StreamTrackEvent} from "../../platform/types/MediaDevices";
import type {StateEvent} from "../storage/types"; import {getStreamVideoTrack, getStreamAudioTrack, MuteSettings} from "./common";
import type {ILogItem} from "../../logging/types";
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices";
import type {LocalMedia} from "./LocalMedia";
import { import {
SDPStreamMetadataKey, SDPStreamMetadataKey,
SDPStreamMetadataPurpose, SDPStreamMetadataPurpose,
EventType, EventType,
CallErrorCode,
} from "./callEventTypes"; } from "./callEventTypes";
import type {Room} from "../room/Room";
import type {StateEvent} from "../storage/types";
import type {ILogItem} from "../../logging/types";
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
import type {LocalMedia} from "./LocalMedia";
import type { import type {
MCallBase, MCallBase,
MCallInvite, MCallInvite,
MCallNegotiate,
MCallAnswer, MCallAnswer,
MCallSDPStreamMetadataChanged, MCallSDPStreamMetadataChanged,
MCallCandidates, MCallCandidates,
@ -66,7 +67,9 @@ export class PeerCall implements IDisposable {
private readonly peerConnection: PeerConnection; private readonly peerConnection: PeerConnection;
private _state = CallState.Fledgling; private _state = CallState.Fledgling;
private direction: CallDirection; private direction: CallDirection;
// we don't own localMedia and should hence not call dispose on it from here
private localMedia?: LocalMedia; private localMedia?: LocalMedia;
private localMuteSettings?: MuteSettings;
private seq: number = 0; private seq: number = 0;
// A queue for candidates waiting to go out. // A queue for candidates waiting to go out.
// We try to amalgamate candidates into a single candidate message where // We try to amalgamate candidates into a single candidate message where
@ -83,7 +86,8 @@ export class PeerCall implements IDisposable {
private hangupParty: CallParty; private hangupParty: CallParty;
private disposables = new Disposables(); private disposables = new Disposables();
private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>(); private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>();
private _remoteTrackToStreamId = new Map<string, string>();
private _remoteStreams = new Map<string, {stream: Stream, disposeListener: Disposable}>();
// perfect negotiation flags // perfect negotiation flags
private makingOffer: boolean = false; private makingOffer: boolean = false;
private ignoreOffer: boolean = false; private ignoreOffer: boolean = false;
@ -94,55 +98,62 @@ export class PeerCall implements IDisposable {
private _dataChannel?: any; private _dataChannel?: any;
private _hangupReason?: CallErrorCode; private _hangupReason?: CallErrorCode;
private _remoteMedia: RemoteMedia; private _remoteMedia: RemoteMedia;
private _remoteMuteSettings = new MuteSettings();
constructor( constructor(
private callId: string, private callId: string,
private readonly options: Options, private readonly options: Options,
private readonly logItem: ILogItem, private readonly logItem: ILogItem,
) { ) {
const outer = this;
this._remoteMedia = new RemoteMedia(); this._remoteMedia = new RemoteMedia();
this.peerConnection = options.webRTC.createPeerConnection({ this.peerConnection = options.webRTC.createPeerConnection(this.options.forceTURN, this.options.turnServers, 0);
onIceConnectionStateChange(state: RTCIceConnectionState) {
outer.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => { const listen = <K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void => {
outer.onIceConnectionStateChange(state, log); this.peerConnection.addEventListener(type, listener);
const dispose = () => {
this.peerConnection.removeEventListener(type, listener);
};
this.disposables.track(dispose);
};
listen("iceconnectionstatechange", () => {
const state = this.peerConnection.iceConnectionState;
this.logItem.wrap({l: "onIceConnectionStateChange", status: state}, log => {
this.onIceConnectionStateChange(state, log);
});
});
listen("icecandidate", event => {
this.logItem.wrap("onLocalIceCandidate", log => {
if (event.candidate) {
this.handleLocalIceCandidate(event.candidate, log);
}
});
});
listen("icegatheringstatechange", () => {
const state = this.peerConnection.iceGatheringState;
this.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => {
this.handleIceGatheringState(state, log);
});
});
listen("track", event => {
this.logItem.wrap("onRemoteTrack", log => {
this.onRemoteTrack(event.track, event.streams, log);
});
});
listen("datachannel", event => {
this.logItem.wrap("onRemoteDataChannel", log => {
this._dataChannel = event.channel;
this.options.emitUpdate(this, undefined);
});
});
listen("negotiationneeded", () => {
const promiseCreator = () => {
return this.logItem.wrap("onNegotiationNeeded", log => {
return this.handleNegotiation(log);
}); });
}, };
onLocalIceCandidate(candidate: RTCIceCandidate) { this.responsePromiseChain = this.responsePromiseChain?.then(promiseCreator) ?? promiseCreator();
outer.logItem.wrap("onLocalIceCandidate", log => { });
outer.handleLocalIceCandidate(candidate, log);
});
},
onIceGatheringStateChange(state: RTCIceGatheringState) {
outer.logItem.wrap({l: "onIceGatheringStateChange", status: state}, log => {
outer.handleIceGatheringState(state, log);
});
},
onRemoteStreamRemoved(stream: Stream) {
outer.logItem.wrap("onRemoteStreamRemoved", log => {
outer.updateRemoteMedia(log);
});
},
onRemoteTracksAdded(trackReceiver: TrackReceiver) {
outer.logItem.wrap("onRemoteTracksAdded", log => {
outer.updateRemoteMedia(log);
});
},
onRemoteDataChannel(dataChannel: any | undefined) {
outer.logItem.wrap("onRemoteDataChannel", log => {
outer._dataChannel = dataChannel;
outer.options.emitUpdate(outer, undefined);
});
},
onNegotiationNeeded() {
const promiseCreator = () => {
return outer.logItem.wrap("onNegotiationNeeded", log => {
return outer.handleNegotiation(log);
});
};
outer.responsePromiseChain = outer.responsePromiseChain?.then(promiseCreator) ?? promiseCreator();
}
}, this.options.forceTURN, this.options.turnServers, 0);
} }
get dataChannel(): any | undefined { return this._dataChannel; } get dataChannel(): any | undefined { return this._dataChannel; }
@ -151,21 +162,25 @@ export class PeerCall implements IDisposable {
get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } get hangupReason(): CallErrorCode | undefined { return this._hangupReason; }
// we should keep an object with streams by purpose ... e.g. RemoteMedia?
get remoteMedia(): Readonly<RemoteMedia> { get remoteMedia(): Readonly<RemoteMedia> {
return this._remoteMedia; return this._remoteMedia;
} }
call(localMedia: LocalMedia): Promise<void> { get remoteMuteSettings(): MuteSettings {
return this._remoteMuteSettings;
}
call(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise<void> {
return this.logItem.wrap("call", async log => { return this.logItem.wrap("call", async log => {
if (this._state !== CallState.Fledgling) { if (this._state !== CallState.Fledgling) {
return; return;
} }
this.direction = CallDirection.Outbound; this.direction = CallDirection.Outbound;
this.setState(CallState.CreateOffer, log); this.setState(CallState.CreateOffer, log);
this.setMedia(localMedia); this.localMuteSettings = localMuteSettings;
await this.updateLocalMedia(localMedia, log);
if (this.localMedia?.dataChannelOptions) { if (this.localMedia?.dataChannelOptions) {
this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); this._dataChannel = this.peerConnection.createDataChannel("channel", this.localMedia.dataChannelOptions);
} }
// after adding the local tracks, and wait for handleNegotiation to be called, // after adding the local tracks, and wait for handleNegotiation to be called,
// or invite glare where we give up our invite and answer instead // or invite glare where we give up our invite and answer instead
@ -173,13 +188,14 @@ export class PeerCall implements IDisposable {
}); });
} }
answer(localMedia: LocalMedia): Promise<void> { answer(localMedia: LocalMedia, localMuteSettings: MuteSettings): Promise<void> {
return this.logItem.wrap("answer", async log => { return this.logItem.wrap("answer", async log => {
if (this._state !== CallState.Ringing) { if (this._state !== CallState.Ringing) {
return; return;
} }
this.setState(CallState.CreateAnswer, log); this.setState(CallState.CreateAnswer, log);
this.setMedia(localMedia, log); this.localMuteSettings = localMuteSettings;
await this.updateLocalMedia(localMedia, log);
let myAnswer: RTCSessionDescriptionInit; let myAnswer: RTCSessionDescriptionInit;
try { try {
myAnswer = await this.peerConnection.createAnswer(); myAnswer = await this.peerConnection.createAnswer();
@ -208,46 +224,47 @@ export class PeerCall implements IDisposable {
}); });
} }
setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise<void> { setMedia(localMedia: LocalMedia): Promise<void> {
return logItem.wrap("setMedia", async log => { return this.logItem.wrap("setMedia", async log => {
const oldMedia = this.localMedia; log.set("userMedia_audio", !!getStreamAudioTrack(localMedia.userMedia));
this.localMedia = localMedia; log.set("userMedia_video", !!getStreamVideoTrack(localMedia.userMedia));
const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { log.set("screenShare_video", !!getStreamVideoTrack(localMedia.screenShare));
const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; log.set("datachannel", !!localMedia.dataChannelOptions);
await this.updateLocalMedia(localMedia, log);
const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { const content: MCallSDPStreamMetadataChanged<MCallBase> = {
if (track) { call_id: this.callId,
if (oldTrack && sender) { version: 1,
log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { seq: this.seq++,
sender.replaceTrack(track); [SDPStreamMetadataKey]: this.getSDPMetadata()
}); };
} else { await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
this.peerConnection.addTrack(track);
});
}
} else {
if (sender) {
log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => {
this.peerConnection.removeTrack(sender);
});
}
}
}
applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack);
applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack);
}
applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia");
applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare");
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
}); });
} }
/** group calls would handle reject at the group call level, not at the peer call level */ setMuted(localMuteSettings: MuteSettings) {
async reject() { return this.logItem.wrap("setMuted", async log => {
this.localMuteSettings = localMuteSettings;
log.set("cameraMuted", localMuteSettings.camera);
log.set("microphoneMuted", localMuteSettings.microphone);
if (this.localMedia) {
const userMediaAudio = getStreamAudioTrack(this.localMedia.userMedia);
if (userMediaAudio) {
this.muteTrack(userMediaAudio, this.localMuteSettings.microphone, log);
}
const userMediaVideo = getStreamVideoTrack(this.localMedia.userMedia);
if (userMediaVideo) {
this.muteTrack(userMediaVideo, this.localMuteSettings.camera, log);
}
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
call_id: this.callId,
version: 1,
seq: this.seq++,
[SDPStreamMetadataKey]: this.getSDPMetadata()
};
await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
}
});
} }
hangup(errorCode: CallErrorCode): Promise<void> { hangup(errorCode: CallErrorCode): Promise<void> {
@ -256,6 +273,22 @@ export class PeerCall implements IDisposable {
}); });
} }
private muteTrack(track: Track, muted: boolean, log: ILogItem): void {
log.wrap({l: "track", kind: track.kind, id: track.id}, log => {
const enabled = !muted;
log.set("enabled", enabled);
const transceiver = this.findTransceiverForTrack(track);
if (transceiver) {
if (transceiver.sender.track) {
transceiver.sender.track.enabled = enabled;
}
log.set("fromDirection", transceiver.direction);
// enableSenderOnTransceiver(transceiver, enabled);
log.set("toDirection", transceiver.direction);
}
});
}
private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise<void> { private async _hangup(errorCode: CallErrorCode, log: ILogItem): Promise<void> {
if (this._state === CallState.Ended) { if (this._state === CallState.Ended) {
return; return;
@ -277,10 +310,20 @@ export class PeerCall implements IDisposable {
case EventType.Answer: case EventType.Answer:
await this.handleAnswer(message.content, partyId, log); await this.handleAnswer(message.content, partyId, log);
break; break;
case EventType.Negotiate:
await this.onNegotiateReceived(message.content, log);
break;
case EventType.Candidates: case EventType.Candidates:
await this.handleRemoteIceCandidates(message.content, partyId, log); await this.handleRemoteIceCandidates(message.content, partyId, log);
break; break;
case EventType.SDPStreamMetadataChanged:
case EventType.SDPStreamMetadataChangedPrefix:
this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log);
break;
case EventType.Hangup: case EventType.Hangup:
// TODO: this is a bit hacky, double check its what we need
this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log);
break;
default: default:
log.log(`Unknown event type for call: ${message.type}`); log.log(`Unknown event type for call: ${message.type}`);
break; break;
@ -305,7 +348,7 @@ export class PeerCall implements IDisposable {
} }
// calls are serialized and deduplicated by responsePromiseChain // calls are serialized and deduplicated by responsePromiseChain
private handleNegotiation = async (log: ILogItem): Promise<void> => { private async handleNegotiation(log: ILogItem): Promise<void> {
this.makingOffer = true; this.makingOffer = true;
try { try {
try { try {
@ -334,22 +377,27 @@ export class PeerCall implements IDisposable {
this.candidateSendQueue = []; this.candidateSendQueue = [];
// need to queue this // need to queue this
const content = {
call_id: this.callId,
offer,
[SDPStreamMetadataKey]: this.getSDPMetadata(),
version: 1,
seq: this.seq++,
lifetime: CALL_TIMEOUT_MS
};
if (this._state === CallState.CreateOffer) { if (this._state === CallState.CreateOffer) {
const content = {
call_id: this.callId,
offer,
[SDPStreamMetadataKey]: this.getSDPMetadata(),
version: 1,
seq: this.seq++,
lifetime: CALL_TIMEOUT_MS
};
await this.sendSignallingMessage({type: EventType.Invite, content}, log); await this.sendSignallingMessage({type: EventType.Invite, content}, log);
this.setState(CallState.InviteSent, log); this.setState(CallState.InviteSent, log);
} else if (this._state === CallState.Connected || this._state === CallState.Connecting) { } else if (this._state === CallState.Connected || this._state === CallState.Connecting) {
log.log("would send renegotiation now but not implemented"); const content = {
// send Negotiate message call_id: this.callId,
//await this.sendSignallingMessage({type: EventType.Invite, content}); description: offer,
//this.setState(CallState.InviteSent); [SDPStreamMetadataKey]: this.getSDPMetadata(),
version: 1,
seq: this.seq++,
lifetime: CALL_TIMEOUT_MS
};
await this.sendSignallingMessage({type: EventType.Negotiate, content}, log);
} }
} finally { } finally {
this.makingOffer = false; this.makingOffer = false;
@ -385,7 +433,7 @@ export class PeerCall implements IDisposable {
} }
await this.handleInvite(content, partyId, log); await this.handleInvite(content, partyId, log);
// TODO: need to skip state check // TODO: need to skip state check
await this.answer(this.localMedia!); await this.answer(this.localMedia!, this.localMuteSettings!);
} else { } else {
log.log( log.log(
"Glare detected: rejecting incoming call " + newCallId + "Glare detected: rejecting incoming call " + newCallId +
@ -444,12 +492,12 @@ export class PeerCall implements IDisposable {
// According to previous comments in this file, firefox at some point did not // According to previous comments in this file, firefox at some point did not
// add streams until media started arriving on them. Testing latest firefox // add streams until media started arriving on them. Testing latest firefox
// (81 at time of writing), this is no longer a problem, so let's do it the correct way. // (81 at time of writing), this is no longer a problem, so let's do it the correct way.
// if (this.peerConnection.remoteTracks.length === 0) { if (this.peerConnection.getReceivers().length === 0) {
// await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => {
// return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
// }); });
// return; return;
// } }
this.setState(CallState.Ringing, log); this.setState(CallState.Ringing, log);
@ -571,55 +619,55 @@ export class PeerCall implements IDisposable {
await this.addIceCandidates(candidates, log); await this.addIceCandidates(candidates, log);
} }
// private async onNegotiateReceived(event: MatrixEvent): Promise<void> { private async onNegotiateReceived(content: MCallNegotiate<MCallBase>, log: ILogItem): Promise<void> {
// const content = event.getContent<MCallNegotiate>(); const description = content.description;
// const description = content.description; if (!description || !description.sdp || !description.type) {
// if (!description || !description.sdp || !description.type) { log.log(`Ignoring invalid m.call.negotiate event`);
// this.logger.info(`Ignoring invalid m.call.negotiate event`); return;
// return; }
// } // Politeness always follows the direction of the call: in a glare situation,
// // Politeness always follows the direction of the call: in a glare situation, // we pick either the inbound or outbound call, so one side will always be
// // we pick either the inbound or outbound call, so one side will always be // inbound and one outbound
// // inbound and one outbound const polite = this.direction === CallDirection.Inbound;
// const polite = this.direction === CallDirection.Inbound;
// // Here we follow the perfect negotiation logic from // Here we follow the perfect negotiation logic from
// // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation
// const offerCollision = ( const offerCollision = (
// (description.type === 'offer') && (description.type === 'offer') &&
// (this.makingOffer || this.peerConnection.signalingState !== 'stable') (this.makingOffer || this.peerConnection.signalingState !== 'stable')
// ); );
// this.ignoreOffer = !polite && offerCollision; this.ignoreOffer = !polite && offerCollision;
// if (this.ignoreOffer) { if (this.ignoreOffer) {
// this.logger.info(`Ignoring colliding negotiate event because we're impolite`); log.log(`Ignoring colliding negotiate event because we're impolite`);
// return; return;
// } }
// const sdpStreamMetadata = content[SDPStreamMetadataKey]; const sdpStreamMetadata = content[SDPStreamMetadataKey];
// if (sdpStreamMetadata) { if (sdpStreamMetadata) {
// this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log);
// } else { } else {
// this.logger.warn(`Received negotiation event without SDPStreamMetadata!`); log.log(`Received negotiation event without SDPStreamMetadata!`);
// } }
// try { try {
// await this.peerConnection.setRemoteDescription(description); await this.peerConnection.setRemoteDescription(description);
if (description.type === 'offer') {
// if (description.type === 'offer') { await this.peerConnection.setLocalDescription();
// await this.peerConnection.setLocalDescription(); const content = {
// await this.sendSignallingMessage({ call_id: this.callId,
// type: EventType.CallNegotiate, description: this.peerConnection.localDescription!,
// content: { [SDPStreamMetadataKey]: this.getSDPMetadata(),
// description: this.peerConnection.localDescription!, version: 1,
// [SDPStreamMetadataKey]: this.getSDPMetadata(), seq: this.seq++,
// } lifetime: CALL_TIMEOUT_MS
// }); };
// } await this.sendSignallingMessage({type: EventType.Negotiate, content}, log);
// } catch (err) { }
// this.logger.warn(`Failed to complete negotiation`, err); } catch (err) {
// } log.log(`Failed to complete negotiation`, err);
// } }
}
private async sendAnswer(log: ILogItem): Promise<void> { private async sendAnswer(log: ILogItem): Promise<void> {
const localDescription = this.peerConnection.localDescription!; const localDescription = this.peerConnection.localDescription!;
@ -719,7 +767,7 @@ export class PeerCall implements IDisposable {
// this will accumulate all updates into one object, so we still have the old stream info when we change stream id // this will accumulate all updates into one object, so we still have the old stream info when we change stream id
this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
this.updateRemoteMedia(log); this.updateRemoteMedia(log);
// TODO: apply muting
} }
private async addBufferedIceCandidates(log: ILogItem): Promise<void> { private async addBufferedIceCandidates(log: ILogItem): Promise<void> {
@ -825,13 +873,11 @@ export class PeerCall implements IDisposable {
const metadata = {}; const metadata = {};
if (this.localMedia?.userMedia) { if (this.localMedia?.userMedia) {
const streamId = this.localMedia.userMedia.id; const streamId = this.localMedia.userMedia.id;
const streamSender = this.peerConnection.localStreams.get(streamId);
metadata[streamId] = { metadata[streamId] = {
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audio_muted: !(streamSender?.audioSender?.enabled), audio_muted: this.localMuteSettings?.microphone || !getStreamAudioTrack(this.localMedia.userMedia),
video_muted: !(streamSender?.videoSender?.enabled), video_muted: this.localMuteSettings?.camera || !getStreamVideoTrack(this.localMedia.userMedia),
}; };
console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata));
} }
if (this.localMedia?.screenShare) { if (this.localMedia?.screenShare) {
const streamId = this.localMedia.screenShare.id; const streamId = this.localMedia.screenShare.id;
@ -842,17 +888,78 @@ export class PeerCall implements IDisposable {
return metadata; return metadata;
} }
private updateRemoteMedia(log: ILogItem) { private findReceiverForStream(kind: TrackKind, streamId: string): Receiver | undefined {
return this.peerConnection.getReceivers().find(r => {
return r.track.kind === kind && this._remoteTrackToStreamId.get(r.track.id) === streamId;
});
}
private findTransceiverForTrack(track: Track): Transceiver | undefined {
return this.peerConnection.getTransceivers().find(t => {
return t.sender.track?.id === track.id;
});
}
private onRemoteTrack(track: Track, streams: ReadonlyArray<Stream>, log: ILogItem) {
if (streams.length === 0) {
log.log({l: `ignoring ${track.kind} streamless track`, id: track.id});
return;
}
const stream = streams[0];
this._remoteTrackToStreamId.set(track.id, stream.id);
if (!this._remoteStreams.has(stream.id)) {
const listener = (event: StreamTrackEvent): void => {
this.logItem.wrap({l: "removetrack", id: event.track.id}, log => {
const streamId = this._remoteTrackToStreamId.get(event.track.id);
if (streamId) {
this._remoteTrackToStreamId.delete(event.track.id);
const streamDetails = this._remoteStreams.get(streamId);
if (streamDetails && streamDetails.stream.getTracks().length === 0) {
this.disposables.disposeTracked(disposeListener);
this._remoteStreams.delete(stream.id);
this.updateRemoteMedia(log);
}
}
})
};
stream.addEventListener("removetrack", listener);
const disposeListener = () => {
stream.removeEventListener("removetrack", listener);
};
this.disposables.track(disposeListener);
this._remoteStreams.set(stream.id, {
disposeListener,
stream
});
this.updateRemoteMedia(log);
}
}
private updateRemoteMedia(log: ILogItem): void {
this._remoteMedia.userMedia = undefined; this._remoteMedia.userMedia = undefined;
this._remoteMedia.screenShare = undefined; this._remoteMedia.screenShare = undefined;
if (this.remoteSDPStreamMetadata) { if (this.remoteSDPStreamMetadata) {
for (const [streamId, streamReceiver] of this.peerConnection.remoteStreams.entries()) { for (const streamDetails of this._remoteStreams.values()) {
const metaData = this.remoteSDPStreamMetadata[streamId]; const {stream} = streamDetails;
const metaData = this.remoteSDPStreamMetadata[stream.id];
if (metaData) { if (metaData) {
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
this._remoteMedia.userMedia = streamReceiver.stream; this._remoteMedia.userMedia = stream;
const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
if (audioReceiver) {
audioReceiver.track.enabled = !metaData.audio_muted;
}
const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
if (videoReceiver) {
videoReceiver.track.enabled = !metaData.video_muted;
}
this._remoteMuteSettings = new MuteSettings(
metaData.audio_muted || !audioReceiver?.track,
metaData.video_muted || !videoReceiver?.track
);
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
this._remoteMedia.screenShare = streamReceiver.stream; this._remoteMedia.screenShare = stream;
} }
} }
} }
@ -860,6 +967,55 @@ export class PeerCall implements IDisposable {
this.options.emitUpdate(this, undefined); this.options.emitUpdate(this, undefined);
} }
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<void> {
return logItem.wrap("updateLocalMedia", async log => {
const oldMedia = this.localMedia;
this.localMedia = localMedia;
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, streamPurpose: SDPStreamMetadataPurpose) => {
const applyTrack = async (oldTrack: Track | undefined, newTrack: Track | undefined) => {
if (!oldTrack && newTrack) {
log.wrap(`adding ${streamPurpose} ${newTrack.kind} track`, log => {
const sender = this.peerConnection.addTrack(newTrack, stream!);
this.options.webRTC.prepareSenderForPurpose(this.peerConnection, sender, streamPurpose);
});
} else if (oldTrack) {
const sender = this.peerConnection.getSenders().find(s => s.track && s.track.id === oldTrack.id);
if (sender) {
if (newTrack && oldTrack.id !== newTrack.id) {
try {
await log.wrap(`replacing ${streamPurpose} ${newTrack.kind} track`, log => {
return sender.replaceTrack(newTrack);
});
} catch (err) {
// can't replace the track without renegotiating{
log.wrap(`adding and removing ${streamPurpose} ${newTrack.kind} track`, log => {
this.peerConnection.removeTrack(sender);
const newSender = this.peerConnection.addTrack(newTrack);
this.options.webRTC.prepareSenderForPurpose(this.peerConnection, newSender, streamPurpose);
});
}
} else if (!newTrack) {
log.wrap(`removing ${streamPurpose} ${sender.track!.kind} track`, log => {
this.peerConnection.removeTrack(sender);
});
} else {
log.log(`${streamPurpose} ${oldTrack.kind} track hasn't changed`);
}
}
// TODO: should we do something if we didn't find the sender? e.g. some other code already removed the sender but didn't update localMedia
}
}
await applyTrack(getStreamAudioTrack(oldStream), getStreamAudioTrack(stream));
await applyTrack(getStreamVideoTrack(oldStream), getStreamVideoTrack(stream));
};
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, SDPStreamMetadataPurpose.Usermedia);
await applyStream(oldMedia?.screenShare, localMedia?.screenShare, SDPStreamMetadataPurpose.Screenshare);
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
});
}
private async delay(timeoutMs: number): Promise<void> { private async delay(timeoutMs: number): Promise<void> {
// Allow a short time for initial candidates to be gathered // Allow a short time for initial candidates to be gathered
const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); const timeout = this.disposables.track(this.options.createTimeout(timeoutMs));
@ -875,7 +1031,7 @@ export class PeerCall implements IDisposable {
public dispose(): void { public dispose(): void {
this.disposables.dispose(); this.disposables.dispose();
this.peerConnection.dispose(); this.peerConnection.close();
} }
public close(reason: CallErrorCode | undefined, log: ILogItem): void { public close(reason: CallErrorCode | undefined, log: ILogItem): void {
@ -915,100 +1071,11 @@ export enum CallDirection {
Outbound = 'outbound', Outbound = 'outbound',
} }
export enum CallErrorCode {
/** The user chose to end the call */
UserHangup = 'user_hangup',
/** An error code when the local client failed to create an offer. */
LocalOfferFailed = 'local_offer_failed',
/**
* An error code when there is no local mic/camera to use. This may be because
* the hardware isn't plugged in, or the user has explicitly denied access.
*/
NoUserMedia = 'no_user_media',
/**
* Error code used when a call event failed to send
* because unknown devices were present in the room
*/
UnknownDevices = 'unknown_devices',
/**
* Error code used when we fail to send the invite
* for some reason other than there being unknown devices
*/
SendInvite = 'send_invite',
/**
* An answer could not be created
*/
CreateAnswer = 'create_answer',
/**
* Error code used when we fail to send the answer
* for some reason other than there being unknown devices
*/
SendAnswer = 'send_answer',
/**
* The session description from the other side could not be set
*/
SetRemoteDescription = 'set_remote_description',
/**
* The session description from this side could not be set
*/
SetLocalDescription = 'set_local_description',
/**
* A different device answered the call
*/
AnsweredElsewhere = 'answered_elsewhere',
/**
* No media connection could be established to the other party
*/
IceFailed = 'ice_failed',
/**
* The invite timed out whilst waiting for an answer
*/
InviteTimeout = 'invite_timeout',
/**
* The call was replaced by another call
*/
Replaced = 'replaced',
/**
* Signalling for the call could not be sent (other than the initial invite)
*/
SignallingFailed = 'signalling_timeout',
/**
* The remote party is busy
*/
UserBusy = 'user_busy',
/**
* We transferred the call off to somewhere else
*/
Transfered = 'transferred',
/**
* A call from the same user was found with a new session id
*/
NewSession = 'new_session',
}
/** /**
* The version field that we set in m.call.* events * The version field that we set in m.call.* events
*/ */
const VOIP_PROTO_VERSION = 1; const VOIP_PROTO_VERSION = 1;
/** The fallback ICE server to use for STUN or TURN protocols. */
const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org';
/** The length of time a call can be ringing for. */ /** The length of time a call can be ringing for. */
const CALL_TIMEOUT_MS = 60000; const CALL_TIMEOUT_MS = 60000;
@ -1029,9 +1096,28 @@ export function handlesEventType(eventType: string): boolean {
return eventType === EventType.Invite || return eventType === EventType.Invite ||
eventType === EventType.Candidates || eventType === EventType.Candidates ||
eventType === EventType.Answer || eventType === EventType.Answer ||
eventType === EventType.Hangup; eventType === EventType.Hangup ||
eventType === EventType.SDPStreamMetadataChanged ||
eventType === EventType.SDPStreamMetadataChangedPrefix ||
eventType === EventType.Negotiate;
} }
export function tests() { function enableSenderOnTransceiver(transceiver: Transceiver, enabled: boolean) {
return enableTransceiver(transceiver, enabled, "sendonly", "recvonly");
}
function enableTransceiver(transceiver: Transceiver, enabled: boolean, exclusiveValue: TransceiverDirection, excludedValue: TransceiverDirection) {
if (enabled) {
if (transceiver.direction === "inactive") {
transceiver.direction = exclusiveValue;
} else {
transceiver.direction = "sendrecv";
}
} else {
if (transceiver.direction === "sendrecv") {
transceiver.direction = excludedValue;
} else {
transceiver.direction = "inactive";
}
}
} }

View file

@ -9,14 +9,14 @@
## TODO ## TODO
- DONE: implement receiving hangup - DONE: implement receiving hangup
- making logging better - DONE: implement cloning the localMedia so it works in safari?
- DONE: implement 3 retries per peer
- implement muting tracks with m.call.sdp_stream_metadata_changed
- implement renegotiation - implement renegotiation
- making logging better
- finish session id support - finish session id support
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
- implement to_device messages arriving before m.call(.member) state event - implement to_device messages arriving before m.call(.member) state event
- implement muting tracks with m.call.sdp_stream_metadata_changed
- implement cloning the localMedia so it works in safari?
- DONE: implement 3 retries per peer
- reeable crypto & implement fetching olm keys before sending encrypted signalling message - reeable crypto & implement fetching olm keys before sending encrypted signalling message
- local echo for join/leave buttons? - local echo for join/leave buttons?
- make UI pretsy - make UI pretsy

View file

@ -1,7 +1,7 @@
// allow non-camelcase as these are events type that go onto the wire // allow non-camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */ /* eslint-disable camelcase */
import type {StateEvent} from "../storage/types"; import type {StateEvent} from "../storage/types";
import type {SessionDescription} from "../../platform/types/WebRTC";
export enum EventType { export enum EventType {
GroupCall = "org.matrix.msc3401.call", GroupCall = "org.matrix.msc3401.call",
GroupCallMember = "org.matrix.msc3401.call.member", GroupCallMember = "org.matrix.msc3401.call.member",
@ -36,11 +36,6 @@ export interface CallMemberContent {
["m.calls"]: CallMembership[]; ["m.calls"]: CallMembership[];
} }
export interface SessionDescription {
sdp?: string;
type: RTCSdpType
}
export enum SDPStreamMetadataPurpose { export enum SDPStreamMetadataPurpose {
Usermedia = "m.usermedia", Usermedia = "m.usermedia",
Screenshare = "m.screenshare", Screenshare = "m.screenshare",
@ -97,6 +92,12 @@ export type MCallInvite<Base extends MCallBase> = Base & {
[SDPStreamMetadataKey]: SDPStreamMetadata; [SDPStreamMetadataKey]: SDPStreamMetadata;
} }
export type MCallNegotiate<Base extends MCallBase> = Base & {
description: SessionDescription;
lifetime: number;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & { export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & {
[SDPStreamMetadataKey]: SDPStreamMetadata; [SDPStreamMetadataKey]: SDPStreamMetadata;
} }
@ -213,6 +214,7 @@ export enum CallErrorCode {
export type SignallingMessage<Base extends MCallBase> = export type SignallingMessage<Base extends MCallBase> =
{type: EventType.Invite, content: MCallInvite<Base>} | {type: EventType.Invite, content: MCallInvite<Base>} |
{type: EventType.Negotiate, content: MCallNegotiate<Base>} |
{type: EventType.Answer, content: MCallAnswer<Base>} | {type: EventType.Answer, content: MCallAnswer<Base>} |
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} | {type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
{type: EventType.Candidates, content: MCallCandidates<Base>} | {type: EventType.Candidates, content: MCallCandidates<Base>} |

View file

@ -0,0 +1,37 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import type {Track, Stream} from "../../platform/types/MediaDevices";
export function getStreamAudioTrack(stream: Stream | undefined): Track | undefined {
return stream?.getAudioTracks()[0];
}
export function getStreamVideoTrack(stream: Stream | undefined): Track | undefined {
return stream?.getVideoTracks()[0];
}
export class MuteSettings {
constructor (public readonly microphone: boolean = false, public readonly camera: boolean = false) {}
toggleCamera(): MuteSettings {
return new MuteSettings(this.microphone, !this.camera);
}
toggleMicrophone(): MuteSettings {
return new MuteSettings(!this.microphone, this.camera);
}
}

View file

@ -17,6 +17,7 @@ limitations under the License.
import {ObservableMap} from "../../../observable/map/ObservableMap"; import {ObservableMap} from "../../../observable/map/ObservableMap";
import {Member} from "./Member"; import {Member} from "./Member";
import {LocalMedia} from "../LocalMedia"; import {LocalMedia} from "../LocalMedia";
import {MuteSettings} from "../common";
import {RoomMember} from "../../room/members/RoomMember"; import {RoomMember} from "../../room/members/RoomMember";
import {EventEmitter} from "../../../utils/EventEmitter"; import {EventEmitter} from "../../../utils/EventEmitter";
import {EventType, CallIntent} from "../callEventTypes"; import {EventType, CallIntent} from "../callEventTypes";
@ -63,6 +64,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
private _localMedia?: LocalMedia = undefined; private _localMedia?: LocalMedia = undefined;
private _memberOptions: MemberOptions; private _memberOptions: MemberOptions;
private _state: GroupCallState; private _state: GroupCallState;
private localMuteSettings: MuteSettings = new MuteSettings(false, false);
private _deviceIndex?: number; private _deviceIndex?: number;
private _eventTimestamp?: number; private _eventTimestamp?: number;
@ -129,11 +131,36 @@ export class GroupCall extends EventEmitter<{change: never}> {
this.emitChange(); this.emitChange();
// send invite to all members that are < my userId // send invite to all members that are < my userId
for (const [,member] of this._members) { for (const [,member] of this._members) {
member.connect(this._localMedia!.clone()); member.connect(this._localMedia!.clone(), this.localMuteSettings);
} }
}); });
} }
async setMedia(localMedia: LocalMedia): Promise<void> {
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) {
const oldMedia = this._localMedia!;
this._localMedia = localMedia;
await Promise.all(Array.from(this._members.values()).map(m => {
return m.setMedia(localMedia, oldMedia);
}));
oldMedia?.stopExcept(localMedia);
}
}
setMuted(muteSettings: MuteSettings) {
this.localMuteSettings = muteSettings;
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
for (const [,member] of this._members) {
member.setMuted(this.localMuteSettings);
}
}
this.emitChange();
}
get muteSettings(): MuteSettings {
return this.localMuteSettings;
}
get hasJoined() { get hasJoined() {
return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined;
} }
@ -168,7 +195,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
} }
/** @internal */ /** @internal */
create(callType: "m.video" | "m.voice"): Promise<void> { create(type: "m.video" | "m.voice"): Promise<void> {
return this.logItem.wrap("create", async log => { return this.logItem.wrap("create", async log => {
if (this._state !== GroupCallState.Fledgling) { if (this._state !== GroupCallState.Fledgling) {
return; return;
@ -176,7 +203,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
this._state = GroupCallState.Creating; this._state = GroupCallState.Creating;
this.emitChange(); this.emitChange();
this.callContent = Object.assign({ this.callContent = Object.assign({
"m.type": callType, "m.type": type,
}, this.callContent); }, this.callContent);
const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log}); const request = this.options.hsApi.sendState(this.roomId, EventType.GroupCall, this.id, this.callContent!, {log});
await request.response(); await request.response();
@ -235,7 +262,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
this._members.add(memberKey, member); this._members.add(memberKey, member);
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) { if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
// Safari can't send a MediaStream to multiple sources, so clone it // Safari can't send a MediaStream to multiple sources, so clone it
member.connect(this._localMedia!.clone()); member.connect(this._localMedia!.clone(), this.localMuteSettings);
} }
} }
} }

View file

@ -19,6 +19,7 @@ import {makeTxnId, makeId} from "../../common";
import {EventType, CallErrorCode} from "../callEventTypes"; import {EventType, CallErrorCode} from "../callEventTypes";
import {formatToDeviceMessagesPayload} from "../../common"; import {formatToDeviceMessagesPayload} from "../../common";
import type {MuteSettings} from "../common";
import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall"; import type {Options as PeerCallOptions, RemoteMedia} from "../PeerCall";
import type {LocalMedia} from "../LocalMedia"; import type {LocalMedia} from "../LocalMedia";
import type {HomeServerApi} from "../../net/HomeServerApi"; import type {HomeServerApi} from "../../net/HomeServerApi";
@ -50,6 +51,7 @@ const errorCodesWithoutRetry = [
export class Member { export class Member {
private peerCall?: PeerCall; private peerCall?: PeerCall;
private localMedia?: LocalMedia; private localMedia?: LocalMedia;
private localMuteSettings?: MuteSettings;
private retryCount: number = 0; private retryCount: number = 0;
constructor( constructor(
@ -65,6 +67,10 @@ export class Member {
return this.peerCall?.remoteMedia; return this.peerCall?.remoteMedia;
} }
get remoteMuteSettings(): MuteSettings | undefined {
return this.peerCall?.remoteMuteSettings;
}
get isConnected(): boolean { get isConnected(): boolean {
return this.peerCall?.state === CallState.Connected; return this.peerCall?.state === CallState.Connected;
} }
@ -90,9 +96,10 @@ export class Member {
} }
/** @internal */ /** @internal */
connect(localMedia: LocalMedia) { connect(localMedia: LocalMedia, localMuteSettings: MuteSettings) {
this.logItem.wrap("connect", () => { this.logItem.wrap("connect", () => {
this.localMedia = localMedia; this.localMedia = localMedia;
this.localMuteSettings = localMuteSettings;
// otherwise wait for it to connect // otherwise wait for it to connect
let shouldInitiateCall; let shouldInitiateCall;
// the lexicographically lower side initiates the call // the lexicographically lower side initiates the call
@ -103,7 +110,7 @@ export class Member {
} }
if (shouldInitiateCall) { if (shouldInitiateCall) {
this.peerCall = this._createPeerCall(makeId("c")); this.peerCall = this._createPeerCall(makeId("c"));
this.peerCall.call(localMedia); this.peerCall.call(localMedia, localMuteSettings);
} }
}); });
} }
@ -133,7 +140,7 @@ export class Member {
/** @internal */ /** @internal */
emitUpdate = (peerCall: PeerCall, params: any) => { emitUpdate = (peerCall: PeerCall, params: any) => {
if (peerCall.state === CallState.Ringing) { if (peerCall.state === CallState.Ringing) {
peerCall.answer(this.localMedia!); peerCall.answer(this.localMedia!, this.localMuteSettings!);
} }
else if (peerCall.state === CallState.Ended) { else if (peerCall.state === CallState.Ended) {
const hangupReason = peerCall.hangupReason; const hangupReason = peerCall.hangupReason;
@ -142,7 +149,7 @@ export class Member {
if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) { if (hangupReason && !errorCodesWithoutRetry.includes(hangupReason)) {
this.retryCount += 1; this.retryCount += 1;
if (this.retryCount <= 3) { if (this.retryCount <= 3) {
this.connect(this.localMedia!); this.connect(this.localMedia!, this.localMuteSettings!);
} }
} }
} }
@ -166,6 +173,8 @@ export class Member {
} }
} }
}; };
// TODO: remove this for release
log.set("payload", groupMessage.content);
const request = this.options.hsApi.sendToDevice( const request = this.options.hsApi.sendToDevice(
message.type, message.type,
//"m.room.encrypted", //"m.room.encrypted",
@ -194,6 +203,17 @@ export class Member {
} }
} }
/** @internal */
async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise<void> {
this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia);
await this.peerCall?.setMedia(this.localMedia);
}
setMuted(muteSettings: MuteSettings) {
this.localMuteSettings = muteSettings;
this.peerCall?.setMuted(muteSettings);
}
private _createPeerCall(callId: string): PeerCall { private _createPeerCall(callId: string): PeerCall {
return new PeerCall(callId, Object.assign({}, this.options, { return new PeerCall(callId, Object.assign({}, this.options, {
emitUpdate: this.emitUpdate, emitUpdate: this.emitUpdate,

View file

@ -29,32 +29,31 @@ export class MediaRepository {
this._platform = platform; this._platform = platform;
} }
mxcUrlThumbnail(url: string, width: number, height: number, method: "crop" | "scale"): string | null { mxcUrlThumbnail(url: string, width: number, height: number, method: "crop" | "scale"): string | undefined {
const parts = this._parseMxcUrl(url); const parts = this._parseMxcUrl(url);
if (parts) { if (parts) {
const [serverName, mediaId] = parts; const [serverName, mediaId] = parts;
const httpUrl = `${this._homeserver}/_matrix/media/r0/thumbnail/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`; const httpUrl = `${this._homeserver}/_matrix/media/r0/thumbnail/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`;
return httpUrl + "?" + encodeQueryParams({width: Math.round(width), height: Math.round(height), method}); return httpUrl + "?" + encodeQueryParams({width: Math.round(width), height: Math.round(height), method});
} }
return null; return undefined;
} }
mxcUrl(url: string): string | null { mxcUrl(url: string): string | undefined {
const parts = this._parseMxcUrl(url); const parts = this._parseMxcUrl(url);
if (parts) { if (parts) {
const [serverName, mediaId] = parts; const [serverName, mediaId] = parts;
return `${this._homeserver}/_matrix/media/r0/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`; return `${this._homeserver}/_matrix/media/r0/download/${encodeURIComponent(serverName)}/${encodeURIComponent(mediaId)}`;
} else {
return null;
} }
return undefined;
} }
private _parseMxcUrl(url: string): string[] | null { private _parseMxcUrl(url: string): string[] | undefined {
const prefix = "mxc://"; const prefix = "mxc://";
if (url.startsWith(prefix)) { if (url.startsWith(prefix)) {
return url.substr(prefix.length).split("/", 2); return url.substr(prefix.length).split("/", 2);
} else { } else {
return null; return undefined;
} }
} }

View file

@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index.js"; import {SortedArray, AsyncMappedList, ConcatList, ObservableArray} from "../../../observable/index";
import {Disposables} from "../../../utils/Disposables"; import {Disposables} from "../../../utils/Disposables";
import {Direction} from "./Direction"; import {Direction} from "./Direction";
import {TimelineReader} from "./persistence/TimelineReader.js"; import {TimelineReader} from "./persistence/TimelineReader.js";

View file

@ -46,3 +46,12 @@ Object.assign(BaseObservableMap.prototype, {
return new JoinedMap([this].concat(otherMaps)); return new JoinedMap([this].concat(otherMaps));
} }
}); });
declare module "./map/BaseObservableMap" {
interface BaseObservableMap<K, V> {
sortValues(comparator: (a: V, b: V) => number): SortedMapList<V>;
mapValues<M>(mapper: (V, emitSpontaneousUpdate: (params: any) => void) => M, updater: (mappedValue: M, params: any, value: V) => void): MappedMap<K, M>;
filterValues(filter: (V, K) => boolean): FilteredMap<K, V>;
join(...otherMaps: BaseObservableMap<K, V>[]): JoinedMap<K, V>;
}
}

View file

@ -0,0 +1,53 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {BaseObservableMap} from "./BaseObservableMap";
import {BaseObservableValue} from "../value/BaseObservableValue";
import {SubscriptionHandle} from "../BaseObservable";
export class ObservableValueMap<K, V> extends BaseObservableMap<K, V> {
private subscription?: SubscriptionHandle;
constructor(private readonly key: K, private readonly observableValue: BaseObservableValue<V>) {
super();
}
onSubscribeFirst() {
this.subscription = this.observableValue.subscribe(value => {
this.emitUpdate(this.key, value, undefined);
});
super.onSubscribeFirst();
}
onUnsubscribeLast() {
this.subscription!();
super.onUnsubscribeLast();
}
*[Symbol.iterator](): Iterator<[K, V]> {
yield [this.key, this.observableValue.get()];
}
get size(): number {
return 1;
}
get(key: K): V | undefined {
if (key == this.key) {
return this.observableValue.get();
}
}
}

View file

@ -0,0 +1,45 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import {BaseObservableValue} from "./BaseObservableValue";
import {EventEmitter} from "../../utils/EventEmitter";
export class EventObservableValue<T, V extends EventEmitter<T>> extends BaseObservableValue<V> {
private eventSubscription: () => void;
constructor(
private readonly value: V,
private readonly eventName: keyof T
) {
super();
}
onSubscribeFirst(): void {
this.eventSubscription = this.value.disposableOn(this.eventName, () => {
this.emit(this.value);
});
super.onSubscribeFirst();
}
onUnsubscribeLast(): void {
this.eventSubscription!();
super.onUnsubscribeLast();
}
get(): V {
return this.value;
}
}

View file

@ -14,19 +14,48 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
export interface Event {}
export interface MediaDevices { export interface MediaDevices {
// filter out audiooutput // filter out audiooutput
enumerate(): Promise<MediaDeviceInfo[]>; enumerate(): Promise<MediaDeviceInfo[]>;
// to assign to a video element, we downcast to WrappedTrack and use the stream property. // to assign to a video element, we downcast to WrappedTrack and use the stream property.
getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream>; getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream>;
getScreenShareTrack(): Promise<Stream | undefined>; getScreenShareTrack(): Promise<Stream | undefined>;
createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer;
}
// Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts
/*! *****************************************************************************
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions
and limitations under the License.
***************************************************************************** */
export interface StreamTrackEvent extends Event {
readonly track: Track;
}
export interface StreamEventMap {
"addtrack": StreamTrackEvent;
"removetrack": StreamTrackEvent;
} }
export interface Stream { export interface Stream {
readonly audioTrack: AudioTrack | undefined; getTracks(): ReadonlyArray<Track>;
readonly videoTrack: Track | undefined; getAudioTracks(): ReadonlyArray<Track>;
getVideoTracks(): ReadonlyArray<Track>;
readonly id: string; readonly id: string;
clone(): Stream; clone(): Stream;
addEventListener<K extends keyof StreamEventMap>(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
removeEventListener<K extends keyof StreamEventMap>(type: K, listener: (this: Stream, ev: StreamEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
} }
export enum TrackKind { export enum TrackKind {
@ -38,12 +67,13 @@ export interface Track {
readonly kind: TrackKind; readonly kind: TrackKind;
readonly label: string; readonly label: string;
readonly id: string; readonly id: string;
readonly settings: MediaTrackSettings; enabled: boolean;
// getSettings(): MediaTrackSettings;
stop(): void; stop(): void;
} }
export interface AudioTrack extends Track { export interface VolumeMeasurer {
// TODO: how to emit updates on this?
get isSpeaking(): boolean; get isSpeaking(): boolean;
setSpeakingThreshold(threshold: number): void;
stop();
} }

View file

@ -14,63 +14,155 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {Track, Stream} from "./MediaDevices"; import {Track, Stream, Event} from "./MediaDevices";
import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes"; import {SDPStreamMetadataPurpose} from "../../matrix/calls/callEventTypes";
export interface WebRTC { export interface WebRTC {
createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection; createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize: number): PeerConnection;
prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void;
} }
export interface StreamSender { // Typescript definitions derived from https://github.com/microsoft/TypeScript/blob/main/lib/lib.dom.d.ts
get stream(): Stream; /*! *****************************************************************************
get audioSender(): TrackSender | undefined; Copyright (c) Microsoft Corporation. All rights reserved.
get videoSender(): TrackSender | undefined; Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions
and limitations under the License.
***************************************************************************** */
export interface DataChannelEventMap {
"bufferedamountlow": Event;
"close": Event;
"error": Event;
"message": MessageEvent;
"open": Event;
} }
export interface StreamReceiver { export interface DataChannel {
get stream(): Stream; binaryType: BinaryType;
get audioReceiver(): TrackReceiver | undefined; readonly id: number | null;
get videoReceiver(): TrackReceiver | undefined; readonly label: string;
} readonly negotiated: boolean;
readonly readyState: DataChannelState;
export interface TrackReceiver {
get track(): Track;
get enabled(): boolean;
enable(enabled: boolean); // this modifies the transceiver direction
}
export interface TrackSender extends TrackReceiver {
/** replaces the track if possible without renegotiation. Can throw. */
replaceTrack(track: Track | undefined): Promise<void>;
/** make any needed adjustments to the sender or transceiver settings
* depending on the purpose, after adding the track to the connection */
prepareForPurpose(purpose: SDPStreamMetadataPurpose): void;
}
export interface PeerConnectionHandler {
onIceConnectionStateChange(state: RTCIceConnectionState);
onLocalIceCandidate(candidate: RTCIceCandidate);
onIceGatheringStateChange(state: RTCIceGatheringState);
onRemoteStreamRemoved(stream: Stream);
onRemoteTracksAdded(receiver: TrackReceiver);
onRemoteDataChannel(dataChannel: any | undefined);
onNegotiationNeeded();
}
export interface PeerConnection {
get iceGatheringState(): RTCIceGatheringState;
get signalingState(): RTCSignalingState;
get localDescription(): RTCSessionDescription | undefined;
get localStreams(): ReadonlyMap<string, StreamSender>;
get remoteStreams(): ReadonlyMap<string, StreamReceiver>;
createOffer(): Promise<RTCSessionDescriptionInit>;
createAnswer(): Promise<RTCSessionDescriptionInit>;
setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void>;
setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void>;
addIceCandidate(candidate: RTCIceCandidate): Promise<void>;
addTrack(track: Track): TrackSender | undefined;
removeTrack(track: TrackSender): void;
createDataChannel(options: RTCDataChannelInit): any;
dispose(): void;
close(): void; close(): void;
send(data: string): void;
send(data: Blob): void;
send(data: ArrayBuffer): void;
send(data: ArrayBufferView): void;
addEventListener<K extends keyof DataChannelEventMap>(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
removeEventListener<K extends keyof DataChannelEventMap>(type: K, listener: (this: DataChannel, ev: DataChannelEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
}
export interface DataChannelInit {
id?: number;
maxPacketLifeTime?: number;
maxRetransmits?: number;
negotiated?: boolean;
ordered?: boolean;
protocol?: string;
}
export interface DataChannelEvent extends Event {
readonly channel: DataChannel;
}
export interface PeerConnectionIceEvent extends Event {
readonly candidate: RTCIceCandidate | null;
}
export interface TrackEvent extends Event {
readonly receiver: Receiver;
readonly streams: ReadonlyArray<Stream>;
readonly track: Track;
readonly transceiver: Transceiver;
}
export interface PeerConnectionEventMap {
"connectionstatechange": Event;
"datachannel": DataChannelEvent;
"icecandidate": PeerConnectionIceEvent;
"iceconnectionstatechange": Event;
"icegatheringstatechange": Event;
"negotiationneeded": Event;
"signalingstatechange": Event;
"track": TrackEvent;
}
export type DataChannelState = "closed" | "closing" | "connecting" | "open";
export type IceConnectionState = "checking" | "closed" | "completed" | "connected" | "disconnected" | "failed" | "new";
export type PeerConnectionState = "closed" | "connected" | "connecting" | "disconnected" | "failed" | "new";
export type SignalingState = "closed" | "have-local-offer" | "have-local-pranswer" | "have-remote-offer" | "have-remote-pranswer" | "stable";
export type IceGatheringState = "complete" | "gathering" | "new";
export type SdpType = "answer" | "offer" | "pranswer" | "rollback";
export type TransceiverDirection = "inactive" | "recvonly" | "sendonly" | "sendrecv" | "stopped";
export interface SessionDescription {
readonly sdp: string;
readonly type: SdpType;
}
export interface AnswerOptions {}
export interface OfferOptions {
iceRestart?: boolean;
offerToReceiveAudio?: boolean;
offerToReceiveVideo?: boolean;
}
export interface SessionDescriptionInit {
sdp?: string;
type: SdpType;
}
export interface LocalSessionDescriptionInit {
sdp?: string;
type?: SdpType;
}
/** A WebRTC connection between the local computer and a remote peer. It provides methods to connect to a remote peer, maintain and monitor the connection, and close the connection once it's no longer needed. */
export interface PeerConnection {
readonly connectionState: PeerConnectionState;
readonly iceConnectionState: IceConnectionState;
readonly iceGatheringState: IceGatheringState;
readonly localDescription: SessionDescription | null;
readonly remoteDescription: SessionDescription | null;
readonly signalingState: SignalingState;
addIceCandidate(candidate?: RTCIceCandidateInit): Promise<void>;
addTrack(track: Track, ...streams: Stream[]): Sender;
close(): void;
createAnswer(options?: AnswerOptions): Promise<SessionDescriptionInit>;
createDataChannel(label: string, dataChannelDict?: DataChannelInit): DataChannel;
createOffer(options?: OfferOptions): Promise<SessionDescriptionInit>;
getReceivers(): Receiver[];
getSenders(): Sender[];
getTransceivers(): Transceiver[];
removeTrack(sender: Sender): void;
restartIce(): void;
setLocalDescription(description?: LocalSessionDescriptionInit): Promise<void>;
setRemoteDescription(description: SessionDescriptionInit): Promise<void>;
addEventListener<K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
removeEventListener<K extends keyof PeerConnectionEventMap>(type: K, listener: (this: PeerConnection, ev: PeerConnectionEventMap[K]) => any, options?: boolean | EventListenerOptions): void;
}
export interface Receiver {
readonly track: Track;
}
export interface Sender {
readonly track: Track | null;
replaceTrack(withTrack: Track | null): Promise<void>;
}
export interface Transceiver {
readonly currentDirection: TransceiverDirection | null;
direction: TransceiverDirection;
readonly mid: string | null;
readonly receiver: Receiver;
readonly sender: Sender;
stop(): void;
} }

View file

@ -19,6 +19,6 @@ import {hkdf} from "../../utils/crypto/hkdf";
import {Platform as ModernPlatform} from "./Platform.js"; import {Platform as ModernPlatform} from "./Platform.js";
export function Platform(container, assetPaths, config, options = null) { export function Platform({ container, assetPaths, config, configURL, options = null }) {
return new ModernPlatform(container, assetPaths, config, options, {aesjs, hkdf}); return new ModernPlatform({ container, assetPaths, config, configURL, options, cryptoExtras: { aesjs, hkdf }});
} }

View file

@ -128,10 +128,11 @@ function adaptUIOnVisualViewportResize(container) {
} }
export class Platform { export class Platform {
constructor(container, assetPaths, config, options = null, cryptoExtras = null) { constructor({ container, assetPaths, config, configURL, options = null, cryptoExtras = null }) {
this._container = container; this._container = container;
this._assetPaths = assetPaths; this._assetPaths = assetPaths;
this._config = config; this._config = config;
this._configURL = configURL;
this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_"); this.settingsStorage = new SettingsStorage("hydrogen_setting_v1_");
this.clock = new Clock(); this.clock = new Clock();
this.encoding = new Encoding(); this.encoding = new Encoding();
@ -144,7 +145,7 @@ export class Platform {
this._serviceWorkerHandler = new ServiceWorkerHandler(); this._serviceWorkerHandler = new ServiceWorkerHandler();
this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker); this._serviceWorkerHandler.registerAndStart(assetPaths.serviceWorker);
} }
this.notificationService = new NotificationService(this._serviceWorkerHandler, config.push); this.notificationService = undefined;
// Only try to use crypto when olm is provided // Only try to use crypto when olm is provided
if(this._assetPaths.olm) { if(this._assetPaths.olm) {
this.crypto = new Crypto(cryptoExtras); this.crypto = new Crypto(cryptoExtras);
@ -169,6 +170,20 @@ export class Platform {
this.webRTC = new DOMWebRTC(); this.webRTC = new DOMWebRTC();
} }
async init() {
if (!this._config) {
if (!this._configURL) {
throw new Error("Neither config nor configURL was provided!");
}
const {body}= await this.request(this._configURL, {method: "GET", format: "json", cache: true}).response();
this._config = body;
}
this.notificationService = new NotificationService(
this._serviceWorkerHandler,
this._config.push
);
}
_createLogger(isDevelopment) { _createLogger(isDevelopment) {
// Make sure that loginToken does not end up in the logs // Make sure that loginToken does not end up in the logs
const transformer = (item) => { const transformer = (item) => {

View file

@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, AudioTrack} from "../../types/MediaDevices"; import {MediaDevices as IMediaDevices, Stream, Track, TrackKind, VolumeMeasurer} from "../../types/MediaDevices";
const POLLING_INTERVAL = 200; // ms const POLLING_INTERVAL = 200; // ms
export const SPEAKING_THRESHOLD = -60; // dB export const SPEAKING_THRESHOLD = -60; // dB
@ -30,12 +30,12 @@ export class MediaDevicesWrapper implements IMediaDevices {
async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream> { async getMediaTracks(audio: true | MediaDeviceInfo, video: boolean | MediaDeviceInfo): Promise<Stream> {
const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video)); const stream = await this.mediaDevices.getUserMedia(this.getUserMediaContraints(audio, video));
return new StreamWrapper(stream); return stream as Stream;
} }
async getScreenShareTrack(): Promise<Stream | undefined> { async getScreenShareTrack(): Promise<Stream | undefined> {
const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints()); const stream = await this.mediaDevices.getDisplayMedia(this.getScreenshareContraints());
return new StreamWrapper(stream); return stream as Stream;
} }
private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints { private getUserMediaContraints(audio: boolean | MediaDeviceInfo, video: boolean | MediaDeviceInfo): MediaStreamConstraints {
@ -68,55 +68,13 @@ export class MediaDevicesWrapper implements IMediaDevices {
video: true, video: true,
}; };
} }
}
export class StreamWrapper implements Stream { createVolumeMeasurer(stream: Stream, callback: () => void): VolumeMeasurer {
return new WebAudioVolumeMeasurer(stream as MediaStream, callback);
public audioTrack: AudioTrackWrapper | undefined = undefined;
public videoTrack: TrackWrapper | undefined = undefined;
constructor(public readonly stream: MediaStream) {
for (const track of stream.getTracks()) {
this.update(track);
}
}
get id(): string { return this.stream.id; }
clone(): Stream {
return new StreamWrapper(this.stream.clone());
}
update(track: MediaStreamTrack): TrackWrapper | undefined {
if (track.kind === "video") {
if (!this.videoTrack || track.id !== this.videoTrack.track.id) {
this.videoTrack = new TrackWrapper(track, this.stream);
}
return this.videoTrack;
} else if (track.kind === "audio") {
if (!this.audioTrack || track.id !== this.audioTrack.track.id) {
this.audioTrack = new AudioTrackWrapper(track, this.stream);
}
return this.audioTrack;
}
} }
} }
export class TrackWrapper implements Track { export class WebAudioVolumeMeasurer implements VolumeMeasurer {
constructor(
public readonly track: MediaStreamTrack,
public readonly stream: MediaStream
) {}
get kind(): TrackKind { return this.track.kind as TrackKind; }
get label(): string { return this.track.label; }
get id(): string { return this.track.id; }
get settings(): MediaTrackSettings { return this.track.getSettings(); }
stop() { this.track.stop(); }
}
export class AudioTrackWrapper extends TrackWrapper {
private measuringVolumeActivity = false; private measuringVolumeActivity = false;
private audioContext?: AudioContext; private audioContext?: AudioContext;
private analyser: AnalyserNode; private analyser: AnalyserNode;
@ -125,9 +83,12 @@ export class AudioTrackWrapper extends TrackWrapper {
private speaking = false; private speaking = false;
private volumeLooperTimeout: number; private volumeLooperTimeout: number;
private speakingVolumeSamples: number[]; private speakingVolumeSamples: number[];
private callback: () => void;
private stream: MediaStream;
constructor(track: MediaStreamTrack, stream: MediaStream) { constructor(stream: MediaStream, callback: () => void) {
super(track, stream); this.stream = stream;
this.callback = callback;
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
this.initVolumeMeasuring(); this.initVolumeMeasuring();
this.measureVolumeActivity(true); this.measureVolumeActivity(true);
@ -147,6 +108,7 @@ export class AudioTrackWrapper extends TrackWrapper {
} else { } else {
this.measuringVolumeActivity = false; this.measuringVolumeActivity = false;
this.speakingVolumeSamples.fill(-Infinity); this.speakingVolumeSamples.fill(-Infinity);
this.callback();
// this.emit(CallFeedEvent.VolumeChanged, -Infinity); // this.emit(CallFeedEvent.VolumeChanged, -Infinity);
} }
} }
@ -167,7 +129,6 @@ export class AudioTrackWrapper extends TrackWrapper {
this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
} }
public setSpeakingThreshold(threshold: number) { public setSpeakingThreshold(threshold: number) {
this.speakingThreshold = threshold; this.speakingThreshold = threshold;
} }
@ -189,6 +150,7 @@ export class AudioTrackWrapper extends TrackWrapper {
this.speakingVolumeSamples.shift(); this.speakingVolumeSamples.shift();
this.speakingVolumeSamples.push(maxVolume); this.speakingVolumeSamples.push(maxVolume);
this.callback();
// this.emit(CallFeedEvent.VolumeChanged, maxVolume); // this.emit(CallFeedEvent.VolumeChanged, maxVolume);
let newSpeaking = false; let newSpeaking = false;
@ -204,267 +166,16 @@ export class AudioTrackWrapper extends TrackWrapper {
if (this.speaking !== newSpeaking) { if (this.speaking !== newSpeaking) {
this.speaking = newSpeaking; this.speaking = newSpeaking;
this.callback();
// this.emit(CallFeedEvent.Speaking, this.speaking); // this.emit(CallFeedEvent.Speaking, this.speaking);
} }
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number; this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number;
}; };
public dispose(): void { public stop(): void {
clearTimeout(this.volumeLooperTimeout); clearTimeout(this.volumeLooperTimeout);
this.analyser.disconnect();
this.audioContext?.close();
} }
} }
// export interface ICallFeedOpts {
// client: MatrixClient;
// roomId: string;
// userId: string;
// stream: MediaStream;
// purpose: SDPStreamMetadataPurpose;
// audioMuted: boolean;
// videoMuted: boolean;
// }
// export enum CallFeedEvent {
// NewStream = "new_stream",
// MuteStateChanged = "mute_state_changed",
// VolumeChanged = "volume_changed",
// Speaking = "speaking",
// }
// export class CallFeed extends EventEmitter {
// public stream: MediaStream;
// public sdpMetadataStreamId: string;
// public userId: string;
// public purpose: SDPStreamMetadataPurpose;
// public speakingVolumeSamples: number[];
// private client: MatrixClient;
// private roomId: string;
// private audioMuted: boolean;
// private videoMuted: boolean;
// private measuringVolumeActivity = false;
// private audioContext: AudioContext;
// private analyser: AnalyserNode;
// private frequencyBinCount: Float32Array;
// private speakingThreshold = SPEAKING_THRESHOLD;
// private speaking = false;
// private volumeLooperTimeout: number;
// constructor(opts: ICallFeedOpts) {
// super();
// this.client = opts.client;
// this.roomId = opts.roomId;
// this.userId = opts.userId;
// this.purpose = opts.purpose;
// this.audioMuted = opts.audioMuted;
// this.videoMuted = opts.videoMuted;
// this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
// this.sdpMetadataStreamId = opts.stream.id;
// this.updateStream(null, opts.stream);
// if (this.hasAudioTrack) {
// this.initVolumeMeasuring();
// }
// }
// private get hasAudioTrack(): boolean {
// return this.stream.getAudioTracks().length > 0;
// }
// private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
// if (newStream === oldStream) return;
// if (oldStream) {
// oldStream.removeEventListener("addtrack", this.onAddTrack);
// this.measureVolumeActivity(false);
// }
// if (newStream) {
// this.stream = newStream;
// newStream.addEventListener("addtrack", this.onAddTrack);
// if (this.hasAudioTrack) {
// this.initVolumeMeasuring();
// } else {
// this.measureVolumeActivity(false);
// }
// }
// this.emit(CallFeedEvent.NewStream, this.stream);
// }
// private initVolumeMeasuring(): void {
// const AudioContext = window.AudioContext || window.webkitAudioContext;
// if (!this.hasAudioTrack || !AudioContext) return;
// this.audioContext = new AudioContext();
// this.analyser = this.audioContext.createAnalyser();
// this.analyser.fftSize = 512;
// this.analyser.smoothingTimeConstant = 0.1;
// const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
// mediaStreamAudioSourceNode.connect(this.analyser);
// this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
// }
// private onAddTrack = (): void => {
// this.emit(CallFeedEvent.NewStream, this.stream);
// };
// /**
// * Returns callRoom member
// * @returns member of the callRoom
// */
// public getMember(): RoomMember {
// const callRoom = this.client.getRoom(this.roomId);
// return callRoom.getMember(this.userId);
// }
// /**
// * Returns true if CallFeed is local, otherwise returns false
// * @returns {boolean} is local?
// */
// public isLocal(): boolean {
// return this.userId === this.client.getUserId();
// }
// /**
// * Returns true if audio is muted or if there are no audio
// * tracks, otherwise returns false
// * @returns {boolean} is audio muted?
// */
// public isAudioMuted(): boolean {
// return this.stream.getAudioTracks().length === 0 || this.audioMuted;
// }
// *
// * Returns true video is muted or if there are no video
// * tracks, otherwise returns false
// * @returns {boolean} is video muted?
// public isVideoMuted(): boolean {
// // We assume only one video track
// return this.stream.getVideoTracks().length === 0 || this.videoMuted;
// }
// public isSpeaking(): boolean {
// return this.speaking;
// }
// /**
// * Replaces the current MediaStream with a new one.
// * This method should be only used by MatrixCall.
// * @param newStream new stream with which to replace the current one
// */
// public setNewStream(newStream: MediaStream): void {
// this.updateStream(this.stream, newStream);
// }
// /**
// * Set feed's internal audio mute state
// * @param muted is the feed's audio muted?
// */
// public setAudioMuted(muted: boolean): void {
// this.audioMuted = muted;
// this.speakingVolumeSamples.fill(-Infinity);
// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
// }
// /**
// * Set feed's internal video mute state
// * @param muted is the feed's video muted?
// */
// public setVideoMuted(muted: boolean): void {
// this.videoMuted = muted;
// this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
// }
// /**
// * Starts emitting volume_changed events where the emitter value is in decibels
// * @param enabled emit volume changes
// */
// public measureVolumeActivity(enabled: boolean): void {
// if (enabled) {
// if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
// this.measuringVolumeActivity = true;
// this.volumeLooper();
// } else {
// this.measuringVolumeActivity = false;
// this.speakingVolumeSamples.fill(-Infinity);
// this.emit(CallFeedEvent.VolumeChanged, -Infinity);
// }
// }
// public setSpeakingThreshold(threshold: number) {
// this.speakingThreshold = threshold;
// }
// private volumeLooper = () => {
// if (!this.analyser) return;
// if (!this.measuringVolumeActivity) return;
// this.analyser.getFloatFrequencyData(this.frequencyBinCount);
// let maxVolume = -Infinity;
// for (let i = 0; i < this.frequencyBinCount.length; i++) {
// if (this.frequencyBinCount[i] > maxVolume) {
// maxVolume = this.frequencyBinCount[i];
// }
// }
// this.speakingVolumeSamples.shift();
// this.speakingVolumeSamples.push(maxVolume);
// this.emit(CallFeedEvent.VolumeChanged, maxVolume);
// let newSpeaking = false;
// for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
// const volume = this.speakingVolumeSamples[i];
// if (volume > this.speakingThreshold) {
// newSpeaking = true;
// break;
// }
// }
// if (this.speaking !== newSpeaking) {
// this.speaking = newSpeaking;
// this.emit(CallFeedEvent.Speaking, this.speaking);
// }
// this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
// };
// public clone(): CallFeed {
// const mediaHandler = this.client.getMediaHandler();
// const stream = this.stream.clone();
// if (this.purpose === SDPStreamMetadataPurpose.Usermedia) {
// mediaHandler.userMediaStreams.push(stream);
// } else {
// mediaHandler.screensharingStreams.push(stream);
// }
// return new CallFeed({
// client: this.client,
// roomId: this.roomId,
// userId: this.userId,
// stream,
// purpose: this.purpose,
// audioMuted: this.audioMuted,
// videoMuted: this.videoMuted,
// });
// }
// public dispose(): void {
// clearTimeout(this.volumeLooperTimeout);
// this.measureVolumeActivity(false);
// }
// }

View file

@ -14,9 +14,8 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {StreamWrapper, TrackWrapper, AudioTrackWrapper} from "./MediaDevices"; import {Stream, Track, TrackKind} from "../../types/MediaDevices";
import {Stream, Track, AudioTrack, TrackKind} from "../../types/MediaDevices"; import {WebRTC, Sender, PeerConnection} from "../../types/WebRTC";
import {WebRTC, PeerConnectionHandler, StreamSender, TrackSender, StreamReceiver, TrackReceiver, PeerConnection} from "../../types/WebRTC";
import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes"; import {SDPStreamMetadataPurpose} from "../../../matrix/calls/callEventTypes";
const POLLING_INTERVAL = 200; // ms const POLLING_INTERVAL = 200; // ms
@ -24,151 +23,21 @@ export const SPEAKING_THRESHOLD = -60; // dB
const SPEAKING_SAMPLE_COUNT = 8; // samples const SPEAKING_SAMPLE_COUNT = 8; // samples
export class DOMWebRTC implements WebRTC { export class DOMWebRTC implements WebRTC {
createPeerConnection(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection { createPeerConnection(forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize): PeerConnection {
return new DOMPeerConnection(handler, forceTURN, turnServers, iceCandidatePoolSize); return new RTCPeerConnection({
} iceTransportPolicy: forceTURN ? 'relay' : undefined,
} iceServers: turnServers,
iceCandidatePoolSize: iceCandidatePoolSize,
export class RemoteStreamWrapper extends StreamWrapper { }) as PeerConnection;
constructor(stream: MediaStream, private readonly emptyCallback: (stream: RemoteStreamWrapper) => void) {
super(stream);
this.stream.addEventListener("removetrack", this.onTrackRemoved);
} }
onTrackRemoved = (evt: MediaStreamTrackEvent) => { prepareSenderForPurpose(peerConnection: PeerConnection, sender: Sender, purpose: SDPStreamMetadataPurpose): void {
if (evt.track.id === this.audioTrack?.track.id) {
this.audioTrack = undefined;
} else if (evt.track.id === this.videoTrack?.track.id) {
this.videoTrack = undefined;
}
if (!this.audioTrack && !this.videoTrack) {
this.emptyCallback(this);
}
};
dispose() {
this.stream.removeEventListener("removetrack", this.onTrackRemoved);
}
}
export class DOMStreamSender implements StreamSender {
public audioSender: DOMTrackSender | undefined;
public videoSender: DOMTrackSender | undefined;
constructor(public readonly stream: StreamWrapper) {}
update(transceivers: ReadonlyArray<RTCRtpTransceiver>, sender: RTCRtpSender): DOMTrackSender | undefined {
const transceiver = transceivers.find(t => t.sender === sender);
if (transceiver && sender.track) {
const trackWrapper = this.stream.update(sender.track);
if (trackWrapper) {
if (trackWrapper.kind === TrackKind.Video && (!this.videoSender || this.videoSender.track.id !== trackWrapper.id)) {
this.videoSender = new DOMTrackSender(trackWrapper, transceiver);
return this.videoSender;
} else if (trackWrapper.kind === TrackKind.Audio && (!this.audioSender || this.audioSender.track.id !== trackWrapper.id)) {
this.audioSender = new DOMTrackSender(trackWrapper, transceiver);
return this.audioSender;
}
}
}
}
}
export class DOMStreamReceiver implements StreamReceiver {
public audioReceiver: DOMTrackReceiver | undefined;
public videoReceiver: DOMTrackReceiver | undefined;
constructor(public readonly stream: RemoteStreamWrapper) {}
update(event: RTCTrackEvent): DOMTrackReceiver | undefined {
const {receiver} = event;
const {track} = receiver;
const trackWrapper = this.stream.update(track);
if (trackWrapper) {
if (trackWrapper.kind === TrackKind.Video) {
this.videoReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver);
return this.videoReceiver;
} else {
this.audioReceiver = new DOMTrackReceiver(trackWrapper, event.transceiver);
return this.audioReceiver;
}
}
}
}
export class DOMTrackSenderOrReceiver implements TrackReceiver {
constructor(
public readonly track: TrackWrapper,
public readonly transceiver: RTCRtpTransceiver,
private readonly exclusiveValue: RTCRtpTransceiverDirection,
private readonly excludedValue: RTCRtpTransceiverDirection
) {}
get enabled(): boolean {
return this.transceiver.direction === "sendrecv" ||
this.transceiver.direction === this.exclusiveValue;
}
enable(enabled: boolean) {
if (enabled !== this.enabled) {
if (enabled) {
if (this.transceiver.direction === "inactive") {
this.transceiver.direction = this.exclusiveValue;
} else {
this.transceiver.direction = "sendrecv";
}
} else {
if (this.transceiver.direction === "sendrecv") {
this.transceiver.direction = this.excludedValue;
} else {
this.transceiver.direction = "inactive";
}
}
}
}
}
export class DOMTrackReceiver extends DOMTrackSenderOrReceiver {
constructor(
track: TrackWrapper,
transceiver: RTCRtpTransceiver,
) {
super(track, transceiver, "recvonly", "sendonly");
}
}
export class DOMTrackSender extends DOMTrackSenderOrReceiver {
constructor(
track: TrackWrapper,
transceiver: RTCRtpTransceiver,
) {
super(track, transceiver, "sendonly", "recvonly");
}
/** replaces the track if possible without renegotiation. Can throw. */
replaceTrack(track: Track | undefined): Promise<void> {
return this.transceiver.sender.replaceTrack(track ? (track as TrackWrapper).track : null);
}
prepareForPurpose(purpose: SDPStreamMetadataPurpose): void {
if (purpose === SDPStreamMetadataPurpose.Screenshare) { if (purpose === SDPStreamMetadataPurpose.Screenshare) {
this.getRidOfRTXCodecs(); this.getRidOfRTXCodecs(peerConnection as RTCPeerConnection, sender as RTCRtpSender);
} }
} }
/** private getRidOfRTXCodecs(peerConnection: RTCPeerConnection, sender: RTCRtpSender): void {
* This method removes all video/rtx codecs from screensharing video
* transceivers. This is necessary since they can cause problems. Without
* this the following steps should produce an error:
* Chromium calls Firefox
* Firefox answers
* Firefox starts screen-sharing
* Chromium starts screen-sharing
* Call crashes for Chromium with:
* [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list.
* [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs.
* [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER)
*/
private getRidOfRTXCodecs(): void {
// RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF
if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return;
@ -182,172 +51,14 @@ export class DOMTrackSender extends DOMTrackSenderOrReceiver {
codecs.splice(rtxCodecIndex, 1); codecs.splice(rtxCodecIndex, 1);
} }
} }
if (this.transceiver.sender.track?.kind === "video" ||
this.transceiver.receiver.track?.kind === "video") { const transceiver = peerConnection.getTransceivers().find(t => t.sender === sender);
this.transceiver.setCodecPreferences(codecs); if (transceiver && (
} transceiver.sender.track?.kind === "video" ||
} transceiver.receiver.track?.kind === "video"
} )
) {
class DOMPeerConnection implements PeerConnection { transceiver.setCodecPreferences(codecs);
private readonly peerConnection: RTCPeerConnection;
private readonly handler: PeerConnectionHandler;
public readonly localStreams: Map<string, DOMStreamSender> = new Map();
public readonly remoteStreams: Map<string, DOMStreamReceiver> = new Map();
constructor(handler: PeerConnectionHandler, forceTURN: boolean, turnServers: RTCIceServer[], iceCandidatePoolSize) {
this.handler = handler;
this.peerConnection = new RTCPeerConnection({
iceTransportPolicy: forceTURN ? 'relay' : undefined,
iceServers: turnServers,
iceCandidatePoolSize: iceCandidatePoolSize,
});
this.registerHandler();
}
get iceGatheringState(): RTCIceGatheringState { return this.peerConnection.iceGatheringState; }
get localDescription(): RTCSessionDescription | undefined { return this.peerConnection.localDescription ?? undefined; }
get signalingState(): RTCSignalingState { return this.peerConnection.signalingState; }
createOffer(): Promise<RTCSessionDescriptionInit> {
return this.peerConnection.createOffer();
}
createAnswer(): Promise<RTCSessionDescriptionInit> {
return this.peerConnection.createAnswer();
}
setLocalDescription(description?: RTCSessionDescriptionInit): Promise<void> {
return this.peerConnection.setLocalDescription(description);
}
setRemoteDescription(description: RTCSessionDescriptionInit): Promise<void> {
return this.peerConnection.setRemoteDescription(description);
}
addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
return this.peerConnection.addIceCandidate(candidate);
}
close(): void {
return this.peerConnection.close();
}
addTrack(track: Track): DOMTrackSender | undefined {
if (!(track instanceof TrackWrapper)) {
throw new Error("Not a TrackWrapper");
}
const sender = this.peerConnection.addTrack(track.track, track.stream);
let streamSender = this.localStreams.get(track.stream.id);
if (!streamSender) {
// TODO: reuse existing stream wrapper here?
streamSender = new DOMStreamSender(new StreamWrapper(track.stream));
this.localStreams.set(track.stream.id, streamSender);
}
const trackSender = streamSender.update(this.peerConnection.getTransceivers(), sender);
return trackSender;
}
removeTrack(sender: TrackSender): void {
if (!(sender instanceof DOMTrackSender)) {
throw new Error("Not a DOMTrackSender");
}
this.peerConnection.removeTrack((sender as DOMTrackSender).transceiver.sender);
// TODO: update localStreams
}
createDataChannel(options: RTCDataChannelInit): any {
return this.peerConnection.createDataChannel("channel", options);
}
private registerHandler() {
const pc = this.peerConnection;
pc.addEventListener('negotiationneeded', this);
pc.addEventListener('icecandidate', this);
pc.addEventListener('iceconnectionstatechange', this);
pc.addEventListener('icegatheringstatechange', this);
pc.addEventListener('signalingstatechange', this);
pc.addEventListener('track', this);
pc.addEventListener('datachannel', this);
}
private deregisterHandler() {
const pc = this.peerConnection;
pc.removeEventListener('negotiationneeded', this);
pc.removeEventListener('icecandidate', this);
pc.removeEventListener('iceconnectionstatechange', this);
pc.removeEventListener('icegatheringstatechange', this);
pc.removeEventListener('signalingstatechange', this);
pc.removeEventListener('track', this);
pc.removeEventListener('datachannel', this);
}
/** @internal */
handleEvent(evt: Event) {
switch (evt.type) {
case "iceconnectionstatechange":
this.handleIceConnectionStateChange();
break;
case "icecandidate":
this.handleLocalIceCandidate(evt as RTCPeerConnectionIceEvent);
break;
case "icegatheringstatechange":
this.handler.onIceGatheringStateChange(this.peerConnection.iceGatheringState);
break;
case "track":
this.handleRemoteTrack(evt as RTCTrackEvent);
break;
case "negotiationneeded":
this.handler.onNegotiationNeeded();
break;
case "datachannel":
this.handler.onRemoteDataChannel((evt as RTCDataChannelEvent).channel);
break;
}
}
dispose(): void {
this.deregisterHandler();
for (const r of this.remoteStreams.values()) {
r.stream.dispose();
}
}
private handleLocalIceCandidate(event: RTCPeerConnectionIceEvent) {
if (event.candidate) {
this.handler.onLocalIceCandidate(event.candidate);
}
};
private handleIceConnectionStateChange() {
const {iceConnectionState} = this.peerConnection;
if (iceConnectionState === "failed" && this.peerConnection.restartIce) {
this.peerConnection.restartIce();
} else {
this.handler.onIceConnectionStateChange(iceConnectionState);
}
}
onRemoteStreamEmpty = (stream: RemoteStreamWrapper): void => {
if (this.remoteStreams.delete(stream.id)) {
this.handler.onRemoteStreamRemoved(stream);
}
}
private handleRemoteTrack(evt: RTCTrackEvent) {
if (evt.streams.length !== 1) {
throw new Error("track in multiple streams is not supported");
}
const stream = evt.streams[0];
const transceivers = this.peerConnection.getTransceivers();
let streamReceiver: DOMStreamReceiver | undefined = this.remoteStreams.get(stream.id);
if (!streamReceiver) {
streamReceiver = new DOMStreamReceiver(new RemoteStreamWrapper(stream, this.onRemoteStreamEmpty));
this.remoteStreams.set(stream.id, streamReceiver);
}
const trackReceiver = streamReceiver.update(evt);
if (trackReceiver) {
this.handler.onRemoteTracksAdded(trackReceiver);
} }
} }
} }

View file

@ -17,17 +17,17 @@
<script id="main" type="module"> <script id="main" type="module">
import {main} from "./main"; import {main} from "./main";
import {Platform} from "./Platform"; import {Platform} from "./Platform";
import configJSON from "./assets/config.json?raw"; import configURL from "./assets/config.json?url";
import assetPaths from "./sdk/paths/vite"; import assetPaths from "./sdk/paths/vite";
if (import.meta.env.PROD) { if (import.meta.env.PROD) {
assetPaths.serviceWorker = "sw.js"; assetPaths.serviceWorker = "sw.js";
} }
const platform = new Platform( const platform = new Platform({
document.body, container: document.body,
assetPaths, assetPaths,
JSON.parse(configJSON), configURL,
{development: import.meta.env.DEV} options: {development: import.meta.env.DEV}
); });
main(platform); main(platform);
</script> </script>
</body> </body>

View file

@ -32,6 +32,7 @@ export async function main(platform) {
// const recorder = new RecordRequester(createFetchRequest(clock.createTimeout)); // const recorder = new RecordRequester(createFetchRequest(clock.createTimeout));
// const request = recorder.request; // const request = recorder.request;
// window.getBrawlFetchLog = () => recorder.log(); // window.getBrawlFetchLog = () => recorder.log();
await platform.init();
const navigation = createNavigation(); const navigation = createNavigation();
platform.setNavigation(navigation); platform.setNavigation(navigation);
const urlRouter = createRouter({navigation, history: platform.history}); const urlRouter = createRouter({navigation, history: platform.history});

View file

@ -92,8 +92,12 @@ function isCacheableThumbnail(url) {
const baseURL = new URL(self.registration.scope); const baseURL = new URL(self.registration.scope);
let pendingFetchAbortController = new AbortController(); let pendingFetchAbortController = new AbortController();
async function handleRequest(request) { async function handleRequest(request) {
try { try {
if (request.url.includes("config.json")) {
return handleConfigRequest(request);
}
const url = new URL(request.url); const url = new URL(request.url);
// rewrite / to /index.html so it hits the cache // rewrite / to /index.html so it hits the cache
if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) { if (url.origin === baseURL.origin && url.pathname === baseURL.pathname) {
@ -119,6 +123,27 @@ async function handleRequest(request) {
} }
} }
async function handleConfigRequest(request) {
let response = await readCache(request);
const networkResponsePromise = fetchAndUpdateConfig(request);
if (response) {
return response;
} else {
return await networkResponsePromise;
}
}
async function fetchAndUpdateConfig(request) {
const response = await fetch(request, {
signal: pendingFetchAbortController.signal,
headers: {
"Cache-Control": "no-cache",
},
});
updateCache(request, response.clone());
return response;
}
async function updateCache(request, response) { async function updateCache(request, response) {
// don't write error responses to the cache // don't write error responses to the cache
if (response.status >= 400) { if (response.status >= 400) {

View file

@ -1155,3 +1155,55 @@ button.RoomDetailsView_row::after {
background-position: center; background-position: center;
background-size: 36px; background-size: 36px;
} }
.CallView {
max-height: 50vh;
overflow-y: auto;
}
.CallView ul {
display: flex;
margin: 0;
gap: 12px;
padding: 0;
flex-wrap: wrap;
justify-content: center;
}
.StreamView {
width: 360px;
min-height: 200px;
border: 2px var(--accent-color) solid;
display: grid;
border-radius: 8px;
overflow: hidden;
background-color: black;
}
.StreamView > * {
grid-column: 1;
grid-row: 1;
}
.StreamView video {
width: 100%;
}
.StreamView_avatar {
align-self: center;
justify-self: center;
}
.StreamView_muteStatus {
align-self: end;
justify-self: start;
}
.StreamView_muteStatus.microphoneMuted::before {
content: "mic muted";
}
.StreamView_muteStatus.cameraMuted::before {
content: "cam muted";
}

View file

@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child} from "./html"; import { setAttribute, text, isChildren, classNames, TAG_NAMES, HTML_NS, ClassNames, Child as NonBoundChild} from "./html";
import {mountView} from "./utils"; import {mountView} from "./utils";
import {BaseUpdateView, IObservableValue} from "./BaseUpdateView"; import {BaseUpdateView, IObservableValue} from "./BaseUpdateView";
import {IMountArgs, ViewNode, IView} from "./types"; import {IMountArgs, ViewNode, IView} from "./types";
@ -30,12 +30,15 @@ function objHasFns(obj: ClassNames<unknown>): obj is { [className: string]: bool
} }
export type RenderFn<T> = (t: Builder<T>, vm: T) => ViewNode; export type RenderFn<T> = (t: Builder<T>, vm: T) => ViewNode;
type TextBinding<T> = (T) => string | number | boolean | undefined | null;
type Child<T> = NonBoundChild | TextBinding<T>;
type Children<T> = Child<T> | Child<T>[];
type EventHandler = ((event: Event) => void); type EventHandler = ((event: Event) => void);
type AttributeStaticValue = string | boolean; type AttributeStaticValue = string | boolean;
type AttributeBinding<T> = (value: T) => AttributeStaticValue; type AttributeBinding<T> = (value: T) => AttributeStaticValue;
export type AttrValue<T> = AttributeStaticValue | AttributeBinding<T> | EventHandler | ClassNames<T>; export type AttrValue<T> = AttributeStaticValue | AttributeBinding<T> | EventHandler | ClassNames<T>;
export type Attributes<T> = { [attribute: string]: AttrValue<T> }; export type Attributes<T> = { [attribute: string]: AttrValue<T> };
type ElementFn<T> = (attributes?: Attributes<T> | Child | Child[], children?: Child | Child[]) => Element; type ElementFn<T> = (attributes?: Attributes<T> | Children<T>, children?: Children<T>) => Element;
export type Builder<T> = TemplateBuilder<T> & { [tagName in typeof TAG_NAMES[string][number]]: ElementFn<T> }; export type Builder<T> = TemplateBuilder<T> & { [tagName in typeof TAG_NAMES[string][number]]: ElementFn<T> };
/** /**
@ -178,7 +181,7 @@ export class TemplateBuilder<T extends IObservableValue> {
this._templateView._addEventListener(node, name, fn, useCapture); this._templateView._addEventListener(node, name, fn, useCapture);
} }
_addAttributeBinding(node: Element, name: string, fn: (value: T) => boolean | string): void { _addAttributeBinding(node: Element, name: string, fn: AttributeBinding<T>): void {
let prevValue: string | boolean | undefined = undefined; let prevValue: string | boolean | undefined = undefined;
const binding = () => { const binding = () => {
const newValue = fn(this._value); const newValue = fn(this._value);
@ -195,15 +198,15 @@ export class TemplateBuilder<T extends IObservableValue> {
this._addAttributeBinding(node, "className", value => classNames(obj, value)); this._addAttributeBinding(node, "className", value => classNames(obj, value));
} }
_addTextBinding(fn: (value: T) => string): Text { _addTextBinding(fn: (value: T) => ReturnType<TextBinding<T>>): Text {
const initialValue = fn(this._value); const initialValue = fn(this._value)+"";
const node = text(initialValue); const node = text(initialValue);
let prevValue = initialValue; let prevValue = initialValue;
const binding = () => { const binding = () => {
const newValue = fn(this._value); const newValue = fn(this._value)+"";
if (prevValue !== newValue) { if (prevValue !== newValue) {
prevValue = newValue; prevValue = newValue;
node.textContent = newValue+""; node.textContent = newValue;
} }
}; };
@ -242,7 +245,7 @@ export class TemplateBuilder<T extends IObservableValue> {
} }
} }
_setNodeChildren(node: Element, children: Child | Child[]): void{ _setNodeChildren(node: Element, children: Children<T>): void{
if (!Array.isArray(children)) { if (!Array.isArray(children)) {
children = [children]; children = [children];
} }
@ -276,14 +279,18 @@ export class TemplateBuilder<T extends IObservableValue> {
return node; return node;
} }
el(name: string, attributes?: Attributes<T> | Child | Child[], children?: Child | Child[]): ViewNode { el(name: string, attributes?: Attributes<T> | Children<T>, children?: Children<T>): ViewNode {
return this.elNS(HTML_NS, name, attributes, children); return this.elNS(HTML_NS, name, attributes, children);
} }
elNS(ns: string, name: string, attributes?: Attributes<T> | Child | Child[], children?: Child | Child[]): ViewNode { elNS(ns: string, name: string, attributesOrChildren?: Attributes<T> | Children<T>, children?: Children<T>): ViewNode {
if (attributes !== undefined && isChildren(attributes)) { let attributes: Attributes<T> | undefined;
children = attributes; if (attributesOrChildren) {
attributes = undefined; if (isChildren(attributesOrChildren)) {
children = attributesOrChildren as Children<T>;
} else {
attributes = attributesOrChildren as Attributes<T>;
}
} }
const node = document.createElementNS(ns, name); const node = document.createElementNS(ns, name);
@ -330,7 +337,7 @@ export class TemplateBuilder<T extends IObservableValue> {
// Special case of mapView for a TemplateView. // Special case of mapView for a TemplateView.
// Always creates a TemplateView, if this is optional depending // Always creates a TemplateView, if this is optional depending
// on mappedValue, use `if` or `mapView` // on mappedValue, use `if` or `mapView`
map<R>(mapFn: (value: T) => R, renderFn: (mapped: R, t: Builder<T>, vm: T) => ViewNode): ViewNode { map<R>(mapFn: (value: T) => R, renderFn: (mapped: R, t: Builder<T>, vm: T) => ViewNode | undefined): ViewNode {
return this.mapView(mapFn, mappedValue => { return this.mapView(mapFn, mappedValue => {
return new InlineTemplateView(this._value, (t, vm) => { return new InlineTemplateView(this._value, (t, vm) => {
const rootNode = renderFn(mappedValue, t, vm); const rootNode = renderFn(mappedValue, t, vm);
@ -364,17 +371,17 @@ export class TemplateBuilder<T extends IObservableValue> {
event handlers, ... event handlers, ...
You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect, You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect,
instead use tags from html.ts to help you construct any DOM you need. */ instead use tags from html.ts to help you construct any DOM you need. */
mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined) => void) { mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined, value: T) => void) {
let prevValue = mapFn(this._value); let prevValue = mapFn(this._value);
const binding = () => { const binding = () => {
const newValue = mapFn(this._value); const newValue = mapFn(this._value);
if (prevValue !== newValue) { if (prevValue !== newValue) {
sideEffect(newValue, prevValue); sideEffect(newValue, prevValue, this._value);
prevValue = newValue; prevValue = newValue;
} }
}; };
this._addBinding(binding); this._addBinding(binding);
sideEffect(prevValue, undefined); sideEffect(prevValue, undefined, this._value);
} }
} }

View file

@ -14,36 +14,50 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
import {TemplateView, TemplateBuilder} from "../../general/TemplateView"; import {TemplateView, Builder} from "../../general/TemplateView";
import {AvatarView} from "../../AvatarView";
import {ListView} from "../../general/ListView"; import {ListView} from "../../general/ListView";
import {Stream} from "../../../../types/MediaDevices"; import {Stream} from "../../../../types/MediaDevices";
import type {StreamWrapper} from "../../../dom/MediaDevices"; import type {CallViewModel, CallMemberViewModel, IStreamViewModel} from "../../../../../domain/session/room/CallViewModel";
import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel";
function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) {
t.mapSideEffect(propSelector, stream => {
if (stream) {
video.srcObject = (stream as StreamWrapper).stream;
}
});
return video;
}
export class CallView extends TemplateView<CallViewModel> { export class CallView extends TemplateView<CallViewModel> {
render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement { render(t: Builder<CallViewModel>, vm: CallViewModel): Element {
return t.div({class: "CallView"}, [ return t.div({class: "CallView"}, [
t.p(vm => `Call ${vm.name} (${vm.id})`), t.p(vm => `Call ${vm.name} (${vm.id})`),
t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), t.view(new ListView({list: vm.memberViewModels}, vm => new StreamView(vm))),
t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))),
t.div({class: "buttons"}, [ t.div({class: "buttons"}, [
t.button({onClick: () => vm.leave()}, "Leave") t.button({onClick: () => vm.leave()}, "Leave"),
t.button({onClick: () => vm.toggleVideo()}, "Toggle video"),
]) ])
]); ]);
} }
} }
class MemberView extends TemplateView<CallMemberViewModel> { class StreamView extends TemplateView<IStreamViewModel> {
render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) { render(t: Builder<IStreamViewModel>, vm: IStreamViewModel): Element {
return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream); const video = t.video({
autoplay: true,
className: {
hidden: vm => vm.isCameraMuted
}
}) as HTMLVideoElement;
t.mapSideEffect(vm => vm.stream, stream => {
video.srcObject = stream as MediaStream;
});
return t.div({className: "StreamView"}, [
video,
t.div({className: {
StreamView_avatar: true,
hidden: vm => !vm.isCameraMuted
}}, t.view(new AvatarView(vm, 64), {parentProvidesUpdates: true})),
t.div({
className: {
StreamView_muteStatus: true,
hidden: vm => !vm.isCameraMuted && !vm.isMicrophoneMuted,
microphoneMuted: vm => vm.isMicrophoneMuted && !vm.isCameraMuted,
cameraMuted: vm => vm.isCameraMuted,
}
})
]);
} }
} }

View file

@ -1,52 +0,0 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
export class AsyncQueue<T, V> {
private isRunning = false;
private queue: T[] = [];
private error?: Error;
constructor(
private readonly reducer: (v: V, t: T) => Promise<V>,
private value: V,
private readonly contains: (t: T, queue: T[]) => boolean = (t, queue) => queue.includes(t)
) {}
push(t: T) {
if (this.contains(t, this.queue)) {
return;
}
this.queue.push(t);
this.runLoopIfNeeded();
}
private async runLoopIfNeeded() {
if (this.isRunning || this.error) {
return;
}
this.isRunning = true;
try {
let item: T | undefined;
while (item = this.queue.shift()) {
this.value = await this.reducer(this.value, item);
}
} catch (err) {
this.error = err;
} finally {
this.isRunning = false;
}
}
}

View file

@ -31,6 +31,7 @@ const commonOptions = {
assetsInlineLimit: 0, assetsInlineLimit: 0,
polyfillModulePreload: false, polyfillModulePreload: false,
}, },
assetsInclude: ['**/config.json'],
define: { define: {
DEFINE_VERSION: JSON.stringify(version), DEFINE_VERSION: JSON.stringify(version),
DEFINE_GLOBAL_HASH: JSON.stringify(null), DEFINE_GLOBAL_HASH: JSON.stringify(null),

View file

@ -14,6 +14,11 @@ export default defineConfig(({mode}) => {
outDir: "../../../target", outDir: "../../../target",
minify: true, minify: true,
sourcemap: true, sourcemap: true,
rollupOptions: {
output: {
assetFileNames: (asset) => asset.name.includes("config.json") ? "assets/[name][extname]": "assets/[name].[hash][extname]",
},
},
}, },
plugins: [ plugins: [
themeBuilder({ themeBuilder({