WIP for muting

This commit is contained in:
Bruno Windels 2022-04-14 23:19:44 +02:00
parent 468a0a9698
commit 382fba88bd
12 changed files with 210 additions and 163 deletions

View file

@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel<Options> {
this.call.leave(); this.call.leave();
} }
} }
async toggleVideo() {
const localMedia = this.call.localMedia!;
const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted);
await this.call.setMedia(toggledMedia);
}
} }
type MemberOptions = BaseOptions & {member: Member}; type MemberOptions = BaseOptions & {member: Member};

View file

@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes";
export class LocalMedia { export class LocalMedia {
constructor( constructor(
public readonly userMedia?: Stream, public readonly userMedia?: Stream,
public readonly microphoneMuted: boolean = false,
public readonly cameraMuted: boolean = false,
public readonly screenShare?: Stream, public readonly screenShare?: Stream,
public readonly dataChannelOptions?: RTCDataChannelInit, public readonly dataChannelOptions?: RTCDataChannelInit,
) {} ) {}
withMuted(microphone: boolean, camera: boolean) {
return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions);
}
withUserMedia(stream: Stream) { withUserMedia(stream: Stream) {
return new LocalMedia(stream, this.screenShare, this.dataChannelOptions); return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions);
} }
withScreenShare(stream: Stream) { withScreenShare(stream: Stream) {
return new LocalMedia(this.userMedia, stream, this.dataChannelOptions); return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions);
} }
withDataChannel(options: RTCDataChannelInit): LocalMedia { withDataChannel(options: RTCDataChannelInit): LocalMedia {
return new LocalMedia(this.userMedia, this.screenShare, options); return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options);
} }
clone(): LocalMedia { clone(): LocalMedia {
return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions); return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions);
} }
dispose() { dispose() {

View file

@ -31,6 +31,7 @@ import {
SDPStreamMetadataKey, SDPStreamMetadataKey,
SDPStreamMetadataPurpose, SDPStreamMetadataPurpose,
EventType, EventType,
CallErrorCode,
} from "./callEventTypes"; } from "./callEventTypes";
import type { import type {
MCallBase, MCallBase,
@ -66,6 +67,7 @@ export class PeerCall implements IDisposable {
private readonly peerConnection: PeerConnection; private readonly peerConnection: PeerConnection;
private _state = CallState.Fledgling; private _state = CallState.Fledgling;
private direction: CallDirection; private direction: CallDirection;
// we don't own localMedia and should hence not call dispose on it from here
private localMedia?: LocalMedia; private localMedia?: LocalMedia;
private seq: number = 0; private seq: number = 0;
// A queue for candidates waiting to go out. // A queue for candidates waiting to go out.
@ -151,7 +153,6 @@ export class PeerCall implements IDisposable {
get hangupReason(): CallErrorCode | undefined { return this._hangupReason; } get hangupReason(): CallErrorCode | undefined { return this._hangupReason; }
// we should keep an object with streams by purpose ... e.g. RemoteMedia?
get remoteMedia(): Readonly<RemoteMedia> { get remoteMedia(): Readonly<RemoteMedia> {
return this._remoteMedia; return this._remoteMedia;
} }
@ -163,7 +164,7 @@ export class PeerCall implements IDisposable {
} }
this.direction = CallDirection.Outbound; this.direction = CallDirection.Outbound;
this.setState(CallState.CreateOffer, log); this.setState(CallState.CreateOffer, log);
this.setMedia(localMedia); this.updateLocalMedia(localMedia, log);
if (this.localMedia?.dataChannelOptions) { if (this.localMedia?.dataChannelOptions) {
this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions); this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions);
} }
@ -179,7 +180,7 @@ export class PeerCall implements IDisposable {
return; return;
} }
this.setState(CallState.CreateAnswer, log); this.setState(CallState.CreateAnswer, log);
this.setMedia(localMedia, log); this.updateLocalMedia(localMedia, log);
let myAnswer: RTCSessionDescriptionInit; let myAnswer: RTCSessionDescriptionInit;
try { try {
myAnswer = await this.peerConnection.createAnswer(); myAnswer = await this.peerConnection.createAnswer();
@ -208,46 +209,30 @@ export class PeerCall implements IDisposable {
}); });
} }
setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise<void> { setMedia(localMedia: LocalMedia): Promise<void> {
return logItem.wrap("setMedia", async log => { return this.logItem.wrap("setMedia", async log => {
const oldMedia = this.localMedia; log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack);
this.localMedia = localMedia; log.set("userMedia_audio_muted", localMedia.microphoneMuted);
const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => { log.set("userMedia_video", !!localMedia.userMedia?.videoTrack);
const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined; log.set("userMedia_video_muted", localMedia.cameraMuted);
log.set("screenShare_video", !!localMedia.screenShare?.videoTrack);
log.set("datachannel", !!localMedia.dataChannelOptions);
const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => { const oldMetaData = this.getSDPMetadata();
if (track) { const willRenegotiate = await this.updateLocalMedia(localMedia, log);
if (oldTrack && sender) { if (!willRenegotiate) {
log.wrap(`replacing ${logLabel} ${track.kind} track`, log => { const newMetaData = this.getSDPMetadata();
sender.replaceTrack(track); if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) {
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
call_id: this.callId,
version: 1,
seq: this.seq++,
[SDPStreamMetadataKey]: newMetaData
};
await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log);
}
}
}); });
} else {
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
this.peerConnection.addTrack(track);
});
}
} else {
if (sender) {
log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => {
this.peerConnection.removeTrack(sender);
});
}
}
}
applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack);
applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack);
}
applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia");
applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare");
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
});
}
/** group calls would handle reject at the group call level, not at the peer call level */
async reject() {
} }
hangup(errorCode: CallErrorCode): Promise<void> { hangup(errorCode: CallErrorCode): Promise<void> {
@ -280,7 +265,14 @@ export class PeerCall implements IDisposable {
case EventType.Candidates: case EventType.Candidates:
await this.handleRemoteIceCandidates(message.content, partyId, log); await this.handleRemoteIceCandidates(message.content, partyId, log);
break; break;
case EventType.SDPStreamMetadataChanged:
case EventType.SDPStreamMetadataChangedPrefix:
this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log);
break;
case EventType.Hangup: case EventType.Hangup:
// TODO: this is a bit hacky, double check its what we need
this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log);
break;
default: default:
log.log(`Unknown event type for call: ${message.type}`); log.log(`Unknown event type for call: ${message.type}`);
break; break;
@ -444,12 +436,12 @@ export class PeerCall implements IDisposable {
// According to previous comments in this file, firefox at some point did not // According to previous comments in this file, firefox at some point did not
// add streams until media started arriving on them. Testing latest firefox // add streams until media started arriving on them. Testing latest firefox
// (81 at time of writing), this is no longer a problem, so let's do it the correct way. // (81 at time of writing), this is no longer a problem, so let's do it the correct way.
// if (this.peerConnection.remoteTracks.length === 0) { if (this.peerConnection.remoteStreams.size === 0) {
// await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => { await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => {
// return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log); return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
// }); });
// return; return;
// } }
this.setState(CallState.Ringing, log); this.setState(CallState.Ringing, log);
@ -719,7 +711,7 @@ export class PeerCall implements IDisposable {
// this will accumulate all updates into one object, so we still have the old stream info when we change stream id // this will accumulate all updates into one object, so we still have the old stream info when we change stream id
this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true); this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
this.updateRemoteMedia(log); this.updateRemoteMedia(log);
// TODO: apply muting
} }
private async addBufferedIceCandidates(log: ILogItem): Promise<void> { private async addBufferedIceCandidates(log: ILogItem): Promise<void> {
@ -828,10 +820,9 @@ export class PeerCall implements IDisposable {
const streamSender = this.peerConnection.localStreams.get(streamId); const streamSender = this.peerConnection.localStreams.get(streamId);
metadata[streamId] = { metadata[streamId] = {
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audio_muted: !(streamSender?.audioSender?.enabled), audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled),
video_muted: !(streamSender?.videoSender?.enabled), video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled),
}; };
console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata));
} }
if (this.localMedia?.screenShare) { if (this.localMedia?.screenShare) {
const streamId = this.localMedia.screenShare.id; const streamId = this.localMedia.screenShare.id;
@ -851,6 +842,8 @@ export class PeerCall implements IDisposable {
if (metaData) { if (metaData) {
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) { if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
this._remoteMedia.userMedia = streamReceiver.stream; this._remoteMedia.userMedia = streamReceiver.stream;
streamReceiver.audioReceiver?.enable(!metaData.audio_muted);
streamReceiver.videoReceiver?.enable(!metaData.video_muted);
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) { } else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
this._remoteMedia.screenShare = streamReceiver.stream; this._remoteMedia.screenShare = streamReceiver.stream;
} }
@ -860,6 +853,77 @@ export class PeerCall implements IDisposable {
this.options.emitUpdate(this, undefined); this.options.emitUpdate(this, undefined);
} }
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<boolean> {
return logItem.wrap("updateLocalMedia", async log => {
let willRenegotiate = false;
const oldMedia = this.localMedia;
this.localMedia = localMedia;
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => {
const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined;
const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => {
const changed = (!track && oldTrack) ||
(track && !oldTrack) ||
(track && oldTrack && !track.equals(oldTrack));
if (changed) {
if (track) {
if (oldTrack && sender && !track.equals(oldTrack)) {
try {
await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => {
return sender.replaceTrack(track);
});
} catch (err) {
// can't replace the track without renegotiating
log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => {
this.peerConnection.removeTrack(sender);
this.peerConnection.addTrack(track);
willRenegotiate = true;
});
}
} else {
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
this.peerConnection.addTrack(track);
willRenegotiate = true;
});
}
} else {
if (sender) {
// this will be used for muting, do we really want to trigger renegotiation here?
// we want to disable the sender, but also remove the track as we don't want to keep
// using the webcam if we don't need to
log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => {
sender.track.enabled = false;
this.peerConnection.removeTrack(sender);
willRenegotiate = true;
});
}
}
} else if (track) {
console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted});
if (sender && muted !== wasMuted) {
// TODO: why does unmuting not work? wasMuted is false
log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => {
sender.track.enabled = !muted;
sender.enable(!muted);
willRenegotiate = true;
});
} else {
log.log(`${logLabel} ${track.kind} track hasn't changed`);
}
}
}
await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted);
await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted);
}
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia");
applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare");
return willRenegotiate;
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
});
}
private async delay(timeoutMs: number): Promise<void> { private async delay(timeoutMs: number): Promise<void> {
// Allow a short time for initial candidates to be gathered // Allow a short time for initial candidates to be gathered
const timeout = this.disposables.track(this.options.createTimeout(timeoutMs)); const timeout = this.disposables.track(this.options.createTimeout(timeoutMs));
@ -915,100 +979,11 @@ export enum CallDirection {
Outbound = 'outbound', Outbound = 'outbound',
} }
export enum CallErrorCode {
/** The user chose to end the call */
UserHangup = 'user_hangup',
/** An error code when the local client failed to create an offer. */
LocalOfferFailed = 'local_offer_failed',
/**
* An error code when there is no local mic/camera to use. This may be because
* the hardware isn't plugged in, or the user has explicitly denied access.
*/
NoUserMedia = 'no_user_media',
/**
* Error code used when a call event failed to send
* because unknown devices were present in the room
*/
UnknownDevices = 'unknown_devices',
/**
* Error code used when we fail to send the invite
* for some reason other than there being unknown devices
*/
SendInvite = 'send_invite',
/**
* An answer could not be created
*/
CreateAnswer = 'create_answer',
/**
* Error code used when we fail to send the answer
* for some reason other than there being unknown devices
*/
SendAnswer = 'send_answer',
/**
* The session description from the other side could not be set
*/
SetRemoteDescription = 'set_remote_description',
/**
* The session description from this side could not be set
*/
SetLocalDescription = 'set_local_description',
/**
* A different device answered the call
*/
AnsweredElsewhere = 'answered_elsewhere',
/**
* No media connection could be established to the other party
*/
IceFailed = 'ice_failed',
/**
* The invite timed out whilst waiting for an answer
*/
InviteTimeout = 'invite_timeout',
/**
* The call was replaced by another call
*/
Replaced = 'replaced',
/**
* Signalling for the call could not be sent (other than the initial invite)
*/
SignallingFailed = 'signalling_timeout',
/**
* The remote party is busy
*/
UserBusy = 'user_busy',
/**
* We transferred the call off to somewhere else
*/
Transfered = 'transferred',
/**
* A call from the same user was found with a new session id
*/
NewSession = 'new_session',
}
/** /**
* The version field that we set in m.call.* events * The version field that we set in m.call.* events
*/ */
const VOIP_PROTO_VERSION = 1; const VOIP_PROTO_VERSION = 1;
/** The fallback ICE server to use for STUN or TURN protocols. */
const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org';
/** The length of time a call can be ringing for. */ /** The length of time a call can be ringing for. */
const CALL_TIMEOUT_MS = 60000; const CALL_TIMEOUT_MS = 60000;
@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean {
return eventType === EventType.Invite || return eventType === EventType.Invite ||
eventType === EventType.Candidates || eventType === EventType.Candidates ||
eventType === EventType.Answer || eventType === EventType.Answer ||
eventType === EventType.Hangup; eventType === EventType.Hangup ||
eventType === EventType.SDPStreamMetadataChanged ||
eventType === EventType.SDPStreamMetadataChangedPrefix ||
eventType === EventType.Negotiate;
} }
export function tests() { export function tests() {

View file

@ -15,7 +15,7 @@
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid). - call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
- implement to_device messages arriving before m.call(.member) state event - implement to_device messages arriving before m.call(.member) state event
- implement muting tracks with m.call.sdp_stream_metadata_changed - implement muting tracks with m.call.sdp_stream_metadata_changed
- implement cloning the localMedia so it works in safari? - DONE: implement cloning the localMedia so it works in safari?
- DONE: implement 3 retries per peer - DONE: implement 3 retries per peer
- reeable crypto & implement fetching olm keys before sending encrypted signalling message - reeable crypto & implement fetching olm keys before sending encrypted signalling message
- local echo for join/leave buttons? - local echo for join/leave buttons?

View file

@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> {
}); });
} }
async setMedia(localMedia: LocalMedia): Promise<void> {
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
const oldMedia = this._localMedia;
this._localMedia = localMedia;
await Promise.all(Array.from(this._members.values()).map(m => {
return m.setMedia(localMedia!.clone());
}));
oldMedia?.dispose();
}
}
get hasJoined() { get hasJoined() {
return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined; return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined;
} }

View file

@ -182,6 +182,14 @@ export class Member {
} }
} }
/** @internal */
async setMedia(localMedia: LocalMedia): Promise<void> {
const oldMedia = this.localMedia;
this.localMedia = localMedia;
await this.peerCall?.setMedia(localMedia);
oldMedia?.dispose();
}
private _createPeerCall(callId: string): PeerCall { private _createPeerCall(callId: string): PeerCall {
return new PeerCall(callId, Object.assign({}, this.options, { return new PeerCall(callId, Object.assign({}, this.options, {
emitUpdate: this.emitUpdate, emitUpdate: this.emitUpdate,

View file

@ -39,6 +39,9 @@ export interface Track {
readonly label: string; readonly label: string;
readonly id: string; readonly id: string;
readonly settings: MediaTrackSettings; readonly settings: MediaTrackSettings;
get enabled(): boolean;
set enabled(value: boolean);
equals(track: Track): boolean;
stop(): void; stop(): void;
} }

View file

@ -75,27 +75,37 @@ export class StreamWrapper implements Stream {
public audioTrack: AudioTrackWrapper | undefined = undefined; public audioTrack: AudioTrackWrapper | undefined = undefined;
public videoTrack: TrackWrapper | undefined = undefined; public videoTrack: TrackWrapper | undefined = undefined;
constructor(public readonly stream: MediaStream) { constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) {
if (clonedTracks) {
this.audioTrack = clonedTracks.audioTrack;
this.videoTrack = clonedTracks.videoTrack;
} else {
for (const track of stream.getTracks()) { for (const track of stream.getTracks()) {
this.update(track); this.update(track);
} }
} }
}
get id(): string { return this.stream.id; } get id(): string { return this.stream.id; }
clone(): Stream { clone(): Stream {
return new StreamWrapper(this.stream.clone()); const clonedStream = this.stream.clone();
const clonedTracks = {
audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined,
videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined,
};
return new StreamWrapper(clonedStream, clonedTracks);
} }
update(track: MediaStreamTrack): TrackWrapper | undefined { update(track: MediaStreamTrack): TrackWrapper | undefined {
if (track.kind === "video") { if (track.kind === "video") {
if (!this.videoTrack || track.id !== this.videoTrack.track.id) { if (!this.videoTrack || track.id !== this.videoTrack.track.id) {
this.videoTrack = new TrackWrapper(track, this.stream); this.videoTrack = new TrackWrapper(track, this.stream, track.id);
} }
return this.videoTrack; return this.videoTrack;
} else if (track.kind === "audio") { } else if (track.kind === "audio") {
if (!this.audioTrack || track.id !== this.audioTrack.track.id) { if (!this.audioTrack || track.id !== this.audioTrack.track.id) {
this.audioTrack = new AudioTrackWrapper(track, this.stream); this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id);
} }
return this.audioTrack; return this.audioTrack;
} }
@ -105,14 +115,18 @@ export class StreamWrapper implements Stream {
export class TrackWrapper implements Track { export class TrackWrapper implements Track {
constructor( constructor(
public readonly track: MediaStreamTrack, public readonly track: MediaStreamTrack,
public readonly stream: MediaStream public readonly stream: MediaStream,
public readonly originalId: string,
) {} ) {}
get kind(): TrackKind { return this.track.kind as TrackKind; } get kind(): TrackKind { return this.track.kind as TrackKind; }
get label(): string { return this.track.label; } get label(): string { return this.track.label; }
get id(): string { return this.track.id; } get id(): string { return this.track.id; }
get settings(): MediaTrackSettings { return this.track.getSettings(); } get settings(): MediaTrackSettings { return this.track.getSettings(); }
get enabled(): boolean { return this.track.enabled; }
set enabled(enabled: boolean) { this.track.enabled = enabled; }
// test equality across clones
equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; }
stop() { this.track.stop(); } stop() { this.track.stop(); }
} }
@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper {
private volumeLooperTimeout: number; private volumeLooperTimeout: number;
private speakingVolumeSamples: number[]; private speakingVolumeSamples: number[];
constructor(track: MediaStreamTrack, stream: MediaStream) { constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) {
super(track, stream); super(track, stream, originalId);
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
this.initVolumeMeasuring(); this.initVolumeMeasuring();
this.measureVolumeActivity(true); this.measureVolumeActivity(true);

View file

@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver {
this.transceiver.direction === this.exclusiveValue; this.transceiver.direction === this.exclusiveValue;
} }
enableWithoutRenegotiation(enabled: boolean) {
this.track.track.enabled = enabled;
}
enable(enabled: boolean) { enable(enabled: boolean) {
if (enabled !== this.enabled) { if (enabled !== this.enabled) {
// do this first, so we stop sending track data immediately.
// this will still consume bandwidth though, so also disable the transceiver,
// which will trigger a renegotiation though.
this.enableWithoutRenegotiation(enabled);
if (enabled) { if (enabled) {
if (this.transceiver.direction === "inactive") { if (this.transceiver.direction === "inactive") {
this.transceiver.direction = this.exclusiveValue; this.transceiver.direction = this.exclusiveValue;

View file

@ -11,7 +11,7 @@
<meta name="description" content="A matrix chat application"> <meta name="description" content="A matrix chat application">
<link rel="apple-touch-icon" href="./assets/icon-maskable.png"> <link rel="apple-touch-icon" href="./assets/icon-maskable.png">
<link rel="icon" type="image/png" href="assets/icon-maskable.png"> <link rel="icon" type="image/png" href="assets/icon-maskable.png">
<script type="module"> import "@theme/default"; </script> <script type="module"> import "@theme/default/dark"; </script>
</head> </head>
<body class="hydrogen"> <body class="hydrogen">
<script id="main" type="module"> <script id="main" type="module">

View file

@ -364,17 +364,17 @@ export class TemplateBuilder<T extends IObservableValue> {
event handlers, ... event handlers, ...
You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect, You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect,
instead use tags from html.ts to help you construct any DOM you need. */ instead use tags from html.ts to help you construct any DOM you need. */
mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined) => void) { mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined, value: T) => void) {
let prevValue = mapFn(this._value); let prevValue = mapFn(this._value);
const binding = () => { const binding = () => {
const newValue = mapFn(this._value); const newValue = mapFn(this._value);
if (prevValue !== newValue) { if (prevValue !== newValue) {
sideEffect(newValue, prevValue); sideEffect(newValue, prevValue, this._value);
prevValue = newValue; prevValue = newValue;
} }
}; };
this._addBinding(binding); this._addBinding(binding);
sideEffect(prevValue, undefined); sideEffect(prevValue, undefined, this._value);
} }
} }

View file

@ -20,10 +20,18 @@ import {Stream} from "../../../../types/MediaDevices";
import type {StreamWrapper} from "../../../dom/MediaDevices"; import type {StreamWrapper} from "../../../dom/MediaDevices";
import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel"; import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel";
function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) { function bindStream<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) {
t.mapSideEffect(propSelector, stream => { t.mapSideEffect(vm => propSelector(vm)?.videoTrack?.enabled, (_,__, vm) => {
const stream = propSelector(vm);
if (stream) { if (stream) {
video.srcObject = (stream as StreamWrapper).stream; video.srcObject = (stream as StreamWrapper).stream;
if (stream.videoTrack?.enabled) {
video.classList.remove("hidden");
} else {
video.classList.add("hidden");
}
} else {
video.classList.add("hidden");
} }
}); });
return video; return video;
@ -33,10 +41,11 @@ export class CallView extends TemplateView<CallViewModel> {
render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement { render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement {
return t.div({class: "CallView"}, [ return t.div({class: "CallView"}, [
t.p(vm => `Call ${vm.name} (${vm.id})`), t.p(vm => `Call ${vm.name} (${vm.id})`),
t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)), t.div({class: "CallView_me"}, bindStream(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)),
t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))), t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))),
t.div({class: "buttons"}, [ t.div({class: "buttons"}, [
t.button({onClick: () => vm.leave()}, "Leave") t.button({onClick: () => vm.leave()}, "Leave"),
t.button({onClick: () => vm.toggleVideo()}, "Toggle video"),
]) ])
]); ]);
} }
@ -44,6 +53,10 @@ export class CallView extends TemplateView<CallViewModel> {
class MemberView extends TemplateView<CallMemberViewModel> { class MemberView extends TemplateView<CallMemberViewModel> {
render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) { render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) {
return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream); return bindStream(t, t.video({autoplay: true, width: 360}), vm => vm.stream);
} }
} }
// class StreamView extends TemplateView<Stream> {
// render(t: TemplateBuilder<Stream)
// }