forked from mystiq/hydrogen-web
WIP for muting
This commit is contained in:
parent
468a0a9698
commit
382fba88bd
12 changed files with 210 additions and 163 deletions
|
@ -55,6 +55,12 @@ export class CallViewModel extends ViewModel<Options> {
|
||||||
this.call.leave();
|
this.call.leave();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async toggleVideo() {
|
||||||
|
const localMedia = this.call.localMedia!;
|
||||||
|
const toggledMedia = localMedia.withMuted(localMedia.microphoneMuted, !localMedia.cameraMuted);
|
||||||
|
await this.call.setMedia(toggledMedia);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type MemberOptions = BaseOptions & {member: Member};
|
type MemberOptions = BaseOptions & {member: Member};
|
||||||
|
|
|
@ -21,24 +21,30 @@ import {SDPStreamMetadata} from "./callEventTypes";
|
||||||
export class LocalMedia {
|
export class LocalMedia {
|
||||||
constructor(
|
constructor(
|
||||||
public readonly userMedia?: Stream,
|
public readonly userMedia?: Stream,
|
||||||
|
public readonly microphoneMuted: boolean = false,
|
||||||
|
public readonly cameraMuted: boolean = false,
|
||||||
public readonly screenShare?: Stream,
|
public readonly screenShare?: Stream,
|
||||||
public readonly dataChannelOptions?: RTCDataChannelInit,
|
public readonly dataChannelOptions?: RTCDataChannelInit,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
withMuted(microphone: boolean, camera: boolean) {
|
||||||
|
return new LocalMedia(this.userMedia, microphone, camera, this.screenShare, this.dataChannelOptions);
|
||||||
|
}
|
||||||
|
|
||||||
withUserMedia(stream: Stream) {
|
withUserMedia(stream: Stream) {
|
||||||
return new LocalMedia(stream, this.screenShare, this.dataChannelOptions);
|
return new LocalMedia(stream, this.microphoneMuted, this.cameraMuted, this.screenShare, this.dataChannelOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
withScreenShare(stream: Stream) {
|
withScreenShare(stream: Stream) {
|
||||||
return new LocalMedia(this.userMedia, stream, this.dataChannelOptions);
|
return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, stream, this.dataChannelOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
withDataChannel(options: RTCDataChannelInit): LocalMedia {
|
withDataChannel(options: RTCDataChannelInit): LocalMedia {
|
||||||
return new LocalMedia(this.userMedia, this.screenShare, options);
|
return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
clone(): LocalMedia {
|
clone(): LocalMedia {
|
||||||
return new LocalMedia(this.userMedia?.clone(), this.screenShare?.clone(), this.dataChannelOptions);
|
return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
dispose() {
|
dispose() {
|
||||||
|
|
|
@ -31,6 +31,7 @@ import {
|
||||||
SDPStreamMetadataKey,
|
SDPStreamMetadataKey,
|
||||||
SDPStreamMetadataPurpose,
|
SDPStreamMetadataPurpose,
|
||||||
EventType,
|
EventType,
|
||||||
|
CallErrorCode,
|
||||||
} from "./callEventTypes";
|
} from "./callEventTypes";
|
||||||
import type {
|
import type {
|
||||||
MCallBase,
|
MCallBase,
|
||||||
|
@ -66,6 +67,7 @@ export class PeerCall implements IDisposable {
|
||||||
private readonly peerConnection: PeerConnection;
|
private readonly peerConnection: PeerConnection;
|
||||||
private _state = CallState.Fledgling;
|
private _state = CallState.Fledgling;
|
||||||
private direction: CallDirection;
|
private direction: CallDirection;
|
||||||
|
// we don't own localMedia and should hence not call dispose on it from here
|
||||||
private localMedia?: LocalMedia;
|
private localMedia?: LocalMedia;
|
||||||
private seq: number = 0;
|
private seq: number = 0;
|
||||||
// A queue for candidates waiting to go out.
|
// A queue for candidates waiting to go out.
|
||||||
|
@ -151,7 +153,6 @@ export class PeerCall implements IDisposable {
|
||||||
|
|
||||||
get hangupReason(): CallErrorCode | undefined { return this._hangupReason; }
|
get hangupReason(): CallErrorCode | undefined { return this._hangupReason; }
|
||||||
|
|
||||||
// we should keep an object with streams by purpose ... e.g. RemoteMedia?
|
|
||||||
get remoteMedia(): Readonly<RemoteMedia> {
|
get remoteMedia(): Readonly<RemoteMedia> {
|
||||||
return this._remoteMedia;
|
return this._remoteMedia;
|
||||||
}
|
}
|
||||||
|
@ -163,7 +164,7 @@ export class PeerCall implements IDisposable {
|
||||||
}
|
}
|
||||||
this.direction = CallDirection.Outbound;
|
this.direction = CallDirection.Outbound;
|
||||||
this.setState(CallState.CreateOffer, log);
|
this.setState(CallState.CreateOffer, log);
|
||||||
this.setMedia(localMedia);
|
this.updateLocalMedia(localMedia, log);
|
||||||
if (this.localMedia?.dataChannelOptions) {
|
if (this.localMedia?.dataChannelOptions) {
|
||||||
this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions);
|
this._dataChannel = this.peerConnection.createDataChannel(this.localMedia.dataChannelOptions);
|
||||||
}
|
}
|
||||||
|
@ -179,7 +180,7 @@ export class PeerCall implements IDisposable {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.setState(CallState.CreateAnswer, log);
|
this.setState(CallState.CreateAnswer, log);
|
||||||
this.setMedia(localMedia, log);
|
this.updateLocalMedia(localMedia, log);
|
||||||
let myAnswer: RTCSessionDescriptionInit;
|
let myAnswer: RTCSessionDescriptionInit;
|
||||||
try {
|
try {
|
||||||
myAnswer = await this.peerConnection.createAnswer();
|
myAnswer = await this.peerConnection.createAnswer();
|
||||||
|
@ -208,46 +209,30 @@ export class PeerCall implements IDisposable {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
setMedia(localMedia: LocalMedia, logItem: ILogItem = this.logItem): Promise<void> {
|
setMedia(localMedia: LocalMedia): Promise<void> {
|
||||||
return logItem.wrap("setMedia", async log => {
|
return this.logItem.wrap("setMedia", async log => {
|
||||||
const oldMedia = this.localMedia;
|
log.set("userMedia_audio", !!localMedia.userMedia?.audioTrack);
|
||||||
this.localMedia = localMedia;
|
log.set("userMedia_audio_muted", localMedia.microphoneMuted);
|
||||||
const applyStream = (oldStream: Stream | undefined, stream: Stream | undefined, logLabel: string) => {
|
log.set("userMedia_video", !!localMedia.userMedia?.videoTrack);
|
||||||
const streamSender = oldMedia ? this.peerConnection.localStreams.get(oldStream!.id) : undefined;
|
log.set("userMedia_video_muted", localMedia.cameraMuted);
|
||||||
|
log.set("screenShare_video", !!localMedia.screenShare?.videoTrack);
|
||||||
|
log.set("datachannel", !!localMedia.dataChannelOptions);
|
||||||
|
|
||||||
const applyTrack = (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined) => {
|
const oldMetaData = this.getSDPMetadata();
|
||||||
if (track) {
|
const willRenegotiate = await this.updateLocalMedia(localMedia, log);
|
||||||
if (oldTrack && sender) {
|
if (!willRenegotiate) {
|
||||||
log.wrap(`replacing ${logLabel} ${track.kind} track`, log => {
|
const newMetaData = this.getSDPMetadata();
|
||||||
sender.replaceTrack(track);
|
if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) {
|
||||||
|
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
|
||||||
|
call_id: this.callId,
|
||||||
|
version: 1,
|
||||||
|
seq: this.seq++,
|
||||||
|
[SDPStreamMetadataKey]: newMetaData
|
||||||
|
};
|
||||||
|
await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChanged, content}, log);
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
} else {
|
|
||||||
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
|
|
||||||
this.peerConnection.addTrack(track);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (sender) {
|
|
||||||
log.wrap(`replacing ${logLabel} ${sender.track.kind} track`, log => {
|
|
||||||
this.peerConnection.removeTrack(sender);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack);
|
|
||||||
applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack);
|
|
||||||
}
|
|
||||||
|
|
||||||
applyStream(oldMedia?.userMedia, localMedia?.userMedia, "userMedia");
|
|
||||||
applyStream(oldMedia?.screenShare, localMedia?.screenShare, "screenShare");
|
|
||||||
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** group calls would handle reject at the group call level, not at the peer call level */
|
|
||||||
async reject() {
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
hangup(errorCode: CallErrorCode): Promise<void> {
|
hangup(errorCode: CallErrorCode): Promise<void> {
|
||||||
|
@ -280,7 +265,14 @@ export class PeerCall implements IDisposable {
|
||||||
case EventType.Candidates:
|
case EventType.Candidates:
|
||||||
await this.handleRemoteIceCandidates(message.content, partyId, log);
|
await this.handleRemoteIceCandidates(message.content, partyId, log);
|
||||||
break;
|
break;
|
||||||
|
case EventType.SDPStreamMetadataChanged:
|
||||||
|
case EventType.SDPStreamMetadataChangedPrefix:
|
||||||
|
this.updateRemoteSDPStreamMetadata(message.content[SDPStreamMetadataKey], log);
|
||||||
|
break;
|
||||||
case EventType.Hangup:
|
case EventType.Hangup:
|
||||||
|
// TODO: this is a bit hacky, double check its what we need
|
||||||
|
this.terminate(CallParty.Remote, message.content.reason ?? CallErrorCode.UserHangup, log);
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
log.log(`Unknown event type for call: ${message.type}`);
|
log.log(`Unknown event type for call: ${message.type}`);
|
||||||
break;
|
break;
|
||||||
|
@ -444,12 +436,12 @@ export class PeerCall implements IDisposable {
|
||||||
// According to previous comments in this file, firefox at some point did not
|
// According to previous comments in this file, firefox at some point did not
|
||||||
// add streams until media started arriving on them. Testing latest firefox
|
// add streams until media started arriving on them. Testing latest firefox
|
||||||
// (81 at time of writing), this is no longer a problem, so let's do it the correct way.
|
// (81 at time of writing), this is no longer a problem, so let's do it the correct way.
|
||||||
// if (this.peerConnection.remoteTracks.length === 0) {
|
if (this.peerConnection.remoteStreams.size === 0) {
|
||||||
// await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => {
|
await log.wrap(`Call no remote stream or no tracks after setting remote description!`, async log => {
|
||||||
// return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
|
return this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
|
||||||
// });
|
});
|
||||||
// return;
|
return;
|
||||||
// }
|
}
|
||||||
|
|
||||||
this.setState(CallState.Ringing, log);
|
this.setState(CallState.Ringing, log);
|
||||||
|
|
||||||
|
@ -719,7 +711,7 @@ export class PeerCall implements IDisposable {
|
||||||
// this will accumulate all updates into one object, so we still have the old stream info when we change stream id
|
// this will accumulate all updates into one object, so we still have the old stream info when we change stream id
|
||||||
this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
|
this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
|
||||||
this.updateRemoteMedia(log);
|
this.updateRemoteMedia(log);
|
||||||
// TODO: apply muting
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async addBufferedIceCandidates(log: ILogItem): Promise<void> {
|
private async addBufferedIceCandidates(log: ILogItem): Promise<void> {
|
||||||
|
@ -828,10 +820,9 @@ export class PeerCall implements IDisposable {
|
||||||
const streamSender = this.peerConnection.localStreams.get(streamId);
|
const streamSender = this.peerConnection.localStreams.get(streamId);
|
||||||
metadata[streamId] = {
|
metadata[streamId] = {
|
||||||
purpose: SDPStreamMetadataPurpose.Usermedia,
|
purpose: SDPStreamMetadataPurpose.Usermedia,
|
||||||
audio_muted: !(streamSender?.audioSender?.enabled),
|
audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled),
|
||||||
video_muted: !(streamSender?.videoSender?.enabled),
|
video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled),
|
||||||
};
|
};
|
||||||
console.log("video_muted", streamSender?.videoSender?.enabled, streamSender?.videoSender?.transceiver?.direction, streamSender?.videoSender?.transceiver?.currentDirection, JSON.stringify(metadata));
|
|
||||||
}
|
}
|
||||||
if (this.localMedia?.screenShare) {
|
if (this.localMedia?.screenShare) {
|
||||||
const streamId = this.localMedia.screenShare.id;
|
const streamId = this.localMedia.screenShare.id;
|
||||||
|
@ -851,6 +842,8 @@ export class PeerCall implements IDisposable {
|
||||||
if (metaData) {
|
if (metaData) {
|
||||||
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
|
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
|
||||||
this._remoteMedia.userMedia = streamReceiver.stream;
|
this._remoteMedia.userMedia = streamReceiver.stream;
|
||||||
|
streamReceiver.audioReceiver?.enable(!metaData.audio_muted);
|
||||||
|
streamReceiver.videoReceiver?.enable(!metaData.video_muted);
|
||||||
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
|
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
|
||||||
this._remoteMedia.screenShare = streamReceiver.stream;
|
this._remoteMedia.screenShare = streamReceiver.stream;
|
||||||
}
|
}
|
||||||
|
@ -860,6 +853,77 @@ export class PeerCall implements IDisposable {
|
||||||
this.options.emitUpdate(this, undefined);
|
this.options.emitUpdate(this, undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<boolean> {
|
||||||
|
return logItem.wrap("updateLocalMedia", async log => {
|
||||||
|
let willRenegotiate = false;
|
||||||
|
const oldMedia = this.localMedia;
|
||||||
|
this.localMedia = localMedia;
|
||||||
|
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => {
|
||||||
|
const streamSender = oldStream ? this.peerConnection.localStreams.get(oldStream.id) : undefined;
|
||||||
|
|
||||||
|
const applyTrack = async (oldTrack: Track | undefined, sender: TrackSender | undefined, track: Track | undefined, wasMuted: boolean | undefined, muted: boolean | undefined) => {
|
||||||
|
const changed = (!track && oldTrack) ||
|
||||||
|
(track && !oldTrack) ||
|
||||||
|
(track && oldTrack && !track.equals(oldTrack));
|
||||||
|
if (changed) {
|
||||||
|
if (track) {
|
||||||
|
if (oldTrack && sender && !track.equals(oldTrack)) {
|
||||||
|
try {
|
||||||
|
await log.wrap(`replacing ${logLabel} ${track.kind} track`, log => {
|
||||||
|
return sender.replaceTrack(track);
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
// can't replace the track without renegotiating
|
||||||
|
log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => {
|
||||||
|
this.peerConnection.removeTrack(sender);
|
||||||
|
this.peerConnection.addTrack(track);
|
||||||
|
willRenegotiate = true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
|
||||||
|
this.peerConnection.addTrack(track);
|
||||||
|
willRenegotiate = true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (sender) {
|
||||||
|
// this will be used for muting, do we really want to trigger renegotiation here?
|
||||||
|
// we want to disable the sender, but also remove the track as we don't want to keep
|
||||||
|
// using the webcam if we don't need to
|
||||||
|
log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => {
|
||||||
|
sender.track.enabled = false;
|
||||||
|
this.peerConnection.removeTrack(sender);
|
||||||
|
willRenegotiate = true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (track) {
|
||||||
|
console.log({muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted});
|
||||||
|
if (sender && muted !== wasMuted) {
|
||||||
|
// TODO: why does unmuting not work? wasMuted is false
|
||||||
|
log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => {
|
||||||
|
sender.track.enabled = !muted;
|
||||||
|
sender.enable(!muted);
|
||||||
|
willRenegotiate = true;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
log.log(`${logLabel} ${track.kind} track hasn't changed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await applyTrack(oldStream?.audioTrack, streamSender?.audioSender, stream?.audioTrack, oldMuteSettings?.microphoneMuted, mutedSettings?.microphoneMuted);
|
||||||
|
await applyTrack(oldStream?.videoTrack, streamSender?.videoSender, stream?.videoTrack, oldMuteSettings?.cameraMuted, mutedSettings?.cameraMuted);
|
||||||
|
}
|
||||||
|
|
||||||
|
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia");
|
||||||
|
applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare");
|
||||||
|
return willRenegotiate;
|
||||||
|
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private async delay(timeoutMs: number): Promise<void> {
|
private async delay(timeoutMs: number): Promise<void> {
|
||||||
// Allow a short time for initial candidates to be gathered
|
// Allow a short time for initial candidates to be gathered
|
||||||
const timeout = this.disposables.track(this.options.createTimeout(timeoutMs));
|
const timeout = this.disposables.track(this.options.createTimeout(timeoutMs));
|
||||||
|
@ -915,100 +979,11 @@ export enum CallDirection {
|
||||||
Outbound = 'outbound',
|
Outbound = 'outbound',
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum CallErrorCode {
|
|
||||||
/** The user chose to end the call */
|
|
||||||
UserHangup = 'user_hangup',
|
|
||||||
|
|
||||||
/** An error code when the local client failed to create an offer. */
|
|
||||||
LocalOfferFailed = 'local_offer_failed',
|
|
||||||
/**
|
|
||||||
* An error code when there is no local mic/camera to use. This may be because
|
|
||||||
* the hardware isn't plugged in, or the user has explicitly denied access.
|
|
||||||
*/
|
|
||||||
NoUserMedia = 'no_user_media',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error code used when a call event failed to send
|
|
||||||
* because unknown devices were present in the room
|
|
||||||
*/
|
|
||||||
UnknownDevices = 'unknown_devices',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error code used when we fail to send the invite
|
|
||||||
* for some reason other than there being unknown devices
|
|
||||||
*/
|
|
||||||
SendInvite = 'send_invite',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An answer could not be created
|
|
||||||
*/
|
|
||||||
CreateAnswer = 'create_answer',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Error code used when we fail to send the answer
|
|
||||||
* for some reason other than there being unknown devices
|
|
||||||
*/
|
|
||||||
SendAnswer = 'send_answer',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The session description from the other side could not be set
|
|
||||||
*/
|
|
||||||
SetRemoteDescription = 'set_remote_description',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The session description from this side could not be set
|
|
||||||
*/
|
|
||||||
SetLocalDescription = 'set_local_description',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A different device answered the call
|
|
||||||
*/
|
|
||||||
AnsweredElsewhere = 'answered_elsewhere',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* No media connection could be established to the other party
|
|
||||||
*/
|
|
||||||
IceFailed = 'ice_failed',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The invite timed out whilst waiting for an answer
|
|
||||||
*/
|
|
||||||
InviteTimeout = 'invite_timeout',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The call was replaced by another call
|
|
||||||
*/
|
|
||||||
Replaced = 'replaced',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Signalling for the call could not be sent (other than the initial invite)
|
|
||||||
*/
|
|
||||||
SignallingFailed = 'signalling_timeout',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The remote party is busy
|
|
||||||
*/
|
|
||||||
UserBusy = 'user_busy',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* We transferred the call off to somewhere else
|
|
||||||
*/
|
|
||||||
Transfered = 'transferred',
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A call from the same user was found with a new session id
|
|
||||||
*/
|
|
||||||
NewSession = 'new_session',
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The version field that we set in m.call.* events
|
* The version field that we set in m.call.* events
|
||||||
*/
|
*/
|
||||||
const VOIP_PROTO_VERSION = 1;
|
const VOIP_PROTO_VERSION = 1;
|
||||||
|
|
||||||
/** The fallback ICE server to use for STUN or TURN protocols. */
|
|
||||||
const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org';
|
|
||||||
|
|
||||||
/** The length of time a call can be ringing for. */
|
/** The length of time a call can be ringing for. */
|
||||||
const CALL_TIMEOUT_MS = 60000;
|
const CALL_TIMEOUT_MS = 60000;
|
||||||
|
|
||||||
|
@ -1029,7 +1004,10 @@ export function handlesEventType(eventType: string): boolean {
|
||||||
return eventType === EventType.Invite ||
|
return eventType === EventType.Invite ||
|
||||||
eventType === EventType.Candidates ||
|
eventType === EventType.Candidates ||
|
||||||
eventType === EventType.Answer ||
|
eventType === EventType.Answer ||
|
||||||
eventType === EventType.Hangup;
|
eventType === EventType.Hangup ||
|
||||||
|
eventType === EventType.SDPStreamMetadataChanged ||
|
||||||
|
eventType === EventType.SDPStreamMetadataChangedPrefix ||
|
||||||
|
eventType === EventType.Negotiate;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function tests() {
|
export function tests() {
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
|
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
|
||||||
- implement to_device messages arriving before m.call(.member) state event
|
- implement to_device messages arriving before m.call(.member) state event
|
||||||
- implement muting tracks with m.call.sdp_stream_metadata_changed
|
- implement muting tracks with m.call.sdp_stream_metadata_changed
|
||||||
- implement cloning the localMedia so it works in safari?
|
- DONE: implement cloning the localMedia so it works in safari?
|
||||||
- DONE: implement 3 retries per peer
|
- DONE: implement 3 retries per peer
|
||||||
- reeable crypto & implement fetching olm keys before sending encrypted signalling message
|
- reeable crypto & implement fetching olm keys before sending encrypted signalling message
|
||||||
- local echo for join/leave buttons?
|
- local echo for join/leave buttons?
|
||||||
|
|
|
@ -123,6 +123,17 @@ export class GroupCall extends EventEmitter<{change: never}> {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async setMedia(localMedia: LocalMedia): Promise<void> {
|
||||||
|
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
|
||||||
|
const oldMedia = this._localMedia;
|
||||||
|
this._localMedia = localMedia;
|
||||||
|
await Promise.all(Array.from(this._members.values()).map(m => {
|
||||||
|
return m.setMedia(localMedia!.clone());
|
||||||
|
}));
|
||||||
|
oldMedia?.dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
get hasJoined() {
|
get hasJoined() {
|
||||||
return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined;
|
return this._state === GroupCallState.Joining || this._state === GroupCallState.Joined;
|
||||||
}
|
}
|
||||||
|
|
|
@ -182,6 +182,14 @@ export class Member {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @internal */
|
||||||
|
async setMedia(localMedia: LocalMedia): Promise<void> {
|
||||||
|
const oldMedia = this.localMedia;
|
||||||
|
this.localMedia = localMedia;
|
||||||
|
await this.peerCall?.setMedia(localMedia);
|
||||||
|
oldMedia?.dispose();
|
||||||
|
}
|
||||||
|
|
||||||
private _createPeerCall(callId: string): PeerCall {
|
private _createPeerCall(callId: string): PeerCall {
|
||||||
return new PeerCall(callId, Object.assign({}, this.options, {
|
return new PeerCall(callId, Object.assign({}, this.options, {
|
||||||
emitUpdate: this.emitUpdate,
|
emitUpdate: this.emitUpdate,
|
||||||
|
|
|
@ -39,6 +39,9 @@ export interface Track {
|
||||||
readonly label: string;
|
readonly label: string;
|
||||||
readonly id: string;
|
readonly id: string;
|
||||||
readonly settings: MediaTrackSettings;
|
readonly settings: MediaTrackSettings;
|
||||||
|
get enabled(): boolean;
|
||||||
|
set enabled(value: boolean);
|
||||||
|
equals(track: Track): boolean;
|
||||||
stop(): void;
|
stop(): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -75,27 +75,37 @@ export class StreamWrapper implements Stream {
|
||||||
public audioTrack: AudioTrackWrapper | undefined = undefined;
|
public audioTrack: AudioTrackWrapper | undefined = undefined;
|
||||||
public videoTrack: TrackWrapper | undefined = undefined;
|
public videoTrack: TrackWrapper | undefined = undefined;
|
||||||
|
|
||||||
constructor(public readonly stream: MediaStream) {
|
constructor(public readonly stream: MediaStream, clonedTracks?: {audioTrack?: AudioTrackWrapper, videoTrack?: TrackWrapper}) {
|
||||||
|
if (clonedTracks) {
|
||||||
|
this.audioTrack = clonedTracks.audioTrack;
|
||||||
|
this.videoTrack = clonedTracks.videoTrack;
|
||||||
|
} else {
|
||||||
for (const track of stream.getTracks()) {
|
for (const track of stream.getTracks()) {
|
||||||
this.update(track);
|
this.update(track);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
get id(): string { return this.stream.id; }
|
get id(): string { return this.stream.id; }
|
||||||
|
|
||||||
clone(): Stream {
|
clone(): Stream {
|
||||||
return new StreamWrapper(this.stream.clone());
|
const clonedStream = this.stream.clone();
|
||||||
|
const clonedTracks = {
|
||||||
|
audioTrack: this.audioTrack ? new AudioTrackWrapper(clonedStream.getAudioTracks()[0], clonedStream, this.audioTrack.id): undefined,
|
||||||
|
videoTrack: this.videoTrack ? new TrackWrapper(clonedStream.getVideoTracks()[0], clonedStream, this.videoTrack.id): undefined,
|
||||||
|
};
|
||||||
|
return new StreamWrapper(clonedStream, clonedTracks);
|
||||||
}
|
}
|
||||||
|
|
||||||
update(track: MediaStreamTrack): TrackWrapper | undefined {
|
update(track: MediaStreamTrack): TrackWrapper | undefined {
|
||||||
if (track.kind === "video") {
|
if (track.kind === "video") {
|
||||||
if (!this.videoTrack || track.id !== this.videoTrack.track.id) {
|
if (!this.videoTrack || track.id !== this.videoTrack.track.id) {
|
||||||
this.videoTrack = new TrackWrapper(track, this.stream);
|
this.videoTrack = new TrackWrapper(track, this.stream, track.id);
|
||||||
}
|
}
|
||||||
return this.videoTrack;
|
return this.videoTrack;
|
||||||
} else if (track.kind === "audio") {
|
} else if (track.kind === "audio") {
|
||||||
if (!this.audioTrack || track.id !== this.audioTrack.track.id) {
|
if (!this.audioTrack || track.id !== this.audioTrack.track.id) {
|
||||||
this.audioTrack = new AudioTrackWrapper(track, this.stream);
|
this.audioTrack = new AudioTrackWrapper(track, this.stream, track.id);
|
||||||
}
|
}
|
||||||
return this.audioTrack;
|
return this.audioTrack;
|
||||||
}
|
}
|
||||||
|
@ -105,14 +115,18 @@ export class StreamWrapper implements Stream {
|
||||||
export class TrackWrapper implements Track {
|
export class TrackWrapper implements Track {
|
||||||
constructor(
|
constructor(
|
||||||
public readonly track: MediaStreamTrack,
|
public readonly track: MediaStreamTrack,
|
||||||
public readonly stream: MediaStream
|
public readonly stream: MediaStream,
|
||||||
|
public readonly originalId: string,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
get kind(): TrackKind { return this.track.kind as TrackKind; }
|
get kind(): TrackKind { return this.track.kind as TrackKind; }
|
||||||
get label(): string { return this.track.label; }
|
get label(): string { return this.track.label; }
|
||||||
get id(): string { return this.track.id; }
|
get id(): string { return this.track.id; }
|
||||||
get settings(): MediaTrackSettings { return this.track.getSettings(); }
|
get settings(): MediaTrackSettings { return this.track.getSettings(); }
|
||||||
|
get enabled(): boolean { return this.track.enabled; }
|
||||||
|
set enabled(enabled: boolean) { this.track.enabled = enabled; }
|
||||||
|
// test equality across clones
|
||||||
|
equals(track: Track): boolean { return (track as TrackWrapper).originalId === this.originalId; }
|
||||||
stop() { this.track.stop(); }
|
stop() { this.track.stop(); }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -126,8 +140,8 @@ export class AudioTrackWrapper extends TrackWrapper {
|
||||||
private volumeLooperTimeout: number;
|
private volumeLooperTimeout: number;
|
||||||
private speakingVolumeSamples: number[];
|
private speakingVolumeSamples: number[];
|
||||||
|
|
||||||
constructor(track: MediaStreamTrack, stream: MediaStream) {
|
constructor(track: MediaStreamTrack, stream: MediaStream, originalId: string) {
|
||||||
super(track, stream);
|
super(track, stream, originalId);
|
||||||
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
|
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
|
||||||
this.initVolumeMeasuring();
|
this.initVolumeMeasuring();
|
||||||
this.measureVolumeActivity(true);
|
this.measureVolumeActivity(true);
|
||||||
|
|
|
@ -109,8 +109,16 @@ export class DOMTrackSenderOrReceiver implements TrackReceiver {
|
||||||
this.transceiver.direction === this.exclusiveValue;
|
this.transceiver.direction === this.exclusiveValue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enableWithoutRenegotiation(enabled: boolean) {
|
||||||
|
this.track.track.enabled = enabled;
|
||||||
|
}
|
||||||
|
|
||||||
enable(enabled: boolean) {
|
enable(enabled: boolean) {
|
||||||
if (enabled !== this.enabled) {
|
if (enabled !== this.enabled) {
|
||||||
|
// do this first, so we stop sending track data immediately.
|
||||||
|
// this will still consume bandwidth though, so also disable the transceiver,
|
||||||
|
// which will trigger a renegotiation though.
|
||||||
|
this.enableWithoutRenegotiation(enabled);
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
if (this.transceiver.direction === "inactive") {
|
if (this.transceiver.direction === "inactive") {
|
||||||
this.transceiver.direction = this.exclusiveValue;
|
this.transceiver.direction = this.exclusiveValue;
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
<meta name="description" content="A matrix chat application">
|
<meta name="description" content="A matrix chat application">
|
||||||
<link rel="apple-touch-icon" href="./assets/icon-maskable.png">
|
<link rel="apple-touch-icon" href="./assets/icon-maskable.png">
|
||||||
<link rel="icon" type="image/png" href="assets/icon-maskable.png">
|
<link rel="icon" type="image/png" href="assets/icon-maskable.png">
|
||||||
<script type="module"> import "@theme/default"; </script>
|
<script type="module"> import "@theme/default/dark"; </script>
|
||||||
</head>
|
</head>
|
||||||
<body class="hydrogen">
|
<body class="hydrogen">
|
||||||
<script id="main" type="module">
|
<script id="main" type="module">
|
||||||
|
|
|
@ -364,17 +364,17 @@ export class TemplateBuilder<T extends IObservableValue> {
|
||||||
event handlers, ...
|
event handlers, ...
|
||||||
You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect,
|
You should not call the TemplateBuilder (e.g. `t.xxx()`) at all from the side effect,
|
||||||
instead use tags from html.ts to help you construct any DOM you need. */
|
instead use tags from html.ts to help you construct any DOM you need. */
|
||||||
mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined) => void) {
|
mapSideEffect<R>(mapFn: (value: T) => R, sideEffect: (newV: R, oldV: R | undefined, value: T) => void) {
|
||||||
let prevValue = mapFn(this._value);
|
let prevValue = mapFn(this._value);
|
||||||
const binding = () => {
|
const binding = () => {
|
||||||
const newValue = mapFn(this._value);
|
const newValue = mapFn(this._value);
|
||||||
if (prevValue !== newValue) {
|
if (prevValue !== newValue) {
|
||||||
sideEffect(newValue, prevValue);
|
sideEffect(newValue, prevValue, this._value);
|
||||||
prevValue = newValue;
|
prevValue = newValue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
this._addBinding(binding);
|
this._addBinding(binding);
|
||||||
sideEffect(prevValue, undefined);
|
sideEffect(prevValue, undefined, this._value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,10 +20,18 @@ import {Stream} from "../../../../types/MediaDevices";
|
||||||
import type {StreamWrapper} from "../../../dom/MediaDevices";
|
import type {StreamWrapper} from "../../../dom/MediaDevices";
|
||||||
import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel";
|
import type {CallViewModel, CallMemberViewModel} from "../../../../../domain/session/room/CallViewModel";
|
||||||
|
|
||||||
function bindVideoTracks<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) {
|
function bindStream<T>(t: TemplateBuilder<T>, video: HTMLVideoElement, propSelector: (vm: T) => Stream | undefined) {
|
||||||
t.mapSideEffect(propSelector, stream => {
|
t.mapSideEffect(vm => propSelector(vm)?.videoTrack?.enabled, (_,__, vm) => {
|
||||||
|
const stream = propSelector(vm);
|
||||||
if (stream) {
|
if (stream) {
|
||||||
video.srcObject = (stream as StreamWrapper).stream;
|
video.srcObject = (stream as StreamWrapper).stream;
|
||||||
|
if (stream.videoTrack?.enabled) {
|
||||||
|
video.classList.remove("hidden");
|
||||||
|
} else {
|
||||||
|
video.classList.add("hidden");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
video.classList.add("hidden");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
return video;
|
return video;
|
||||||
|
@ -33,10 +41,11 @@ export class CallView extends TemplateView<CallViewModel> {
|
||||||
render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement {
|
render(t: TemplateBuilder<CallViewModel>, vm: CallViewModel): HTMLElement {
|
||||||
return t.div({class: "CallView"}, [
|
return t.div({class: "CallView"}, [
|
||||||
t.p(vm => `Call ${vm.name} (${vm.id})`),
|
t.p(vm => `Call ${vm.name} (${vm.id})`),
|
||||||
t.div({class: "CallView_me"}, bindVideoTracks(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)),
|
t.div({class: "CallView_me"}, bindStream(t, t.video({autoplay: true, width: 240}), vm => vm.localStream)),
|
||||||
t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))),
|
t.view(new ListView({list: vm.memberViewModels}, vm => new MemberView(vm))),
|
||||||
t.div({class: "buttons"}, [
|
t.div({class: "buttons"}, [
|
||||||
t.button({onClick: () => vm.leave()}, "Leave")
|
t.button({onClick: () => vm.leave()}, "Leave"),
|
||||||
|
t.button({onClick: () => vm.toggleVideo()}, "Toggle video"),
|
||||||
])
|
])
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
@ -44,6 +53,10 @@ export class CallView extends TemplateView<CallViewModel> {
|
||||||
|
|
||||||
class MemberView extends TemplateView<CallMemberViewModel> {
|
class MemberView extends TemplateView<CallMemberViewModel> {
|
||||||
render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) {
|
render(t: TemplateBuilder<CallMemberViewModel>, vm: CallMemberViewModel) {
|
||||||
return bindVideoTracks(t, t.video({autoplay: true, width: 360}), vm => vm.stream);
|
return bindStream(t, t.video({autoplay: true, width: 360}), vm => vm.stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// class StreamView extends TemplateView<Stream> {
|
||||||
|
// render(t: TemplateBuilder<Stream)
|
||||||
|
// }
|
||||||
|
|
Loading…
Reference in a new issue