This commit is contained in:
Bruno Windels 2022-02-15 17:05:20 +01:00 committed by Bruno Windels
parent 46ebd55092
commit b12bc52c4a
4 changed files with 2860 additions and 0 deletions

2493
src/matrix/calls/Call.ts Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,274 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { SDPStreamMetadataPurpose } from "./callEventTypes";
const POLLING_INTERVAL = 200; // ms
export const SPEAKING_THRESHOLD = -60; // dB
const SPEAKING_SAMPLE_COUNT = 8; // samples
export interface ICallFeedOpts {
client: MatrixClient;
roomId: string;
userId: string;
stream: MediaStream;
purpose: SDPStreamMetadataPurpose;
audioMuted: boolean;
videoMuted: boolean;
}
export enum CallFeedEvent {
NewStream = "new_stream",
MuteStateChanged = "mute_state_changed",
VolumeChanged = "volume_changed",
Speaking = "speaking",
}
export class CallFeed extends EventEmitter {
public stream: MediaStream;
public sdpMetadataStreamId: string;
public userId: string;
public purpose: SDPStreamMetadataPurpose;
public speakingVolumeSamples: number[];
private client: MatrixClient;
private roomId: string;
private audioMuted: boolean;
private videoMuted: boolean;
private measuringVolumeActivity = false;
private audioContext: AudioContext;
private analyser: AnalyserNode;
private frequencyBinCount: Float32Array;
private speakingThreshold = SPEAKING_THRESHOLD;
private speaking = false;
private volumeLooperTimeout: number;
constructor(opts: ICallFeedOpts) {
super();
this.client = opts.client;
this.roomId = opts.roomId;
this.userId = opts.userId;
this.purpose = opts.purpose;
this.audioMuted = opts.audioMuted;
this.videoMuted = opts.videoMuted;
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
this.sdpMetadataStreamId = opts.stream.id;
this.updateStream(null, opts.stream);
if (this.hasAudioTrack) {
this.initVolumeMeasuring();
}
}
private get hasAudioTrack(): boolean {
return this.stream.getAudioTracks().length > 0;
}
private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
if (newStream === oldStream) return;
if (oldStream) {
oldStream.removeEventListener("addtrack", this.onAddTrack);
this.measureVolumeActivity(false);
}
if (newStream) {
this.stream = newStream;
newStream.addEventListener("addtrack", this.onAddTrack);
if (this.hasAudioTrack) {
this.initVolumeMeasuring();
} else {
this.measureVolumeActivity(false);
}
}
this.emit(CallFeedEvent.NewStream, this.stream);
}
private initVolumeMeasuring(): void {
const AudioContext = window.AudioContext || window.webkitAudioContext;
if (!this.hasAudioTrack || !AudioContext) return;
this.audioContext = new AudioContext();
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512;
this.analyser.smoothingTimeConstant = 0.1;
const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
mediaStreamAudioSourceNode.connect(this.analyser);
this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
}
private onAddTrack = (): void => {
this.emit(CallFeedEvent.NewStream, this.stream);
};
/**
* Returns callRoom member
* @returns member of the callRoom
*/
public getMember(): RoomMember {
const callRoom = this.client.getRoom(this.roomId);
return callRoom.getMember(this.userId);
}
/**
* Returns true if CallFeed is local, otherwise returns false
* @returns {boolean} is local?
*/
public isLocal(): boolean {
return this.userId === this.client.getUserId();
}
/**
* Returns true if audio is muted or if there are no audio
* tracks, otherwise returns false
* @returns {boolean} is audio muted?
*/
public isAudioMuted(): boolean {
return this.stream.getAudioTracks().length === 0 || this.audioMuted;
}
/**
* Returns true video is muted or if there are no video
* tracks, otherwise returns false
* @returns {boolean} is video muted?
*/
public isVideoMuted(): boolean {
// We assume only one video track
return this.stream.getVideoTracks().length === 0 || this.videoMuted;
}
public isSpeaking(): boolean {
return this.speaking;
}
/**
* Replaces the current MediaStream with a new one.
* This method should be only used by MatrixCall.
* @param newStream new stream with which to replace the current one
*/
public setNewStream(newStream: MediaStream): void {
this.updateStream(this.stream, newStream);
}
/**
* Set feed's internal audio mute state
* @param muted is the feed's audio muted?
*/
public setAudioMuted(muted: boolean): void {
this.audioMuted = muted;
this.speakingVolumeSamples.fill(-Infinity);
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
}
/**
* Set feed's internal video mute state
* @param muted is the feed's video muted?
*/
public setVideoMuted(muted: boolean): void {
this.videoMuted = muted;
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
}
/**
* Starts emitting volume_changed events where the emitter value is in decibels
* @param enabled emit volume changes
*/
public measureVolumeActivity(enabled: boolean): void {
if (enabled) {
if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
this.measuringVolumeActivity = true;
this.volumeLooper();
} else {
this.measuringVolumeActivity = false;
this.speakingVolumeSamples.fill(-Infinity);
this.emit(CallFeedEvent.VolumeChanged, -Infinity);
}
}
public setSpeakingThreshold(threshold: number) {
this.speakingThreshold = threshold;
}
private volumeLooper = () => {
if (!this.analyser) return;
if (!this.measuringVolumeActivity) return;
this.analyser.getFloatFrequencyData(this.frequencyBinCount);
let maxVolume = -Infinity;
for (let i = 0; i < this.frequencyBinCount.length; i++) {
if (this.frequencyBinCount[i] > maxVolume) {
maxVolume = this.frequencyBinCount[i];
}
}
this.speakingVolumeSamples.shift();
this.speakingVolumeSamples.push(maxVolume);
this.emit(CallFeedEvent.VolumeChanged, maxVolume);
let newSpeaking = false;
for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
const volume = this.speakingVolumeSamples[i];
if (volume > this.speakingThreshold) {
newSpeaking = true;
break;
}
}
if (this.speaking !== newSpeaking) {
this.speaking = newSpeaking;
this.emit(CallFeedEvent.Speaking, this.speaking);
}
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
};
public clone(): CallFeed {
const mediaHandler = this.client.getMediaHandler();
const stream = this.stream.clone();
if (this.purpose === SDPStreamMetadataPurpose.Usermedia) {
mediaHandler.userMediaStreams.push(stream);
} else {
mediaHandler.screensharingStreams.push(stream);
}
return new CallFeed({
client: this.client,
roomId: this.roomId,
userId: this.userId,
stream,
purpose: this.purpose,
audioMuted: this.audioMuted,
videoMuted: this.videoMuted,
});
}
public dispose(): void {
clearTimeout(this.volumeLooperTimeout);
}
}

View file

@ -34,6 +34,7 @@ const CALL_ID = "m.call_id";
const CALL_TERMINATED = "m.terminated"; const CALL_TERMINATED = "m.terminated";
export class CallHandler { export class CallHandler {
// group calls by call id
public readonly groupCalls: ObservableMap<string, GroupCall> = new ObservableMap<string, GroupCall>(); public readonly groupCalls: ObservableMap<string, GroupCall> = new ObservableMap<string, GroupCall>();
constructor() { constructor() {

View file

@ -0,0 +1,92 @@
// allow non-camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */
import { CallErrorCode } from "./Call";
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
export enum SDPStreamMetadataPurpose {
Usermedia = "m.usermedia",
Screenshare = "m.screenshare",
}
export interface SDPStreamMetadataObject {
purpose: SDPStreamMetadataPurpose;
audio_muted: boolean;
video_muted: boolean;
}
export interface SDPStreamMetadata {
[key: string]: SDPStreamMetadataObject;
}
export interface CallCapabilities {
'm.call.transferee': boolean;
'm.call.dtmf': boolean;
}
export interface CallReplacesTarget {
id: string;
display_name: string;
avatar_url: string;
}
export interface MCallBase {
call_id: string;
version: string | number;
party_id?: string;
sender_session_id?: string;
dest_session_id?: string;
}
export interface MCallAnswer extends MCallBase {
answer: RTCSessionDescription;
capabilities?: CallCapabilities;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallSelectAnswer extends MCallBase {
selected_party_id: string;
}
export interface MCallInviteNegotiate extends MCallBase {
offer: RTCSessionDescription;
description: RTCSessionDescription;
lifetime: number;
capabilities?: CallCapabilities;
invitee?: string;
sender_session_id?: string;
dest_session_id?: string;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallSDPStreamMetadataChanged extends MCallBase {
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallReplacesEvent extends MCallBase {
replacement_id: string;
target_user: CallReplacesTarget;
create_call: string;
await_call: string;
target_room: string;
}
export interface MCAllAssertedIdentity extends MCallBase {
asserted_identity: {
id: string;
display_name: string;
avatar_url: string;
};
}
export interface MCallCandidates extends MCallBase {
candidates: RTCIceCandidate[];
}
export interface MCallHangupReject extends MCallBase {
reason?: CallErrorCode;
}
/* eslint-enable camelcase */