forked from mystiq/hydrogen-web
WIP10
This commit is contained in:
parent
6fe90e60db
commit
60da85d641
13 changed files with 347 additions and 3126 deletions
|
@ -45,7 +45,7 @@
|
||||||
"postcss-flexbugs-fixes": "^5.0.2",
|
"postcss-flexbugs-fixes": "^5.0.2",
|
||||||
"regenerator-runtime": "^0.13.7",
|
"regenerator-runtime": "^0.13.7",
|
||||||
"text-encoding": "^0.7.0",
|
"text-encoding": "^0.7.0",
|
||||||
"typescript": "^4.3.5",
|
"typescript": "^4.4",
|
||||||
"vite": "^2.6.14",
|
"vite": "^2.6.14",
|
||||||
"xxhashjs": "^0.2.2"
|
"xxhashjs": "^0.2.2"
|
||||||
},
|
},
|
||||||
|
|
|
@ -59,7 +59,7 @@ export class DeviceMessageHandler {
|
||||||
}));
|
}));
|
||||||
// TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)?
|
// TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)?
|
||||||
for (const dr of callMessages) {
|
for (const dr of callMessages) {
|
||||||
this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log);
|
this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log);
|
||||||
}
|
}
|
||||||
// TODO: somehow include rooms that received a call to_device message in the sync state?
|
// TODO: somehow include rooms that received a call to_device message in the sync state?
|
||||||
// or have updates flow through event emitter?
|
// or have updates flow through event emitter?
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,274 +0,0 @@
|
||||||
/*
|
|
||||||
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { SDPStreamMetadataPurpose } from "./callEventTypes";
|
|
||||||
|
|
||||||
const POLLING_INTERVAL = 200; // ms
|
|
||||||
export const SPEAKING_THRESHOLD = -60; // dB
|
|
||||||
const SPEAKING_SAMPLE_COUNT = 8; // samples
|
|
||||||
|
|
||||||
export interface ICallFeedOpts {
|
|
||||||
client: MatrixClient;
|
|
||||||
roomId: string;
|
|
||||||
userId: string;
|
|
||||||
stream: MediaStream;
|
|
||||||
purpose: SDPStreamMetadataPurpose;
|
|
||||||
audioMuted: boolean;
|
|
||||||
videoMuted: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export enum CallFeedEvent {
|
|
||||||
NewStream = "new_stream",
|
|
||||||
MuteStateChanged = "mute_state_changed",
|
|
||||||
VolumeChanged = "volume_changed",
|
|
||||||
Speaking = "speaking",
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CallFeed extends EventEmitter {
|
|
||||||
public stream: MediaStream;
|
|
||||||
public sdpMetadataStreamId: string;
|
|
||||||
public userId: string;
|
|
||||||
public purpose: SDPStreamMetadataPurpose;
|
|
||||||
public speakingVolumeSamples: number[];
|
|
||||||
|
|
||||||
private client: MatrixClient;
|
|
||||||
private roomId: string;
|
|
||||||
private audioMuted: boolean;
|
|
||||||
private videoMuted: boolean;
|
|
||||||
private measuringVolumeActivity = false;
|
|
||||||
private audioContext: AudioContext;
|
|
||||||
private analyser: AnalyserNode;
|
|
||||||
private frequencyBinCount: Float32Array;
|
|
||||||
private speakingThreshold = SPEAKING_THRESHOLD;
|
|
||||||
private speaking = false;
|
|
||||||
private volumeLooperTimeout: number;
|
|
||||||
|
|
||||||
constructor(opts: ICallFeedOpts) {
|
|
||||||
super();
|
|
||||||
|
|
||||||
this.client = opts.client;
|
|
||||||
this.roomId = opts.roomId;
|
|
||||||
this.userId = opts.userId;
|
|
||||||
this.purpose = opts.purpose;
|
|
||||||
this.audioMuted = opts.audioMuted;
|
|
||||||
this.videoMuted = opts.videoMuted;
|
|
||||||
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
|
|
||||||
this.sdpMetadataStreamId = opts.stream.id;
|
|
||||||
|
|
||||||
this.updateStream(null, opts.stream);
|
|
||||||
|
|
||||||
if (this.hasAudioTrack) {
|
|
||||||
this.initVolumeMeasuring();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private get hasAudioTrack(): boolean {
|
|
||||||
return this.stream.getAudioTracks().length > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
|
|
||||||
if (newStream === oldStream) return;
|
|
||||||
|
|
||||||
if (oldStream) {
|
|
||||||
oldStream.removeEventListener("addtrack", this.onAddTrack);
|
|
||||||
this.measureVolumeActivity(false);
|
|
||||||
}
|
|
||||||
if (newStream) {
|
|
||||||
this.stream = newStream;
|
|
||||||
newStream.addEventListener("addtrack", this.onAddTrack);
|
|
||||||
|
|
||||||
if (this.hasAudioTrack) {
|
|
||||||
this.initVolumeMeasuring();
|
|
||||||
} else {
|
|
||||||
this.measureVolumeActivity(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.emit(CallFeedEvent.NewStream, this.stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
private initVolumeMeasuring(): void {
|
|
||||||
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
|
||||||
if (!this.hasAudioTrack || !AudioContext) return;
|
|
||||||
|
|
||||||
this.audioContext = new AudioContext();
|
|
||||||
|
|
||||||
this.analyser = this.audioContext.createAnalyser();
|
|
||||||
this.analyser.fftSize = 512;
|
|
||||||
this.analyser.smoothingTimeConstant = 0.1;
|
|
||||||
|
|
||||||
const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
|
|
||||||
mediaStreamAudioSourceNode.connect(this.analyser);
|
|
||||||
|
|
||||||
this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
private onAddTrack = (): void => {
|
|
||||||
this.emit(CallFeedEvent.NewStream, this.stream);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns callRoom member
|
|
||||||
* @returns member of the callRoom
|
|
||||||
*/
|
|
||||||
public getMember(): RoomMember {
|
|
||||||
const callRoom = this.client.getRoom(this.roomId);
|
|
||||||
return callRoom.getMember(this.userId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns true if CallFeed is local, otherwise returns false
|
|
||||||
* @returns {boolean} is local?
|
|
||||||
*/
|
|
||||||
public isLocal(): boolean {
|
|
||||||
return this.userId === this.client.getUserId();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns true if audio is muted or if there are no audio
|
|
||||||
* tracks, otherwise returns false
|
|
||||||
* @returns {boolean} is audio muted?
|
|
||||||
*/
|
|
||||||
public isAudioMuted(): boolean {
|
|
||||||
return this.stream.getAudioTracks().length === 0 || this.audioMuted;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns true video is muted or if there are no video
|
|
||||||
* tracks, otherwise returns false
|
|
||||||
* @returns {boolean} is video muted?
|
|
||||||
*/
|
|
||||||
public isVideoMuted(): boolean {
|
|
||||||
// We assume only one video track
|
|
||||||
return this.stream.getVideoTracks().length === 0 || this.videoMuted;
|
|
||||||
}
|
|
||||||
|
|
||||||
public isSpeaking(): boolean {
|
|
||||||
return this.speaking;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Replaces the current MediaStream with a new one.
|
|
||||||
* This method should be only used by MatrixCall.
|
|
||||||
* @param newStream new stream with which to replace the current one
|
|
||||||
*/
|
|
||||||
public setNewStream(newStream: MediaStream): void {
|
|
||||||
this.updateStream(this.stream, newStream);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set feed's internal audio mute state
|
|
||||||
* @param muted is the feed's audio muted?
|
|
||||||
*/
|
|
||||||
public setAudioMuted(muted: boolean): void {
|
|
||||||
this.audioMuted = muted;
|
|
||||||
this.speakingVolumeSamples.fill(-Infinity);
|
|
||||||
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set feed's internal video mute state
|
|
||||||
* @param muted is the feed's video muted?
|
|
||||||
*/
|
|
||||||
public setVideoMuted(muted: boolean): void {
|
|
||||||
this.videoMuted = muted;
|
|
||||||
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Starts emitting volume_changed events where the emitter value is in decibels
|
|
||||||
* @param enabled emit volume changes
|
|
||||||
*/
|
|
||||||
public measureVolumeActivity(enabled: boolean): void {
|
|
||||||
if (enabled) {
|
|
||||||
if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
|
|
||||||
|
|
||||||
this.measuringVolumeActivity = true;
|
|
||||||
this.volumeLooper();
|
|
||||||
} else {
|
|
||||||
this.measuringVolumeActivity = false;
|
|
||||||
this.speakingVolumeSamples.fill(-Infinity);
|
|
||||||
this.emit(CallFeedEvent.VolumeChanged, -Infinity);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public setSpeakingThreshold(threshold: number) {
|
|
||||||
this.speakingThreshold = threshold;
|
|
||||||
}
|
|
||||||
|
|
||||||
private volumeLooper = () => {
|
|
||||||
if (!this.analyser) return;
|
|
||||||
|
|
||||||
if (!this.measuringVolumeActivity) return;
|
|
||||||
|
|
||||||
this.analyser.getFloatFrequencyData(this.frequencyBinCount);
|
|
||||||
|
|
||||||
let maxVolume = -Infinity;
|
|
||||||
for (let i = 0; i < this.frequencyBinCount.length; i++) {
|
|
||||||
if (this.frequencyBinCount[i] > maxVolume) {
|
|
||||||
maxVolume = this.frequencyBinCount[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.speakingVolumeSamples.shift();
|
|
||||||
this.speakingVolumeSamples.push(maxVolume);
|
|
||||||
|
|
||||||
this.emit(CallFeedEvent.VolumeChanged, maxVolume);
|
|
||||||
|
|
||||||
let newSpeaking = false;
|
|
||||||
|
|
||||||
for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
|
|
||||||
const volume = this.speakingVolumeSamples[i];
|
|
||||||
|
|
||||||
if (volume > this.speakingThreshold) {
|
|
||||||
newSpeaking = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.speaking !== newSpeaking) {
|
|
||||||
this.speaking = newSpeaking;
|
|
||||||
this.emit(CallFeedEvent.Speaking, this.speaking);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
|
|
||||||
};
|
|
||||||
|
|
||||||
public clone(): CallFeed {
|
|
||||||
const mediaHandler = this.client.getMediaHandler();
|
|
||||||
const stream = this.stream.clone();
|
|
||||||
|
|
||||||
if (this.purpose === SDPStreamMetadataPurpose.Usermedia) {
|
|
||||||
mediaHandler.userMediaStreams.push(stream);
|
|
||||||
} else {
|
|
||||||
mediaHandler.screensharingStreams.push(stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
return new CallFeed({
|
|
||||||
client: this.client,
|
|
||||||
roomId: this.roomId,
|
|
||||||
userId: this.userId,
|
|
||||||
stream,
|
|
||||||
purpose: this.purpose,
|
|
||||||
audioMuted: this.audioMuted,
|
|
||||||
videoMuted: this.videoMuted,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public dispose(): void {
|
|
||||||
clearTimeout(this.volumeLooperTimeout);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -22,6 +22,8 @@ import type {ILogItem} from "../../logging/types";
|
||||||
|
|
||||||
import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC";
|
import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC";
|
||||||
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
|
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
|
||||||
|
import type {SignallingMessage} from "./PeerCall";
|
||||||
|
import type {MGroupCallBase} from "./callEventTypes";
|
||||||
|
|
||||||
const GROUP_CALL_TYPE = "m.call";
|
const GROUP_CALL_TYPE = "m.call";
|
||||||
const GROUP_CALL_MEMBER_TYPE = "m.call.member";
|
const GROUP_CALL_MEMBER_TYPE = "m.call.member";
|
||||||
|
@ -33,7 +35,7 @@ enum CallSetupMessageType {
|
||||||
Hangup = "m.call.hangup",
|
Hangup = "m.call.hangup",
|
||||||
}
|
}
|
||||||
|
|
||||||
const CALL_ID = "m.call_id";
|
const CONF_ID = "conf_id";
|
||||||
const CALL_TERMINATED = "m.terminated";
|
const CALL_TERMINATED = "m.terminated";
|
||||||
|
|
||||||
export class GroupCallHandler {
|
export class GroupCallHandler {
|
||||||
|
@ -69,7 +71,7 @@ export class GroupCallHandler {
|
||||||
const participant = event.state_key;
|
const participant = event.state_key;
|
||||||
const sources = event.content["m.sources"];
|
const sources = event.content["m.sources"];
|
||||||
for (const source of sources) {
|
for (const source of sources) {
|
||||||
const call = this.calls.get(source[CALL_ID]);
|
const call = this.calls.get(source[CONF_ID]);
|
||||||
if (call && !call.isTerminated) {
|
if (call && !call.isTerminated) {
|
||||||
call.addParticipant(participant, source);
|
call.addParticipant(participant, source);
|
||||||
}
|
}
|
||||||
|
@ -85,110 +87,9 @@ export class GroupCallHandler {
|
||||||
eventType === CallSetupMessageType.Hangup;
|
eventType === CallSetupMessageType.Hangup;
|
||||||
}
|
}
|
||||||
|
|
||||||
handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) {
|
handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage<MGroupCallBase>, log: ILogItem) {
|
||||||
const callId = content[CALL_ID];
|
const call = this.calls.get(event.content.conf_id);
|
||||||
const call = this.calls.get(callId);
|
call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log);
|
||||||
call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function participantId(senderUserId: string, senderDeviceId: string | null) {
|
|
||||||
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
|
|
||||||
}
|
|
||||||
|
|
||||||
class GroupParticipant implements PeerCallHandler {
|
|
||||||
private peerCall?: PeerCall;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private readonly userId: string,
|
|
||||||
private readonly deviceId: string,
|
|
||||||
private localMedia: LocalMedia | undefined,
|
|
||||||
private readonly webRTC: WebRTC,
|
|
||||||
private readonly hsApi: HomeServerApi
|
|
||||||
) {}
|
|
||||||
|
|
||||||
sendInvite() {
|
|
||||||
this.peerCall = new PeerCall(this, this.webRTC);
|
|
||||||
this.peerCall.call(this.localMedia);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** From PeerCallHandler
|
|
||||||
* @internal */
|
|
||||||
override emitUpdate() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/** From PeerCallHandler
|
|
||||||
* @internal */
|
|
||||||
override onSendSignallingMessage() {
|
|
||||||
// TODO: this needs to be encrypted with olm first
|
|
||||||
this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class GroupCall {
|
|
||||||
private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
|
|
||||||
private localMedia?: LocalMedia;
|
|
||||||
|
|
||||||
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
get id(): string { return this.callEvent.state_key; }
|
|
||||||
|
|
||||||
async participate(tracks: Track[]) {
|
|
||||||
this.localMedia = LocalMedia.fromTracks(tracks);
|
|
||||||
for (const [,participant] of this.participants) {
|
|
||||||
participant.setMedia(this.localMedia.clone());
|
|
||||||
}
|
|
||||||
// send m.call.member state event
|
|
||||||
|
|
||||||
// send invite to all participants that are < my userId
|
|
||||||
for (const [,participant] of this.participants) {
|
|
||||||
if (participant.userId < this.ownUserId) {
|
|
||||||
participant.sendInvite();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
updateCallEvent(callEvent: StateEvent) {
|
|
||||||
this.callEvent = callEvent;
|
|
||||||
}
|
|
||||||
|
|
||||||
addParticipant(userId, source) {
|
|
||||||
const participantId = getParticipantId(userId, source.device_id);
|
|
||||||
const participant = this.participants.get(participantId);
|
|
||||||
if (participant) {
|
|
||||||
participant.updateSource(source);
|
|
||||||
} else {
|
|
||||||
participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) {
|
|
||||||
const participantId = getParticipantId(senderUserId, senderDeviceId);
|
|
||||||
let peerCall = this.participants.get(participantId);
|
|
||||||
let hasDeviceInKey = true;
|
|
||||||
if (!peerCall) {
|
|
||||||
hasDeviceInKey = false;
|
|
||||||
peerCall = this.participants.get(getParticipantId(senderUserId, null))
|
|
||||||
}
|
|
||||||
if (peerCall) {
|
|
||||||
peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId);
|
|
||||||
if (!hasDeviceInKey && peerCall.opponentPartyId) {
|
|
||||||
this.participants.delete(getParticipantId(senderUserId, null));
|
|
||||||
this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId));
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// create peerCall
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
get id(): string {
|
|
||||||
return this.callEvent.state_key;
|
|
||||||
}
|
|
||||||
|
|
||||||
get isTerminated(): boolean {
|
|
||||||
return !!this.callEvent.content[CALL_TERMINATED];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -17,26 +17,41 @@ limitations under the License.
|
||||||
import {ObservableMap} from "../../observable/map/ObservableMap";
|
import {ObservableMap} from "../../observable/map/ObservableMap";
|
||||||
import {recursivelyAssign} from "../../utils/recursivelyAssign";
|
import {recursivelyAssign} from "../../utils/recursivelyAssign";
|
||||||
import {AsyncQueue} from "../../utils/AsyncQueue";
|
import {AsyncQueue} from "../../utils/AsyncQueue";
|
||||||
import {Disposables, Disposable} from "../../utils/Disposables";
|
import {Disposables, IDisposable} from "../../utils/Disposables";
|
||||||
import type {Room} from "../room/Room";
|
import type {Room} from "../room/Room";
|
||||||
import type {StateEvent} from "../storage/types";
|
import type {StateEvent} from "../storage/types";
|
||||||
import type {ILogItem} from "../../logging/types";
|
import type {ILogItem} from "../../logging/types";
|
||||||
|
|
||||||
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
|
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
|
||||||
import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC";
|
import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC";
|
||||||
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
|
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
|
||||||
import type {LocalMedia} from "./LocalMedia";
|
import type {LocalMedia} from "./LocalMedia";
|
||||||
|
|
||||||
|
import {
|
||||||
|
SDPStreamMetadataKey,
|
||||||
|
SDPStreamMetadataPurpose
|
||||||
|
} from "./callEventTypes";
|
||||||
|
import type {
|
||||||
|
MCallBase,
|
||||||
|
MCallInvite,
|
||||||
|
MCallAnswer,
|
||||||
|
MCallSDPStreamMetadataChanged,
|
||||||
|
MCallCandidates,
|
||||||
|
MCallHangupReject,
|
||||||
|
SDPStreamMetadata,
|
||||||
|
} from "./callEventTypes";
|
||||||
|
|
||||||
// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already
|
// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already
|
||||||
// do for sharing keys will be best as that already deals with room tracking.
|
// do for sharing keys will be best as that already deals with room tracking.
|
||||||
/**
|
/**
|
||||||
* Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform
|
* Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform
|
||||||
* */
|
* */
|
||||||
/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/
|
/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/
|
||||||
class PeerCall {
|
export class PeerCall implements IDisposable {
|
||||||
private readonly peerConnection: PeerConnection;
|
private readonly peerConnection: PeerConnection;
|
||||||
private state = CallState.Fledgling;
|
private state = CallState.Fledgling;
|
||||||
private direction: CallDirection;
|
private direction: CallDirection;
|
||||||
|
private localMedia?: LocalMedia;
|
||||||
// A queue for candidates waiting to go out.
|
// A queue for candidates waiting to go out.
|
||||||
// We try to amalgamate candidates into a single candidate message where
|
// We try to amalgamate candidates into a single candidate message where
|
||||||
// possible
|
// possible
|
||||||
|
@ -54,9 +69,13 @@ class PeerCall {
|
||||||
private disposables = new Disposables();
|
private disposables = new Disposables();
|
||||||
private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>();
|
private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>();
|
||||||
|
|
||||||
|
// perfect negotiation flags
|
||||||
|
private makingOffer: boolean = false;
|
||||||
|
private ignoreOffer: boolean = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
|
private callId: string, // generated or from invite
|
||||||
private readonly handler: PeerCallHandler,
|
private readonly handler: PeerCallHandler,
|
||||||
private localMedia: LocalMedia,
|
|
||||||
private readonly createTimeout: TimeoutCreator,
|
private readonly createTimeout: TimeoutCreator,
|
||||||
webRTC: WebRTC
|
webRTC: WebRTC
|
||||||
) {
|
) {
|
||||||
|
@ -83,29 +102,8 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) {
|
get remoteTracks(): Track[] {
|
||||||
switch (message.type) {
|
return this.peerConnection.remoteTracks;
|
||||||
case EventType.Invite:
|
|
||||||
// determining whether or not an incoming invite glares
|
|
||||||
// with an instance of PeerCall is different for group calls
|
|
||||||
// and 1:1 calls, so done outside of this class.
|
|
||||||
// If you pass an event for another call id in here it will assume it glares.
|
|
||||||
|
|
||||||
//const newCallId = message.content.call_id;
|
|
||||||
//if (this.id && newCallId !== this.id) {
|
|
||||||
// this.handleInviteGlare(message.content);
|
|
||||||
//} else {
|
|
||||||
this.handleInvite(message.content, partyId);
|
|
||||||
//}
|
|
||||||
break;
|
|
||||||
case EventType.Answer:
|
|
||||||
this.handleAnswer(message.content, partyId);
|
|
||||||
break;
|
|
||||||
case EventType.Candidates:
|
|
||||||
this.handleRemoteIceCandidates(message.content, partyId);
|
|
||||||
break;
|
|
||||||
case EventType.Hangup:
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async call(localMediaPromise: Promise<LocalMedia>): Promise<void> {
|
async call(localMediaPromise: Promise<LocalMedia>): Promise<void> {
|
||||||
|
@ -125,7 +123,9 @@ class PeerCall {
|
||||||
for (const t of this.localMedia.tracks) {
|
for (const t of this.localMedia.tracks) {
|
||||||
this.peerConnection.addTrack(t);
|
this.peerConnection.addTrack(t);
|
||||||
}
|
}
|
||||||
await this.waitForState(CallState.InviteSent);
|
// TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet
|
||||||
|
// but we would go straight to CreateAnswer, so also need to wait for that state
|
||||||
|
await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]);
|
||||||
}
|
}
|
||||||
|
|
||||||
async answer(localMediaPromise: Promise<LocalMedia>): Promise<void> {
|
async answer(localMediaPromise: Promise<LocalMedia>): Promise<void> {
|
||||||
|
@ -166,15 +166,11 @@ class PeerCall {
|
||||||
this.sendAnswer();
|
this.sendAnswer();
|
||||||
}
|
}
|
||||||
|
|
||||||
async hangup() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
async setMedia(localMediaPromise: Promise<LocalMedia>) {
|
async setMedia(localMediaPromise: Promise<LocalMedia>) {
|
||||||
const oldMedia = this.localMedia;
|
const oldMedia = this.localMedia;
|
||||||
this.localMedia = await localMediaPromise;
|
this.localMedia = await localMediaPromise;
|
||||||
|
|
||||||
const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => {
|
const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => {
|
||||||
const oldTrack = selectTrack(oldMedia);
|
const oldTrack = selectTrack(oldMedia);
|
||||||
const newTrack = selectTrack(this.localMedia);
|
const newTrack = selectTrack(this.localMedia);
|
||||||
if (oldTrack && newTrack) {
|
if (oldTrack && newTrack) {
|
||||||
|
@ -187,15 +183,57 @@ class PeerCall {
|
||||||
};
|
};
|
||||||
|
|
||||||
// add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called
|
// add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called
|
||||||
applyTrack(m => m.microphoneTrack);
|
applyTrack(m => m?.microphoneTrack);
|
||||||
applyTrack(m => m.cameraTrack);
|
applyTrack(m => m?.cameraTrack);
|
||||||
applyTrack(m => m.screenShareTrack);
|
applyTrack(m => m?.screenShareTrack);
|
||||||
|
}
|
||||||
|
|
||||||
|
async reject() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
async hangup(errorCode: CallErrorCode) {
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleIncomingSignallingMessage<B extends MCallBase>(message: SignallingMessage<B>, partyId: PartyId): Promise<void> {
|
||||||
|
switch (message.type) {
|
||||||
|
case EventType.Invite:
|
||||||
|
if (this.callId !== message.content.call_id) {
|
||||||
|
await this.handleInviteGlare(message.content, partyId);
|
||||||
|
} else {
|
||||||
|
await this.handleFirstInvite(message.content, partyId);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case EventType.Answer:
|
||||||
|
await this.handleAnswer(message.content, partyId);
|
||||||
|
break;
|
||||||
|
//case EventType.Candidates:
|
||||||
|
// await this.handleRemoteIceCandidates(message.content, partyId);
|
||||||
|
// break;
|
||||||
|
case EventType.Hangup:
|
||||||
|
default:
|
||||||
|
throw new Error(`Unknown event type for call: ${message.type}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise<void> {
|
||||||
|
const content = {
|
||||||
|
call_id: callId,
|
||||||
|
version: 1,
|
||||||
|
};
|
||||||
|
if (reason) {
|
||||||
|
content["reason"] = reason;
|
||||||
|
}
|
||||||
|
return this.handler.sendSignallingMessage({
|
||||||
|
type: EventType.Hangup,
|
||||||
|
content
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// calls are serialized and deduplicated by responsePromiseChain
|
// calls are serialized and deduplicated by responsePromiseChain
|
||||||
private handleNegotiation = async (): Promise<void> => {
|
private handleNegotiation = async (): Promise<void> => {
|
||||||
// TODO: does this make sense to have this state if we're already connected?
|
this.makingOffer = true;
|
||||||
this.setState(CallState.MakingOffer)
|
try {
|
||||||
try {
|
try {
|
||||||
await this.peerConnection.setLocalDescription();
|
await this.peerConnection.setLocalDescription();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
@ -222,15 +260,24 @@ class PeerCall {
|
||||||
|
|
||||||
// need to queue this
|
// need to queue this
|
||||||
const content = {
|
const content = {
|
||||||
|
call_id: this.callId,
|
||||||
offer,
|
offer,
|
||||||
[SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
|
[SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(),
|
||||||
version: 1,
|
version: 1,
|
||||||
lifetime: CALL_TIMEOUT_MS
|
lifetime: CALL_TIMEOUT_MS
|
||||||
};
|
};
|
||||||
if (this.state === CallState.CreateOffer) {
|
if (this.state === CallState.CreateOffer) {
|
||||||
await this.handler.sendSignallingMessage({type: EventType.Invite, content});
|
await this.handler.sendSignallingMessage({type: EventType.Invite, content});
|
||||||
this.setState(CallState.InviteSent);
|
this.setState(CallState.InviteSent);
|
||||||
|
} else if (this.state === CallState.Connected || this.state === CallState.Connecting) {
|
||||||
|
// send Negotiate message
|
||||||
|
//await this.handler.sendSignallingMessage({type: EventType.Invite, content});
|
||||||
|
//this.setState(CallState.InviteSent);
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
this.makingOffer = false;
|
||||||
|
}
|
||||||
|
|
||||||
this.sendCandidateQueue();
|
this.sendCandidateQueue();
|
||||||
|
|
||||||
if (this.state === CallState.InviteSent) {
|
if (this.state === CallState.InviteSent) {
|
||||||
|
@ -242,11 +289,47 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
private async handleInvite(content: InviteContent, partyId: PartyId): Promise<void> {
|
private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise<void> {
|
||||||
|
// this is only called when the ids are different
|
||||||
|
const newCallId = content.call_id;
|
||||||
|
if (this.callId! > newCallId) {
|
||||||
|
this.logger.log(
|
||||||
|
"Glare detected: answering incoming call " + newCallId +
|
||||||
|
" and canceling outgoing call " + this.callId,
|
||||||
|
);
|
||||||
|
|
||||||
|
/*
|
||||||
|
first, we should set CallDirection
|
||||||
|
we should anser the call
|
||||||
|
*/
|
||||||
|
|
||||||
|
// TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer?
|
||||||
|
// How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer?
|
||||||
|
if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) {
|
||||||
|
|
||||||
|
} else {
|
||||||
|
await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced);
|
||||||
|
}
|
||||||
|
await this.handleInvite(content, partyId);
|
||||||
|
await this.answer(Promise.resolve(this.localMedia!));
|
||||||
|
} else {
|
||||||
|
this.logger.log(
|
||||||
|
"Glare detected: rejecting incoming call " + newCallId +
|
||||||
|
" and keeping outgoing call " + this.callId,
|
||||||
|
);
|
||||||
|
await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise<void> {
|
||||||
if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) {
|
if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) {
|
||||||
// TODO: hangup or ignore?
|
// TODO: hangup or ignore?
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
await this.handleInvite(content, partyId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async handleInvite(content: MCallInvite, partyId: PartyId): Promise<void> {
|
||||||
|
|
||||||
// we must set the party ID before await-ing on anything: the call event
|
// we must set the party ID before await-ing on anything: the call event
|
||||||
// handler will start giving us more call events (eg. candidates) so if
|
// handler will start giving us more call events (eg. candidates) so if
|
||||||
|
@ -296,8 +379,8 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise<void> {
|
private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise<void> {
|
||||||
this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`);
|
this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`);
|
||||||
|
|
||||||
if (this.state === CallState.Ended) {
|
if (this.state === CallState.Ended) {
|
||||||
this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`);
|
this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`);
|
||||||
|
@ -307,7 +390,7 @@ class PeerCall {
|
||||||
if (this.opponentPartyId !== undefined) {
|
if (this.opponentPartyId !== undefined) {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Call ${this.callId} ` +
|
`Call ${this.callId} ` +
|
||||||
`Ignoring answer from party ID ${content.party_id}: ` +
|
`Ignoring answer from party ID ${partyId}: ` +
|
||||||
`we already have an answer/reject from ${this.opponentPartyId}`,
|
`we already have an answer/reject from ${this.opponentPartyId}`,
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
|
@ -334,16 +417,66 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// private async onNegotiateReceived(event: MatrixEvent): Promise<void> {
|
||||||
|
// const content = event.getContent<MCallNegotiate>();
|
||||||
|
// const description = content.description;
|
||||||
|
// if (!description || !description.sdp || !description.type) {
|
||||||
|
// this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`);
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// // Politeness always follows the direction of the call: in a glare situation,
|
||||||
|
// // we pick either the inbound or outbound call, so one side will always be
|
||||||
|
// // inbound and one outbound
|
||||||
|
// const polite = this.direction === CallDirection.Inbound;
|
||||||
|
|
||||||
|
// // Here we follow the perfect negotiation logic from
|
||||||
|
// // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation
|
||||||
|
// const offerCollision = (
|
||||||
|
// (description.type === 'offer') &&
|
||||||
|
// (this.makingOffer || this.peerConnection.signalingState !== 'stable')
|
||||||
|
// );
|
||||||
|
|
||||||
|
// this.ignoreOffer = !polite && offerCollision;
|
||||||
|
// if (this.ignoreOffer) {
|
||||||
|
// this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`);
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const sdpStreamMetadata = content[SDPStreamMetadataKey];
|
||||||
|
// if (sdpStreamMetadata) {
|
||||||
|
// this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
|
||||||
|
// } else {
|
||||||
|
// this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// try {
|
||||||
|
// await this.peerConnection.setRemoteDescription(description);
|
||||||
|
|
||||||
|
// if (description.type === 'offer') {
|
||||||
|
// await this.peerConnection.setLocalDescription();
|
||||||
|
// await this.handler.sendSignallingMessage({
|
||||||
|
// type: EventType.CallNegotiate,
|
||||||
|
// content: {
|
||||||
|
// description: this.peerConnection.localDescription!,
|
||||||
|
// [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// } catch (err) {
|
||||||
|
// this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
private async sendAnswer(): Promise<void> {
|
private async sendAnswer(): Promise<void> {
|
||||||
const answerMessage: AnswerMessage = {
|
const localDescription = this.peerConnection.localDescription!;
|
||||||
type: EventType.Answer,
|
const answerContent: MCallAnswer = {
|
||||||
content: {
|
call_id: this.callId,
|
||||||
|
version: 1,
|
||||||
answer: {
|
answer: {
|
||||||
sdp: this.peerConnection.localDescription!.sdp,
|
sdp: localDescription.sdp,
|
||||||
type: this.peerConnection.localDescription!.type,
|
type: localDescription.type,
|
||||||
},
|
},
|
||||||
[SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
|
[SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(),
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// We have just taken the local description from the peerConn which will
|
// We have just taken the local description from the peerConn which will
|
||||||
|
@ -354,7 +487,7 @@ class PeerCall {
|
||||||
this.candidateSendQueue = [];
|
this.candidateSendQueue = [];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.handler.sendSignallingMessage(answerMessage);
|
await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false);
|
this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false);
|
||||||
throw error;
|
throw error;
|
||||||
|
@ -387,7 +520,6 @@ class PeerCall {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async sendCandidateQueue(): Promise<void> {
|
private async sendCandidateQueue(): Promise<void> {
|
||||||
if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) {
|
if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) {
|
||||||
return;
|
return;
|
||||||
|
@ -395,15 +527,16 @@ class PeerCall {
|
||||||
|
|
||||||
const candidates = this.candidateSendQueue;
|
const candidates = this.candidateSendQueue;
|
||||||
this.candidateSendQueue = [];
|
this.candidateSendQueue = [];
|
||||||
const candidatesMessage: CandidatesMessage = {
|
|
||||||
type: EventType.Candidates,
|
|
||||||
content: {
|
|
||||||
candidates: candidates,
|
|
||||||
}
|
|
||||||
};
|
|
||||||
this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`);
|
this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`);
|
||||||
try {
|
try {
|
||||||
await this.handler.sendSignallingMessage(candidatesMessage);
|
await this.handler.sendSignallingMessage({
|
||||||
|
type: EventType.Candidates,
|
||||||
|
content: {
|
||||||
|
call_id: this.callId,
|
||||||
|
version: 1,
|
||||||
|
candidates
|
||||||
|
}
|
||||||
|
});
|
||||||
// Try to send candidates again just in case we received more candidates while sending.
|
// Try to send candidates again just in case we received more candidates while sending.
|
||||||
this.sendCandidateQueue();
|
this.sendCandidateQueue();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
@ -430,7 +563,6 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async addBufferedIceCandidates(): Promise<void> {
|
private async addBufferedIceCandidates(): Promise<void> {
|
||||||
if (this.remoteCandidateBuffer && this.opponentPartyId) {
|
if (this.remoteCandidateBuffer && this.opponentPartyId) {
|
||||||
const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId);
|
const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId);
|
||||||
|
@ -463,7 +595,6 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private setState(state: CallState): void {
|
private setState(state: CallState): void {
|
||||||
const oldState = this.state;
|
const oldState = this.state;
|
||||||
this.state = state;
|
this.state = state;
|
||||||
|
@ -475,7 +606,9 @@ class PeerCall {
|
||||||
this.handler.emitUpdate(this, undefined);
|
this.handler.emitUpdate(this, undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
private waitForState(state: CallState): Promise<void> {
|
private waitForState(states: CallState[]): Promise<void> {
|
||||||
|
// TODO: rework this, do we need to clean up the promises?
|
||||||
|
return Promise.race(states.map(state => {
|
||||||
let deferred = this.statePromiseMap.get(state);
|
let deferred = this.statePromiseMap.get(state);
|
||||||
if (!deferred) {
|
if (!deferred) {
|
||||||
let resolve;
|
let resolve;
|
||||||
|
@ -486,6 +619,7 @@ class PeerCall {
|
||||||
this.statePromiseMap.set(state, deferred);
|
this.statePromiseMap.set(state, deferred);
|
||||||
}
|
}
|
||||||
return deferred.promise;
|
return deferred.promise;
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise<void> {
|
private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise<void> {
|
||||||
|
@ -493,10 +627,12 @@ class PeerCall {
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopAllMedia(): void {
|
private stopAllMedia(): void {
|
||||||
|
if (this.localMedia) {
|
||||||
for (const track of this.localMedia.tracks) {
|
for (const track of this.localMedia.tracks) {
|
||||||
track.stop();
|
track.stop();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async delay(timeoutMs: number): Promise<void> {
|
private async delay(timeoutMs: number): Promise<void> {
|
||||||
// Allow a short time for initial candidates to be gathered
|
// Allow a short time for initial candidates to be gathered
|
||||||
|
@ -514,21 +650,6 @@ class PeerCall {
|
||||||
|
|
||||||
|
|
||||||
//import { randomString } from '../randomstring';
|
//import { randomString } from '../randomstring';
|
||||||
import {
|
|
||||||
MCallReplacesEvent,
|
|
||||||
MCallAnswer,
|
|
||||||
MCallInviteNegotiate,
|
|
||||||
CallCapabilities,
|
|
||||||
SDPStreamMetadataPurpose,
|
|
||||||
SDPStreamMetadata,
|
|
||||||
SDPStreamMetadataKey,
|
|
||||||
MCallSDPStreamMetadataChanged,
|
|
||||||
MCallSelectAnswer,
|
|
||||||
MCAllAssertedIdentity,
|
|
||||||
MCallCandidates,
|
|
||||||
MCallBase,
|
|
||||||
MCallHangupReject,
|
|
||||||
} from './callEventTypes';
|
|
||||||
|
|
||||||
// null is used as a special value meaning that the we're in a legacy 1:1 call
|
// null is used as a special value meaning that the we're in a legacy 1:1 call
|
||||||
// without MSC2746 that doesn't provide an id which device sent the message.
|
// without MSC2746 that doesn't provide an id which device sent the message.
|
||||||
|
@ -681,46 +802,18 @@ export class CallError extends Error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type InviteContent = {
|
export type SignallingMessage<Base extends MCallBase> =
|
||||||
offer: RTCSessionDescriptionInit,
|
{type: EventType.Invite, content: MCallInvite<Base>} |
|
||||||
[SDPStreamMetadataKey]: SDPStreamMetadata,
|
{type: EventType.Answer, content: MCallAnswer<Base>} |
|
||||||
version?: number,
|
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
|
||||||
lifetime?: number
|
{type: EventType.Candidates, content: MCallCandidates<Base>} |
|
||||||
}
|
{type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};
|
||||||
|
|
||||||
export type InviteMessage = {
|
|
||||||
type: EventType.Invite,
|
|
||||||
content: InviteContent
|
|
||||||
}
|
|
||||||
|
|
||||||
type AnwserContent = {
|
|
||||||
answer: {
|
|
||||||
sdp: string,
|
|
||||||
// type is now deprecated as of Matrix VoIP v1, but
|
|
||||||
// required to still be sent for backwards compat
|
|
||||||
type: RTCSdpType,
|
|
||||||
},
|
|
||||||
[SDPStreamMetadataKey]: SDPStreamMetadata,
|
|
||||||
}
|
|
||||||
|
|
||||||
export type AnswerMessage = {
|
|
||||||
type: EventType.Answer,
|
|
||||||
content: AnwserContent
|
|
||||||
}
|
|
||||||
|
|
||||||
type CandidatesContent = {
|
|
||||||
candidates: RTCIceCandidate[]
|
|
||||||
}
|
|
||||||
|
|
||||||
export type CandidatesMessage = {
|
|
||||||
type: EventType.Candidates,
|
|
||||||
content: CandidatesContent
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage;
|
|
||||||
|
|
||||||
export interface PeerCallHandler {
|
export interface PeerCallHandler {
|
||||||
emitUpdate(peerCall: PeerCall, params: any);
|
emitUpdate(peerCall: PeerCall, params: any);
|
||||||
sendSignallingMessage(message: SignallingMessage);
|
sendSignallingMessage(message: SignallingMessage<MCallBase>);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tests() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -130,55 +130,32 @@ write view
|
||||||
|
|
||||||
I think we need to synchronize the negotiation needed because we don't use a CallState to guard it...
|
I think we need to synchronize the negotiation needed because we don't use a CallState to guard it...
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Thursday 3-3 notes
|
## Thursday 3-3 notes
|
||||||
|
|
||||||
we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags?
|
we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags?
|
||||||
|
|
||||||
|
|
||||||
List state transitions
|
## Peer call state transitions
|
||||||
|
|
||||||
FROM CALLER FROM CALLEE
|
FROM CALLER FROM CALLEE
|
||||||
|
|
||||||
Fledgling Fledgling
|
Fledgling Fledgling
|
||||||
V calling `call()` V handleInvite
|
V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates
|
||||||
WaitLocalMedia Ringing
|
WaitLocalMedia Ringing
|
||||||
V media promise resolves V answer()
|
V media promise resolves V `answer()`
|
||||||
CreateOffer WaitLocalMedia
|
V add local tracks WaitLocalMedia
|
||||||
V add tracks V media promise resolves
|
CreateOffer V media promise resolves
|
||||||
V wait for negotionneeded events CreateAnswer
|
V wait for negotionneeded events V add local tracks
|
||||||
V setLocalDescription() V
|
V setLocalDescription() CreateAnswer
|
||||||
V send invite events
|
V send invite event V setLocalDescription(createAnswer())
|
||||||
InviteSent
|
InviteSent |
|
||||||
V receive anwser, setRemoteDescription() |
|
V receive anwser, setRemoteDescription() |
|
||||||
\__________________________________________________/
|
\___________________________________________________/
|
||||||
V
|
V
|
||||||
Connecting
|
Connecting
|
||||||
V receive ice candidates and
|
V receive ice candidates and iceConnectionState becomes 'connected'
|
||||||
iceConnectionState becomes 'connected'
|
|
||||||
Connected
|
Connected
|
||||||
V hangup for some reason
|
V `hangup()` or some terminate condition
|
||||||
Ended
|
|
||||||
|
|
||||||
## From callee
|
|
||||||
|
|
||||||
Fledgling
|
|
||||||
Ringing
|
|
||||||
WaitLocalMedia
|
|
||||||
CreateAnswer
|
|
||||||
Connecting
|
|
||||||
Connected
|
|
||||||
Ended
|
|
||||||
|
|
||||||
Fledgling
|
|
||||||
WaitLocalMedia
|
|
||||||
CreateOffer
|
|
||||||
InviteSent
|
|
||||||
CreateAnswer
|
|
||||||
Connecting
|
|
||||||
Connected
|
|
||||||
Ringing
|
|
||||||
Ended
|
Ended
|
||||||
|
|
||||||
so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection.
|
so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection.
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
// allow non-camelcase as these are events type that go onto the wire
|
// allow non-camelcase as these are events type that go onto the wire
|
||||||
/* eslint-disable camelcase */
|
/* eslint-disable camelcase */
|
||||||
|
|
||||||
import { CallErrorCode } from "./Call";
|
|
||||||
|
|
||||||
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
|
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
|
||||||
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
|
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
|
||||||
|
|
||||||
|
export interface SessionDescription {
|
||||||
|
sdp?: string;
|
||||||
|
type: RTCSdpType
|
||||||
|
}
|
||||||
|
|
||||||
export enum SDPStreamMetadataPurpose {
|
export enum SDPStreamMetadataPurpose {
|
||||||
Usermedia = "m.usermedia",
|
Usermedia = "m.usermedia",
|
||||||
Screenshare = "m.screenshare",
|
Screenshare = "m.screenshare",
|
||||||
|
@ -32,40 +35,36 @@ export interface CallReplacesTarget {
|
||||||
avatar_url: string;
|
avatar_url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallBase {
|
export type MCallBase = {
|
||||||
call_id: string;
|
call_id: string;
|
||||||
version: string | number;
|
version: string | number;
|
||||||
party_id?: string;
|
|
||||||
sender_session_id?: string;
|
|
||||||
dest_session_id?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallAnswer extends MCallBase {
|
export type MGroupCallBase = MCallBase & {
|
||||||
answer: RTCSessionDescription;
|
conf_id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MCallAnswer<Base extends MCallBase> = Base & {
|
||||||
|
answer: SessionDescription;
|
||||||
capabilities?: CallCapabilities;
|
capabilities?: CallCapabilities;
|
||||||
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallSelectAnswer extends MCallBase {
|
export type MCallSelectAnswer<Base extends MCallBase> = Base & {
|
||||||
selected_party_id: string;
|
selected_party_id: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallInviteNegotiate extends MCallBase {
|
export type MCallInvite<Base extends MCallBase> = Base & {
|
||||||
offer: RTCSessionDescription;
|
offer: SessionDescription;
|
||||||
description: RTCSessionDescription;
|
|
||||||
lifetime: number;
|
lifetime: number;
|
||||||
capabilities?: CallCapabilities;
|
|
||||||
invitee?: string;
|
|
||||||
sender_session_id?: string;
|
|
||||||
dest_session_id?: string;
|
|
||||||
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallSDPStreamMetadataChanged extends MCallBase {
|
export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & {
|
||||||
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
[SDPStreamMetadataKey]: SDPStreamMetadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallReplacesEvent extends MCallBase {
|
export type MCallReplacesEvent<Base extends MCallBase> = Base & {
|
||||||
replacement_id: string;
|
replacement_id: string;
|
||||||
target_user: CallReplacesTarget;
|
target_user: CallReplacesTarget;
|
||||||
create_call: string;
|
create_call: string;
|
||||||
|
@ -73,7 +72,7 @@ export interface MCallReplacesEvent extends MCallBase {
|
||||||
target_room: string;
|
target_room: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCAllAssertedIdentity extends MCallBase {
|
export type MCAllAssertedIdentity<Base extends MCallBase> = Base & {
|
||||||
asserted_identity: {
|
asserted_identity: {
|
||||||
id: string;
|
id: string;
|
||||||
display_name: string;
|
display_name: string;
|
||||||
|
@ -81,11 +80,11 @@ export interface MCAllAssertedIdentity extends MCallBase {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallCandidates extends MCallBase {
|
export type MCallCandidates<Base extends MCallBase> = Base & {
|
||||||
candidates: RTCIceCandidate[];
|
candidates: RTCIceCandidate[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MCallHangupReject extends MCallBase {
|
export type MCallHangupReject<Base extends MCallBase> = Base & {
|
||||||
reason?: CallErrorCode;
|
reason?: CallErrorCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -15,14 +15,17 @@ limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {ObservableMap} from "../../../observable/map/ObservableMap";
|
import {ObservableMap} from "../../../observable/map/ObservableMap";
|
||||||
|
import {Participant} from "./Participant";
|
||||||
|
import {LocalMedia} from "../LocalMedia";
|
||||||
|
import type {Track} from "../../../platform/types/MediaDevices";
|
||||||
|
|
||||||
function participantId(senderUserId: string, senderDeviceId: string | null) {
|
function getParticipantId(senderUserId: string, senderDeviceId: string | null) {
|
||||||
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
|
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
class Call {
|
export class GroupCall {
|
||||||
private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
|
private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
|
||||||
private localMedia?: LocalMedia;
|
private localMedia?: Promise<LocalMedia>;
|
||||||
|
|
||||||
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
|
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
|
||||||
|
|
||||||
|
@ -30,17 +33,17 @@ class Call {
|
||||||
|
|
||||||
get id(): string { return this.callEvent.state_key; }
|
get id(): string { return this.callEvent.state_key; }
|
||||||
|
|
||||||
async participate(tracks: Track[]) {
|
async participate(tracks: Promise<Track[]>) {
|
||||||
this.localMedia = LocalMedia.fromTracks(tracks);
|
this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks));
|
||||||
for (const [,participant] of this.participants) {
|
for (const [,participant] of this.participants) {
|
||||||
participant.setLocalMedia(this.localMedia.clone());
|
participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone()));
|
||||||
}
|
}
|
||||||
// send m.call.member state event
|
// send m.call.member state event
|
||||||
|
|
||||||
// send invite to all participants that are < my userId
|
// send invite to all participants that are < my userId
|
||||||
for (const [,participant] of this.participants) {
|
for (const [,participant] of this.participants) {
|
||||||
if (participant.userId < this.ownUserId) {
|
if (participant.userId < this.ownUserId) {
|
||||||
participant.sendInvite();
|
participant.call();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -78,10 +81,6 @@ class Call {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
get id(): string {
|
|
||||||
return this.callEvent.state_key;
|
|
||||||
}
|
|
||||||
|
|
||||||
get isTerminated(): boolean {
|
get isTerminated(): boolean {
|
||||||
return !!this.callEvent.content[CALL_TERMINATED];
|
return !!this.callEvent.content[CALL_TERMINATED];
|
||||||
}
|
}
|
|
@ -14,35 +14,54 @@ See the License for the specific language governing permissions and
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {EventType} from "../PeerCall";
|
import {EventType, PeerCall, SignallingMessage} from "../PeerCall";
|
||||||
|
import {makeTxnId} from "../../common";
|
||||||
|
|
||||||
import type {PeerCallHandler} from "../PeerCall";
|
import type {PeerCallHandler} from "../PeerCall";
|
||||||
|
import type {LocalMedia} from "../LocalMedia";
|
||||||
|
import type {HomeServerApi} from "../../net/HomeServerApi";
|
||||||
|
import type {Track} from "../../../platform/types/MediaDevices";
|
||||||
|
import type {MCallBase, MGroupCallBase} from "../callEventTypes";
|
||||||
|
import type {GroupCall} from "./GroupCall";
|
||||||
|
import type {RoomMember} from "../../room/members/RoomMember";
|
||||||
|
|
||||||
class Participant implements PeerCallHandler {
|
export class Participant implements PeerCallHandler {
|
||||||
private peerCall?: PeerCall;
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly userId: string,
|
public readonly member: RoomMember,
|
||||||
private readonly deviceId: string,
|
private readonly deviceId: string | undefined,
|
||||||
private localMedia: LocalMedia | undefined,
|
private readonly peerCall: PeerCall,
|
||||||
private readonly webRTC: WebRTC,
|
private readonly hsApi: HomeServerApi,
|
||||||
private readonly hsApi: HomeServerApi
|
private readonly groupCall: GroupCall
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
sendInvite() {
|
/* @internal */
|
||||||
this.peerCall = new PeerCall(this, this.webRTC);
|
call(localMedia: Promise<LocalMedia>) {
|
||||||
this.peerCall.call(this.localMedia);
|
this.peerCall.call(localMedia);
|
||||||
|
}
|
||||||
|
|
||||||
|
get remoteTracks(): Track[] {
|
||||||
|
return this.peerCall.remoteTracks;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** From PeerCallHandler
|
/** From PeerCallHandler
|
||||||
* @internal */
|
* @internal */
|
||||||
emitUpdate(params: any) {
|
emitUpdate(params: any) {
|
||||||
|
this.groupCall.emitParticipantUpdate(this, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** From PeerCallHandler
|
/** From PeerCallHandler
|
||||||
* @internal */
|
* @internal */
|
||||||
onSendSignallingMessage(type: EventType, content: Record<string, any>) {
|
async sendSignallingMessage(message: SignallingMessage<MCallBase>) {
|
||||||
|
const groupMessage = message as SignallingMessage<MGroupCallBase>;
|
||||||
|
groupMessage.content.conf_id = this.groupCall.id;
|
||||||
// TODO: this needs to be encrypted with olm first
|
// TODO: this needs to be encrypted with olm first
|
||||||
this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
|
|
||||||
|
const request = this.hsApi.sendToDevice(
|
||||||
|
groupMessage.type,
|
||||||
|
{[this.member.userId]: {
|
||||||
|
[this.deviceId ?? "*"]: groupMessage.content
|
||||||
|
}
|
||||||
|
}, makeTxnId());
|
||||||
|
await request.response();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -144,7 +144,7 @@ export class AudioTrackWrapper extends TrackWrapper {
|
||||||
} else {
|
} else {
|
||||||
this.measuringVolumeActivity = false;
|
this.measuringVolumeActivity = false;
|
||||||
this.speakingVolumeSamples.fill(-Infinity);
|
this.speakingVolumeSamples.fill(-Infinity);
|
||||||
this.emit(CallFeedEvent.VolumeChanged, -Infinity);
|
// this.emit(CallFeedEvent.VolumeChanged, -Infinity);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -186,7 +186,7 @@ export class AudioTrackWrapper extends TrackWrapper {
|
||||||
this.speakingVolumeSamples.shift();
|
this.speakingVolumeSamples.shift();
|
||||||
this.speakingVolumeSamples.push(maxVolume);
|
this.speakingVolumeSamples.push(maxVolume);
|
||||||
|
|
||||||
this.emit(CallFeedEvent.VolumeChanged, maxVolume);
|
// this.emit(CallFeedEvent.VolumeChanged, maxVolume);
|
||||||
|
|
||||||
let newSpeaking = false;
|
let newSpeaking = false;
|
||||||
|
|
||||||
|
@ -201,7 +201,7 @@ export class AudioTrackWrapper extends TrackWrapper {
|
||||||
|
|
||||||
if (this.speaking !== newSpeaking) {
|
if (this.speaking !== newSpeaking) {
|
||||||
this.speaking = newSpeaking;
|
this.speaking = newSpeaking;
|
||||||
this.emit(CallFeedEvent.Speaking, this.speaking);
|
// this.emit(CallFeedEvent.Speaking, this.speaking);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number;
|
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number;
|
||||||
|
|
|
@ -114,7 +114,7 @@ class DOMPeerConnection implements PeerConnection {
|
||||||
}
|
}
|
||||||
|
|
||||||
createDataChannel(): DataChannel {
|
createDataChannel(): DataChannel {
|
||||||
return new DataChannel(this.peerConnection.createDataChannel());
|
return undefined as any;// new DataChannel(this.peerConnection.createDataChannel());
|
||||||
}
|
}
|
||||||
|
|
||||||
private registerHandler() {
|
private registerHandler() {
|
||||||
|
|
|
@ -1486,9 +1486,9 @@ type-fest@^0.20.2:
|
||||||
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
|
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
|
||||||
|
|
||||||
typescript@^4.3.5:
|
typescript@^4.3.5:
|
||||||
version "4.3.5"
|
version "4.6.2"
|
||||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4"
|
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
|
||||||
integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==
|
integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
|
||||||
|
|
||||||
typeson-registry@^1.0.0-alpha.20:
|
typeson-registry@^1.0.0-alpha.20:
|
||||||
version "1.0.0-alpha.39"
|
version "1.0.0-alpha.39"
|
||||||
|
|
Loading…
Reference in a new issue