mystiq/hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity

WIP10

Browse source
This commit is contained in:
Bruno Windels 2022-03-09 11:29:39 +01:00 committed by Bruno Windels
parent 6fe90e60db
commit 60da85d641
13 changed files with 347 additions and 3126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -45,7 +45,7 @@
"postcss-flexbugs-fixes": "^5.0.2",
"regenerator-runtime": "^0.13.7",
"text-encoding": "^0.7.0",
"typescript": "^4.3.5",
"typescript": "^4.4",
"vite": "^2.6.14",
"xxhashjs": "^0.2.2"
},

2
src/matrix/DeviceMessageHandler.js
View file

@ -59,7 +59,7 @@ export class DeviceMessageHandler {
}));
// TODO: pass this in the prep and run it in afterSync or afterSyncComplete (as callHandler can send events as well)?
for (const dr of callMessages) {
this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event.type, dr.event.content, log);
this._callHandler.handleDeviceMessage(dr.device.userId, dr.device.deviceId, dr.event, log);
}
// TODO: somehow include rooms that received a call to_device message in the sync state?
// or have updates flow through event emitter?

2493
src/matrix/calls/Call.ts
View file

File diff suppressed because it is too large Load diff

274
src/matrix/calls/CallFeed.ts
View file

@ -1,274 +0,0 @@
/*
Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { SDPStreamMetadataPurpose } from "./callEventTypes";
const POLLING_INTERVAL = 200; // ms
export const SPEAKING_THRESHOLD = -60; // dB
const SPEAKING_SAMPLE_COUNT = 8; // samples
export interface ICallFeedOpts {
client: MatrixClient;
roomId: string;
userId: string;
stream: MediaStream;
purpose: SDPStreamMetadataPurpose;
audioMuted: boolean;
videoMuted: boolean;
}
export enum CallFeedEvent {
NewStream = "new_stream",
MuteStateChanged = "mute_state_changed",
VolumeChanged = "volume_changed",
Speaking = "speaking",
}
export class CallFeed extends EventEmitter {
public stream: MediaStream;
public sdpMetadataStreamId: string;
public userId: string;
public purpose: SDPStreamMetadataPurpose;
public speakingVolumeSamples: number[];
private client: MatrixClient;
private roomId: string;
private audioMuted: boolean;
private videoMuted: boolean;
private measuringVolumeActivity = false;
private audioContext: AudioContext;
private analyser: AnalyserNode;
private frequencyBinCount: Float32Array;
private speakingThreshold = SPEAKING_THRESHOLD;
private speaking = false;
private volumeLooperTimeout: number;
constructor(opts: ICallFeedOpts) {
super();
this.client = opts.client;
this.roomId = opts.roomId;
this.userId = opts.userId;
this.purpose = opts.purpose;
this.audioMuted = opts.audioMuted;
this.videoMuted = opts.videoMuted;
this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
this.sdpMetadataStreamId = opts.stream.id;
this.updateStream(null, opts.stream);
if (this.hasAudioTrack) {
this.initVolumeMeasuring();
}
}
private get hasAudioTrack(): boolean {
return this.stream.getAudioTracks().length > 0;
}
private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
if (newStream === oldStream) return;
if (oldStream) {
oldStream.removeEventListener("addtrack", this.onAddTrack);
this.measureVolumeActivity(false);
}
if (newStream) {
this.stream = newStream;
newStream.addEventListener("addtrack", this.onAddTrack);
if (this.hasAudioTrack) {
this.initVolumeMeasuring();
} else {
this.measureVolumeActivity(false);
}
}
this.emit(CallFeedEvent.NewStream, this.stream);
}
private initVolumeMeasuring(): void {
const AudioContext = window.AudioContext || window.webkitAudioContext;
if (!this.hasAudioTrack || !AudioContext) return;
this.audioContext = new AudioContext();
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 512;
this.analyser.smoothingTimeConstant = 0.1;
const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
mediaStreamAudioSourceNode.connect(this.analyser);
this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
}
private onAddTrack = (): void => {
this.emit(CallFeedEvent.NewStream, this.stream);
};
/**
* Returns callRoom member
* @returns member of the callRoom
*/
public getMember(): RoomMember {
const callRoom = this.client.getRoom(this.roomId);
return callRoom.getMember(this.userId);
}
/**
* Returns true if CallFeed is local, otherwise returns false
* @returns {boolean} is local?
*/
public isLocal(): boolean {
return this.userId === this.client.getUserId();
}
/**
* Returns true if audio is muted or if there are no audio
* tracks, otherwise returns false
* @returns {boolean} is audio muted?
*/
public isAudioMuted(): boolean {
return this.stream.getAudioTracks().length === 0 || this.audioMuted;
}
/**
* Returns true video is muted or if there are no video
* tracks, otherwise returns false
* @returns {boolean} is video muted?
*/
public isVideoMuted(): boolean {
// We assume only one video track
return this.stream.getVideoTracks().length === 0 || this.videoMuted;
}
public isSpeaking(): boolean {
return this.speaking;
}
/**
* Replaces the current MediaStream with a new one.
* This method should be only used by MatrixCall.
* @param newStream new stream with which to replace the current one
*/
public setNewStream(newStream: MediaStream): void {
this.updateStream(this.stream, newStream);
}
/**
* Set feed's internal audio mute state
* @param muted is the feed's audio muted?
*/
public setAudioMuted(muted: boolean): void {
this.audioMuted = muted;
this.speakingVolumeSamples.fill(-Infinity);
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
}
/**
* Set feed's internal video mute state
* @param muted is the feed's video muted?
*/
public setVideoMuted(muted: boolean): void {
this.videoMuted = muted;
this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
}
/**
* Starts emitting volume_changed events where the emitter value is in decibels
* @param enabled emit volume changes
*/
public measureVolumeActivity(enabled: boolean): void {
if (enabled) {
if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
this.measuringVolumeActivity = true;
this.volumeLooper();
} else {
this.measuringVolumeActivity = false;
this.speakingVolumeSamples.fill(-Infinity);
this.emit(CallFeedEvent.VolumeChanged, -Infinity);
}
}
public setSpeakingThreshold(threshold: number) {
this.speakingThreshold = threshold;
}
private volumeLooper = () => {
if (!this.analyser) return;
if (!this.measuringVolumeActivity) return;
this.analyser.getFloatFrequencyData(this.frequencyBinCount);
let maxVolume = -Infinity;
for (let i = 0; i < this.frequencyBinCount.length; i++) {
if (this.frequencyBinCount[i] > maxVolume) {
maxVolume = this.frequencyBinCount[i];
}
}
this.speakingVolumeSamples.shift();
this.speakingVolumeSamples.push(maxVolume);
this.emit(CallFeedEvent.VolumeChanged, maxVolume);
let newSpeaking = false;
for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
const volume = this.speakingVolumeSamples[i];
if (volume > this.speakingThreshold) {
newSpeaking = true;
break;
}
}
if (this.speaking !== newSpeaking) {
this.speaking = newSpeaking;
this.emit(CallFeedEvent.Speaking, this.speaking);
}
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
};
public clone(): CallFeed {
const mediaHandler = this.client.getMediaHandler();
const stream = this.stream.clone();
if (this.purpose === SDPStreamMetadataPurpose.Usermedia) {
mediaHandler.userMediaStreams.push(stream);
} else {
mediaHandler.screensharingStreams.push(stream);
}
return new CallFeed({
client: this.client,
roomId: this.roomId,
userId: this.userId,
stream,
purpose: this.purpose,
audioMuted: this.audioMuted,
videoMuted: this.videoMuted,
});
}
public dispose(): void {
clearTimeout(this.volumeLooperTimeout);
}
}

113
src/matrix/calls/CallHandler.ts
View file

@ -22,6 +22,8 @@ import type {ILogItem} from "../../logging/types";
import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
import type {SignallingMessage} from "./PeerCall";
import type {MGroupCallBase} from "./callEventTypes";
const GROUP_CALL_TYPE = "m.call";
const GROUP_CALL_MEMBER_TYPE = "m.call.member";
@ -33,7 +35,7 @@ enum CallSetupMessageType {
Hangup = "m.call.hangup",
}
const CALL_ID = "m.call_id";
const CONF_ID = "conf_id";
const CALL_TERMINATED = "m.terminated";
export class GroupCallHandler {
@ -69,7 +71,7 @@ export class GroupCallHandler {
const participant = event.state_key;
const sources = event.content["m.sources"];
for (const source of sources) {
const call = this.calls.get(source[CALL_ID]);
const call = this.calls.get(source[CONF_ID]);
if (call && !call.isTerminated) {
call.addParticipant(participant, source);
}
@ -85,110 +87,9 @@ export class GroupCallHandler {
eventType === CallSetupMessageType.Hangup;
}
handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) {
const callId = content[CALL_ID];
const call = this.calls.get(callId);
call?.handleDeviceMessage(senderUserId, senderDeviceId, eventType, content, log);
handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage<MGroupCallBase>, log: ILogItem) {
const call = this.calls.get(event.content.conf_id);
call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log);
}
}
function participantId(senderUserId: string, senderDeviceId: string | null) {
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
}
class GroupParticipant implements PeerCallHandler {
private peerCall?: PeerCall;
constructor(
private readonly userId: string,
private readonly deviceId: string,
private localMedia: LocalMedia | undefined,
private readonly webRTC: WebRTC,
private readonly hsApi: HomeServerApi
) {}
sendInvite() {
this.peerCall = new PeerCall(this, this.webRTC);
this.peerCall.call(this.localMedia);
}
/** From PeerCallHandler
* @internal */
override emitUpdate() {
}
/** From PeerCallHandler
* @internal */
override onSendSignallingMessage() {
// TODO: this needs to be encrypted with olm first
this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
}
}
class GroupCall {
private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
private localMedia?: LocalMedia;
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
}
get id(): string { return this.callEvent.state_key; }
async participate(tracks: Track[]) {
this.localMedia = LocalMedia.fromTracks(tracks);
for (const [,participant] of this.participants) {
participant.setMedia(this.localMedia.clone());
}
// send m.call.member state event
// send invite to all participants that are < my userId
for (const [,participant] of this.participants) {
if (participant.userId < this.ownUserId) {
participant.sendInvite();
}
}
}
updateCallEvent(callEvent: StateEvent) {
this.callEvent = callEvent;
}
addParticipant(userId, source) {
const participantId = getParticipantId(userId, source.device_id);
const participant = this.participants.get(participantId);
if (participant) {
participant.updateSource(source);
} else {
participant.add(participantId, new GroupParticipant(userId, source.device_id, this.localMedia?.clone(), this.webRTC));
}
}
handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) {
const participantId = getParticipantId(senderUserId, senderDeviceId);
let peerCall = this.participants.get(participantId);
let hasDeviceInKey = true;
if (!peerCall) {
hasDeviceInKey = false;
peerCall = this.participants.get(getParticipantId(senderUserId, null))
}
if (peerCall) {
peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId);
if (!hasDeviceInKey && peerCall.opponentPartyId) {
this.participants.delete(getParticipantId(senderUserId, null));
this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId));
}
} else {
// create peerCall
}
}
get id(): string {
return this.callEvent.state_key;
}
get isTerminated(): boolean {
return !!this.callEvent.content[CALL_TERMINATED];
}
}

415
src/matrix/calls/PeerCall.ts
View file

@ -17,26 +17,41 @@ limitations under the License.
import {ObservableMap} from "../../observable/map/ObservableMap";
import {recursivelyAssign} from "../../utils/recursivelyAssign";
import {AsyncQueue} from "../../utils/AsyncQueue";
import {Disposables, Disposable} from "../../utils/Disposables";
import {Disposables, IDisposable} from "../../utils/Disposables";
import type {Room} from "../room/Room";
import type {StateEvent} from "../storage/types";
import type {ILogItem} from "../../logging/types";
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
import {WebRTC, PeerConnection, PeerConnectionHandler} from "../../platform/types/WebRTC";
import {WebRTC, PeerConnection, PeerConnectionHandler, DataChannel} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
import type {LocalMedia} from "./LocalMedia";
import {
SDPStreamMetadataKey,
SDPStreamMetadataPurpose
} from "./callEventTypes";
import type {
MCallBase,
MCallInvite,
MCallAnswer,
MCallSDPStreamMetadataChanged,
MCallCandidates,
MCallHangupReject,
SDPStreamMetadata,
} from "./callEventTypes";
// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already
// do for sharing keys will be best as that already deals with room tracking.
/**
* Does WebRTC signalling for a single PeerConnection, and deals with WebRTC wrappers from platform
* */
/** Implements a call between two peers with the signalling state keeping, while still delegating the signalling message sending. Used by GroupCall.*/
class PeerCall {
export class PeerCall implements IDisposable {
private readonly peerConnection: PeerConnection;
private state = CallState.Fledgling;
private direction: CallDirection;
private localMedia?: LocalMedia;
// A queue for candidates waiting to go out.
// We try to amalgamate candidates into a single candidate message where
// possible
@ -54,9 +69,13 @@ class PeerCall {
private disposables = new Disposables();
private statePromiseMap = new Map<CallState, {resolve: () => void, promise: Promise<void>}>();
// perfect negotiation flags
private makingOffer: boolean = false;
private ignoreOffer: boolean = false;
constructor(
private callId: string, // generated or from invite
private readonly handler: PeerCallHandler,
private localMedia: LocalMedia,
private readonly createTimeout: TimeoutCreator,
webRTC: WebRTC
) {
@ -83,29 +102,8 @@ class PeerCall {
}
}
handleIncomingSignallingMessage(message: SignallingMessage, partyId: PartyId) {
switch (message.type) {
case EventType.Invite:
// determining whether or not an incoming invite glares
// with an instance of PeerCall is different for group calls
// and 1:1 calls, so done outside of this class.
// If you pass an event for another call id in here it will assume it glares.
//const newCallId = message.content.call_id;
//if (this.id && newCallId !== this.id) {
// this.handleInviteGlare(message.content);
//} else {
this.handleInvite(message.content, partyId);
//}
break;
case EventType.Answer:
this.handleAnswer(message.content, partyId);
break;
case EventType.Candidates:
this.handleRemoteIceCandidates(message.content, partyId);
break;
case EventType.Hangup:
}
get remoteTracks(): Track[] {
return this.peerConnection.remoteTracks;
}
async call(localMediaPromise: Promise<LocalMedia>): Promise<void> {
@ -125,7 +123,9 @@ class PeerCall {
for (const t of this.localMedia.tracks) {
this.peerConnection.addTrack(t);
}
await this.waitForState(CallState.InviteSent);
// TODO: in case of glare, we would not go to InviteSent if we haven't started sending yet
// but we would go straight to CreateAnswer, so also need to wait for that state
await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]);
}
async answer(localMediaPromise: Promise<LocalMedia>): Promise<void> {
@ -166,15 +166,11 @@ class PeerCall {
this.sendAnswer();
}
async hangup() {
}
async setMedia(localMediaPromise: Promise<LocalMedia>) {
const oldMedia = this.localMedia;
this.localMedia = await localMediaPromise;
const applyTrack = (selectTrack: (media: LocalMedia) => Track | undefined) => {
const applyTrack = (selectTrack: (media: LocalMedia | undefined) => Track | undefined) => {
const oldTrack = selectTrack(oldMedia);
const newTrack = selectTrack(this.localMedia);
if (oldTrack && newTrack) {
@ -187,50 +183,101 @@ class PeerCall {
};
// add the local tracks, and wait for onNegotiationNeeded and handleNegotiation to be called
applyTrack(m => m.microphoneTrack);
applyTrack(m => m.cameraTrack);
applyTrack(m => m.screenShareTrack);
applyTrack(m => m?.microphoneTrack);
applyTrack(m => m?.cameraTrack);
applyTrack(m => m?.screenShareTrack);
}
async reject() {
}
async hangup(errorCode: CallErrorCode) {
}
async handleIncomingSignallingMessage<B extends MCallBase>(message: SignallingMessage<B>, partyId: PartyId): Promise<void> {
switch (message.type) {
case EventType.Invite:
if (this.callId !== message.content.call_id) {
await this.handleInviteGlare(message.content, partyId);
} else {
await this.handleFirstInvite(message.content, partyId);
}
break;
case EventType.Answer:
await this.handleAnswer(message.content, partyId);
break;
//case EventType.Candidates:
// await this.handleRemoteIceCandidates(message.content, partyId);
// break;
case EventType.Hangup:
default:
throw new Error(`Unknown event type for call: ${message.type}`);
}
}
private sendHangupWithCallId(callId: string, reason?: CallErrorCode): Promise<void> {
const content = {
call_id: callId,
version: 1,
};
if (reason) {
content["reason"] = reason;
}
return this.handler.sendSignallingMessage({
type: EventType.Hangup,
content
});
}
// calls are serialized and deduplicated by responsePromiseChain
private handleNegotiation = async (): Promise<void> => {
// TODO: does this make sense to have this state if we're already connected?
this.setState(CallState.MakingOffer)
this.makingOffer = true;
try {
await this.peerConnection.setLocalDescription();
} catch (err) {
this.logger.debug(`Call ${this.callId} Error setting local description!`, err);
this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
return;
try {
await this.peerConnection.setLocalDescription();
} catch (err) {
this.logger.debug(`Call ${this.callId} Error setting local description!`, err);
this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
return;
}
if (this.peerConnection.iceGatheringState === 'gathering') {
// Allow a short time for initial candidates to be gathered
await this.delay(200);
}
if (this.state === CallState.Ended) {
return;
}
const offer = this.peerConnection.localDescription!;
// Get rid of any candidates waiting to be sent: they'll be included in the local
// description we just got and will send in the offer.
this.logger.info(`Call ${this.callId} Discarding ${
this.candidateSendQueue.length} candidates that will be sent in offer`);
this.candidateSendQueue = [];
// need to queue this
const content = {
call_id: this.callId,
offer,
[SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(),
version: 1,
lifetime: CALL_TIMEOUT_MS
};
if (this.state === CallState.CreateOffer) {
await this.handler.sendSignallingMessage({type: EventType.Invite, content});
this.setState(CallState.InviteSent);
} else if (this.state === CallState.Connected || this.state === CallState.Connecting) {
// send Negotiate message
//await this.handler.sendSignallingMessage({type: EventType.Invite, content});
//this.setState(CallState.InviteSent);
}
} finally {
this.makingOffer = false;
}
if (this.peerConnection.iceGatheringState === 'gathering') {
// Allow a short time for initial candidates to be gathered
await this.delay(200);
}
if (this.state === CallState.Ended) {
return;
}
const offer = this.peerConnection.localDescription!;
// Get rid of any candidates waiting to be sent: they'll be included in the local
// description we just got and will send in the offer.
this.logger.info(`Call ${this.callId} Discarding ${
this.candidateSendQueue.length} candidates that will be sent in offer`);
this.candidateSendQueue = [];
// need to queue this
const content = {
offer,
[SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
version: 1,
lifetime: CALL_TIMEOUT_MS
};
if (this.state === CallState.CreateOffer) {
await this.handler.sendSignallingMessage({type: EventType.Invite, content});
this.setState(CallState.InviteSent);
}
this.sendCandidateQueue();
if (this.state === CallState.InviteSent) {
@ -242,11 +289,47 @@ class PeerCall {
}
};
private async handleInvite(content: InviteContent, partyId: PartyId): Promise<void> {
private async handleInviteGlare(content: MCallInvite, partyId: PartyId): Promise<void> {
// this is only called when the ids are different
const newCallId = content.call_id;
if (this.callId! > newCallId) {
this.logger.log(
"Glare detected: answering incoming call " + newCallId +
" and canceling outgoing call " + this.callId,
);
/*
first, we should set CallDirection
we should anser the call
*/
// TODO: review states to be unambigous, WaitLocalMedia for sending offer or answer?
// How do we interrupt `call()`? well, perhaps we need to not just await InviteSent but also CreateAnswer?
if (this.state === CallState.Fledgling || this.state === CallState.CreateOffer || this.state === CallState.WaitLocalMedia) {
} else {
await this.sendHangupWithCallId(this.callId, CallErrorCode.Replaced);
}
await this.handleInvite(content, partyId);
await this.answer(Promise.resolve(this.localMedia!));
} else {
this.logger.log(
"Glare detected: rejecting incoming call " + newCallId +
" and keeping outgoing call " + this.callId,
);
await this.sendHangupWithCallId(newCallId, CallErrorCode.Replaced);
}
}
private async handleFirstInvite(content: MCallInvite, partyId: PartyId): Promise<void> {
if (this.state !== CallState.Fledgling || this.opponentPartyId !== undefined) {
// TODO: hangup or ignore?
return;
}
await this.handleInvite(content, partyId);
}
private async handleInvite(content: MCallInvite, partyId: PartyId): Promise<void> {
// we must set the party ID before await-ing on anything: the call event
// handler will start giving us more call events (eg. candidates) so if
@ -296,8 +379,8 @@ class PeerCall {
}
}
private async handleAnswer(content: AnwserContent, partyId: PartyId): Promise<void> {
this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${content.party_id}`);
private async handleAnswer(content: MCallAnswer, partyId: PartyId): Promise<void> {
this.logger.debug(`Got answer for call ID ${this.callId} from party ID ${partyId}`);
if (this.state === CallState.Ended) {
this.logger.debug(`Ignoring answer because call ID ${this.callId} has ended`);
@ -307,7 +390,7 @@ class PeerCall {
if (this.opponentPartyId !== undefined) {
this.logger.info(
`Call ${this.callId} ` +
`Ignoring answer from party ID ${content.party_id}: ` +
`Ignoring answer from party ID ${partyId}: ` +
`we already have an answer/reject from ${this.opponentPartyId}`,
);
return;
@ -334,16 +417,66 @@ class PeerCall {
}
}
// private async onNegotiateReceived(event: MatrixEvent): Promise<void> {
// const content = event.getContent<MCallNegotiate>();
// const description = content.description;
// if (!description || !description.sdp || !description.type) {
// this.logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`);
// return;
// }
// // Politeness always follows the direction of the call: in a glare situation,
// // we pick either the inbound or outbound call, so one side will always be
// // inbound and one outbound
// const polite = this.direction === CallDirection.Inbound;
// // Here we follow the perfect negotiation logic from
// // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation
// const offerCollision = (
// (description.type === 'offer') &&
// (this.makingOffer || this.peerConnection.signalingState !== 'stable')
// );
// this.ignoreOffer = !polite && offerCollision;
// if (this.ignoreOffer) {
// this.logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`);
// return;
// }
// const sdpStreamMetadata = content[SDPStreamMetadataKey];
// if (sdpStreamMetadata) {
// this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
// } else {
// this.logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`);
// }
// try {
// await this.peerConnection.setRemoteDescription(description);
// if (description.type === 'offer') {
// await this.peerConnection.setLocalDescription();
// await this.handler.sendSignallingMessage({
// type: EventType.CallNegotiate,
// content: {
// description: this.peerConnection.localDescription!,
// [SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
// }
// });
// }
// } catch (err) {
// this.logger.warn(`Call ${this.callId} Failed to complete negotiation`, err);
// }
// }
private async sendAnswer(): Promise<void> {
const answerMessage: AnswerMessage = {
type: EventType.Answer,
content: {
answer: {
sdp: this.peerConnection.localDescription!.sdp,
type: this.peerConnection.localDescription!.type,
},
[SDPStreamMetadataKey]: this.localMedia.getSDPMetadata(),
}
const localDescription = this.peerConnection.localDescription!;
const answerContent: MCallAnswer = {
call_id: this.callId,
version: 1,
answer: {
sdp: localDescription.sdp,
type: localDescription.type,
},
[SDPStreamMetadataKey]: this.localMedia!.getSDPMetadata(),
};
// We have just taken the local description from the peerConn which will
@ -354,7 +487,7 @@ class PeerCall {
this.candidateSendQueue = [];
try {
await this.handler.sendSignallingMessage(answerMessage);
await this.handler.sendSignallingMessage({type: EventType.Answer, content: answerContent});
} catch (error) {
this.terminate(CallParty.Local, CallErrorCode.SendAnswer, false);
throw error;
@ -387,7 +520,6 @@ class PeerCall {
});
}
private async sendCandidateQueue(): Promise<void> {
if (this.candidateSendQueue.length === 0 || this.state === CallState.Ended) {
return;
@ -395,15 +527,16 @@ class PeerCall {
const candidates = this.candidateSendQueue;
this.candidateSendQueue = [];
const candidatesMessage: CandidatesMessage = {
type: EventType.Candidates,
content: {
candidates: candidates,
}
};
this.logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`);
try {
await this.handler.sendSignallingMessage(candidatesMessage);
await this.handler.sendSignallingMessage({
type: EventType.Candidates,
content: {
call_id: this.callId,
version: 1,
candidates
}
});
// Try to send candidates again just in case we received more candidates while sending.
this.sendCandidateQueue();
} catch (error) {
@ -430,7 +563,6 @@ class PeerCall {
}
}
private async addBufferedIceCandidates(): Promise<void> {
if (this.remoteCandidateBuffer && this.opponentPartyId) {
const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId);
@ -463,7 +595,6 @@ class PeerCall {
}
}
private setState(state: CallState): void {
const oldState = this.state;
this.state = state;
@ -475,17 +606,20 @@ class PeerCall {
this.handler.emitUpdate(this, undefined);
}
private waitForState(state: CallState): Promise<void> {
let deferred = this.statePromiseMap.get(state);
if (!deferred) {
let resolve;
const promise = new Promise<void>(r => {
resolve = r;
});
deferred = {resolve, promise};
this.statePromiseMap.set(state, deferred);
}
return deferred.promise;
private waitForState(states: CallState[]): Promise<void> {
// TODO: rework this, do we need to clean up the promises?
return Promise.race(states.map(state => {
let deferred = this.statePromiseMap.get(state);
if (!deferred) {
let resolve;
const promise = new Promise<void>(r => {
resolve = r;
});
deferred = {resolve, promise};
this.statePromiseMap.set(state, deferred);
}
return deferred.promise;
}));
}
private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise<void> {
@ -493,8 +627,10 @@ class PeerCall {
}
private stopAllMedia(): void {
for (const track of this.localMedia.tracks) {
track.stop();
if (this.localMedia) {
for (const track of this.localMedia.tracks) {
track.stop();
}
}
}
@ -514,21 +650,6 @@ class PeerCall {
//import { randomString } from '../randomstring';
import {
MCallReplacesEvent,
MCallAnswer,
MCallInviteNegotiate,
CallCapabilities,
SDPStreamMetadataPurpose,
SDPStreamMetadata,
SDPStreamMetadataKey,
MCallSDPStreamMetadataChanged,
MCallSelectAnswer,
MCAllAssertedIdentity,
MCallCandidates,
MCallBase,
MCallHangupReject,
} from './callEventTypes';
// null is used as a special value meaning that the we're in a legacy 1:1 call
// without MSC2746 that doesn't provide an id which device sent the message.
@ -681,46 +802,18 @@ export class CallError extends Error {
}
}
type InviteContent = {
offer: RTCSessionDescriptionInit,
[SDPStreamMetadataKey]: SDPStreamMetadata,
version?: number,
lifetime?: number
}
export type InviteMessage = {
type: EventType.Invite,
content: InviteContent
}
type AnwserContent = {
answer: {
sdp: string,
// type is now deprecated as of Matrix VoIP v1, but
// required to still be sent for backwards compat
type: RTCSdpType,
},
[SDPStreamMetadataKey]: SDPStreamMetadata,
}
export type AnswerMessage = {
type: EventType.Answer,
content: AnwserContent
}
type CandidatesContent = {
candidates: RTCIceCandidate[]
}
export type CandidatesMessage = {
type: EventType.Candidates,
content: CandidatesContent
}
export type SignallingMessage = InviteMessage | AnswerMessage | CandidatesMessage;
export type SignallingMessage<Base extends MCallBase> =
{type: EventType.Invite, content: MCallInvite<Base>} |
{type: EventType.Answer, content: MCallAnswer<Base>} |
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
{type: EventType.Candidates, content: MCallCandidates<Base>} |
{type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};
export interface PeerCallHandler {
emitUpdate(peerCall: PeerCall, params: any);
sendSignallingMessage(message: SignallingMessage);
sendSignallingMessage(message: SignallingMessage<MCallBase>);
}
export function tests() {
}

49
src/matrix/calls/TODO.md
View file

@ -130,57 +130,34 @@ write view
I think we need to synchronize the negotiation needed because we don't use a CallState to guard it...
## Thursday 3-3 notes
we probably best keep the perfect negotiation flags, as they are needed for both starting the call AND renegotiation? if only for the former, it would make sense as it is a step in setting up the call, but if the call is ongoing, does it make sense to have a MakingOffer state? it actually looks like they are only needed for renegotiation! for call setup we compare the call_ids. What does that mean for these flags?
List state transitions
## Peer call state transitions
FROM CALLER FROM CALLEE
Fledgling Fledgling
V calling `call()` V handleInvite
V `call()` V `handleInvite()`: setRemoteDescription(event.offer), add buffered candidates
WaitLocalMedia Ringing
V media promise resolves V answer()
CreateOffer WaitLocalMedia
V add tracks V media promise resolves
V wait for negotionneeded events CreateAnswer
V setLocalDescription() V
V send invite events
InviteSent
V receive anwser, setRemoteDescription() |
\__________________________________________________/
V media promise resolves V `answer()`
V add local tracks WaitLocalMedia
CreateOffer V media promise resolves
V wait for negotionneeded events V add local tracks
V setLocalDescription() CreateAnswer
V send invite event V setLocalDescription(createAnswer())
InviteSent |
V receive anwser, setRemoteDescription() |
\___________________________________________________/
V
Connecting
V receive ice candidates and
iceConnectionState becomes 'connected'
V receive ice candidates and iceConnectionState becomes 'connected'
Connected
V hangup for some reason
V `hangup()` or some terminate condition
Ended
## From callee
Fledgling
Ringing
WaitLocalMedia
CreateAnswer
Connecting
Connected
Ended
Fledgling
WaitLocalMedia
CreateOffer
InviteSent
CreateAnswer
Connecting
Connected
Ringing
Ended
so if we don't want to bother with having two call objects, we can make the existing call hangup his old call_id? That way we keep the old peerConnection.

41
src/matrix/calls/callEventTypes.ts
View file

@ -1,11 +1,14 @@
// allow non-camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */
import { CallErrorCode } from "./Call";
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
export interface SessionDescription {
sdp?: string;
type: RTCSdpType
}
export enum SDPStreamMetadataPurpose {
Usermedia = "m.usermedia",
Screenshare = "m.screenshare",
@ -32,40 +35,36 @@ export interface CallReplacesTarget {
avatar_url: string;
}
export interface MCallBase {
export type MCallBase = {
call_id: string;
version: string | number;
party_id?: string;
sender_session_id?: string;
dest_session_id?: string;
}
export interface MCallAnswer extends MCallBase {
answer: RTCSessionDescription;
export type MGroupCallBase = MCallBase & {
conf_id: string;
}
export type MCallAnswer<Base extends MCallBase> = Base & {
answer: SessionDescription;
capabilities?: CallCapabilities;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallSelectAnswer extends MCallBase {
export type MCallSelectAnswer<Base extends MCallBase> = Base & {
selected_party_id: string;
}
export interface MCallInviteNegotiate extends MCallBase {
offer: RTCSessionDescription;
description: RTCSessionDescription;
export type MCallInvite<Base extends MCallBase> = Base & {
offer: SessionDescription;
lifetime: number;
capabilities?: CallCapabilities;
invitee?: string;
sender_session_id?: string;
dest_session_id?: string;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallSDPStreamMetadataChanged extends MCallBase {
export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & {
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallReplacesEvent extends MCallBase {
export type MCallReplacesEvent<Base extends MCallBase> = Base & {
replacement_id: string;
target_user: CallReplacesTarget;
create_call: string;
@ -73,7 +72,7 @@ export interface MCallReplacesEvent extends MCallBase {
target_room: string;
}
export interface MCAllAssertedIdentity extends MCallBase {
export type MCAllAssertedIdentity<Base extends MCallBase> = Base & {
asserted_identity: {
id: string;
display_name: string;
@ -81,11 +80,11 @@ export interface MCAllAssertedIdentity extends MCallBase {
};
}
export interface MCallCandidates extends MCallBase {
export type MCallCandidates<Base extends MCallBase> = Base & {
candidates: RTCIceCandidate[];
}
export interface MCallHangupReject extends MCallBase {
export type MCallHangupReject<Base extends MCallBase> = Base & {
reason?: CallErrorCode;
}

21
src/matrix/calls/group/Call.ts → src/matrix/calls/group/GroupCall.ts
View file

@ -15,14 +15,17 @@ limitations under the License.
*/
import {ObservableMap} from "../../../observable/map/ObservableMap";
import {Participant} from "./Participant";
import {LocalMedia} from "../LocalMedia";
import type {Track} from "../../../platform/types/MediaDevices";
function participantId(senderUserId: string, senderDeviceId: string | null) {
function getParticipantId(senderUserId: string, senderDeviceId: string | null) {
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId);
}
class Call {
export class GroupCall {
private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
private localMedia?: LocalMedia;
private localMedia?: Promise<LocalMedia>;
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) {
@ -30,17 +33,17 @@ class Call {
get id(): string { return this.callEvent.state_key; }
async participate(tracks: Track[]) {
this.localMedia = LocalMedia.fromTracks(tracks);
async participate(tracks: Promise<Track[]>) {
this.localMedia = tracks.then(tracks => LocalMedia.fromTracks(tracks));
for (const [,participant] of this.participants) {
participant.setLocalMedia(this.localMedia.clone());
participant.setLocalMedia(this.localMedia.then(localMedia => localMedia.clone()));
}
// send m.call.member state event
// send invite to all participants that are < my userId
for (const [,participant] of this.participants) {
if (participant.userId < this.ownUserId) {
participant.sendInvite();
participant.call();
}
}
}
@ -78,10 +81,6 @@ class Call {
}
}
get id(): string {
return this.callEvent.state_key;
}
get isTerminated(): boolean {
return !!this.callEvent.content[CALL_TERMINATED];
}

49
src/matrix/calls/group/Participant.ts
View file

@ -14,35 +14,54 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import {EventType} from "../PeerCall";
import {EventType, PeerCall, SignallingMessage} from "../PeerCall";
import {makeTxnId} from "../../common";
import type {PeerCallHandler} from "../PeerCall";
import type {LocalMedia} from "../LocalMedia";
import type {HomeServerApi} from "../../net/HomeServerApi";
import type {Track} from "../../../platform/types/MediaDevices";
import type {MCallBase, MGroupCallBase} from "../callEventTypes";
import type {GroupCall} from "./GroupCall";
import type {RoomMember} from "../../room/members/RoomMember";
class Participant implements PeerCallHandler {
private peerCall?: PeerCall;
export class Participant implements PeerCallHandler {
constructor(
private readonly userId: string,
private readonly deviceId: string,
private localMedia: LocalMedia | undefined,
private readonly webRTC: WebRTC,
private readonly hsApi: HomeServerApi
public readonly member: RoomMember,
private readonly deviceId: string | undefined,
private readonly peerCall: PeerCall,
private readonly hsApi: HomeServerApi,
private readonly groupCall: GroupCall
) {}
sendInvite() {
this.peerCall = new PeerCall(this, this.webRTC);
this.peerCall.call(this.localMedia);
/* @internal */
call(localMedia: Promise<LocalMedia>) {
this.peerCall.call(localMedia);
}
get remoteTracks(): Track[] {
return this.peerCall.remoteTracks;
}
/** From PeerCallHandler
* @internal */
emitUpdate(params: any) {
this.groupCall.emitParticipantUpdate(this, params);
}
/** From PeerCallHandler
* @internal */
onSendSignallingMessage(type: EventType, content: Record<string, any>) {
async sendSignallingMessage(message: SignallingMessage<MCallBase>) {
const groupMessage = message as SignallingMessage<MGroupCallBase>;
groupMessage.content.conf_id = this.groupCall.id;
// TODO: this needs to be encrypted with olm first
this.hsApi.sendToDevice(type, {[this.userId]: {[this.deviceId ?? "*"]: content}});
const request = this.hsApi.sendToDevice(
groupMessage.type,
{[this.member.userId]: {
[this.deviceId ?? "*"]: groupMessage.content
}
}, makeTxnId());
await request.response();
}
}

6
src/platform/web/dom/MediaDevices.ts
View file

@ -144,7 +144,7 @@ export class AudioTrackWrapper extends TrackWrapper {
} else {
this.measuringVolumeActivity = false;
this.speakingVolumeSamples.fill(-Infinity);
this.emit(CallFeedEvent.VolumeChanged, -Infinity);
// this.emit(CallFeedEvent.VolumeChanged, -Infinity);
}
}
@ -186,7 +186,7 @@ export class AudioTrackWrapper extends TrackWrapper {
this.speakingVolumeSamples.shift();
this.speakingVolumeSamples.push(maxVolume);
this.emit(CallFeedEvent.VolumeChanged, maxVolume);
// this.emit(CallFeedEvent.VolumeChanged, maxVolume);
let newSpeaking = false;
@ -201,7 +201,7 @@ export class AudioTrackWrapper extends TrackWrapper {
if (this.speaking !== newSpeaking) {
this.speaking = newSpeaking;
this.emit(CallFeedEvent.Speaking, this.speaking);
// this.emit(CallFeedEvent.Speaking, this.speaking);
}
this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL) as unknown as number;

2
src/platform/web/dom/WebRTC.ts
View file

@ -114,7 +114,7 @@ class DOMPeerConnection implements PeerConnection {
}
createDataChannel(): DataChannel {
return new DataChannel(this.peerConnection.createDataChannel());
return undefined as any;// new DataChannel(this.peerConnection.createDataChannel());
}
private registerHandler() {

6
yarn.lock
View file

@ -1486,9 +1486,9 @@ type-fest@^0.20.2:
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
typescript@^4.3.5:
version "4.3.5"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4"
integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==
version "4.6.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.2.tgz#fe12d2727b708f4eef40f51598b3398baa9611d4"
integrity sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg==
typeson-registry@^1.0.0-alpha.20:
version "1.0.0-alpha.39"