mystiq/hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'bwindels/calls-wip' into bwindels/calls-thinner-abstraction

Browse source
This commit is contained in:
Bruno Windels 2022-04-21 10:20:03 +02:00
commit baa884e9d0
6 changed files with 105 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/matrix/calls/LocalMedia.ts
View file

@ -43,13 +43,43 @@ export class LocalMedia {
return new LocalMedia(this.userMedia, this.microphoneMuted, this.cameraMuted, this.screenShare, options);
}
replaceClone(oldClone: LocalMedia | undefined, oldOriginal: LocalMedia | undefined): LocalMedia {
let userMedia;
let screenShare;
const cloneOrAdoptStream = (oldOriginalStream: Stream | undefined, oldCloneStream: Stream | undefined, newStream: Stream | undefined): Stream | undefined => {
let stream;
if (oldOriginalStream?.id === newStream?.id) {
stream = oldCloneStream;
} else {
stream = newStream?.clone();
oldCloneStream?.audioTrack?.stop();
oldCloneStream?.videoTrack?.stop();
}
return stream;
}
return new LocalMedia(
cloneOrAdoptStream(oldOriginal?.userMedia, oldClone?.userMedia, this.userMedia),
this.microphoneMuted, this.cameraMuted,
cloneOrAdoptStream(oldOriginal?.screenShare, oldClone?.screenShare, this.screenShare),
this.dataChannelOptions
);
}
clone(): LocalMedia {
return new LocalMedia(this.userMedia?.clone(), this.microphoneMuted, this.cameraMuted, this.screenShare?.clone(), this.dataChannelOptions);
}
dispose() {
this.userMedia?.audioTrack?.stop();
this.userMedia?.videoTrack?.stop();
this.screenShare?.videoTrack?.stop();
this.stopExcept(undefined);
}
stopExcept(newMedia: LocalMedia | undefined) {
if(newMedia?.userMedia?.id !== this.userMedia?.id) {
this.userMedia?.audioTrack?.stop();
this.userMedia?.videoTrack?.stop();
}
if(newMedia?.screenShare?.id !== this.screenShare?.id) {
this.screenShare?.videoTrack?.stop();
}
}
}

92
src/matrix/calls/PeerCall.ts
View file

@ -18,24 +18,24 @@ import {ObservableMap} from "../../observable/map/ObservableMap";
import {recursivelyAssign} from "../../utils/recursivelyAssign";
import {AsyncQueue} from "../../utils/AsyncQueue";
import {Disposables, IDisposable} from "../../utils/Disposables";
import type {Room} from "../room/Room";
import type {StateEvent} from "../storage/types";
import type {ILogItem} from "../../logging/types";
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
import {WebRTC, PeerConnection, PeerConnectionHandler, TrackSender, TrackReceiver} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack, Stream} from "../../platform/types/MediaDevices";
import type {LocalMedia} from "./LocalMedia";
import {
SDPStreamMetadataKey,
SDPStreamMetadataPurpose,
EventType,
CallErrorCode,
} from "./callEventTypes";
import type {Room} from "../room/Room";
import type {StateEvent} from "../storage/types";
import type {ILogItem} from "../../logging/types";
import type {TimeoutCreator, Timeout} from "../../platform/types/types";
import type {LocalMedia} from "./LocalMedia";
import type {
MCallBase,
MCallInvite,
MCallNegotiate,
MCallAnswer,
MCallSDPStreamMetadataChanged,
MCallCandidates,
@ -217,24 +217,14 @@ export class PeerCall implements IDisposable {
log.set("userMedia_video_muted", localMedia.cameraMuted);
log.set("screenShare_video", !!localMedia.screenShare?.videoTrack);
log.set("datachannel", !!localMedia.dataChannelOptions);
const oldMetaData = this.getSDPMetadata();
const willRenegotiate = await this.updateLocalMedia(localMedia, log);
// TODO: if we will renegotiate, we don't bother sending the metadata changed event
// because the renegotiate event will send new metadata anyway, but is that the right
// call?
if (!willRenegotiate) {
const newMetaData = this.getSDPMetadata();
if (JSON.stringify(oldMetaData) !== JSON.stringify(newMetaData)) {
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
call_id: this.callId,
version: 1,
seq: this.seq++,
[SDPStreamMetadataKey]: newMetaData
};
await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
}
}
await this.updateLocalMedia(localMedia, log);
const content: MCallSDPStreamMetadataChanged<MCallBase> = {
call_id: this.callId,
version: 1,
seq: this.seq++,
[SDPStreamMetadataKey]: this.getSDPMetadata()
};
await this.sendSignallingMessage({type: EventType.SDPStreamMetadataChangedPrefix, content}, log);
});
}
@ -265,6 +255,9 @@ export class PeerCall implements IDisposable {
case EventType.Answer:
await this.handleAnswer(message.content, partyId, log);
break;
case EventType.Negotiate:
await this.handleRemoteNegotiate(message.content, partyId, log);
break;
case EventType.Candidates:
await this.handleRemoteIceCandidates(message.content, partyId, log);
break;
@ -341,10 +334,10 @@ export class PeerCall implements IDisposable {
await this.sendSignallingMessage({type: EventType.Invite, content}, log);
this.setState(CallState.InviteSent, log);
} else if (this._state === CallState.Connected || this._state === CallState.Connecting) {
log.log("would send renegotiation now but not implemented");
// send Negotiate message
//await this.sendSignallingMessage({type: EventType.Invite, content});
//this.setState(CallState.InviteSent);
content.description = content.offer;
delete content.offer;
await this.sendSignallingMessage({type: EventType.Negotiate, content}, log);
}
} finally {
this.makingOffer = false;
@ -497,6 +490,36 @@ export class PeerCall implements IDisposable {
}
}
private async handleRemoteNegotiate(content: MCallNegotiate<MCallBase>, partyId: PartyId, log: ILogItem): Promise<void> {
if (this._state !== CallState.Connected) {
log.log({l: `Ignoring renegotiate because not connected`, status: this._state});
return;
}
if (this.opponentPartyId !== partyId) {
log.log(`Ignoring answer: we already have an answer/reject from ${this.opponentPartyId}`);
return;
}
const sdpStreamMetadata = content[SDPStreamMetadataKey];
if (sdpStreamMetadata) {
this.updateRemoteSDPStreamMetadata(sdpStreamMetadata, log);
} else {
log.log(`Did not get any SDPStreamMetadata! Can not send/receive multiple streams`);
}
try {
await this.peerConnection.setRemoteDescription(content.description);
} catch (e) {
await log.wrap(`Failed to set remote description`, log => {
log.catch(e);
this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, log);
});
return;
}
}
private handleIceGatheringState(state: RTCIceGatheringState, log: ILogItem) {
if (state === 'complete' && !this.sentEndOfCandidates) {
// If we didn't get an empty-string candidate to signal the end of candidates,
@ -823,8 +846,8 @@ export class PeerCall implements IDisposable {
const streamSender = this.peerConnection.localStreams.get(streamId);
metadata[streamId] = {
purpose: SDPStreamMetadataPurpose.Usermedia,
audio_muted: !(streamSender?.audioSender?.enabled && streamSender?.audioSender?.track?.enabled),
video_muted: !(streamSender?.videoSender?.enabled && streamSender?.videoSender?.track?.enabled),
audio_muted: this.localMedia.microphoneMuted || !this.localMedia.userMedia.audioTrack,
video_muted: this.localMedia.cameraMuted || !this.localMedia.userMedia.videoTrack,
};
}
if (this.localMedia?.screenShare) {
@ -856,9 +879,8 @@ export class PeerCall implements IDisposable {
this.options.emitUpdate(this, undefined);
}
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<boolean> {
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<void> {
return logItem.wrap("updateLocalMedia", async log => {
let willRenegotiate = false;
const oldMedia = this.localMedia;
this.localMedia = localMedia;
const applyStream = async (oldStream: Stream | undefined, stream: Stream | undefined, oldMuteSettings: LocalMedia | undefined, mutedSettings: LocalMedia | undefined, logLabel: string) => {
@ -887,13 +909,11 @@ export class PeerCall implements IDisposable {
log.wrap(`adding and removing ${logLabel} ${track.kind} track`, log => {
this.peerConnection.removeTrack(sender);
this.peerConnection.addTrack(track);
willRenegotiate = true;
});
}
} else {
log.wrap(`adding ${logLabel} ${track.kind} track`, log => {
this.peerConnection.addTrack(track);
willRenegotiate = true;
});
}
} else {
@ -904,20 +924,17 @@ export class PeerCall implements IDisposable {
log.wrap(`removing ${logLabel} ${sender.track.kind} track`, log => {
sender.track.enabled = false;
this.peerConnection.removeTrack(sender);
willRenegotiate = true;
});
}
}
} else if (track) {
log.log({l: "checking mute status", muted, wasMuted, wasCameraMuted: oldMedia?.cameraMuted, sender: !!sender, streamSender: !!streamSender, oldStream: oldStream?.id, stream: stream?.id});
if (sender && muted !== wasMuted) {
// TODO: why does unmuting not work? wasMuted is false
log.wrap(`${logLabel} ${track.kind} ${muted ? "muting" : "unmuting"}`, log => {
// sender.track.enabled = !muted;
// This doesn't always seem to trigger renegotiation??
// We should probably always send the new metadata first ...
sender.enable(!muted);
willRenegotiate = true;
});
} else {
log.log(`${logLabel} ${track.kind} track hasn't changed`);
@ -931,7 +948,6 @@ export class PeerCall implements IDisposable {
await applyStream(oldMedia?.userMedia, localMedia?.userMedia, oldMedia, localMedia, "userMedia");
await applyStream(oldMedia?.screenShare, localMedia?.screenShare, undefined, undefined, "screenShare");
return willRenegotiate;
// TODO: datachannel, but don't do it here as we don't want to do it from answer, rather in different method
});
}

8
src/matrix/calls/TODO.md
View file

@ -9,14 +9,14 @@
## TODO
- DONE: implement receiving hangup
- making logging better
- DONE: implement cloning the localMedia so it works in safari?
- DONE: implement 3 retries per peer
- implement muting tracks with m.call.sdp_stream_metadata_changed
- implement renegotiation
- making logging better
- finish session id support
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
- implement to_device messages arriving before m.call(.member) state event
- implement muting tracks with m.call.sdp_stream_metadata_changed
- DONE: implement cloning the localMedia so it works in safari?
- DONE: implement 3 retries per peer
- reeable crypto & implement fetching olm keys before sending encrypted signalling message
- local echo for join/leave buttons?
- make UI pretsy

7
src/matrix/calls/callEventTypes.ts
View file

@ -97,6 +97,12 @@ export type MCallInvite<Base extends MCallBase> = Base & {
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export type MCallNegotiate<Base extends MCallBase> = Base & {
description: SessionDescription;
lifetime: number;
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export type MCallSDPStreamMetadataChanged<Base extends MCallBase> = Base & {
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
@ -213,6 +219,7 @@ export enum CallErrorCode {
export type SignallingMessage<Base extends MCallBase> =
{type: EventType.Invite, content: MCallInvite<Base>} |
{type: EventType.Negotiate, content: MCallNegotiate<Base>} |
{type: EventType.Answer, content: MCallAnswer<Base>} |
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
{type: EventType.Candidates, content: MCallCandidates<Base>} |

8
src/matrix/calls/group/GroupCall.ts
View file

@ -124,13 +124,13 @@ export class GroupCall extends EventEmitter<{change: never}> {
}
async setMedia(localMedia: LocalMedia): Promise<void> {
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
const oldMedia = this._localMedia;
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined && this._localMedia) {
const oldMedia = this._localMedia!;
this._localMedia = localMedia;
await Promise.all(Array.from(this._members.values()).map(m => {
return m.setMedia(localMedia!.clone());
return m.setMedia(localMedia, oldMedia);
}));
oldMedia?.dispose();
oldMedia?.stopExcept(localMedia);
}
}

8
src/matrix/calls/group/Member.ts
View file

@ -185,11 +185,9 @@ export class Member {
}
/** @internal */
async setMedia(localMedia: LocalMedia): Promise<void> {
const oldMedia = this.localMedia;
this.localMedia = localMedia;
await this.peerCall?.setMedia(localMedia);
oldMedia?.dispose();
async setMedia(localMedia: LocalMedia, previousMedia: LocalMedia): Promise<void> {
this.localMedia = localMedia.replaceClone(this.localMedia, previousMedia);
await this.peerCall?.setMedia(this.localMedia);
}
private _createPeerCall(callId: string): PeerCall {