This commit is contained in:
Bruno Windels 2022-03-09 18:53:51 +01:00
parent 60da85d641
commit 4bedd4737b
5 changed files with 213 additions and 89 deletions

View file

@ -17,33 +17,34 @@ limitations under the License.
import {ObservableMap} from "../../observable/map/ObservableMap"; import {ObservableMap} from "../../observable/map/ObservableMap";
import type {Room} from "../room/Room"; import type {Room} from "../room/Room";
import type {MemberChange} from "../room/members/RoomMember";
import type {StateEvent} from "../storage/types"; import type {StateEvent} from "../storage/types";
import type {ILogItem} from "../../logging/types"; import type {ILogItem} from "../../logging/types";
import type {Platform} from "../../platform/web/Platform";
import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC"; import {WebRTC, PeerConnection, PeerConnectionHandler, StreamPurpose} from "../../platform/types/WebRTC";
import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices"; import {MediaDevices, Track, AudioTrack, TrackType} from "../../platform/types/MediaDevices";
import type {SignallingMessage} from "./PeerCall"; import {handlesEventType, PeerCall, PeerCallHandler} from "./PeerCall";
import type {MGroupCallBase} from "./callEventTypes"; import {EventType} from "./callEventTypes";
import type {SignallingMessage, MGroupCallBase} from "./callEventTypes";
import type {GroupCall} from "./group/GroupCall";
const GROUP_CALL_TYPE = "m.call"; const GROUP_CALL_TYPE = "m.call";
const GROUP_CALL_MEMBER_TYPE = "m.call.member"; const GROUP_CALL_MEMBER_TYPE = "m.call.member";
enum CallSetupMessageType {
Invite = "m.call.invite",
Answer = "m.call.answer",
Candidates = "m.call.candidates",
Hangup = "m.call.hangup",
}
const CONF_ID = "conf_id";
const CALL_TERMINATED = "m.terminated"; const CALL_TERMINATED = "m.terminated";
export class GroupCallHandler { export class GroupCallHandler {
private createPeerCall: (callId: string, handler: PeerCallHandler) => PeerCall;
// group calls by call id // group calls by call id
public readonly calls: ObservableMap<string, GroupCall> = new ObservableMap<string, GroupCall>(); public readonly calls: ObservableMap<string, GroupCall> = new ObservableMap<string, GroupCall>();
// map of userId to set of conf_id's they are in
private memberToCallIds: Map<string, Set<string>> = new Map();
constructor() { constructor(hsApi: HomeServerApi, platform: Platform, ownUserId: string, ownDeviceId: string) {
this.createPeerCall = (callId: string, handler: PeerCallHandler) => {
return new PeerCall(callId, handler, platform.createTimeout, platform.webRTC);
}
} }
// TODO: check and poll turn server credentials here // TODO: check and poll turn server credentials here
@ -51,7 +52,23 @@ export class GroupCallHandler {
handleRoomState(room: Room, events: StateEvent[], log: ILogItem) { handleRoomState(room: Room, events: StateEvent[], log: ILogItem) {
// first update call events // first update call events
for (const event of events) { for (const event of events) {
if (event.type === GROUP_CALL_TYPE) { if (event.type === EventType.GroupCall) {
this.handleCallEvent(event);
}
}
// then update participants
for (const event of events) {
if (event.type === EventType.GroupCallMember) {
this.handleCallMemberEvent(event);
}
}
}
updateRoomMembers(room: Room, memberChanges: Map<string, MemberChange>) {
}
private handleCallEvent(event: StateEvent) {
const callId = event.state_key; const callId = event.state_key;
let call = this.calls.get(callId); let call = this.calls.get(callId);
if (call) { if (call) {
@ -60,34 +77,44 @@ export class GroupCallHandler {
this.calls.remove(call.id); this.calls.remove(call.id);
} }
} else { } else {
call = new GroupCall(event, room); call = new GroupCall(event, room, this.createPeerCall);
this.calls.set(call.id, call); this.calls.set(call.id, call);
} }
} }
}
// then update participants private handleCallMemberEvent(event: StateEvent) {
for (const event of events) {
if (event.type === GROUP_CALL_MEMBER_TYPE) {
const participant = event.state_key; const participant = event.state_key;
const sources = event.content["m.sources"]; const calls = event.content["m.calls"] ?? [];
for (const source of sources) { const newCallIdsMemberOf = new Set<string>(calls.map(call => {
const call = this.calls.get(source[CONF_ID]); const callId = call["m.call_id"];
if (call && !call.isTerminated) { const groupCall = this.calls.get(callId);
call.addParticipant(participant, source); // TODO: also check the participant when receiving the m.call event
groupCall?.addParticipant(participant, call);
return callId;
}));
let previousCallIdsMemberOf = this.memberToCallIds.get(participant);
// remove user as participant of any calls not present anymore
if (previousCallIdsMemberOf) {
for (const previousCallId of previousCallIdsMemberOf) {
if (!newCallIdsMemberOf.has(previousCallId)) {
const groupCall = this.calls.get(previousCallId);
groupCall?.removeParticipant(participant);
} }
} }
} }
if (newCallIdsMemberOf.size === 0) {
this.memberToCallIds.delete(participant);
} else {
this.memberToCallIds.set(participant, newCallIdsMemberOf);
} }
} }
handlesDeviceMessageEventType(eventType: string | undefined): boolean { handlesDeviceMessageEventType(eventType: string): boolean {
return eventType === CallSetupMessageType.Invite || return handlesEventType(eventType);
eventType === CallSetupMessageType.Candidates ||
eventType === CallSetupMessageType.Answer ||
eventType === CallSetupMessageType.Hangup;
} }
handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage<MGroupCallBase>, log: ILogItem) { handleDeviceMessage(senderUserId: string, senderDeviceId: string, event: SignallingMessage<MGroupCallBase>, log: ILogItem) {
// TODO: buffer messages for calls we haven't received the state event for yet?
const call = this.calls.get(event.content.conf_id); const call = this.calls.get(event.content.conf_id);
call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log); call?.handleDeviceMessage(senderUserId, senderDeviceId, event, log);
} }

View file

@ -29,7 +29,8 @@ import type {LocalMedia} from "./LocalMedia";
import { import {
SDPStreamMetadataKey, SDPStreamMetadataKey,
SDPStreamMetadataPurpose SDPStreamMetadataPurpose,
EventType,
} from "./callEventTypes"; } from "./callEventTypes";
import type { import type {
MCallBase, MCallBase,
@ -39,6 +40,7 @@ import type {
MCallCandidates, MCallCandidates,
MCallHangupReject, MCallHangupReject,
SDPStreamMetadata, SDPStreamMetadata,
SignallingMessage
} from "./callEventTypes"; } from "./callEventTypes";
// when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already // when sending, we need to encrypt message with olm. I think the flow of room => roomEncryption => olmEncryption as we already
@ -677,21 +679,6 @@ export enum CallDirection {
Outbound = 'outbound', Outbound = 'outbound',
} }
export enum EventType {
Invite = "m.call.invite",
Candidates = "m.call.candidates",
Answer = "m.call.answer",
Hangup = "m.call.hangup",
Reject = "m.call.reject",
SelectAnswer = "m.call.select_answer",
Negotiate = "m.call.negotiate",
SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed",
SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed",
Replaces = "m.call.replaces",
AssertedIdentity = "m.call.asserted_identity",
AssertedIdentityPrefix = "org.matrix.call.asserted_identity",
}
export enum CallErrorCode { export enum CallErrorCode {
/** The user chose to end the call */ /** The user chose to end the call */
UserHangup = 'user_hangup', UserHangup = 'user_hangup',
@ -802,18 +789,18 @@ export class CallError extends Error {
} }
} }
export type SignallingMessage<Base extends MCallBase> =
{type: EventType.Invite, content: MCallInvite<Base>} |
{type: EventType.Answer, content: MCallAnswer<Base>} |
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
{type: EventType.Candidates, content: MCallCandidates<Base>} |
{type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};
export interface PeerCallHandler { export interface PeerCallHandler {
emitUpdate(peerCall: PeerCall, params: any); emitUpdate(peerCall: PeerCall, params: any);
sendSignallingMessage(message: SignallingMessage<MCallBase>); sendSignallingMessage(message: SignallingMessage<MCallBase>);
} }
export function handlesEventType(eventType: string): boolean {
return eventType === EventType.Invite ||
eventType === EventType.Candidates ||
eventType === EventType.Answer ||
eventType === EventType.Hangup;
}
export function tests() { export function tests() {
} }

View file

@ -1,6 +1,24 @@
// allow non-camelcase as these are events type that go onto the wire // allow non-camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */ /* eslint-disable camelcase */
export enum EventType {
GroupCall = "m.call",
GroupCallMember = "m.call.member",
Invite = "m.call.invite",
Candidates = "m.call.candidates",
Answer = "m.call.answer",
Hangup = "m.call.hangup",
Reject = "m.call.reject",
SelectAnswer = "m.call.select_answer",
Negotiate = "m.call.negotiate",
SDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed",
SDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed",
Replaces = "m.call.replaces",
AssertedIdentity = "m.call.asserted_identity",
AssertedIdentityPrefix = "org.matrix.call.asserted_identity",
}
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
@ -88,4 +106,95 @@ export type MCallHangupReject<Base extends MCallBase> = Base & {
reason?: CallErrorCode; reason?: CallErrorCode;
} }
/* eslint-enable camelcase */ export enum CallErrorCode {
/** The user chose to end the call */
UserHangup = 'user_hangup',
/** An error code when the local client failed to create an offer. */
LocalOfferFailed = 'local_offer_failed',
/**
* An error code when there is no local mic/camera to use. This may be because
* the hardware isn't plugged in, or the user has explicitly denied access.
*/
NoUserMedia = 'no_user_media',
/**
* Error code used when a call event failed to send
* because unknown devices were present in the room
*/
UnknownDevices = 'unknown_devices',
/**
* Error code used when we fail to send the invite
* for some reason other than there being unknown devices
*/
SendInvite = 'send_invite',
/**
* An answer could not be created
*/
CreateAnswer = 'create_answer',
/**
* Error code used when we fail to send the answer
* for some reason other than there being unknown devices
*/
SendAnswer = 'send_answer',
/**
* The session description from the other side could not be set
*/
SetRemoteDescription = 'set_remote_description',
/**
* The session description from this side could not be set
*/
SetLocalDescription = 'set_local_description',
/**
* A different device answered the call
*/
AnsweredElsewhere = 'answered_elsewhere',
/**
* No media connection could be established to the other party
*/
IceFailed = 'ice_failed',
/**
* The invite timed out whilst waiting for an answer
*/
InviteTimeout = 'invite_timeout',
/**
* The call was replaced by another call
*/
Replaced = 'replaced',
/**
* Signalling for the call could not be sent (other than the initial invite)
*/
SignallingFailed = 'signalling_timeout',
/**
* The remote party is busy
*/
UserBusy = 'user_busy',
/**
* We transferred the call off to somewhere else
*/
Transfered = 'transferred',
/**
* A call from the same user was found with a new session id
*/
NewSession = 'new_session',
}
export type SignallingMessage<Base extends MCallBase> =
{type: EventType.Invite, content: MCallInvite<Base>} |
{type: EventType.Answer, content: MCallAnswer<Base>} |
{type: EventType.SDPStreamMetadataChanged | EventType.SDPStreamMetadataChangedPrefix, content: MCallSDPStreamMetadataChanged<Base>} |
{type: EventType.Candidates, content: MCallCandidates<Base>} |
{type: EventType.Hangup | EventType.Reject, content: MCallHangupReject<Base>};

View file

@ -18,16 +18,21 @@ import {ObservableMap} from "../../../observable/map/ObservableMap";
import {Participant} from "./Participant"; import {Participant} from "./Participant";
import {LocalMedia} from "../LocalMedia"; import {LocalMedia} from "../LocalMedia";
import type {Track} from "../../../platform/types/MediaDevices"; import type {Track} from "../../../platform/types/MediaDevices";
import type {SignallingMessage, MGroupCallBase} from "../callEventTypes";
function getParticipantId(senderUserId: string, senderDeviceId: string | null) { import type {Room} from "../../room/Room";
return JSON.stringify(senderUserId) + JSON.stringify(senderDeviceId); import type {StateEvent} from "../../storage/types";
} import type {Platform} from "../../../platform/web/Platform";
export class GroupCall { export class GroupCall {
private readonly participants: ObservableMap<string, Participant> = new ObservableMap(); private readonly participants: ObservableMap<string, Participant> = new ObservableMap();
private localMedia?: Promise<LocalMedia>; private localMedia?: Promise<LocalMedia>;
constructor(private readonly ownUserId: string, private callEvent: StateEvent, private readonly room: Room, private readonly webRTC: WebRTC) { constructor(
private readonly ownUserId: string,
private callEvent: StateEvent,
private readonly room: Room,
private readonly platform: Platform
) {
} }
@ -52,32 +57,25 @@ export class GroupCall {
this.callEvent = callEvent; this.callEvent = callEvent;
} }
addParticipant(userId, source) { addParticipant(userId, memberCallInfo) {
const participantId = getParticipantId(userId, source.device_id); let participant = this.participants.get(userId);
const participant = this.participants.get(participantId);
if (participant) { if (participant) {
participant.updateSource(source); participant.updateCallInfo(memberCallInfo);
} else { } else {
participant.add(participantId, new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC)); participant = new Participant(userId, source.device_id, this.localMedia?.clone(), this.webRTC);
participant.updateCallInfo(memberCallInfo);
this.participants.add(userId, participant);
} }
} }
handleDeviceMessage(senderUserId: string, senderDeviceId: string, eventType: string, content: Record<string, any>, log: ILogItem) { removeParticipant(userId) {
const participantId = getParticipantId(senderUserId, senderDeviceId);
let peerCall = this.participants.get(participantId);
let hasDeviceInKey = true;
if (!peerCall) {
hasDeviceInKey = false;
peerCall = this.participants.get(getParticipantId(senderUserId, null))
} }
if (peerCall) {
peerCall.handleIncomingSignallingMessage(eventType, content, senderDeviceId); handleDeviceMessage(userId: string, senderDeviceId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) {
if (!hasDeviceInKey && peerCall.opponentPartyId) { let participant = this.participants.get(userId);
this.participants.delete(getParticipantId(senderUserId, null)); if (participant) {
this.participants.add(getParticipantId(senderUserId, peerCall.opponentPartyId)); participant.handleIncomingSignallingMessage(message, senderDeviceId);
}
} else {
// create peerCall
} }
} }

View file

@ -218,6 +218,9 @@ export class Room extends BaseRoom {
if (this._memberList) { if (this._memberList) {
this._memberList.afterSync(memberChanges); this._memberList.afterSync(memberChanges);
} }
if (this._callHandler) {
this._callHandler.updateRoomMembers(this, memberChanges);
}
if (this._observedMembers) { if (this._observedMembers) {
this._updateObservedMembers(memberChanges); this._updateObservedMembers(memberChanges);
} }