mystiq
/
hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support multiple devices in call per user

This commit is contained in:
Bruno Windels 2022-03-29 17:13:33 +02:00
parent ba45178e04
commit c54ffd4fc3
7 changed files with 117 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/matrix/calls/CallHandler.ts
View File

@ -55,7 +55,6 @@ export class CallHandler {
async createCall(roomId: string, localMedia: LocalMedia, name: string): Promise<GroupCall> {
const logItem = this.options.logger.child({l: "call", incoming: false});
const call = new GroupCall(undefined, undefined, roomId, this.groupCallOptions, logItem);
console.log("created call with id", call.id);
this._calls.set(call.id, call);
try {
await call.create(localMedia, name);
@ -67,9 +66,7 @@ export class CallHandler {
}
throw err;
}
console.log("joining call I just created");
await call.join(localMedia);
console.log("joined!");
return call;
}
@ -79,7 +76,6 @@ export class CallHandler {
/** @internal */
handleRoomState(room: Room, events: StateEvent[], log: ILogItem) {
console.log("handling room state");
// first update call events
for (const event of events) {
if (event.type === EventType.GroupCall) {
@ -135,7 +131,7 @@ export class CallHandler {
const callId = call["m.call_id"];
const groupCall = this._calls.get(callId);
// TODO: also check the member when receiving the m.call event
groupCall?.addMember(userId, call, log);
groupCall?.updateMember(userId, call, log);
};
const newCallIdsMemberOf = new Set<string>(calls.map(call => call["m.call_id"]));
let previousCallIdsMemberOf = this.memberToCallIds.get(userId);

5
src/matrix/calls/LocalMedia.ts
View File

@ -61,6 +61,11 @@ export class LocalMedia {
return metadata;
}
clone() {
// TODO: implement
return this;
}
dispose() {
this.cameraTrack?.stop();
this.microphoneTrack?.stop();

1
src/matrix/calls/TODO.md
View File

@ -108,6 +108,7 @@ Expose call objects
expose volume events from audiotrack to group call
Write view model
write view
- handle glare edge-cases (not yet sent): https://spec.matrix.org/latest/client-server-api/#glare
## Calls questions
- how do we handle glare between group calls (e.g. different state events with different call ids?)

13
src/matrix/calls/callEventTypes.ts
View File

@ -22,6 +22,19 @@ export enum EventType {
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata";
export interface CallDeviceMembership {
device_id: string
}
export interface CallMembership {
["m.call_id"]: string,
["m.devices"]: CallDeviceMembership[]
}
export interface CallMemberContent {
["m.calls"]: CallMembership[];
}
export interface SessionDescription {
sdp?: string;
type: RTCSdpType

92
src/matrix/calls/group/GroupCall.ts
View File

@ -24,7 +24,7 @@ import {EventEmitter} from "../../../utils/EventEmitter";
import type {Options as MemberOptions} from "./Member";
import type {BaseObservableMap} from "../../../observable/map/BaseObservableMap";
import type {Track} from "../../../platform/types/MediaDevices";
import type {SignallingMessage, MGroupCallBase} from "../callEventTypes";
import type {SignallingMessage, MGroupCallBase, CallMembership} from "../callEventTypes";
import type {Room} from "../../room/Room";
import type {StateEvent} from "../../storage/types";
import type {Platform} from "../../../platform/web/Platform";
@ -43,11 +43,22 @@ export enum GroupCallState {
Joined = "joined",
}
function getMemberKey(userId: string, deviceId: string) {
return JSON.stringify(userId)+`,`+JSON.stringify(deviceId);
}
function memberKeyIsForUser(key: string, userId: string) {
return key.startsWith(JSON.stringify(userId)+`,`);
}
function getDeviceFromMemberKey(key: string): string {
return JSON.parse(`[${key}]`)[1];
}
export type Options = Omit<MemberOptions, "emitUpdate" | "confId" | "encryptDeviceMessage"> & {
emitUpdate: (call: GroupCall, params?: any) => void;
encryptDeviceMessage: (roomId: string, userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
storage: Storage,
ownDeviceId: string
};
export class GroupCall extends EventEmitter<{change: never}> {
@ -70,7 +81,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
this._state = id ? GroupCallState.Created : GroupCallState.Fledgling;
this._memberOptions = Object.assign({}, options, {
confId: this.id,
emitUpdate: member => this._members.update(member.member.userId, member),
emitUpdate: member => this._members.update(getMemberKey(member.userId, member.deviceId), member),
encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log) => {
return this.options.encryptDeviceMessage(this.roomId, userId, message, log);
}
@ -173,26 +184,42 @@ export class GroupCall extends EventEmitter<{change: never}> {
}
/** @internal */
addMember(userId: string, memberCallInfo, syncLog: ILogItem) {
this.logItem.wrap({l: "addMember", id: userId}, log => {
updateMember(userId: string, callMembership: CallMembership, syncLog: ILogItem) {
this.logItem.wrap({l: "updateMember", id: userId}, log => {
syncLog.refDetached(log);
if (userId === this.options.ownUserId) {
if (this._state === GroupCallState.Joining) {
this._state = GroupCallState.Joined;
this.emitChange();
const devices = callMembership["m.devices"];
const previousDeviceIds = this.getDeviceIdsForUserId(userId);
for (const device of devices) {
const deviceId = device.device_id;
const memberKey = getMemberKey(userId, deviceId);
if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
if (this._state === GroupCallState.Joining) {
this._state = GroupCallState.Joined;
this.emitChange();
}
return;
}
let member = this._members.get(memberKey);
if (member) {
member.updateCallInfo(device);
} else {
const logItem = this.logItem.child("member");
member = new Member(
RoomMember.fromUserId(this.roomId, userId, "join"),
device, this._memberOptions, logItem
);
this._members.add(memberKey, member);
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
member.connect(this._localMedia!.clone());
}
}
return;
}
let member = this._members.get(userId);
if (member) {
member.updateCallInfo(memberCallInfo);
} else {
const logItem = this.logItem.child("member");
member = new Member(RoomMember.fromUserId(this.roomId, userId, "join"), memberCallInfo, this._memberOptions, logItem);
this._members.add(userId, member);
if (this._state === GroupCallState.Joining || this._state === GroupCallState.Joined) {
member.connect(this._localMedia!);
const newDeviceIds = new Set<string>(devices.map(call => call.device_id));
// remove user as member of any calls not present anymore
for (const previousDeviceId of previousDeviceIds) {
if (!newDeviceIds.has(previousDeviceId)) {
this.removeMemberDevice(userId, previousDeviceId, syncLog);
}
}
});
@ -200,9 +227,24 @@ export class GroupCall extends EventEmitter<{change: never}> {
/** @internal */
removeMember(userId: string, syncLog: ILogItem) {
this.logItem.wrap({l: "removeMember", id: userId}, log => {
const deviceIds = this.getDeviceIdsForUserId(userId);
for (const deviceId of deviceIds) {
this.removeMemberDevice(userId, deviceId, syncLog);
}
}
private getDeviceIdsForUserId(userId: string): string[] {
return Array.from(this._members.keys())
.filter(key => memberKeyIsForUser(key, userId))
.map(key => getDeviceFromMemberKey(key));
}
/** @internal */
private removeMemberDevice(userId: string, deviceId: string, syncLog: ILogItem) {
const memberKey = getMemberKey(userId, deviceId);
this.logItem.wrap({l: "removeMemberDevice", id: memberKey}, log => {
syncLog.refDetached(log);
if (userId === this.options.ownUserId) {
if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
if (this._state === GroupCallState.Joined) {
this._localMedia?.dispose();
this._localMedia = undefined;
@ -212,9 +254,9 @@ export class GroupCall extends EventEmitter<{change: never}> {
this._state = GroupCallState.Created;
}
} else {
const member = this._members.get(userId);
const member = this._members.get(memberKey);
if (member) {
this._members.remove(userId);
this._members.remove(memberKey);
member.disconnect();
}
}
@ -225,7 +267,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
/** @internal */
handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, userId: string, deviceId: string, syncLog: ILogItem) {
// TODO: return if we are not membering to the call
let member = this._members.get(userId);
let member = this._members.get(getMemberKey(userId, deviceId));
if (member) {
member.handleDeviceMessage(message, deviceId, syncLog);
} else {

38
src/matrix/calls/group/Member.ts
View File

@ -23,7 +23,7 @@ import type {Options as PeerCallOptions} from "../PeerCall";
import type {LocalMedia} from "../LocalMedia";
import type {HomeServerApi} from "../../net/HomeServerApi";
import type {Track} from "../../../platform/types/MediaDevices";
import type {MCallBase, MGroupCallBase, SignallingMessage} from "../callEventTypes";
import type {MCallBase, MGroupCallBase, SignallingMessage, CallDeviceMembership} from "../callEventTypes";
import type {GroupCall} from "./GroupCall";
import type {RoomMember} from "../../room/members/RoomMember";
import type {EncryptedMessage} from "../../e2ee/olm/Encryption";
@ -32,6 +32,7 @@ import type {ILogItem} from "../../../logging/types";
export type Options = Omit<PeerCallOptions, "emitUpdate" | "sendSignallingMessage"> & {
confId: string,
ownUserId: string,
ownDeviceId: string,
hsApi: HomeServerApi,
encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
emitUpdate: (participant: Member, params?: any) => void,
@ -43,7 +44,7 @@ export class Member {
constructor(
public readonly member: RoomMember,
private memberCallInfo: Record<string, any>,
private callDeviceMembership: CallDeviceMembership,
private readonly options: Options,
private readonly logItem: ILogItem,
) {
@ -58,15 +59,30 @@ export class Member {
return this.peerCall?.state === CallState.Connected;
}
get userId(): string {
return this.member.userId;
}
get deviceId(): string {
return this.callDeviceMembership.device_id;
}
/** @internal */
connect(localMedia: LocalMedia) {
this.logItem.log("connect");
this.localMedia = localMedia;
// otherwise wait for it to connect
if (this.member.userId < this.options.ownUserId) {
this.peerCall = this._createPeerCall(makeId("c"));
this.peerCall.call(localMedia);
}
this.logItem.wrap("connect", () => {
this.localMedia = localMedia;
// otherwise wait for it to connect
let shouldInitiateCall;
if (this.member.userId === this.options.ownUserId) {
shouldInitiateCall = this.deviceId < this.options.ownDeviceId;
} else {
shouldInitiateCall = this.member.userId < this.options.ownUserId;
}
if (shouldInitiateCall) {
this.peerCall = this._createPeerCall(makeId("c"));
this.peerCall.call(localMedia);
}
});
}
/** @internal */
@ -80,8 +96,8 @@ export class Member {
}
/** @internal */
updateCallInfo(memberCallInfo) {
// m.calls object from the m.call.member event
updateCallInfo(callDeviceMembership: CallDeviceMembership) {
this.callDeviceMembership = callDeviceMembership;
}
/** @internal */

6
src/observable/map/ObservableMap.ts
View File

@ -80,15 +80,15 @@ export class ObservableMap<K, V> extends BaseObservableMap<K, V> {
return this._values.size;
}
[Symbol.iterator](): Iterator<[K, V]> {
[Symbol.iterator](): IterableIterator<[K, V]> {
return this._values.entries();
}
values(): Iterator<V> {
values(): IterableIterator<V> {
return this._values.values();
}
keys(): Iterator<K> {
keys(): IterableIterator<K> {
return this._values.keys();
}
}