mystiq
/
hydrogen-web
Archived
4
1
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'bwindels/calls' into thirdroom/dev

This commit is contained in:
Robert Long 2022-04-27 11:08:25 -07:00
parent ae29adb4e4 aa709ee6e9
commit bf0638b2f3
6 changed files with 129 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/domain/session/room/CallViewModel.ts
View File

@ -67,14 +67,6 @@ export class CallViewModel extends ViewModel<Options> {
}
}
get isCameraMuted(): boolean {
return this.call.muteSettings.camera;
}
get isMicrophoneMuted(): boolean {
return this.call.muteSettings.microphone;
}
async toggleVideo() {
this.call.setMuted(this.call.muteSettings.toggleCamera());
}
@ -95,11 +87,11 @@ class OwnMemberViewModel extends ViewModel<OwnMemberOptions> implements IStreamV
}
get isCameraMuted(): boolean {
return this.call.muteSettings.camera ?? !!getStreamVideoTrack(this.stream);
return isMuted(this.call.muteSettings.camera, !!getStreamVideoTrack(this.stream));
}
get isMicrophoneMuted(): boolean {
return this.call.muteSettings.microphone ?? !!getStreamAudioTrack(this.stream);
return isMuted(this.call.muteSettings.microphone, !!getStreamAudioTrack(this.stream));
}
get avatarLetter(): string {
@ -135,11 +127,11 @@ export class CallMemberViewModel extends ViewModel<MemberOptions> implements ISt
}
get isCameraMuted(): boolean {
return this.member.remoteMuteSettings?.camera ?? !getStreamVideoTrack(this.stream);
return isMuted(this.member.remoteMuteSettings?.camera, !!getStreamVideoTrack(this.stream));
}
get isMicrophoneMuted(): boolean {
return this.member.remoteMuteSettings?.microphone ?? !getStreamAudioTrack(this.stream);
return isMuted(this.member.remoteMuteSettings?.microphone, !!getStreamAudioTrack(this.stream));
}
get avatarLetter(): string {
@ -178,3 +170,11 @@ export interface IStreamViewModel extends AvatarSource, ViewModel {
get isCameraMuted(): boolean;
get isMicrophoneMuted(): boolean;
}
function isMuted(muted: boolean | undefined, hasTrack: boolean) {
if (muted) {
return true;
} else {
return !hasTrack;
}
}

64
src/matrix/calls/PeerCall.ts
View File

@ -184,6 +184,7 @@ export class PeerCall implements IDisposable {
}
// after adding the local tracks, and wait for handleNegotiation to be called,
// or invite glare where we give up our invite and answer instead
// TODO: we don't actually use this
await this.waitForState([CallState.InviteSent, CallState.CreateAnswer]);
});
}
@ -902,6 +903,9 @@ export class PeerCall implements IDisposable {
private onRemoteTrack(track: Track, streams: ReadonlyArray<Stream>, log: ILogItem) {
log.set("kind", track.kind);
log.set("id", track.id);
log.set("streams", streams.map(s => s.id));
if (streams.length === 0) {
log.log({l: `ignoring ${track.kind} streamless track`, id: track.id});
return;
@ -932,39 +936,49 @@ export class PeerCall implements IDisposable {
disposeListener,
stream
});
this.updateRemoteMedia(log);
}
this.updateRemoteMedia(log);
}
private updateRemoteMedia(log: ILogItem): void {
this._remoteMedia.userMedia = undefined;
this._remoteMedia.screenShare = undefined;
if (this.remoteSDPStreamMetadata) {
for (const streamDetails of this._remoteStreams.values()) {
const {stream} = streamDetails;
const metaData = this.remoteSDPStreamMetadata[stream.id];
if (metaData) {
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
this._remoteMedia.userMedia = stream;
const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
if (audioReceiver) {
audioReceiver.track.enabled = !metaData.audio_muted;
log.wrap("reevaluating remote media", log => {
this._remoteMedia.userMedia = undefined;
this._remoteMedia.screenShare = undefined;
if (this.remoteSDPStreamMetadata) {
for (const streamDetails of this._remoteStreams.values()) {
const {stream} = streamDetails;
const metaData = this.remoteSDPStreamMetadata[stream.id];
if (metaData) {
if (metaData.purpose === SDPStreamMetadataPurpose.Usermedia) {
this._remoteMedia.userMedia = stream;
const audioReceiver = this.findReceiverForStream(TrackKind.Audio, stream.id);
if (audioReceiver) {
audioReceiver.track.enabled = !metaData.audio_muted;
}
const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
if (videoReceiver) {
videoReceiver.track.enabled = !metaData.video_muted;
}
this._remoteMuteSettings = new MuteSettings(
metaData.audio_muted || !audioReceiver?.track,
metaData.video_muted || !videoReceiver?.track
);
log.log({
l: "setting userMedia",
micMuted: this._remoteMuteSettings.microphone,
cameraMuted: this._remoteMuteSettings.camera
});
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
this._remoteMedia.screenShare = stream;
log.log("setting screenShare");
}
const videoReceiver = this.findReceiverForStream(TrackKind.Video, stream.id);
if (videoReceiver) {
videoReceiver.track.enabled = !metaData.video_muted;
}
this._remoteMuteSettings = new MuteSettings(
metaData.audio_muted || !audioReceiver?.track,
metaData.video_muted || !videoReceiver?.track
);
} else if (metaData.purpose === SDPStreamMetadataPurpose.Screenshare) {
this._remoteMedia.screenShare = stream;
} else {
log.log({l: "no metadata yet for stream, ignoring for now", id: stream.id});
}
}
}
}
this.options.emitUpdate(this, undefined);
this.options.emitUpdate(this, undefined);
});
}
private updateLocalMedia(localMedia: LocalMedia, logItem: ILogItem): Promise<void> {

2
src/matrix/calls/TODO.md
View File

@ -14,7 +14,7 @@
- implement muting tracks with m.call.sdp_stream_metadata_changed
- implement renegotiation
- making logging better
- finish session id support
- DONE: finish session id support
- call peers are essentially identified by (userid, deviceid, sessionid). If see a new session id, we first disconnect from the current member so we're ready to connect with a clean slate again (in a member event, also in to_device? no harm I suppose, given olm encryption ensures you can't spoof the deviceid).
- implement to_device messages arriving before m.call(.member) state event
- reeable crypto & implement fetching olm keys before sending encrypted signalling message

49
src/matrix/calls/group/GroupCall.ts
View File

@ -65,6 +65,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
private _memberOptions: MemberOptions;
private _state: GroupCallState;
private localMuteSettings: MuteSettings = new MuteSettings(false, false);
private bufferedDeviceMessages = new Map<string, Set<SignallingMessage<MGroupCallBase>>>();
private _deviceIndex?: number;
private _eventTimestamp?: number;
@ -235,7 +236,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
const device = devices[deviceIndex];
const deviceId = device.device_id;
const memberKey = getMemberKey(userId, deviceId);
log.wrap({l: "update device member", id: memberKey}, log => {
log.wrap({l: "update device member", id: memberKey, sessionId: device.session_id}, log => {
if (userId === this.options.ownUserId && deviceId === this.options.ownDeviceId) {
this._deviceIndex = deviceIndex;
@ -250,7 +251,7 @@ export class GroupCall extends EventEmitter<{change: never}> {
let member = this._members.get(memberKey);
if (member) {
log.set("update", true);
member!.updateCallInfo(device, deviceIndex, eventTimestamp);
member!.updateCallInfo(device, deviceIndex, eventTimestamp, log);
} else {
const logItem = this.logItem.child({l: "member", id: memberKey});
log.set("add", true);
@ -265,6 +266,9 @@ export class GroupCall extends EventEmitter<{change: never}> {
member.connect(this._localMedia!.clone(), this.localMuteSettings);
}
}
// flush pending messages, either after having created the member,
// or updated the session id with updateCallInfo
this.flushPendingDeviceMessages(member, log);
}
});
}
@ -298,6 +302,23 @@ export class GroupCall extends EventEmitter<{change: never}> {
});
}
private flushPendingDeviceMessages(member: Member, log: ILogItem) {
const memberKey = getMemberKey(member.userId, member.deviceId);
const bufferedMessages = this.bufferedDeviceMessages.get(memberKey);
// check if we have any pending message for the member with (userid, deviceid, sessionid)
if (bufferedMessages) {
for (const message of bufferedMessages) {
if (message.content.sender_session_id === member.sessionId) {
member.handleDeviceMessage(message, log);
bufferedMessages.delete(message);
}
}
if (bufferedMessages.size === 0) {
this.bufferedDeviceMessages.delete(memberKey);
}
}
}
private getDeviceIdsForUserId(userId: string): string[] {
return Array.from(this._members.keys())
.filter(key => memberKeyIsForUser(key, userId))
@ -338,13 +359,27 @@ export class GroupCall extends EventEmitter<{change: never}> {
/** @internal */
handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, userId: string, deviceId: string, syncLog: ILogItem) {
// TODO: return if we are not membering to the call
let member = this._members.get(getMemberKey(userId, deviceId));
if (member) {
member.handleDeviceMessage(message, deviceId, syncLog);
const key = getMemberKey(userId, deviceId);
let member = this._members.get(key);
if (member && message.content.sender_session_id === member.sessionId) {
member.handleDeviceMessage(message, syncLog);
} else {
const item = this.logItem.log({l: "could not find member for signalling message", userId, deviceId});
const item = this.logItem.log({
l: "buffering to_device message, member not found",
userId,
deviceId,
sessionId: message.content.sender_session_id,
type: message.type
});
syncLog.refDetached(item);
// we haven't received the m.call.member yet for this caller. buffer the device messages or create the member/call anyway?
// we haven't received the m.call.member yet for this caller (or with this session id).
// buffer the device messages or create the member/call as it should arrive in a moment
let messages = this.bufferedDeviceMessages.get(key);
if (!messages) {
messages = new Set();
this.bufferedDeviceMessages.set(key, messages);
}
messages.add(message);
}
}

41
src/matrix/calls/group/Member.ts
View File

@ -33,6 +33,7 @@ export type Options = Omit<PeerCallOptions, "emitUpdate" | "sendSignallingMessag
confId: string,
ownUserId: string,
ownDeviceId: string,
// local session id of our client
sessionId: string,
hsApi: HomeServerApi,
encryptDeviceMessage: (userId: string, message: SignallingMessage<MGroupCallBase>, log: ILogItem) => Promise<EncryptedMessage>,
@ -83,6 +84,11 @@ export class Member {
return this.callDeviceMembership.device_id;
}
/** session id of the member */
get sessionId(): string {
return this.callDeviceMembership.session_id;
}
get dataChannel(): any | undefined {
return this.peerCall?.dataChannel;
}
@ -127,14 +133,35 @@ export class Member {
this.peerCall = undefined;
this.localMedia?.dispose();
this.localMedia = undefined;
this.retryCount = 0;
});
}
/** @internal */
updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number) {
this.callDeviceMembership = callDeviceMembership;
this._deviceIndex = deviceIndex;
this._eventTimestamp = eventTimestamp;
updateCallInfo(callDeviceMembership: CallDeviceMembership, deviceIndex: number, eventTimestamp: number, log: ILogItem) {
log.wrap({l: "updateing device membership", deviceId: this.deviceId}, log => {
// session id is changing, disconnect so we start with a new slate for the new session
if (callDeviceMembership.session_id !== this.sessionId) {
log.wrap({
l: "member event changes session id",
oldSessionId: this.sessionId,
newSessionId: callDeviceMembership.session_id
}, log => {
// prevent localMedia from being stopped
// as connect won't be called again when reconnecting
// to the new session
const localMedia = this.localMedia;
this.localMedia = undefined;
this.disconnect(false);
// connect again, as the other side might be waiting for our invite
// after refreshing
this.connect(localMedia!, this.localMuteSettings!);
});
}
this.callDeviceMembership = callDeviceMembership;
this._deviceIndex = deviceIndex;
this._eventTimestamp = eventTimestamp;
});
}
/** @internal */
@ -163,7 +190,7 @@ export class Member {
groupMessage.content.device_id = this.options.ownDeviceId;
groupMessage.content.party_id = this.options.ownDeviceId;
groupMessage.content.sender_session_id = this.options.sessionId;
groupMessage.content.dest_session_id = this.callDeviceMembership.session_id;
groupMessage.content.dest_session_id = this.sessionId;
// const encryptedMessages = await this.options.encryptDeviceMessage(this.member.userId, groupMessage, log);
// const payload = formatToDeviceMessagesPayload(encryptedMessages);
const payload = {
@ -186,7 +213,7 @@ export class Member {
}
/** @internal */
handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, deviceId: string, syncLog: ILogItem) {
handleDeviceMessage(message: SignallingMessage<MGroupCallBase>, syncLog: ILogItem): void {
syncLog.refDetached(this.logItem);
const destSessionId = message.content.dest_session_id;
if (destSessionId !== this.options.sessionId) {
@ -197,7 +224,7 @@ export class Member {
this.peerCall = this._createPeerCall(message.content.call_id);
}
if (this.peerCall) {
this.peerCall.handleIncomingSignallingMessage(message, deviceId);
this.peerCall.handleIncomingSignallingMessage(message, this.deviceId);
} else {
// TODO: need to buffer events until invite comes?
}

1
src/platform/web/ui/css/themes/element/theme.css
View File

@ -1197,6 +1197,7 @@ button.RoomDetailsView_row::after {
.StreamView_muteStatus {
align-self: end;
justify-self: start;
color: var(--text-color--lighter-80);
}
.StreamView_muteStatus.microphoneMuted::before {