From 4874af969cac380f91e0006019ade8b33351eceb Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Fri, 23 Jun 2023 15:25:48 +0200 Subject: [PATCH 1/6] force to allow calls without video and audio in embedded mode --- src/components/views/voip/CallView.tsx | 63 ++++++++++++++------------ src/models/Call.ts | 49 ++++++++++++++------ 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 6472bf12e13..cb1983a025b 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -151,38 +151,41 @@ export const Lobby: FC = ({ room, joinCallButtonDisabledTooltip, con const [videoStream, audioInputs, videoInputs] = useAsyncMemo( async (): Promise<[MediaStream | null, MediaDeviceInfo[], MediaDeviceInfo[]]> => { - let devices = await MediaDeviceHandler.getDevices(); - - // We get the preview stream before requesting devices: this is because - // we need (in some browsers) an active media stream in order to get - // non-blank labels for the devices. - let stream: MediaStream | null = null; - try { - if (devices!.audioinput.length > 0) { - // Holding just an audio stream will be enough to get us all device labels, so - // if video is muted, don't bother requesting video. - stream = await navigator.mediaDevices.getUserMedia({ - audio: true, - video: !videoMuted && devices!.videoinput.length > 0 && { deviceId: videoInputId }, - }); - } else if (devices!.videoinput.length > 0) { - // We have to resort to a video stream, even if video is supposed to be muted. - stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: videoInputId } }); - } - } catch (e) { - logger.error(`Failed to get stream for device ${videoInputId}`, e); - } + let devices = []; + let stream: MediaStream | null = new MediaStream(); + // let devices = await MediaDeviceHandler.getDevices(); + // + // // We get the preview stream before requesting devices: this is because + // // we need (in some browsers) an active media stream in order to get + // // non-blank labels for the devices. + // let stream: MediaStream | null = null; + // try { + // if (devices!.audioinput.length > 0) { + // // Holding just an audio stream will be enough to get us all device labels, so + // // if video is muted, don't bother requesting video. + // stream = await navigator.mediaDevices.getUserMedia({ + // audio: true, + // video: !videoMuted && devices!.videoinput.length > 0 && { deviceId: videoInputId }, + // }); + // } else if (devices!.videoinput.length > 0) { + // // We have to resort to a video stream, even if video is supposed to be muted. + // stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: videoInputId } }); + // } + // } catch (e) { + // logger.error(`Failed to get stream for device ${videoInputId}`, e); + // } // Refresh the devices now that we hold a stream - if (stream !== null) devices = await MediaDeviceHandler.getDevices(); - - // If video is muted, we don't actually want the stream, so we can get rid of it now. - if (videoMuted) { - stream?.getTracks().forEach((t) => t.stop()); - stream = null; - } - - return [stream, devices?.audioinput ?? [], devices?.videoinput ?? []]; + // if (stream !== null) devices = await MediaDeviceHandler.getDevices(); + // + // // If video is muted, we don't actually want the stream, so we can get rid of it now. + // if (videoMuted) { + // stream?.getTracks().forEach((t) => t.stop()); + // stream = null; + // } + // + // return [stream, devices?.audioinput ?? [], devices?.videoinput ?? []]; + return [null, [], []]; }, [videoInputId, videoMuted], [null, [], []], diff --git a/src/models/Call.ts b/src/models/Call.ts index f4123f826a1..4182a28355b 100644 --- a/src/models/Call.ts +++ b/src/models/Call.ts @@ -212,19 +212,40 @@ export abstract class Call extends TypedEventEmitter { this.connectionState = ConnectionState.Connecting; - const { [MediaDeviceKindEnum.AudioInput]: audioInputs, [MediaDeviceKindEnum.VideoInput]: videoInputs } = - (await MediaDeviceHandler.getDevices())!; - - let audioInput: MediaDeviceInfo | null = null; - if (!MediaDeviceHandler.startWithAudioMuted) { - const deviceId = MediaDeviceHandler.getAudioInput(); - audioInput = audioInputs.find((d) => d.deviceId === deviceId) ?? audioInputs[0] ?? null; - } - let videoInput: MediaDeviceInfo | null = null; - if (!MediaDeviceHandler.startWithVideoMuted) { - const deviceId = MediaDeviceHandler.getVideoInput(); - videoInput = videoInputs.find((d) => d.deviceId === deviceId) ?? videoInputs[0] ?? null; - } + // const { [MediaDeviceKindEnum.AudioInput]: audioInputs, [MediaDeviceKindEnum.VideoInput]: videoInputs } = + // (await MediaDeviceHandler.getDevices())!; + // + // let audioInput: MediaDeviceInfo | null = null; + // if (!MediaDeviceHandler.startWithAudioMuted) { + // const deviceId = MediaDeviceHandler.getAudioInput(); + // audioInput = audioInputs.find((d) => d.deviceId === deviceId) ?? audioInputs[0] ?? null; + // } + // let videoInput: MediaDeviceInfo | null = null; + // if (!MediaDeviceHandler.startWithVideoMuted) { + // const deviceId = MediaDeviceHandler.getVideoInput(); + // videoInput = videoInputs.find((d) => d.deviceId === deviceId) ?? videoInputs[0] ?? null; + // } + // + // const messagingStore = WidgetMessagingStore.instance; + // this.messaging = messagingStore.getMessagingForUid(this.widgetUid) ?? null; + // if (!this.messaging) { + // // The widget might still be initializing, so wait for it + // try { + // await waitForEvent( + // messagingStore, + // WidgetMessagingStoreEvent.StoreMessaging, + // (uid: string, widgetApi: ClientWidgetApi) => { + // if (uid === this.widgetUid) { + // this.messaging = widgetApi; + // return true; + // } + // return false; + // }, + // ); + // } catch (e) { + // throw new Error(`Failed to bind call widget in room ${this.roomId}: ${e}`); + // } + // } const messagingStore = WidgetMessagingStore.instance; this.messaging = messagingStore.getMessagingForUid(this.widgetUid) ?? null; @@ -248,7 +269,7 @@ export abstract class Call extends TypedEventEmitter Date: Wed, 28 Jun 2023 17:49:20 +0200 Subject: [PATCH 2/6] Check device access permission and introduce a only screen share call mode --- src/components/views/voip/CallView.tsx | 86 ++++++++++++++++---------- src/i18n/strings/en_EN.json | 1 + src/models/Call.ts | 50 +++++---------- src/settings/Settings.tsx | 9 +++ 4 files changed, 77 insertions(+), 69 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index cb1983a025b..2ab20f48ef9 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -32,7 +32,7 @@ import MatrixClientContext from "../../../contexts/MatrixClientContext"; import AppTile from "../elements/AppTile"; import { _t } from "../../../languageHandler"; import { useAsyncMemo } from "../../../hooks/useAsyncMemo"; -import MediaDeviceHandler from "../../../MediaDeviceHandler"; +import MediaDeviceHandler, { IMediaDevices } from "../../../MediaDeviceHandler"; import { CallStore } from "../../../stores/CallStore"; import IconizedContextMenu, { IconizedContextMenuOption, @@ -149,43 +149,61 @@ export const Lobby: FC = ({ room, joinCallButtonDisabledTooltip, con setVideoMuted(!videoMuted); }, [videoMuted, setVideoMuted]); + // In case we can not fetch media devices we should mute the devices + const handleMediaDeviceFailing = (message: string): void => { + MediaDeviceHandler.startWithAudioMuted = true; + MediaDeviceHandler.startWithVideoMuted = true; + logger.warn(message); + }; + const [videoStream, audioInputs, videoInputs] = useAsyncMemo( async (): Promise<[MediaStream | null, MediaDeviceInfo[], MediaDeviceInfo[]]> => { - let devices = []; - let stream: MediaStream | null = new MediaStream(); - // let devices = await MediaDeviceHandler.getDevices(); - // - // // We get the preview stream before requesting devices: this is because - // // we need (in some browsers) an active media stream in order to get - // // non-blank labels for the devices. - // let stream: MediaStream | null = null; - // try { - // if (devices!.audioinput.length > 0) { - // // Holding just an audio stream will be enough to get us all device labels, so - // // if video is muted, don't bother requesting video. - // stream = await navigator.mediaDevices.getUserMedia({ - // audio: true, - // video: !videoMuted && devices!.videoinput.length > 0 && { deviceId: videoInputId }, - // }); - // } else if (devices!.videoinput.length > 0) { - // // We have to resort to a video stream, even if video is supposed to be muted. - // stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: videoInputId } }); - // } - // } catch (e) { - // logger.error(`Failed to get stream for device ${videoInputId}`, e); - // } + let devices: IMediaDevices; + try { + const mediaDevices = await MediaDeviceHandler.getDevices(); + if (mediaDevices === undefined) { + handleMediaDeviceFailing("Could not access devices!"); + return [null, [], []]; + } else { + devices = mediaDevices; + } + } catch (error) { + handleMediaDeviceFailing(`Unable to get Media Devices: ${error}`); + return [null, [], []]; + } + + // We get the preview stream before requesting devices: this is because + // we need (in some browsers) an active media stream in order to get + // non-blank labels for the devices. + let stream: MediaStream | null = null; + + try { + if (devices!.audioinput.length > 0) { + // Holding just an audio stream will be enough to get us all device labels, so + // if video is muted, don't bother requesting video. + stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: !videoMuted && devices!.videoinput.length > 0 && { deviceId: videoInputId }, + }); + } else if (devices!.videoinput.length > 0) { + // We have to resort to a video stream, even if video is supposed to be muted. + stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: videoInputId } }); + } + } catch (e) { + logger.warn(`Failed to get stream for device ${videoInputId}`, e); + handleMediaDeviceFailing(`Have access to Device list but unable to read from Media Devices`); + } // Refresh the devices now that we hold a stream - // if (stream !== null) devices = await MediaDeviceHandler.getDevices(); - // - // // If video is muted, we don't actually want the stream, so we can get rid of it now. - // if (videoMuted) { - // stream?.getTracks().forEach((t) => t.stop()); - // stream = null; - // } - // - // return [stream, devices?.audioinput ?? [], devices?.videoinput ?? []]; - return [null, [], []]; + if (stream !== null) devices = await MediaDeviceHandler.getDevices(); + + // If video is muted, we don't actually want the stream, so we can get rid of it now. + if (videoMuted) { + stream?.getTracks().forEach((t) => t.stop()); + stream = null; + } + + return [stream, devices?.audioinput ?? [], devices?.videoinput ?? []]; }, [videoInputId, videoMuted], [null, [], []], diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index d9f74d40a56..eadaef889e5 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -980,6 +980,7 @@ "Under active development, cannot be disabled.": "Under active development, cannot be disabled.", "Element Call video rooms": "Element Call video rooms", "New group call experience": "New group call experience", + "Allow screen share only mode": "Allow screen share only mode", "Live Location Sharing": "Live Location Sharing", "Temporary implementation. Locations persist in room history.": "Temporary implementation. Locations persist in room history.", "Dynamic room predecessors": "Dynamic room predecessors", diff --git a/src/models/Call.ts b/src/models/Call.ts index 4182a28355b..e959a777280 100644 --- a/src/models/Call.ts +++ b/src/models/Call.ts @@ -212,40 +212,19 @@ export abstract class Call extends TypedEventEmitter { this.connectionState = ConnectionState.Connecting; - // const { [MediaDeviceKindEnum.AudioInput]: audioInputs, [MediaDeviceKindEnum.VideoInput]: videoInputs } = - // (await MediaDeviceHandler.getDevices())!; - // - // let audioInput: MediaDeviceInfo | null = null; - // if (!MediaDeviceHandler.startWithAudioMuted) { - // const deviceId = MediaDeviceHandler.getAudioInput(); - // audioInput = audioInputs.find((d) => d.deviceId === deviceId) ?? audioInputs[0] ?? null; - // } - // let videoInput: MediaDeviceInfo | null = null; - // if (!MediaDeviceHandler.startWithVideoMuted) { - // const deviceId = MediaDeviceHandler.getVideoInput(); - // videoInput = videoInputs.find((d) => d.deviceId === deviceId) ?? videoInputs[0] ?? null; - // } - // - // const messagingStore = WidgetMessagingStore.instance; - // this.messaging = messagingStore.getMessagingForUid(this.widgetUid) ?? null; - // if (!this.messaging) { - // // The widget might still be initializing, so wait for it - // try { - // await waitForEvent( - // messagingStore, - // WidgetMessagingStoreEvent.StoreMessaging, - // (uid: string, widgetApi: ClientWidgetApi) => { - // if (uid === this.widgetUid) { - // this.messaging = widgetApi; - // return true; - // } - // return false; - // }, - // ); - // } catch (e) { - // throw new Error(`Failed to bind call widget in room ${this.roomId}: ${e}`); - // } - // } + const { [MediaDeviceKindEnum.AudioInput]: audioInputs, [MediaDeviceKindEnum.VideoInput]: videoInputs } = + (await MediaDeviceHandler.getDevices())!; + + let audioInput: MediaDeviceInfo | null = null; + if (!MediaDeviceHandler.startWithAudioMuted) { + const deviceId = MediaDeviceHandler.getAudioInput(); + audioInput = audioInputs.find((d) => d.deviceId === deviceId) ?? audioInputs[0] ?? null; + } + let videoInput: MediaDeviceInfo | null = null; + if (!MediaDeviceHandler.startWithVideoMuted) { + const deviceId = MediaDeviceHandler.getVideoInput(); + videoInput = videoInputs.find((d) => d.deviceId === deviceId) ?? videoInputs[0] ?? null; + } const messagingStore = WidgetMessagingStore.instance; this.messaging = messagingStore.getMessagingForUid(this.widgetUid) ?? null; @@ -269,7 +248,7 @@ export abstract class Call extends TypedEventEmitter Date: Wed, 28 Jun 2023 18:01:49 +0200 Subject: [PATCH 3/6] Fix strict typ check issue --- src/components/views/voip/CallView.tsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 2ab20f48ef9..3f1f3b7d45e 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -158,14 +158,12 @@ export const Lobby: FC = ({ room, joinCallButtonDisabledTooltip, con const [videoStream, audioInputs, videoInputs] = useAsyncMemo( async (): Promise<[MediaStream | null, MediaDeviceInfo[], MediaDeviceInfo[]]> => { - let devices: IMediaDevices; + let devices: IMediaDevices | undefined; try { - const mediaDevices = await MediaDeviceHandler.getDevices(); - if (mediaDevices === undefined) { + devices = await MediaDeviceHandler.getDevices(); + if (devices === undefined) { handleMediaDeviceFailing("Could not access devices!"); return [null, [], []]; - } else { - devices = mediaDevices; } } catch (error) { handleMediaDeviceFailing(`Unable to get Media Devices: ${error}`); From a3bd00dadf9a8793b264d65ad7bf482a65e62ac8 Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Wed, 28 Jun 2023 18:13:04 +0200 Subject: [PATCH 4/6] Fix i18n check issue --- src/i18n/strings/en_EN.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index eadaef889e5..bfe6e602557 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -980,6 +980,7 @@ "Under active development, cannot be disabled.": "Under active development, cannot be disabled.", "Element Call video rooms": "Element Call video rooms", "New group call experience": "New group call experience", + "Under active development.": "Under active development.", "Allow screen share only mode": "Allow screen share only mode", "Live Location Sharing": "Live Location Sharing", "Temporary implementation. Locations persist in room history.": "Temporary implementation. Locations persist in room history.", @@ -988,7 +989,6 @@ "Force 15s voice broadcast chunk length": "Force 15s voice broadcast chunk length", "Enable new native OIDC flows (Under active development)": "Enable new native OIDC flows (Under active development)", "Rust cryptography implementation": "Rust cryptography implementation", - "Under active development.": "Under active development.", "Font size": "Font size", "Use custom size": "Use custom size", "Enable Emoji suggestions while typing": "Enable Emoji suggestions while typing", From 4f8ca313acc318628489787b4b488114afe18240 Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Thu, 29 Jun 2023 13:27:35 +0200 Subject: [PATCH 5/6] Add unit tests for device selection --- test/components/views/voip/CallView-test.tsx | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/components/views/voip/CallView-test.tsx b/test/components/views/voip/CallView-test.tsx index 9c276babed4..58fe6cad793 100644 --- a/test/components/views/voip/CallView-test.tsx +++ b/test/components/views/voip/CallView-test.tsx @@ -39,6 +39,7 @@ import { WidgetMessagingStore } from "../../../../src/stores/widgets/WidgetMessa import { CallStore } from "../../../../src/stores/CallStore"; import { Call, ConnectionState } from "../../../../src/models/Call"; import SdkConfig from "../../../../src/SdkConfig"; +import MediaDeviceHandler from "../../../../src/MediaDeviceHandler"; const CallView = wrapInMatrixClientContext(_CallView); @@ -247,6 +248,24 @@ describe("CallLobby", () => { expect(screen.queryByRole("button", { name: /camera/ })).toBe(null); }); + it("hide when no access to device list", async () => { + mocked(navigator.mediaDevices.enumerateDevices).mockRejectedValue("permission denied"); + await renderView(); + expect(MediaDeviceHandler.startWithAudioMuted).toBeTruthy(); + expect(MediaDeviceHandler.startWithVideoMuted).toBeTruthy(); + expect(screen.queryByRole("button", { name: /microphone/ })).toBe(null); + expect(screen.queryByRole("button", { name: /camera/ })).toBe(null); + }); + + it("hide when unknown error with device list", async () => { + MediaDeviceHandler.getDevices = () => Promise.reject("unknown error"); + await renderView(); + expect(MediaDeviceHandler.startWithAudioMuted).toBeTruthy(); + expect(MediaDeviceHandler.startWithVideoMuted).toBeTruthy(); + expect(screen.queryByRole("button", { name: /microphone/ })).toBe(null); + expect(screen.queryByRole("button", { name: /camera/ })).toBe(null); + }); + it("show without dropdown when only one device is available", async () => { mocked(navigator.mediaDevices.enumerateDevices).mockResolvedValue([fakeVideoInput1]); @@ -286,5 +305,21 @@ describe("CallLobby", () => { expect(client.getMediaHandler().setAudioInput).toHaveBeenCalledWith(fakeAudioInput2.deviceId); }); + + it("set media muted if no access to audio device", async () => { + mocked(navigator.mediaDevices.enumerateDevices).mockResolvedValue([fakeAudioInput1, fakeAudioInput2]); + mocked(navigator.mediaDevices.getUserMedia).mockRejectedValue("permission rejected"); + await renderView(); + expect(MediaDeviceHandler.startWithAudioMuted).toBeTruthy(); + expect(MediaDeviceHandler.startWithVideoMuted).toBeTruthy(); + }); + + it("set media muted if no access to video device", async () => { + mocked(navigator.mediaDevices.enumerateDevices).mockResolvedValue([fakeVideoInput1, fakeVideoInput2]); + mocked(navigator.mediaDevices.getUserMedia).mockRejectedValue("permission rejected"); + await renderView(); + expect(MediaDeviceHandler.startWithAudioMuted).toBeTruthy(); + expect(MediaDeviceHandler.startWithVideoMuted).toBeTruthy(); + }); }); }); From b83a798102da771a873f17bb675ccef47f44887c Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Thu, 29 Jun 2023 14:03:06 +0200 Subject: [PATCH 6/6] Fix mocked media device query --- test/components/views/voip/CallView-test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/components/views/voip/CallView-test.tsx b/test/components/views/voip/CallView-test.tsx index 58fe6cad793..ec2bca712c6 100644 --- a/test/components/views/voip/CallView-test.tsx +++ b/test/components/views/voip/CallView-test.tsx @@ -258,12 +258,14 @@ describe("CallLobby", () => { }); it("hide when unknown error with device list", async () => { + const originalGetDevices = MediaDeviceHandler.getDevices; MediaDeviceHandler.getDevices = () => Promise.reject("unknown error"); await renderView(); expect(MediaDeviceHandler.startWithAudioMuted).toBeTruthy(); expect(MediaDeviceHandler.startWithVideoMuted).toBeTruthy(); expect(screen.queryByRole("button", { name: /microphone/ })).toBe(null); expect(screen.queryByRole("button", { name: /camera/ })).toBe(null); + MediaDeviceHandler.getDevices = originalGetDevices; }); it("show without dropdown when only one device is available", async () => {