Display call member speaking status on bottom bar (#2742)

* add mutation observer hok

* add hook to read speaking member by observing iframe content

* display speaking member name in call status bar and improve layout

* fix shrining

src/app/features/call-status/CallControl.tsx

@@ -20,12 +20,12 @@ function MicrophoneButton({ enabled, onToggle }: MicrophoneButtonProps) {
       {(anchorRef) => (
         <IconButton
           ref={anchorRef}
-          variant={enabled ? 'Background' : 'Warning'}
+          variant={enabled ? 'Surface' : 'Warning'}
           fill="Soft"
           radii="300"
           size="300"
           onClick={() => onToggle()}
-          outlined={!enabled}
+          outlined
         >
           <Icon size="100" src={enabled ? Icons.Mic : Icons.MicMute} filled={!enabled} />
         </IconButton>
@@ -51,12 +51,12 @@ function SoundButton({ enabled, onToggle }: SoundButtonProps) {
       {(anchorRef) => (
         <IconButton
           ref={anchorRef}
-          variant={enabled ? 'Background' : 'Warning'}
+          variant={enabled ? 'Surface' : 'Warning'}
           fill="Soft"
           radii="300"
           size="300"
           onClick={() => onToggle()}
-          outlined={!enabled}
+          outlined
         >
           <Icon
             size="100"
@@ -86,12 +86,12 @@ function VideoButton({ enabled, onToggle }: VideoButtonProps) {
       {(anchorRef) => (
         <IconButton
           ref={anchorRef}
-          variant={enabled ? 'Success' : 'Background'}
+          variant={enabled ? 'Success' : 'Surface'}
           fill="Soft"
           radii="300"
           size="300"
           onClick={() => onToggle()}
-          outlined={enabled}
+          outlined
         >
           <Icon
             size="100"
@@ -119,12 +119,12 @@ function ScreenShareButton() {
       {(anchorRef) => (
         <IconButton
           ref={anchorRef}
-          variant={enabled ? 'Success' : 'Background'}
+          variant={enabled ? 'Success' : 'Surface'}
           fill="Soft"
           radii="300"
           size="300"
           onClick={() => setEnabled(!enabled)}
-          outlined={enabled}
+          outlined
         >
           <Icon size="100" src={Icons.ScreenShare} filled={enabled} />
         </IconButton>
@@ -144,9 +144,6 @@ export function CallControl({ callEmbed }: { callEmbed: CallEmbed }) {
           onToggle={() => callEmbed.control.toggleMicrophone()}
         />
         <SoundButton enabled={sound} onToggle={() => callEmbed.control.toggleSound()} />
-      </Box>
-      <StatusDivider />
-      <Box alignItems="Inherit" gap="200">
         <VideoButton enabled={video} onToggle={() => callEmbed.control.toggleVideo()} />
         {false && <ScreenShareButton />}
       </Box>

src/app/features/call-status/CallStatus.tsx

@@ -12,6 +12,8 @@ import { MemberGlance } from './MemberGlance';
 import { StatusDivider } from './components';
 import { CallEmbed } from '../../plugins/call/CallEmbed';
 import { useCallJoined } from '../../hooks/useCallEmbed';
+import { useCallSpeakers } from '../../hooks/useCallSpeakers';
+import { MemberSpeaking } from './MemberSpeaking';
 
 type CallStatusProps = {
   callEmbed: CallEmbed;
@@ -23,28 +25,51 @@ export function CallStatus({ callEmbed }: CallStatusProps) {
   const callMembers = useCallMembers(room, callSession);
   const screenSize = useScreenSize();
   const callJoined = useCallJoined(callEmbed);
+  const speakers = useCallSpeakers(callEmbed);
+
+  const compact = screenSize === ScreenSize.Mobile;
+
+  const memberVisible = callJoined && callMembers.length > 0;
 
   return (
     <Box
       className={classNames(css.CallStatus, ContainerColor({ variant: 'Background' }))}
       shrink="No"
       gap="400"
-      alignItems="Center"
+      alignItems={compact ? undefined : 'Center'}
-      direction={screenSize === ScreenSize.Mobile ? 'Column' : 'Row'}
+      direction={compact ? 'Column' : 'Row'}
     >
-      <Box grow="Yes" alignItems="Inherit" gap="200">
+      <Box grow="Yes" alignItems="Center" gap="200">
-        {callJoined && callMembers.length > 0 ? (
+        {memberVisible ? (
-          <Box shrink="No" gap="Inherit" alignItems="Inherit">
+          <Box shrink="No">
-            <MemberGlance room={room} members={callMembers} />
             <LiveChip count={callMembers.length} room={room} members={callMembers} />
           </Box>
         ) : (
           <Spinner variant="Secondary" size="200" />
         )}
-        <StatusDivider />
+        <Box
+          grow="Yes"
+          alignItems="Center"
+          gap="Inherit"
+          justifyContent={compact ? 'Center' : undefined}
+        >
           <CallRoomName room={room} />
+          {speakers.size > 0 && !compact && (
+            <>
+              <StatusDivider />
+              <span data-spacing-node />
+              <MemberSpeaking room={room} speakers={speakers} />
+            </>
+          )}
         </Box>
-      <Box shrink="No" alignItems="Inherit" gap="Inherit">
+        {memberVisible && (
+          <Box shrink="No">
+            <MemberGlance room={room} members={callMembers} speakers={speakers} />
+          </Box>
+        )}
+      </Box>
+      {memberVisible && !compact && <StatusDivider />}
+      <Box shrink="No" alignItems="Center" justifyContent="Center" gap="Inherit">
         <CallControl callEmbed={callEmbed} />
       </Box>
     </Box>

src/app/features/call-status/MemberGlance.tsx

@@ -10,13 +10,15 @@ import { useMediaAuthentication } from '../../hooks/useMediaAuthentication';
 import { StackedAvatar } from '../../components/stacked-avatar';
 import { useOpenUserRoomProfile } from '../../state/hooks/userRoomProfile';
 import { getMouseEventCords } from '../../utils/dom';
+import * as css from './styles.css';
 
 type MemberGlanceProps = {
   room: Room;
   members: CallMembership[];
+  speakers: Set<string>;
   max?: number;
 };
-export function MemberGlance({ room, members, max = 6 }: MemberGlanceProps) {
+export function MemberGlance({ room, members, speakers, max = 6 }: MemberGlanceProps) {
   const mx = useMatrixClient();
   const useAuthentication = useMediaAuthentication();
   const openUserProfile = useOpenUserRoomProfile();
@@ -38,6 +40,7 @@ export function MemberGlance({ room, members, max = 6 }: MemberGlanceProps) {
         return (
           <StackedAvatar
             key={callMember.membershipID}
+            className={speakers.has(callMember.sender) ? css.SpeakerAvatarOutline : undefined}
             title={name}
             as="button"
             variant="Background"

src/app/features/call-status/MemberSpeaking.tsx

@@ -0,0 +1,78 @@
+import { Room } from 'matrix-js-sdk';
+import React from 'react';
+import { Box, Icon, Icons, Text } from 'folds';
+import { getMemberDisplayName } from '../../utils/room';
+import { getMxIdLocalPart } from '../../utils/matrix';
+
+type MemberSpeakingProps = {
+  room: Room;
+  speakers: Set<string>;
+};
+export function MemberSpeaking({ room, speakers }: MemberSpeakingProps) {
+  const speakingNames = Array.from(speakers).map(
+    (userId) => getMemberDisplayName(room, userId) ?? getMxIdLocalPart(userId) ?? userId
+  );
+  return (
+    <Box alignItems="Center" gap="100">
+      <Icon size="100" src={Icons.Phone} filled />
+      <Text size="T200" truncate>
+        {speakingNames.length === 1 && (
+          <>
+            <b>{speakingNames[0]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' is speaking...'}
+            </Text>
+          </>
+        )}
+        {speakingNames.length === 2 && (
+          <>
+            <b>{speakingNames[0]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' and '}
+            </Text>
+            <b>{speakingNames[1]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' are speaking...'}
+            </Text>
+          </>
+        )}
+        {speakingNames.length === 3 && (
+          <>
+            <b>{speakingNames[0]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {', '}
+            </Text>
+            <b>{speakingNames[1]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' and '}
+            </Text>
+            <b>{speakingNames[2]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' are speaking...'}
+            </Text>
+          </>
+        )}
+        {speakingNames.length > 3 && (
+          <>
+            <b>{speakingNames[0]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {', '}
+            </Text>
+            <b>{speakingNames[1]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {', '}
+            </Text>
+            <b>{speakingNames[2]}</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' and '}
+            </Text>
+            <b>{speakingNames.length - 3} others</b>
+            <Text as="span" size="Inherit" priority="300">
+              {' are speaking...'}
+            </Text>
+          </>
+        )}
+      </Text>
+    </Box>
+  );
+}

src/app/features/call-status/styles.css.ts

@@ -15,3 +15,7 @@ export const CallStatus = style([
 export const ControlDivider = style({
   height: toRem(16),
 });
+
+export const SpeakerAvatarOutline = style({
+  boxShadow: `0 0 0 ${config.borderWidth.B600} ${color.Success.Main}`,
+});

src/app/hooks/useCallSpeakers.ts

@@ -0,0 +1,60 @@
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { CallEmbed } from '../plugins/call';
+import { useMutationObserver } from './useMutationObserver';
+import { isUserId } from '../utils/matrix';
+import { useCallMembers, useCallSession } from './useCall';
+import { useCallJoined } from './useCallEmbed';
+
+export const useCallSpeakers = (callEmbed: CallEmbed): Set<string> => {
+  const [speakers, setSpeakers] = useState(new Set<string>());
+  const callSession = useCallSession(callEmbed.room);
+  const callMembers = useCallMembers(callEmbed.room, callSession);
+  const joined = useCallJoined(callEmbed);
+
+  const videoContainers = useMemo(() => {
+    if (callMembers && joined) return callEmbed.document?.querySelectorAll('[data-video-fit]');
+    return undefined;
+  }, [callEmbed, callMembers, joined]);
+
+  const mutationObserver = useMutationObserver(
+    useCallback(
+      (mutations) => {
+        const s = new Set<string>();
+
+        mutations.forEach((mutation) => {
+          if (mutation.type !== 'attributes') return;
+          const el = mutation.target as HTMLElement;
+
+          const style = callEmbed.iframe.contentWindow?.getComputedStyle(el, '::before');
+          if (!style) return;
+          const tileBackgroundImage = style.getPropertyValue('background-image');
+          const speaking = tileBackgroundImage !== 'none';
+          if (!speaking) return;
+
+          const speakerId = el.querySelector('[aria-label]')?.getAttribute('aria-label');
+          if (speakerId && isUserId(speakerId)) {
+            s.add(speakerId);
+          }
+        });
+
+        setSpeakers(s);
+      },
+      [callEmbed]
+    )
+  );
+
+  useEffect(() => {
+    videoContainers?.forEach((element) => {
+      mutationObserver.observe(element, {
+        attributes: true,
+        attributeFilter: ['class', 'style'],
+      });
+    });
+
+    return () => {
+      mutationObserver.disconnect();
+    };
+  }, [videoContainers, mutationObserver]);
+
+  return speakers;
+};

src/app/hooks/useMutationObserver.ts

@@ -0,0 +1,37 @@
+import { useEffect, useMemo } from 'react';
+
+export type OnMutationCallback = (mutations: MutationRecord[]) => void;
+
+export const getMutationRecord = (
+  target: Node,
+  mutations: MutationRecord[]
+): MutationRecord | undefined => mutations.find((mutation) => mutation.target === target);
+
+export const useMutationObserver = (
+  onMutationCallback: OnMutationCallback,
+  observeElement?: Node | null | (() => Node | null),
+  options?: MutationObserverInit
+): MutationObserver => {
+  const mutationObserver = useMemo(
+    () => new MutationObserver(onMutationCallback),
+    [onMutationCallback]
+  );
+
+  useEffect(() => () => mutationObserver?.disconnect(), [mutationObserver]);
+
+  useEffect(() => {
+    const element = typeof observeElement === 'function' ? observeElement() : observeElement;
+
+    if (element) {
+      mutationObserver.observe(element, options);
+    }
+
+    return () => {
+      if (element) {
+        mutationObserver.disconnect();
+      }
+    };
+  }, [mutationObserver, observeElement, options]);
+
+  return mutationObserver;
+};

src/app/plugins/call/CallEmbed.ts

@@ -163,6 +163,10 @@ export class CallEmbed {
     return this.room.roomId;
   }
 
+  get document(): Document | undefined {
+    return this.iframe.contentDocument ?? this.iframe.contentWindow?.document;
+  }
+
   public setTheme(theme: ElementCallThemeKind) {
     return this.call.transport.send(WidgetApiToWidgetAction.ThemeChange, {
       name: theme,