feat: blur with new media pipe version

package.json

@@ -5,6 +5,7 @@
     "@emotion/react": "11.11.4",
     "@lexical/history": "0.13.1",
     "@lexical/react": "0.13.1",
+    "@mediapipe/tasks-vision": "^0.10.12",
     "@wireapp/avs": "9.6.12",
     "@wireapp/commons": "5.2.7",
     "@wireapp/core": "45.2.11",

src/i18n/en-US.json

@@ -1512,6 +1512,9 @@
   "videoCallaudioInputMicrophone": "Microphone",
   "videoCallaudioOutputSpeaker": "Speaker",
   "videoCallvideoInputCamera": "Camera",
+  "videoCallbackgroundBlurHeadline": "Background",
+  "videoCallbackgroundBlur": "Blur my background",
+  "videoCallbackgroundNotBlurred": "Don't blur my background",
   "videoSpeakersTabAll": "All ({{count}})",
   "videoSpeakersTabSpeakers": "Speakers",
   "warningCallIssues": "This version of {{brandName}} can not participate in the call. Please use",
@@ -1538,4 +1541,4 @@
   "wireMacos": "{{brandName}} for macOS",
   "wireWindows": "{{brandName}} for Windows",
   "wire_for_web": "{{brandName}} for Web"
-}
+}

src/script/calling/Participant.ts

@@ -22,6 +22,7 @@ import ko from 'knockout';
 
 import {VIDEO_STATE} from '@wireapp/avs';
 
+import {backgroundBlur} from 'Components/calling/FullscreenVideoCall';
 import {matchQualifiedIds} from 'Util/QualifiedId';
 
 import {User} from '../entity/User';
@@ -41,7 +42,7 @@ export class Participant {
   public isActivelySpeaking: ko.Observable<boolean>;
   public isSendingVideo: ko.PureComputed<boolean>;
   public isAudioEstablished: ko.Observable<boolean>;
-
+  public isBlurred: ko.Observable<backgroundBlur>;
   // Audio
   public audioStream: ko.Observable<MediaStream | undefined>;
   public isMuted: ko.Observable<boolean>;
@@ -63,6 +64,9 @@ export class Participant {
     this.hasPausedVideo = ko.pureComputed(() => {
       return this.videoState() === VIDEO_STATE.PAUSED;
     });
+    this.isBlurred = ko.observable(
+      localStorage.getItem('blurState') === 'true' ? backgroundBlur.isBlurred : backgroundBlur.isNotBlurred,
+    );
     this.videoStream = ko.observable();
     this.audioStream = ko.observable();
     this.isActivelySpeaking = ko.observable(false);
@@ -87,6 +91,11 @@ export class Participant {
     this.videoStream(videoStream);
   }
 
+  async setBlur(blurState: backgroundBlur): Promise<void> {
+    this.isBlurred(blurState);
+    localStorage.setItem('blurState', blurState === backgroundBlur.isBlurred ? 'true' : 'false');
+  }
+
   updateMediaStream(newStream: MediaStream, stopTracks: boolean): MediaStream {
     if (newStream.getVideoTracks().length) {
       this.setVideoStream(new MediaStream(newStream.getVideoTracks()), stopTracks);

src/script/components/calling/FullscreenVideoCall.tsx

@@ -83,6 +83,11 @@ export interface FullscreenVideoCallProps {
   videoGrid: Grid;
 }
 
+export enum backgroundBlur {
+  isNotBlurred,
+  isBlurred,
+}
+
 const FullscreenVideoCall: React.FC<FullscreenVideoCallProps> = ({
   call,
   canShareScreen,
@@ -108,10 +113,11 @@ const FullscreenVideoCall: React.FC<FullscreenVideoCallProps> = ({
   teamState = container.resolve(TeamState),
 }) => {
   const selfParticipant = call.getSelfParticipant();
-  const {sharesScreen: selfSharesScreen, sharesCamera: selfSharesCamera} = useKoSubscribableChildren(selfParticipant, [
-    'sharesScreen',
-    'sharesCamera',
-  ]);
+  const {
+    sharesScreen: selfSharesScreen,
+    sharesCamera: selfSharesCamera,
+    isBlurred,
+  } = useKoSubscribableChildren(selfParticipant, ['sharesScreen', 'sharesCamera', 'isBlurred']);
 
   const {
     activeSpeakers,
@@ -150,7 +156,7 @@ const FullscreenVideoCall: React.FC<FullscreenVideoCallProps> = ({
     (call.initialType === CALL_TYPE.VIDEO || conversation.supportsVideoCall(call.isConference));
 
   const showSwitchMicrophone = audioinput.length > 1;
-  const showSwitchVideo = videoinput.length > 1;
+  // const showSwitchVideo = videoinput.length > 1; //TODO: should we show video switch list item when there is only one camera?
 
   const audioOptions = [
     {
@@ -227,17 +233,41 @@ const FullscreenVideoCall: React.FC<FullscreenVideoCallProps> = ({
             };
       }),
     },
+    {
+      label: t('videoCallbackgroundBlurHeadline'),
+      options: [
+        {
+          label: t('videoCallbackgroundBlur'),
+          value: backgroundBlur.isBlurred,
+          dataUieName: backgroundBlur.isBlurred,
+          id: backgroundBlur.isBlurred,
+        },
+        {
+          label: t('videoCallbackgroundNotBlurred'),
+          value: backgroundBlur.isNotBlurred,
+          dataUieName: backgroundBlur.isNotBlurred,
+          id: backgroundBlur.isNotBlurred,
+        },
+      ],
+    },
   ];
 
   const [selectedVideoOptions, setSelectedVideoOptions] = React.useState(() =>
-    [currentCameraDevice].flatMap(
+    [currentCameraDevice, isBlurred].flatMap(
       device => videoOptions.flatMap(options => options.options.filter(item => item.id === device)) ?? [],
     ),
   );
-  const updateVideoOptions = (selectedOption: string) => {
+
+  const updateVideoOptions = async (selectedOption: string | backgroundBlur) => {
     const camera = videoOptions[0].options.find(item => item.value === selectedOption) ?? selectedVideoOptions[0];
-    setSelectedVideoOptions([camera]);
-    switchCameraInput(camera.id);
+    const blur = videoOptions[1].options.find(item => item.value == selectedOption) ?? selectedVideoOptions[1];
+
+    if (blur.id !== isBlurred) {
+      await selfParticipant.setBlur(blur.id as backgroundBlur);
+    }
+
+    setSelectedVideoOptions([camera, blur]);
+    switchCameraInput(String(camera.id));
   };
 
   const unreadMessagesCount = useAppState(state => state.unreadMessagesCount);
@@ -510,44 +540,42 @@ const FullscreenVideoCall: React.FC<FullscreenVideoCallProps> = ({
                     </span>
                   </button>
 
-                  {showSwitchVideo && (
-                    <button
-                      className="device-toggle-button"
-                      css={videoOptionsOpen ? videoControlActiveStyles : videoControlInActiveStyles}
-                      onClick={() => setVideoOptionsOpen(prev => !prev)}
-                      onKeyDown={event => handleKeyDown(event, () => setVideoOptionsOpen(prev => !prev))}
-                      onBlur={event => {
-                        if (!event.currentTarget.contains(event.relatedTarget)) {
-                          setVideoOptionsOpen(false);
-                        }
-                      }}
-                    >
-                      {videoOptionsOpen ? (
-                        <>
-                          <Select
-                            // eslint-disable-next-line jsx-a11y/no-autofocus
-                            autoFocus
-                            value={selectedVideoOptions}
-                            onChange={selectedOption => updateVideoOptions(String(selectedOption?.value))}
-                            onKeyDown={event => isEscapeKey(event) && setVideoOptionsOpen(false)}
-                            id="select-camera"
-                            dataUieName="select-camera"
-                            controlShouldRenderValue={false}
-                            isClearable={false}
-                            backspaceRemovesValue={false}
-                            hideSelectedOptions={false}
-                            options={videoOptions}
-                            menuPlacement="top"
-                            menuIsOpen
-                            wrapperCSS={{marginBottom: 0}}
-                          />
-                          <Icon.Chevron css={{height: '16px'}} />
-                        </>
-                      ) : (
-                        <Icon.Chevron css={{rotate: '180deg', height: '16px'}} />
-                      )}
-                    </button>
-                  )}
+                  <button
+                    className="device-toggle-button"
+                    css={videoOptionsOpen ? videoControlActiveStyles : videoControlInActiveStyles}
+                    onClick={() => setVideoOptionsOpen(prev => !prev)}
+                    onKeyDown={event => handleKeyDown(event, () => setVideoOptionsOpen(prev => !prev))}
+                    onBlur={event => {
+                      if (!event.currentTarget.contains(event.relatedTarget)) {
+                        setVideoOptionsOpen(false);
+                      }
+                    }}
+                  >
+                    {videoOptionsOpen ? (
+                      <>
+                        <Select
+                          // eslint-disable-next-line jsx-a11y/no-autofocus
+                          autoFocus
+                          value={selectedVideoOptions}
+                          onChange={selectedOption => updateVideoOptions(String(selectedOption?.value))}
+                          onKeyDown={event => isEscapeKey(event) && setVideoOptionsOpen(false)}
+                          id="select-camera"
+                          dataUieName="select-camera"
+                          controlShouldRenderValue={false}
+                          isClearable={false}
+                          backspaceRemovesValue={false}
+                          hideSelectedOptions={false}
+                          options={videoOptions}
+                          menuPlacement="top"
+                          menuIsOpen
+                          wrapperCSS={{marginBottom: 0}}
+                        />
+                        <Icon.Chevron css={{height: '16px'}} />
+                      </>
+                    ) : (
+                      <Icon.Chevron css={{rotate: '180deg', height: '16px'}} />
+                    )}
+                  </button>
                 </li>
               )}
 

src/script/components/calling/GroupVideoGrid.tsx

@@ -161,6 +161,8 @@ const GroupVideoGrid: React.FunctionComponent<GroupVideoGripProps> = ({
       {thumbnail.videoStream && !maximizedParticipant && (
         <GroupVideoThumbnailWrapper minimized={minimized}>
           <Video
+            blur={!!selfParticipant.isBlurred()}
+            handleBlur={selfParticipant.setVideoStream}
             className="group-video__thumbnail-video"
             autoPlay
             playsInline

src/script/components/calling/GroupVideoGridTile.tsx

@@ -147,6 +147,8 @@ const GroupVideoGridTile: React.FC<GroupVideoGridTileProps> = ({
       {hasActiveVideo ? (
         <div className="tile-wrapper">
           <Video
+            blur={participant === selfParticipant && !!selfParticipant.isBlurred()}
+            handleBlur={participant === selfParticipant ? selfParticipant.setVideoStream : undefined}
             autoPlay
             playsInline
             srcObject={videoStream}

src/script/components/calling/Video.tsx

@@ -18,14 +18,20 @@
  */
 
 // https://github.com/facebook/react/issues/11163#issuecomment-628379291
+
 import {VideoHTMLAttributes, useEffect, useRef} from 'react';
 
+import {applyBlur} from 'Util/applyBlur';
+
 type VideoProps = VideoHTMLAttributes<HTMLVideoElement> & {
   srcObject: MediaStream;
+  blur: boolean;
+  handleBlur?: (stream: MediaStream, stopTracks: boolean) => void;
 };
 
-const Video = ({srcObject, ...props}: VideoProps) => {
+const Video = ({srcObject, blur, handleBlur, ...props}: VideoProps) => {
   const refVideo = useRef<HTMLVideoElement>(null);
+  const blurRef = useRef<HTMLVideoElement>(null);
 
   useEffect(() => {
     if (!refVideo.current) {
@@ -36,14 +42,51 @@ const Video = ({srcObject, ...props}: VideoProps) => {
 
   useEffect(
     () => () => {
+      if (blurRef.current) {
+        blurRef.current.srcObject = null;
+      }
       if (refVideo.current) {
         refVideo.current.srcObject = null;
       }
     },
     [],
   );
 
-  return <video ref={refVideo} {...props} />;
+  useEffect(() => {
+    const asyncBlur = async () => {
+      if (!refVideo.current) {
+        throw new Error('No ref video');
+      }
+      return await applyBlur(refVideo.current, props);
+    };
+
+    if (blur) {
+      asyncBlur()
+        .then(stream => {
+          if (!stream) {
+            throw new Error('Failed to apply blur');
+          }
+          // handleBlur?.(stream, false);
+          blurRef.current!.srcObject = stream;
+        })
+        .catch(console.error);
+    }
+  }, [blur, handleBlur, props]);
+
+  return (
+    <>
+      <video
+        ref={refVideo}
+        {...props}
+        css={{visibility: blur ? 'hidden' : 'visible', display: blur ? 'none' : 'inline block'}}
+      />
+      <video
+        ref={blurRef}
+        {...props}
+        css={{visibility: !blur ? 'hidden' : 'visible', display: !blur ? 'none' : 'inline block'}}
+      />
+    </>
+  );
 };
 
 export {Video};