Classifier: Custom Video Controls

docs/arch/adr-45.md

@@ -0,0 +1,20 @@
+# ADR 45: Custom Video Controls
+
+10 Oct 2022
+
+## Context
+The video subject viewer was built as part of the migration of projects with video subjects from PFE to FEM. In PFE, video subjects are displayed with native browser controls plus custom playback speed buttons.
+
+In FEM, custom video controls are planned for building drawing tools layers on top of a video subject. The `react-player` used to display a video subject has built-in (native browser) controls, but their position overlaps some of the video. When an svg is placed on top of the player to record annotations, the built-in controls become unusable, hence the need for custom controls displayed below the subject.
+
+## Decision
+The end goal for the video subject viewer is to always display custom video controls for consistency - regardless if a project uses drawing tools with a video subject. However, migration of projects from PFE to FEM requires seemlessly moving video subjects to FEM's video subject viewer. There are no launched projects that use video subjects + drawing tools and the custom controls are incomplete. Therefore, further development of custom video controls will be paused and FEM's video subject viewer will use the same native browser controls as PFE for already-launched non-drawing-tools projects.
+
+See [PR 3684](https://github.com/zooniverse/front-end-monorepo/pull/3684) for more discussion.
+
+## Consequences
+The video subject viewer will be deployed without custom video controls for now. Future work toward projects with video subjects + drawing tools is tracked in the [Video Annotations](https://github.com/zooniverse/front-end-monorepo/projects/13) project board.
+
+## Status
+Accepted
+

packages/lib-classifier/package.json

@@ -66,6 +66,7 @@
     "@storybook/builder-webpack5": "~6.5.0",
     "@storybook/manager-webpack5": "~6.5.0",
     "@storybook/react": "~6.5.0",
+    "@storybook/testing-react": "~1.3.0",
     "@testing-library/dom": "~8.19.0",
     "@testing-library/react": "~12.1.2",
     "@testing-library/react-hooks": "~8.0.0",

packages/lib-classifier/src/components/Classifier/components/SubjectViewer/components/SingleVideoViewer/README.md

@@ -7,7 +7,7 @@ The only allowed video media type is mp4.
 
 ## Features
 
-SingleVideoViewerContainer handles state for SingleVideoViewer and VideoController. It's also contains an svg and InteractionLayer for drawing on a video subject.
+SingleVideoViewerContainer handles state for the `react-player` component and VideoController. It also contains an svg and InteractionLayer for drawing on a video subject.
 
 Refs
 - `interactionLayerSVG`: Reference to svg element displayed on top of video subject. Needed for drawing tools' InteractionLayer.
@@ -24,12 +24,14 @@ Props
 State Variables
 - `clientWidth`: (number) Returned from `getBoundingClientRect()` on the <video> element in `react-player`.
 - `duration`: (number) Duration of the video subject. Seconds rounded to 3 decimal places.
+- `fullscreen`: (boolean) Whether or not the video is displayed fullscreen.
 - `isPlaying`: (boolean) Whether or not the video subject is playing.
-- `isSeeking`: (boolean) Whether or not a user is interacting with the VideoController > Slider.
-- `playbackRate`: (number) 1, 0.5, or 0.25 ratio determines the speed of video playback.
-- `timeStamp`: (number) Current played timestamp of video subject.
+- `playbackSpeed`: (string) 1x, 0.5x, or 0.25x ratio determines the speed of video playback.
+- `timeStamp`: (number) Represented by percent of subject played (0 to 1).
 - `videoHeight`: (number) Natural height of video subject file.
 - `videoWidth`: (number) Natural width of the video subject file.
+- `volume`: (number) Number between 0 and 1. It's passed to the react-player to control volume.
+- `volumeOpen`: (boolean) Determines whether the VideoController's volume range input is displayed or not.
 
 ## External Setup: Workflows and Subjects
 

packages/lib-classifier/src/components/Classifier/components/SubjectViewer/components/SingleVideoViewer/SingleVideoViewerContainer.js

@@ -2,17 +2,17 @@ import PropTypes from 'prop-types'
 import styled from 'styled-components'
 import { Box } from 'grommet'
 import { MobXProviderContext } from 'mobx-react'
-import React, { useContext, useState, useRef } from 'react'
+import React, { useContext, useMemo, useState, useRef } from 'react'
 import { useTranslation } from 'react-i18next'
 import asyncStates from '@zooniverse/async-states'
+import ReactPlayer from 'react-player'
 
 import SVGContext from '@plugins/drawingTools/shared/SVGContext'
 
+import getFixedNumber from '../../helpers/getFixedNumber'
 import InteractionLayer from '../InteractionLayer'
 import locationValidator from '../../helpers/locationValidator'
-import SingleVideoViewer from './SingleVideoViewer'
-import VideoController from '../VideoController/VideoController'
-import getFixedNumber from '../../helpers/getFixedNumber'
+import VideoController from './components/VideoController'
 
 const SubjectContainer = styled.div`
   position: relative;
@@ -40,12 +40,14 @@ function SingleVideoViewerContainer({
 }) {
   const [clientWidth, setClientWidth] = useState(0)
   const [duration, setDuration] = useState(0)
+  const [fullscreen, setFullscreen] = useState(false)
   const [isPlaying, setIsPlaying] = useState(false)
-  const [isSeeking, setIsSeeking] = useState(false)
-  const [playbackRate, setPlaybackRate] = useState(1)
+  const [playbackSpeed, setPlaybackSpeed] = useState('1x')
   const [timeStamp, setTimeStamp] = useState(0)
   const [videoHeight, setVideoHeight] = useState(0)
   const [videoWidth, setVideoWidth] = useState(0)
+  const [volume, setVolume] = useState(1)
+  const [volumeOpen, toggleVolumeOpen] = useState(false)
 
   const interactionLayerSVG = useRef()
   const playerRef = useRef()
@@ -83,17 +85,14 @@ function SingleVideoViewerContainer({
     }
   }
 
-  const enableDrawing = loadingState === asyncStates.success && enableInteractionLayer
+  const enableDrawing = enableInteractionLayer && loadingState === asyncStates.success
 
-  /* ==================== SingleVideoViewer react-player ==================== */
+  /* ==================== react-player ==================== */
 
   const handleVideoProgress = reactPlayerState => {
-    const { played } = reactPlayerState
-    const fixedNumber = getFixedNumber(played, 5)
-    // TO DO: Why wouldn't you set timestamp when seeking?
-    if (!isSeeking) {
-      setTimeStamp(fixedNumber)
-    }
+    const { played } = reactPlayerState // percentage of video played (0 to 1)
+    const fixedNumber = getFixedNumber(played, 3)
+    setTimeStamp(fixedNumber)
   }
 
   const handleVideoDuration = duration => {
@@ -109,29 +108,79 @@ function SingleVideoViewerContainer({
     setIsPlaying(!prevStatePlaying)
   }
 
-  const handleSpeedChange = rate => {
-    setPlaybackRate(rate)
+  const handleSpeedChange = speed => {
+    setPlaybackSpeed(speed)
   }
 
-  const handleSliderMouseUp = () => {
-    setIsSeeking(false)
+  const handleSliderChange = e => {
+    const newTimeStamp = e.target.value
+    playerRef?.current?.seekTo(newTimeStamp, 'seconds')
   }
 
-  const handleSliderMouseDown = () => {
-    setIsPlaying(false)
-    setIsSeeking(true)
+  const handleVolume = e => {
+    setVolume(e.target.value)
   }
 
-  /* When VideoController > Slider is clicked or scrubbed */
-  const handleSliderChange = e => {
-    const played = getFixedNumber(e.target.value, 5)
-    playerRef?.current.seekTo(played)
+  const handleVolumeOpen = () => {
+    const prevVolumeOpen = volumeOpen
+    toggleVolumeOpen(!prevVolumeOpen)
+  }
+
+  const handleFullscreen = () => {
+    if (fullscreen) {
+      try {
+        document.exitFullscreen()
+        setFullscreen(false)
+      } catch (error) {
+        console.log(error)
+      }
+    } else {
+      try {
+        playerRef.current?.getInternalPlayer().requestFullscreen()
+        setFullscreen(true)
+      } catch (error) {
+        console.log(error)
+      }
+    }
   }
 
   const handlePlayerError = (error) => {
     onError(error)
   }
 
+  const sanitizedSpeed = Number(playbackSpeed.slice(0, -1))
+
+  /* Memoized so onProgress() and setTimeStamp() don't trigger each other */
+  const memoizedViewer = useMemo(() => (
+    <ReactPlayer
+      controls={!enableDrawing}
+      height='100%'
+      onDuration={handleVideoDuration}
+      onEnded={handleVideoEnded}
+      onError={handlePlayerError}
+      onReady={onReactPlayerReady}
+      onProgress={handleVideoProgress}
+      playing={isPlaying}
+      playbackSpeed={sanitizedSpeed}
+      progressInterval={100} // milliseconds
+      ref={playerRef}
+      width='100%'
+      volume={volume}
+      url={videoSrc}
+      config={{
+        file: { // styling the <video> element
+          attributes: {
+            style: {
+              display: 'block',
+              height: '100%',
+              width: '100%'
+            }
+          }
+        }
+      }}
+    />
+  ), [enableDrawing, isPlaying, playbackSpeed, videoSrc, volume])
+
   const canvas = transformLayer?.current
   const interactionLayerScale = clientWidth / videoWidth
   const svgStyle = {}
@@ -144,17 +193,7 @@ function SingleVideoViewerContainer({
       {videoSrc
         ? (
           <SubjectContainer>
-            <SingleVideoViewer
-              isPlaying={isPlaying}
-              onDuration={handleVideoDuration}
-              onEnded={handleVideoEnded}
-              onError={handlePlayerError}
-              onReactPlayerReady={onReactPlayerReady}
-              onProgress={handleVideoProgress}
-              playbackRate={playbackRate}
-              playerRef={playerRef}
-              url={videoSrc}
-            />
+            {memoizedViewer}
             {enableDrawing && (
               <DrawingLayer>
                 <Box overflow='hidden'>
@@ -187,17 +226,25 @@ function SingleVideoViewerContainer({
         : (
           <Box>{t('SubjectViewer.SingleVideoViewerContainer.error')}</Box>
           )}
-      <VideoController
-        isPlaying={isPlaying}
-        played={timeStamp}
-        playbackRate={playbackRate}
-        duration={duration}
-        onPlayPause={handlePlayPause}
-        onSpeedChange={handleSpeedChange}
-        onSliderMouseUp={handleSliderMouseUp}
-        onSliderMouseDown={handleSliderMouseDown}
-        onSliderChange={handleSliderChange}
-      />
+
+      {/** See ADR 45 for notes on custom video controls */}
+      {enableDrawing && (
+        <VideoController
+          duration={duration}
+          enableDrawing={enableDrawing}
+          isPlaying={isPlaying}
+          handleFullscreen={handleFullscreen}
+          handleVolumeOpen={handleVolumeOpen}
+          onPlayPause={handlePlayPause}
+          onSliderChange={handleSliderChange}
+          onSpeedChange={handleSpeedChange}
+          onVolumeChange={handleVolume}
+          playbackSpeed={playbackSpeed}
+          timeStamp={timeStamp}
+          volume={volume}
+          volumeOpen={volumeOpen}
+        />
+      )}
     </>
   )
 }