Autoplay and Media Component Improvements

packages/client-core/package.json

@@ -79,6 +79,7 @@
   "devDependencies": {
     "@emotion/react": "11.9.0",
     "@emotion/styled": "11.8.1",
+    "@types/hark": "1.2.2",
     "@types/node": "18.11.18",
     "@types/react": "18.0.19",
     "@types/react-router-dom": "5.3.3",

packages/client-core/src/common/services/LocationInstanceConnectionService.ts

@@ -50,13 +50,9 @@ export const LocationInstanceState = defineState({
 })
 
 export function useWorldInstance() {
-  const [state, setState] = React.useState(null as null | State<InstanceState>)
   const worldInstanceState = useState(getMutableState(LocationInstanceState).instances)
   const worldHostId = useState(getMutableState(NetworkState).hostIds.world)
-  useEffect(() => {
-    setState(worldHostId.value ? worldInstanceState[worldHostId.value] : null)
-  }, [worldInstanceState, worldHostId])
-  return state
+  return worldHostId.value ? worldInstanceState[worldHostId.value] : null
 }
 
 export const LocationInstanceConnectionServiceReceptor = (action) => {

packages/client-core/src/common/services/MediaInstanceConnectionService.ts

@@ -50,13 +50,9 @@ export const MediaInstanceState = defineState({
 })
 
 export function useMediaInstance() {
-  const [state, setState] = React.useState(null as null | State<InstanceState>)
   const mediaInstanceState = useState(getMutableState(MediaInstanceState).instances)
   const mediaHostId = useState(getMutableState(NetworkState).hostIds.media)
-  useEffect(() => {
-    setState(mediaHostId.value ? mediaInstanceState[mediaHostId.value] : null)
-  }, [mediaInstanceState, mediaHostId])
-  return state
+  return mediaHostId.value ? mediaInstanceState[mediaHostId.value] : null
 }
 
 export const MediaInstanceConnectionServiceReceptor = (action) => {

packages/client-core/src/components/UserMediaWindow/index.tsx

@@ -76,7 +76,7 @@ export const useUserMediaWindowHook = ({ peerID, type }: Props) => {
   const [videoTrackId, setVideoTrackId] = useState('')
   const [audioTrackId, setAudioTrackId] = useState('')
 
-  const [harkListener, setHarkListener] = useState(null)
+  const [harkListener, setHarkListener] = useState(null as ReturnType<typeof hark> | null)
   const [soundIndicatorOn, setSoundIndicatorOn] = useState(false)
   const [isPiP, setPiP] = useState(false)
   const [videoDisplayReady, setVideoDisplayReady] = useState<boolean>(false)
@@ -90,7 +90,6 @@ export const useUserMediaWindowHook = ({ peerID, type }: Props) => {
 
   const [_volume, _setVolume] = useState(1)
 
-  const userHasInteracted = useEngineState().userHasInteracted
   const selfUser = useAuthState().user.value
   const currentLocation = useLocationState().currentLocation.location
   const enableGlobalMute =
@@ -122,12 +121,16 @@ export const useUserMediaWindowHook = ({ peerID, type }: Props) => {
   }, [peerMediaChannelState.audioStream])
 
   useEffect(() => {
-    if (userHasInteracted.value && !isSelf) {
+    function onUserInteraction() {
       videoElement?.play()
       audioElement?.play()
-      if (harkListener) (harkListener as any).resume()
+      harkListener?.resume()
     }
-  }, [userHasInteracted])
+    window.addEventListener('pointerdown', onUserInteraction)
+    return () => {
+      window.removeEventListener('pointerdown', onUserInteraction)
+    }
+  }, [videoElement, audioElement, harkListener])
 
   useEffect(() => {
     if (audioElement != null) {
@@ -160,7 +163,7 @@ export const useUserMediaWindowHook = ({ peerID, type }: Props) => {
       audioTrackClones.forEach((track) => track.stop())
       if (harkListener) (harkListener as any).stop()
     }
-  }, [audioTrackId])
+  }, [audioTrackId, harkListener])
 
   useEffect(() => {
     videoElement.id = `${peerID}_video`
@@ -432,7 +435,7 @@ const UserMediaWindow = ({ peerID, type }: Props): JSX.Element => {
           className={classNames({
             [styles['video-wrapper']]: !isScreen,
             [styles['screen-video-wrapper']]: isScreen,
-            [styles['border-lit']]: soundIndicatorOn
+            [styles['border-lit']]: soundIndicatorOn && !audioStreamPaused
           })}
         >
           {(videoStream == null ||

packages/engine/src/audio/systems/MediaSystem.ts

@@ -39,6 +39,13 @@ import {
 export class AudioEffectPlayer {
   static instance = new AudioEffectPlayer()
 
+  constructor() {
+    // only init when running in client
+    if (isClient) {
+      this.#init()
+    }
+  }
+
   static SOUNDS = {
     notification: '/sfx/notification.mp3',
     message: '/sfx/message.mp3',
@@ -56,7 +63,7 @@ export class AudioEffectPlayer {
   // pool of elements
   #els: HTMLAudioElement[] = []
 
-  _init() {
+  #init() {
     if (this.#els.length) return
     for (let i = 0; i < 20; i++) {
       const audioElement = document.createElement('audio')
@@ -117,7 +124,6 @@ export default async function MediaSystem() {
 
   const enableAudioContext = () => {
     if (audioContext.state === 'suspended') audioContext.resume()
-    AudioEffectPlayer.instance._init()
   }
 
   if (isClient && !getMutableState(EngineState).isEditor.value) {
@@ -126,13 +132,10 @@ export default async function MediaSystem() {
     const mediaQuery = defineQuery([MediaComponent, MediaElementComponent])
     function handleAutoplay() {
       enableAudioContext()
-
       for (const entity of mediaQuery()) {
-        const media = getComponentState(entity, MediaComponent)
-        if (media.playing.value) return
-        if (media.paused.value && media.autoplay.value) media.paused.set(false)
-        // safari requires play() to be called in the DOM handle function
-        getComponent(entity, MediaElementComponent)?.element.play()
+        const mediaElement = getComponent(entity, MediaElementComponent)
+        const media = getComponent(entity, MediaComponent)
+        if (!media.paused && mediaElement?.element.paused) mediaElement.element.play()
       }
     }
     // TODO: add destroy callbacks
@@ -230,8 +233,6 @@ export default async function MediaSystem() {
   addActionReceptor(AudioSettingReceptor)
   addActionReceptor(MediaSettingReceptor)
 
-  const userInteractActionQueue = createActionQueue(EngineActions.setUserHasInteracted.matches)
-
   const mediaQuery = defineQuery([MediaComponent])
   const videoQuery = defineQuery([VideoComponent])
   const volumetricQuery = defineQuery([VolumetricComponent, MediaElementComponent])
@@ -273,8 +274,6 @@ export default async function MediaSystem() {
     audioState.gainNodeMixBuses.soundEffects.value.disconnect()
     audioState.gainNodeMixBuses.soundEffects.set(null!)
 
-    removeActionQueue(userInteractActionQueue)
-
     removeQuery(mediaQuery)
     removeQuery(videoQuery)
     removeQuery(volumetricQuery)

packages/engine/src/ecs/classes/EngineState.ts

@@ -23,7 +23,6 @@ export const EngineState = defineState({
     isTeleporting: false,
     leaveWorld: false,
     socketInstance: false,
-    userHasInteracted: false,
     spectating: false,
     usersTyping: {} as { [key: string]: true },
     avatarLoadingEffect: true,
@@ -58,7 +57,6 @@ export function EngineEventReceptor(a) {
     .when(EngineActions.sceneLoadingProgress.matches, (action) => s.merge({ loadingProgress: action.progress }))
     .when(EngineActions.connectToWorld.matches, (action) => s.merge({ connectedWorld: action.connectedWorld }))
     .when(EngineActions.setTeleporting.matches, (action) => s.merge({ isTeleporting: action.isTeleporting }))
-    .when(EngineActions.setUserHasInteracted.matches, (action) => s.merge({ userHasInteracted: true }))
     .when(EngineActions.spectateUser.matches, (action) => s.spectating.set(!!action.user))
 }
 

packages/engine/src/scene/components/MediaComponent.ts

@@ -105,25 +105,24 @@ export const MediaComponent = defineComponent({
     return {
       controls: false,
       synchronize: true,
-      autoplay: true,
-      /**
-       * TODO: refactor this into a ScheduleComponent for invoking callbacks at scheduled times
-       * The auto start time for the playlist, in Unix/Epoch time (milliseconds).
-       * If this value is negative, media playback must be explicitly started.
-       * If this value is non-negative and in the past, playback as soon as possible.
-       * If this value is in the future, begin playback at the appointed time.
-       */
-      // autoStartTime: -1,
       paths: [] as string[],
+      paused: true,
+      volume: 1,
       playMode: PlayMode.loop as PlayMode,
       isMusic: false,
-      volume: 1,
       // runtime props
-      paused: true,
-      playing: false,
+      waiting: false,
       track: 0,
       trackDurations: [] as number[],
       helper: null as Mesh<PlaneGeometry, MeshBasicMaterial> | null
+      /**
+       * TODO: refactor this into a ScheduleComponent for invoking callbacks at scheduled times
+       * The auto start time for the playlist, in Unix/Epoch time (milliseconds).
+       * If this value is negative, media playback must be explicitly started.
+       * If this value is non-negative and in the past, playback as soon as possible.
+       * If this value is in the future, begin playback at the appointed time.
+       */
+      // autoStartTime: -1
     }
   },
 
@@ -134,7 +133,7 @@ export const MediaComponent = defineComponent({
   toJSON: (entity, component) => {
     return {
       controls: component.controls.value,
-      autoplay: component.autoplay.value,
+      paused: component.paused.value,
       paths: component.paths.value,
       volume: component.volume.value,
       synchronize: component.synchronize.value,
@@ -155,9 +154,6 @@ export const MediaComponent = defineComponent({
       if (typeof json.controls === 'boolean' && json.controls !== component.controls.value)
         component.controls.set(json.controls)
 
-      if (typeof json.autoplay === 'boolean' && json.autoplay !== component.autoplay.value)
-        component.autoplay.set(json.autoplay)
-
       // backwars-compat: convert from number enums to strings
       if (
         (typeof json.playMode === 'number' || typeof json.playMode === 'string') &&
@@ -185,6 +181,9 @@ export const MediaComponent = defineComponent({
 
       if (typeof json.isMusic === 'boolean' && component.isMusic.value !== json.isMusic)
         component.isMusic.set(json.isMusic)
+
+      // @ts-ignore deprecated autoplay field
+      if (json.autoplay) component.paused.set(false)
     })
 
     return component
@@ -201,7 +200,6 @@ export function MediaReactor({ root }: EntityReactorProps) {
 
   const media = useComponent(entity, MediaComponent)
   const mediaElement = useOptionalComponent(entity, MediaElementComponent)
-  const userHasInteracted = useHookstate(getMutableState(EngineState).userHasInteracted)
   const audioContext = getState(AudioState).audioContext
   const gainNodeMixBuses = getState(AudioState).gainNodeMixBuses
 
@@ -243,9 +241,6 @@ export function MediaReactor({ root }: EntityReactorProps) {
         tempElement.src = path
       }
 
-      // handle autoplay
-      if (media.autoplay.value && userHasInteracted.value) media.paused.set(false)
-
       return () => {
         for (const { tempElement, listener } of metadataListeners) {
           tempElement.removeEventListener('loadedmetadata', listener)
@@ -257,13 +252,6 @@ export function MediaReactor({ root }: EntityReactorProps) {
     [media.paths]
   )
 
-  useEffect(
-    function updatePausedUponInteract() {
-      if (userHasInteracted.value && media.autoplay.value) media.paused.set(false)
-    },
-    [userHasInteracted]
-  )
-
   useEffect(
     function updateMediaElement() {
       if (!isClient) return
@@ -302,16 +290,18 @@ export function MediaReactor({ root }: EntityReactorProps) {
         element.addEventListener(
           'playing',
           () => {
-            media.playing.set(true), clearErrors(entity, MediaElementComponent)
+            media.waiting.set(false)
+            clearErrors(entity, MediaElementComponent)
           },
           { signal }
         )
-        element.addEventListener('waiting', () => media.playing.set(false), { signal })
+        element.addEventListener('waiting', () => media.waiting.set(true), { signal })
         element.addEventListener(
           'error',
           (err) => {
             addError(entity, MediaElementComponent, 'MEDIA_ERROR', err.message)
-            if (media.playing.value) media.track.set(getNextTrack(media.value))
+            if (!media.paused.value) media.track.set(getNextTrack(media.value))
+            media.waiting.set(false)
           },
           { signal }
         )
@@ -321,6 +311,7 @@ export function MediaReactor({ root }: EntityReactorProps) {
           () => {
             if (media.playMode.value === PlayMode.single) return
             media.track.set(getNextTrack(media.value))
+            media.waiting.set(false)
           },
           { signal }
         )

packages/engine/src/scene/functions/loaders/VolumetricFunctions.ts

@@ -125,8 +125,6 @@ export const updateVolumetric = async (entity: Entity) => {
       } else {
         volumetric.loadingEffectActive = false
         endLoadingEffect(volumetric.entity, player.mesh)
-        const media = getComponentState(entity, MediaComponent)
-        if (media.autoplay.value && getMutableState(EngineState).userHasInteracted.value) media.paused.set(false)
       }
     }
   }

packages/projects/default-project/apartment.scene.json

@@ -897,7 +897,7 @@
           "name": "media",
           "props": {
             "controls": false,
-            "autoplay": true,
+            "paused": false,
             "paths": [
               "__$project$__/default-project/assets/apartment-ambience.mp3"
             ],

packages/projects/default-project/sky-station.scene.json

@@ -314,7 +314,7 @@
           "name": "media",
           "props": {
             "controls": true,
-            "autoplay": true,
+            "paused": false,
             "paths": [
               "__$project$__/default-project/assets/SampleVideo.mp4"
             ],
@@ -581,7 +581,7 @@
           "name": "media",
           "props": {
             "controls": true,
-            "autoplay": false,
+            "paused": true,
             "paths": [
               "__$project$__/default-project/assets/SampleAudio.mp3"
             ],
@@ -785,7 +785,7 @@
           "name": "media",
           "props": {
             "controls": false,
-            "autoplay": true,
+            "paused": false,
             "paths": [
               "__$project$__/default-project/assets/sky-station-ambience-loop.mp3"
             ],