diff --git a/packages/engine/src/audio/components/PositionalAudioComponent.ts b/packages/engine/src/audio/components/PositionalAudioComponent.ts index ed4b7a59af2..6134d29b459 100755 --- a/packages/engine/src/audio/components/PositionalAudioComponent.ts +++ b/packages/engine/src/audio/components/PositionalAudioComponent.ts @@ -134,6 +134,7 @@ export const PositionalAudioComponent = defineComponent({ audioNodes.panner.coneOuterAngle = audio.coneOuterAngle.value audioNodes.panner.coneOuterGain = audio.coneOuterGain.value }, [ + mediaElement?.element, audio.refDistance, audio.rolloffFactor, audio.maxDistance, diff --git a/packages/engine/src/audio/systems/MediaSystem.ts b/packages/engine/src/audio/systems/MediaSystem.ts index 561bee8baba..84109b16b8b 100755 --- a/packages/engine/src/audio/systems/MediaSystem.ts +++ b/packages/engine/src/audio/systems/MediaSystem.ts @@ -44,13 +44,6 @@ import { PositionalAudioComponent } from '../components/PositionalAudioComponent export class AudioEffectPlayer { static instance = new AudioEffectPlayer() - constructor() { - // only init when running in client - if (isClient) { - this.#init() - } - } - static SOUNDS = { notification: '/sfx/notification.mp3', message: '/sfx/message.mp3', @@ -65,37 +58,22 @@ export class AudioEffectPlayer { return buffer } - // pool of elements - #els: HTMLAudioElement[] = [] - - #init() { - if (this.#els.length) return - for (let i = 0; i < 20; i++) { - const audioElement = document.createElement('audio') - audioElement.crossOrigin = 'anonymous' - audioElement.loop = false - this.#els.push(audioElement) - } - } - play = async (sound: string, volumeMultiplier = getState(AudioState).notificationVolume) => { await Promise.resolve() - if (!this.#els.length) return - if (!this.bufferMap[sound]) { // create buffer if doesn't exist this.bufferMap[sound] = await AudioEffectPlayer?.instance?.loadBuffer(sound) } - const source = getState(AudioState).audioContext.createBufferSource() + const audioContext = getState(AudioState).audioContext + const source = audioContext.createBufferSource() + const gain = audioContext.createGain() + gain.gain.value = getState(AudioState).masterVolume * volumeMultiplier source.buffer = this.bufferMap[sound] - const el = this.#els.find((el) => el.paused) ?? this.#els[0] - el.volume = getState(AudioState).masterVolume * volumeMultiplier - if (el.src !== sound) el.src = sound - el.currentTime = 0 + source.connect(gain) + gain.connect(audioContext.destination) source.start() - source.connect(getState(AudioState).audioContext.destination) } } diff --git a/packages/engine/src/scene/components/MediaComponent.ts b/packages/engine/src/scene/components/MediaComponent.ts index 446ed7a5014..eb3f30174f9 100644 --- a/packages/engine/src/scene/components/MediaComponent.ts +++ b/packages/engine/src/scene/components/MediaComponent.ts @@ -27,9 +27,18 @@ import Hls from 'hls.js' import { startTransition, useEffect } from 'react' import { DoubleSide, Mesh, MeshBasicMaterial, PlaneGeometry } from 'three' -import { NO_PROXY, State, getMutableState, getState, none, useHookstate } from '@etherealengine/hyperflux' +import { + NO_PROXY, + State, + getMutableState, + getState, + none, + useHookstate, + useMutableState +} from '@etherealengine/hyperflux' import { isClient } from '@etherealengine/common/src/utils/getEnvironment' +import { defineQuery } from '@etherealengine/ecs' import { defineComponent, getComponent, @@ -51,6 +60,8 @@ import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components import { setVisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { requestXRSession } from '@etherealengine/spatial/src/xr/XRSessionFunctions' +import { XRState } from '@etherealengine/spatial/src/xr/XRState' import { AssetLoader } from '../../assets/classes/AssetLoader' import { useTexture } from '../../assets/functions/resourceHooks' import { AudioState } from '../../audio/AudioState' @@ -88,6 +99,11 @@ export const createAudioNodeGroup = ( export const MediaElementComponent = defineComponent({ name: 'MediaElement', + // elementPool: { + // video: [] as HTMLVideoElement[], + // audio: [] as HTMLAudioElement[] + // }, + onInit: (entity) => { return { element: undefined! as HTMLMediaElement, @@ -124,6 +140,51 @@ export const MediaElementComponent = defineComponent({ errors: ['MEDIA_ERROR', 'HLS_ERROR'] }) +// if ('HTMLMediaElement' in globalThis) { +// for (let i = 0; i < 20; i++) { +// MediaElementComponent.elementPool.video.push(document.createElement('video')) +// MediaElementComponent.elementPool.audio.push(document.createElement('audio')) +// } +// } + +// In Safari on Apple Vision Pro, all media looses autoplay permissions after entering XR, so +// we need to trigger the play() method on the media element (and resume the audio context) +// during a user activation event, which necessarily happens when starting an XR session. +// play-puase all media elements to tickle the user activation +// autoplay policy in Safari on Apple Vision Pro in the perfect way + +const elementsQuery = defineQuery([MediaElementComponent]) +let playPausePromises = [] as Promise[] + +requestXRSession.beforeHooks.push(() => { + alert('test') + console.log('requestXRSession.beforeHooks') + + playPausePromises = elementsQuery().map((eid) => { + const el = getComponent(eid, MediaElementComponent).element + return el + .play() + .then(() => el.pause()) + .then(() => el) + }) + + getState(AudioState).audioContext.resume() +}) + +requestXRSession.afterHooks.push((ctx) => { + ctx.result.then(() => { + for (const p of playPausePromises) { + p.then((mediaElement) => { + mediaElement.play() + console.log('Did resume media playback: ' + mediaElement.src) + }) + } + playPausePromises.length = 0 + getState(AudioState).audioContext.resume() + console.log('Did resume audio context') + }) +}) + export const MediaComponent = defineComponent({ name: 'MediaComponent', jsonID: 'EE_media', @@ -240,9 +301,10 @@ export const MediaComponent = defineComponent({ export function MediaReactor() { const entity = useEntityContext() const media = useComponent(entity, MediaComponent) - const mediaElement = useOptionalComponent(entity, MediaElementComponent) + const mediaElementComponent = useOptionalComponent(entity, MediaElementComponent) const audioContext = getState(AudioState).audioContext const gainNodeMixBuses = getState(AudioState).gainNodeMixBuses + const xrSession = useMutableState(XRState).session if (!isClient) return null @@ -250,9 +312,12 @@ export function MediaReactor() { // This must be outside of the normal ECS flow by necessity, since we have to respond to user-input synchronously // in order to ensure media will play programmatically const handleAutoplay = () => { + console.log('handleAutoplay') const mediaComponent = getComponent(entity, MediaElementComponent) // handle when we dont have autoplay enabled but have programatically started playback - if (!media.autoplay.value && !media.paused.value) mediaComponent?.element.play() + mediaComponent?.element.play().then(() => { + if (!media.autoplay.value && !media.paused.value) mediaComponent?.element.pause() + }) // handle when we have autoplay enabled but have paused playback if (media.autoplay.value && media.paused.value) media.paused.set(false) // handle when we have autoplay and mediaComponent is paused @@ -272,6 +337,9 @@ export function MediaReactor() { document.body.addEventListener('touchend', handleAutoplay) EngineRenderer.instance.renderer.domElement.addEventListener('pointerup', handleAutoplay) EngineRenderer.instance.renderer.domElement.addEventListener('touchend', handleAutoplay) + const mediaElement = mediaElementComponent?.element.value + mediaElement?.addEventListener('pause', handleAutoplay) + xrSession.value?.addEventListener('squeeze', handleAutoplay) return () => { window.removeEventListener('pointerup', handleAutoplay) @@ -281,16 +349,18 @@ export function MediaReactor() { document.body.removeEventListener('touchend', handleAutoplay) EngineRenderer.instance.renderer.domElement.removeEventListener('pointerup', handleAutoplay) EngineRenderer.instance.renderer.domElement.removeEventListener('touchend', handleAutoplay) + mediaElement?.removeEventListener('pause', handleAutoplay) + xrSession.value?.removeEventListener('squeeze', handleAutoplay) } - }, []) + }, [mediaElementComponent, xrSession]) useEffect( function updatePlay() { - if (!mediaElement) return + if (!mediaElementComponent) return if (media.paused.value) { - mediaElement.element.value.pause() + mediaElementComponent.element.value.pause() } else { - const promise = mediaElement.element.value.play() + const promise = mediaElementComponent.element.value.play() if (promise) { promise.catch((error) => { console.error(error) @@ -298,16 +368,16 @@ export function MediaReactor() { } } }, - [media.paused, mediaElement] + [media.paused, mediaElementComponent] ) useEffect( function updateSeekTime() { - if (!mediaElement) return - setTime(mediaElement.element, media.seekTime.value) - if (!mediaElement.element.paused.value) mediaElement.element.value.play() // if not paused, start play again + if (!mediaElementComponent) return + setTime(mediaElementComponent.element, media.seekTime.value) + if (!mediaElementComponent.element.paused.value) mediaElementComponent.element.value.play() // if not paused, start play again }, - [media.seekTime, mediaElement] + [media.seekTime, mediaElementComponent] ) useEffect( @@ -469,8 +539,8 @@ export function MediaReactor() { useEffect( function updateMixbus() { - if (!mediaElement?.value) return - const element = mediaElement.element.get({ noproxy: true }) + if (!mediaElementComponent?.value) return + const element = mediaElementComponent.element.get({ noproxy: true }) const audioNodes = AudioNodeGroups.get(element) if (audioNodes) { audioNodes.gain.disconnect(audioNodes.mixbus) @@ -478,7 +548,7 @@ export function MediaReactor() { audioNodes.gain.connect(audioNodes.mixbus) } }, - [mediaElement, media.isMusic] + [mediaElementComponent, media.isMusic] ) const debugEnabled = useHookstate(getMutableState(RendererState).nodeHelperVisibility) diff --git a/packages/engine/src/scene/components/UVOL1Component.ts b/packages/engine/src/scene/components/UVOL1Component.ts index af9b5e9338d..4fca3ff8510 100644 --- a/packages/engine/src/scene/components/UVOL1Component.ts +++ b/packages/engine/src/scene/components/UVOL1Component.ts @@ -51,7 +51,9 @@ import { PlaneGeometry, SRGBColorSpace, ShaderMaterial, - Texture + Sphere, + Texture, + Vector3 } from 'three' import { CORTOLoader } from '../../assets/loaders/corto/CORTOLoader' import { AssetLoaderState } from '../../assets/state/AssetLoaderState' @@ -319,6 +321,7 @@ function UVOL1Reactor() { throw new Error('VDEBUG Entity ${entity} Invalid geometry frame: ' + i.toString()) } + geometry.boundingSphere = new Sphere().set(new Vector3(), Infinity) meshBuffer.set(i, geometry) pendingRequests.current -= 1 diff --git a/packages/server/key.pem b/packages/server/key.pem deleted file mode 100644 index 8a5da712823..00000000000 --- a/packages/server/key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmECBfai56GaTE -5YSV8LRS/PwjqDXr5wq/snj5F424K5+bE1WIxNaPgqV+07NryudSEUkivVqg9Tez -9n2q+bbl3ZqRsErpiTa4RQMr+LIc9/BYOH3qM4kSoNDgVCYWY7Is2+ujQwCFY2Wo -HO0AW3iCUX64MIvhTB5dycdFA3g+A0nmHpwgqJYlwwiugXUyFKG+PL9AGUNIDv2j -RLdWerLAlwt3JhmE66eqaZ913staxDA52cEQkjv2PG48coY+34dALl5wkL/5qxlk -jGpN0G0kouDW/mNzLsvO9EBbolfO2G2/31/9jAef6ktIHwhkYKwfoHEM4U9Fj2Dn -iVYXjBztAgMBAAECggEBAKU7XIqzwyT4iQrk+IScqT/9uv/bqjNcg0R0V4w51GhQ -95E8rpnUiUzoCLVl/Zuem4X77xJi6F9JPzCQbUAHEeYsFlUp3Y/7WHgHKv3AibDr -Prj5FMhImPXxD4R2FiOm9S1PcWnUDBus6ARbK7J0UTcR0Y2BuFzm+TwGvoC70BOa -ZL+LvhznPN0xu4r9O/s1nEsykn6sJGifsfwV9ZafJAZDeytCGhBWdQwchx0rJMq3 -C4MMjL1ehHMbOAW+WBpHXsO2K68ZDUSccknpnwDr9IzIi5veQJcDVFr/RSVsCA8z -mtT3/iPjD1o5BkIL8zYtiwrOuDnVqpXtcs++Pr36M2ECgYEA3VvhvWco4O8rXiCx -8xB+OQGgxZL36zYQ/Tk31rBGgnOs/BC8mg0QxR/gSzrzIpIK28Vg/7OKO/v/2rKH -ZDTmt43sxttwLBKXxc2Slq3doGfV+h3Add/VfoGGoZcU523tqoiLGWICNqTLYNZ4 -MJe4beF/Kz1aIbArdbFAuTTV6TkCgYEAwAz4XpEm0efPcA5d8Rjis6+JaUVuPzh8 -ENU92ojP6pJxXczRCGVRr+hKgsOqEgtRjqy+ksGULTZkv7llaWJ7ZXR3u1eJKyKW -oWkFN0GrahULhlKxEzMkfnrrGegmu7TnYNwe8mER7puuoYl0CaUzFl/aln0G51AJ -S7dcbreEFVUCgYBOR5fk6v5BzVKAzv8e/c4lSrYYKIkT3OLVKc7dVSkaKN4bpa+M -quIrU8J12DrzFsJQRdSvmEZiQBOSu1+1yB9u+fmpuSBJ9alGghQ8xO+DMjUxZiQR -iz5splF+A3eY//70N6U5LLerq0tgy3dld8H42a2nFOMy1qIH1M8Wr+CVwQKBgQCR -LnznyGUHU21OcZ30r/JZEb8YFMOmCUZI11s+BCThWDlZRodTCHz7NOh2+AFuSJ4r -9EzQ1oP0teTtxvJx+1/7L1OADUmFkU070g9+WSeDN0uSDJsOP6A7+SIXYJc/WR98 -6op+goEy1v/p3+YXkIoRRP8Suotoe+m7Em9Ox26TTQKBgCXs7gA9x2QfOe22wfJ6 -bqvj6/EoyD9sHs5SV8BT+0lIX5qiNPwf6jRWe34xGH1e2xscoNk3/p2azEGlpMJS -aQVh6ivOnTlD6K0iZyim7EgBCdxDSibg8RLIVQt+LjpnE4lXCHlRyDr/uLXwv1nq -pYgCYm/gGIM7aH4Y2l6zXpSj ------END PRIVATE KEY-----