bitecs audio-target and zone-audio-source

src/app.ts

@@ -5,7 +5,7 @@ import MediaSearchStore from "./storage/media-search-store";
 import Store from "./storage/store";
 import qsTruthy from "./utils/qs_truthy";
 
-import type { AElement, AScene } from "aframe";
+import type { AComponent, AScene } from "aframe";
 import HubChannel from "./utils/hub-channel";
 import MediaDevicesManager from "./utils/media-devices-manager";
 
@@ -29,6 +29,7 @@ import { waitForPreloads } from "./utils/preload";
 import SceneEntryManager from "./scene-entry-manager";
 import { store } from "./utils/store-instance";
 import { addObject3DComponent } from "./utils/jsx-entity";
+import { ElOrEid } from "./utils/bit-utils";
 
 declare global {
   interface Window {
@@ -74,21 +75,22 @@ export class App {
   entryManager?: SceneEntryManager;
   messageDispatch?: any;
   store: Store;
+  componentRegistry: { [key: string]: AComponent[] };
 
   mediaSearchStore = new MediaSearchStore();
 
-  audios = new Map<AElement | number, PositionalAudio | Audio>();
-  sourceType = new Map<AElement | number, SourceType>();
-  audioOverrides = new Map<AElement | number, Partial<AudioSettings>>();
-  zoneOverrides = new Map<AElement | number, Partial<AudioSettings>>();
-  gainMultipliers = new Map<AElement | number, number>();
-  supplementaryAttenuation = new Map<AElement | number, number>();
-  clippingState = new Set<AElement | number>();
-  mutedState = new Set<AElement | number>();
-  isAudioPaused = new Set<AElement | number>();
+  audios = new Map<ElOrEid, PositionalAudio | Audio>();
+  sourceType = new Map<ElOrEid, SourceType>();
+  audioOverrides = new Map<ElOrEid, Partial<AudioSettings>>();
+  zoneOverrides = new Map<ElOrEid, Partial<AudioSettings>>();
+  gainMultipliers = new Map<ElOrEid, number>();
+  supplementaryAttenuation = new Map<ElOrEid, number>();
+  clippingState = new Set<ElOrEid>();
+  mutedState = new Set<ElOrEid>();
+  isAudioPaused = new Set<ElOrEid>();
   audioDebugPanelOverrides = new Map<SourceType, Partial<AudioSettings>>();
   sceneAudioDefaults = new Map<SourceType, Partial<AudioSettings>>();
-  moderatorAudioSource = new Set<AElement | number>();
+  moderatorAudioSource = new Set<ElOrEid>();
 
   world: HubsWorld = createWorld();
 

src/bit-components.js

@@ -280,6 +280,15 @@ export const ParticleEmitterTag = defineComponent({
 export const AudioZone = defineComponent({
   flags: Types.ui8
 });
+export const AudioTarget = defineComponent({
+  minDelay: Types.ui32,
+  maxDelay: Types.ui32,
+  source: Types.eid,
+  flags: Types.ui8
+});
+export const AudioSource = defineComponent({
+  flags: Types.ui8
+});
 export const AudioParams = defineComponent();
 export const ScenePreviewCamera = defineComponent({
   duration: Types.f32,

src/bit-systems/audio-debug-system.ts

@@ -2,12 +2,13 @@ import audioDebugVert from "../systems/audio-debug.vert";
 import audioDebugFrag from "../systems/audio-debug.frag";
 import { defineQuery, enterQuery, exitQuery } from "bitecs";
 import { getScene, HubsWorld } from "../app";
-import { AudioEmitter, NavMesh } from "../bit-components";
+import { NavMesh } from "../bit-components";
 import { DistanceModelType } from "../components/audio-params";
 import { getWebGLVersion } from "../utils/webgl";
-import { EMITTER_FLAGS, isPositionalAudio } from "./audio-emitter-system";
+import { AudioObject3D, isPositionalAudio } from "./audio-emitter-system";
 import { Mesh, Material, Vector3, ShaderMaterial } from "three";
 import { disposeMaterial } from "../utils/three-utils";
+import { ElOrEid } from "../utils/bit-utils";
 
 const fakePanner = {
   distanceModel: DistanceModelType.Inverse,
@@ -145,7 +146,6 @@ export const cleanupAudioDebugNavMesh = (navEid: number) => removeDebugMaterial(
 
 const emitterPos = new THREE.Vector3();
 const emitterDir = new THREE.Vector3();
-const audioEmittersQuery = defineQuery([AudioEmitter]);
 const navMeshQuery = defineQuery([NavMesh]);
 const navMeshEnterQuery = enterQuery(navMeshQuery);
 const navMeshExitQuery = exitQuery(navMeshQuery);
@@ -159,17 +159,12 @@ export function audioDebugSystem(world: HubsWorld) {
       isEnabled && addDebugMaterial(world, navEid);
     });
     let idx = 0;
-    audioEmittersQuery(world).forEach(emitterEid => {
-      if (
-        AudioEmitter.flags[emitterEid] & EMITTER_FLAGS.PAUSED ||
-        AudioEmitter.flags[emitterEid] & EMITTER_FLAGS.MUTED
-      ) {
+    APP.audios.forEach((audio: AudioObject3D, audioEmitterId: ElOrEid) => {
+      if (APP.isAudioPaused.has(audioEmitterId) || APP.mutedState.has(audioEmitterId)) {
         return;
       }
       if (idx >= maxDebugEmitters) return;
 
-      const audio = APP.audios.get(emitterEid)!;
-
       audio.getWorldPosition(emitterPos);
       audio.getWorldDirection(emitterDir);
 
@@ -191,7 +186,7 @@ export function audioDebugSystem(world: HubsWorld) {
       uniforms.coneInnerAngles[idx] = panner.coneInnerAngle;
       uniforms.coneOuterAngles[idx] = panner.coneOuterAngle;
       uniforms.gains[idx] = audio.gain.gain.value;
-      uniforms.clipped[idx] = APP.clippingState.has(emitterEid) ? 1 : 0;
+      uniforms.clipped[idx] = APP.clippingState.has(audioEmitterId) ? 1 : 0;
 
       idx++;
     });

src/bit-systems/audio-emitter-system.ts

@@ -1,5 +1,11 @@
 import { addComponent, addEntity, defineQuery, removeComponent } from "bitecs";
-import { PositionalAudio, Audio as StereoAudio, AudioListener as ThreeAudioListener } from "three";
+import {
+  PositionalAudio,
+  Audio as StereoAudio,
+  AudioListener as ThreeAudioListener,
+  MeshStandardMaterial,
+  Mesh
+} from "three";
 import { HubsWorld } from "../app";
 import { AudioEmitter, AudioSettingsChanged } from "../bit-components";
 import { AudioType, SourceType } from "../components/audio-params";
@@ -13,12 +19,6 @@ type AudioConstructor<T> = new (listener: ThreeAudioListener) => T;
 export const Emitter2Audio = (AudioEmitter as any).audios as Map<number, number>;
 export const Emitter2Params = (AudioEmitter as any).params as Map<number, number>;
 
-export const EMITTER_FLAGS = {
-  MUTED: 1 << 0,
-  PAUSED: 1 << 1,
-  CLIPPED: 1 << 2
-};
-
 export function isPositionalAudio(node: AudioObject3D): node is PositionalAudio {
   return (node as any).panner !== undefined;
 }
@@ -57,14 +57,9 @@ function swapAudioType<T extends AudioObject3D>(
   swapObject3DComponent(world, eid, newAudio);
 }
 
-export function makeAudioSourceEntity(world: HubsWorld, video: HTMLVideoElement, audioSystem: AudioSystem) {
+export function makeAudioEntity(world: HubsWorld, source: number, sourceType: SourceType, audioSystem: AudioSystem) {
   const eid = addEntity(world);
-  APP.sourceType.set(eid, SourceType.MEDIA_VIDEO);
-  if (video.paused) {
-    AudioEmitter.flags[eid] |= EMITTER_FLAGS.PAUSED;
-  } else {
-    AudioEmitter.flags[eid] &= ~EMITTER_FLAGS.PAUSED;
-  }
+  APP.sourceType.set(eid, sourceType);
 
   let audio;
   const { audioType } = getCurrentAudioSettings(eid);
@@ -74,20 +69,30 @@ export function makeAudioSourceEntity(world: HubsWorld, video: HTMLVideoElement,
   } else {
     audio = new StereoAudio(audioListener);
   }
+
+  if (sourceType === SourceType.MEDIA_VIDEO) {
+    const videoObj = world.eid2obj.get(source) as Mesh;
+    const video = (videoObj.material as MeshStandardMaterial).map!.image as HTMLVideoElement;
+    if (video.paused) {
+      APP.isAudioPaused.add(eid);
+    } else {
+      APP.isAudioPaused.delete(eid);
+    }
+    const audioSrcEl = video;
+    audio.setMediaElementSource(audioSrcEl);
+    // Original audio source volume can now be restored as audio systems will take over
+    audioSrcEl.volume = 1;
+    audio.gain.gain.value = 0;
+  }
+
   addComponent(world, AudioEmitter, eid);
   addObject3DComponent(world, eid, audio);
 
-  audio.gain.gain.value = 0;
-  audioSystem.addAudio({ sourceType: SourceType.MEDIA_VIDEO, node: audio });
-
-  const audioSrcEl = video;
-  audio.setMediaElementSource(audioSrcEl);
+  audioSystem.addAudio({ sourceType, node: audio });
 
   APP.audios.set(eid, audio);
   updateAudioSettings(eid, audio);
 
-  // Original audio source volume can now be restored as audio systems will take over
-  audioSrcEl.volume = 1;
   return eid;
 }