Webaudio migration

src/app.ts

@@ -1,6 +1,6 @@
-import { addEntity, createWorld, IWorld } from "bitecs";
+import { addComponent, addEntity, createWorld, IWorld } from "bitecs";
 import "./aframe-to-bit-components";
-import { AEntity, Networked, Object3DTag, Owned } from "./bit-components";
+import { AEntity, AudioListenerTag, Networked, Object3DTag, Owned } from "./bit-components";
 import MediaSearchStore from "./storage/media-search-store";
 import Store from "./storage/store";
 import qsTruthy from "./utils/qs_truthy";
@@ -10,26 +10,16 @@ import HubChannel from "./utils/hub-channel";
 import MediaDevicesManager from "./utils/media-devices-manager";
 
 import { EffectComposer, EffectPass } from "postprocessing";
-import {
-  Audio,
-  AudioListener,
-  Material,
-  Object3D,
-  PerspectiveCamera,
-  PositionalAudio,
-  Scene,
-  sRGBEncoding,
-  WebGLRenderer
-} from "three";
+import { Camera, Material, Object3D, PerspectiveCamera, Scene, sRGBEncoding, WebGLRenderer } from "three";
 import { AudioSettings, SourceType } from "./components/audio-params";
 import { createEffectsComposer } from "./effects";
 import { DialogAdapter } from "./naf-dialog-adapter";
 import { mainTick } from "./systems/hubs-systems";
 import { waitForPreloads } from "./utils/preload";
 import SceneEntryManager from "./scene-entry-manager";
 import { store } from "./utils/store-instance";
-import { addObject3DComponent } from "./utils/jsx-entity";
 import { ElOrEid } from "./utils/bit-utils";
+import { addObject3DComponent } from "./utils/jsx-entity";
 
 declare global {
   interface Window {
@@ -79,7 +69,8 @@ export class App {
 
   mediaSearchStore = new MediaSearchStore();
 
-  audios = new Map<ElOrEid, PositionalAudio | Audio>();
+  audios = new Map<ElOrEid, AudioNode>();
+  gains = new Map<ElOrEid, GainNode>();
   sourceType = new Map<ElOrEid, SourceType>();
   audioOverrides = new Map<ElOrEid, Partial<AudioSettings>>();
   zoneOverrides = new Map<ElOrEid, Partial<AudioSettings>>();
@@ -91,14 +82,16 @@ export class App {
   audioDebugPanelOverrides = new Map<SourceType, Partial<AudioSettings>>();
   sceneAudioDefaults = new Map<SourceType, Partial<AudioSettings>>();
   moderatorAudioSource = new Set<ElOrEid>();
+  audioSources = new Map<ElOrEid, AudioNode>();
 
   world: HubsWorld = createWorld();
 
   str2sid: Map<string | null, number>;
   sid2str: Map<number, string | null>;
   nextSid = 1;
 
-  audioListener: AudioListener;
+  audioCtx: AudioContext;
+  camera: Camera;
 
   dialog = new DialogAdapter();
 
@@ -124,6 +117,8 @@ export class App {
     this.world.deletedNids = new Set();
     this.world.ignoredNids = new Set();
 
+    this.audioCtx = new AudioContext();
+
     // used in aframe and networked aframe to avoid imports
     this.world.nameToComponent = {
       object3d: Object3DTag,
@@ -166,7 +161,7 @@ export class App {
     canvas.dataset.aframeCanvas = "true";
 
     // TODO this comes from aframe and prevents zoom on ipad.
-    // This should alreeady be handleed by disable-ios-zoom but it does not appear to work
+    // This should already be handled by disable-ios-zoom but it does not appear to work
     canvas.addEventListener("touchmove", function (event) {
       event.preventDefault();
     });
@@ -196,13 +191,11 @@ export class App {
     sceneEl.appendChild(renderer.domElement);
 
     const camera = new PerspectiveCamera(80, window.innerWidth / window.innerHeight, 0.05, 10000);
-
-    const audioListener = new AudioListener();
-    this.audioListener = audioListener;
-    const audioListenerEid = addEntity(this.world);
-    addObject3DComponent(this.world, audioListenerEid, this.audioListener);
-
-    camera.add(audioListener);
+    const listenerEid = addEntity(this.world);
+    addObject3DComponent(this.world, listenerEid, camera);
+    addComponent(this.world, AudioListenerTag, listenerEid);
+    const audioListener = this.audioCtx.listener;
+    this.camera = camera;
 
     this.world.time = {
       delta: 0,
@@ -215,7 +208,7 @@ export class App {
     this.world.scene = scene;
     resolvePromiseToScene(scene);
 
-    // We manually call scene.updateMatrixWolrd in mainTick
+    // We manually call scene.updateMatrixWorld in mainTick
     scene.autoUpdate = false;
 
     if (enablePostEffects) {

src/bit-components.js

@@ -222,7 +222,6 @@ export const VideoMenu = defineComponent({
 export const AudioEmitter = defineComponent({
   flags: Types.ui8
 });
-AudioEmitter.audios = new Map();
 AudioEmitter.params = new Map();
 export const AudioSettingsChanged = defineComponent();
 export const Deletable = defineComponent();
@@ -341,3 +340,4 @@ export const LinearScale = defineComponent({
   targetY: Types.f32,
   targetZ: Types.f32
 });
+export const AudioListenerTag = defineComponent();

src/bit-systems/audio-debug-system.ts

@@ -5,7 +5,7 @@ import { getScene, HubsWorld } from "../app";
 import { NavMesh } from "../bit-components";
 import { DistanceModelType } from "../components/audio-params";
 import { getWebGLVersion } from "../utils/webgl";
-import { AudioObject3D, isPositionalAudio } from "./audio-emitter-system";
+import { getAudioOrientation, getAudioPosition, isPositionalAudio } from "./audio-emitter-system";
 import { Mesh, Material, Vector3, ShaderMaterial } from "three";
 import { disposeMaterial } from "../utils/three-utils";
 import { ElOrEid } from "../utils/bit-utils";
@@ -159,16 +159,17 @@ export function audioDebugSystem(world: HubsWorld) {
       isEnabled && addDebugMaterial(world, navEid);
     });
     let idx = 0;
-    APP.audios.forEach((audio: AudioObject3D, audioEmitterId: ElOrEid) => {
+    APP.audios.forEach((audio: AudioNode, audioEmitterId: ElOrEid) => {
       if (APP.isAudioPaused.has(audioEmitterId) || APP.mutedState.has(audioEmitterId)) {
         return;
       }
       if (idx >= maxDebugEmitters) return;
 
-      audio.getWorldPosition(emitterPos);
-      audio.getWorldDirection(emitterDir);
+      const panner = isPositionalAudio(audio) ? audio : fakePanner;
+      const gain = APP.gains.get(audioEmitterId)!;
 
-      const panner = isPositionalAudio(audio) ? audio.panner : fakePanner;
+      getAudioPosition(audioEmitterId, emitterPos);
+      getAudioOrientation(audioEmitterId, emitterDir);
 
       uniforms.sourcePositions[idx] = emitterPos.clone();
       uniforms.sourceOrientations[idx] = emitterDir.clone();
@@ -185,7 +186,7 @@ export function audioDebugSystem(world: HubsWorld) {
       uniforms.rolloffFactors[idx] = panner.rolloffFactor;
       uniforms.coneInnerAngles[idx] = panner.coneInnerAngle;
       uniforms.coneOuterAngles[idx] = panner.coneOuterAngle;
-      uniforms.gains[idx] = audio.gain.gain.value;
+      uniforms.gains[idx] = gain.gain.value;
       uniforms.clipped[idx] = APP.clippingState.has(audioEmitterId) ? 1 : 0;
 
       idx++;