C2LC-253: Ensure that Tone is started on the first user action.

flow-typed/npm/tone_v14.x.x.js

@@ -1,4 +1,7 @@
 declare module "tone" {
+    // https://tonejs.github.io/docs/14.7.39/fn/start
+    declare export function start(): Promise<void>;
+
     // There are two things called "Frequency" in Tone.js
     //
     //       https://tonejs.github.io/docs/14.7.58/type/Frequency
@@ -13,7 +16,7 @@ declare module "tone" {
     }
 
     declare export function Frequency(value: TimeValue | FrequencyType ): FrequencyClass;
-
+    
     // https://tonejs.github.io/docs/14.7.58/type/InputNode
     declare type InputNode = ToneAudioNode;
 

src/App.js

@@ -419,6 +419,9 @@ export default class App extends React.Component<{}, AppState> {
     };
 
     handleRootClick = (e: SyntheticInputEvent<HTMLInputElement>) => {
+        // Ensure that the audio starts on any user interaction.
+        this.audioManager.startTone();
+
         let element = e.target;
         // Walk up the document tree until we hit the top, or we find that
         // we are within an action panel group area
@@ -436,6 +439,9 @@ export default class App extends React.Component<{}, AppState> {
     };
 
     handleRootKeyDown = (e: SyntheticKeyboardEvent<HTMLInputElement>) => {
+        // Ensure that the audio starts on any user interaction.
+        this.audioManager.startTone();
+
         this.focusTrapManager.handleKeyDown(e);
     };
 

src/AudioManager.js

@@ -1,27 +1,68 @@
 // @flow
 
-import { Midi, Panner, Player, Sampler } from 'tone';
+import { Midi, Panner, Player, Sampler, start as ToneStart} from 'tone';
 import CharacterState from './CharacterState';
 import type {AnnouncedSoundName} from './types';
 
-type AnnouncementLookupTable = {
-    forward1: Player,
-    forward2: Player,
-    forward3: Player,
-    left45: Player,
-    left90: Player,
-    left180: Player,
-    right45: Player,
-    right90: Player,
-    right180: Player,
-    add: Player,
-    deleteAll: Player,
-    delete: Player,
-    moveToPrevious: Player,
-    moveToNext: Player,
-    replace: Player
+class WrappedPlayer extends Player {
+    isLoadedPromise: Promise<any>;
+    isLoadedResolve: function;
+    isLoadedReject: function; // TODO: Discuss adding safety checks so that this eventually resolves or resolves on error.
+    loadedCheckInterval: IntervalID;
+    constructor (value: string) {
+        super(value);
+        this.isLoadedPromise = new Promise((resolve, reject) => {
+            this.isLoadedResolve = resolve;
+            this.isLoadedReject = reject;
+        });
+
+        this.loadedCheckInterval = setInterval(() => {
+            if (this.loaded) {
+                clearInterval(this.loadedCheckInterval);
+                this.isLoadedResolve();
+            }
+        }, 10);
+    }
+}
+
+class WrappedSampler extends Sampler {
+    isLoadedPromise: Promise<any>;
+    isLoadedResolve: function;
+    isLoadedReject: function; // TODO: Discuss adding safety checks so that this eventually resolves or resolves on error.
+    loadedCheckInterval: IntervalID;
+    constructor (samplerOptions: any) { // TODO: Need a better type for thiss.
+        super(samplerOptions);
+        this.isLoadedPromise = new Promise((resolve, reject) => {
+            this.isLoadedResolve = resolve;
+            this.isLoadedReject = reject;
+        });
+
+        this.loadedCheckInterval = setInterval(() => {
+            if (this.loaded) {
+                clearInterval(this.loadedCheckInterval);
+                this.isLoadedResolve();
+            }
+        }, 10);
+    }
 }
 
+type AnnouncementLookupTable = {
+    forward1: WrappedPlayer,
+    forward2: WrappedPlayer,
+    forward3: WrappedPlayer,
+    left45: WrappedPlayer,
+    left90: WrappedPlayer,
+    left180: WrappedPlayer,
+    right45: WrappedPlayer,
+    right90: WrappedPlayer,
+    right180: WrappedPlayer,
+    add: WrappedPlayer,
+    deleteAll: WrappedPlayer,
+    delete: WrappedPlayer,
+    moveToPrevious: WrappedPlayer,
+    moveToNext: WrappedPlayer,
+    replace: WrappedPlayer
+}
 
 const AnnouncementDefs = new Map<string, string>([
     ['forward1', '/audio/Move.wav'],
@@ -79,122 +120,158 @@ export function getNoteForState (characterState: CharacterState) : string {
 export default class AudioManager {
     audioEnabled: boolean;
     announcementLookUpTable: AnnouncementLookupTable;
-    panner: Panner;
     samplers: {
-        movement: Sampler,
-        left: Sampler,
-        right: Sampler
+        movement: WrappedSampler,
+        left: WrappedSampler,
+        right: WrappedSampler
     };
+    panner: Panner;
+    toneStartHasBeenCalled: boolean;
+
+    startPromise: Promise<void>;
+    startResolve: function;
+    startReject: function;
 
     constructor(audioEnabled: boolean) {
         this.audioEnabled = audioEnabled;
 
-        this.buildAnnouncementLookUpTable();
-
-        this.panner = new Panner();
-        this.panner.toDestination();
-
-        this.samplers = {};
-
-        // TODO: Make a sammplerDef for all variations.
-        this.samplers.left = new Sampler({
-            // The percussion instrument we used actually dooesn't vary it's pitch, we use the same sample at different
-            // pitches so that we can scale relative to the octave without ending up with wildy different tempos.
-            urls: {
-                "C0": "C6.wav",
-                "C1": "C6.wav",
-                "C2": "C6.wav",
-                "C3": "C6.wav",
-                "C4": "C6.wav",
-                "C5": "C6.wav",
-                "C6": "C6.wav"
-            },
-            baseUrl: "/audio/left-turn/"
+        // Flag our audio as not having been started.
+        this.toneStartHasBeenCalled = false;
+
+        this.startPromise = new Promise((resolve, reject) => {
+            this.startResolve = resolve;
+            this.startReject = reject;
         });
+    }
 
-        this.samplers.left.connect(this.panner);
+    createSoundInfrastructure = () => {
+        try {
+            this.buildAnnouncementLookUpTable();
 
-        this.samplers.right = new Sampler({
-            urls: {
-                // The percussion instrument we used actually dooesn't vary it's pitch, we use the same sample at different
-                // pitches so that we can scale relative to the octave without ending up with wildy different tempos.
-                "C0": "C6.wav",
-                "C1": "C6.wav",
-                "C2": "C6.wav",
-                "C3": "C6.wav",
-                "C4": "C6.wav",
-                "C5": "C6.wav",
-                "C6": "C6.wav"
-            },
-            baseUrl: "/audio/right-turn/"
-        });
+            this.panner = new Panner();
+            this.panner.toDestination();
 
-        this.samplers.right.connect(this.panner);
-
-        this.samplers.movement = new Sampler({
-            urls: {
-                "C0": "C0.wav",
-                "C1": "C1.wav",
-                "C2": "C2.wav",
-                "C3": "C3.wav",
-                "C4": "C4.wav",
-                "C5": "C5.wav",
-                "C6": "C6.wav"
-            },
-            baseUrl: "/audio/long-bell/"
-        });
+            this.samplers = {};
 
-        this.samplers.movement.connect(this.panner);
+            // TODO: Make a sammplerDef for all variations.
+            this.samplers.left = new WrappedSampler({
+                // The percussion instrument we used actually dooesn't vary it's pitch, we use the same sample at different
+                // pitches so that we can scale relative to the octave without ending up with wildy different tempos.
+                urls: {
+                    "C0": "C6.wav",
+                    "C1": "C6.wav",
+                    "C2": "C6.wav",
+                    "C3": "C6.wav",
+                    "C4": "C6.wav",
+                    "C5": "C6.wav",
+                    "C6": "C6.wav"
+                },
+                baseUrl: "/audio/left-turn/"
+            });
+
+            this.samplers.left.connect(this.panner);
+
+            this.samplers.right = new WrappedSampler({
+                urls: {
+                    // The percussion instrument we used actually dooesn't vary it's pitch, we use the same sample at different
+                    // pitches so that we can scale relative to the octave without ending up with wildy different tempos.
+                    "C0": "C6.wav",
+                    "C1": "C6.wav",
+                    "C2": "C6.wav",
+                    "C3": "C6.wav",
+                    "C4": "C6.wav",
+                    "C5": "C6.wav",
+                    "C6": "C6.wav"
+                },
+                baseUrl: "/audio/right-turn/"
+            });
+
+            this.samplers.right.connect(this.panner);
+
+            this.samplers.movement = new WrappedSampler({
+                urls: {
+                    "C0": "C0.wav",
+                    "C1": "C1.wav",
+                    "C2": "C2.wav",
+                    "C3": "C3.wav",
+                    "C4": "C4.wav",
+                    "C5": "C5.wav",
+                    "C6": "C6.wav"
+                },
+                baseUrl: "/audio/long-bell/"
+            });
+
+            this.samplers.movement.connect(this.panner);
+
+            this.startResolve();
+        }
+        catch (error) {
+            this.startReject(error);
+        }
     }
 
     buildAnnouncementLookUpTable() {
         this.announcementLookUpTable = {};
         AnnouncementDefs.forEach((value, key) => {
-            const player = new Player(value);
+            const player = new WrappedPlayer(value);
             player.toDestination();
             this.announcementLookUpTable[key] = player;
         });
     }
 
     playAnnouncement(soundName: AnnouncedSoundName) {
         if (this.audioEnabled) {
-            const player = this.announcementLookUpTable[soundName];
-            if (player.loaded) {
-                player.start();
-            }
+            this.startPromise.then(() => {
+                const player = this.announcementLookUpTable[soundName];
+                player.isLoadedPromise.then(() => {
+                    player.start();
+                });
+            });
         }
     }
 
     // TODO: Add a better type for pitch.
     // TODO: Make this private, as it doesn't respect the audioEnabled setting.
-    playPitchedSample(sampler: Sampler, pitch: string, releaseTime: number) {
-        // We can only play the sound if it's already loaded.
-        if (sampler.loaded) {
-            sampler.triggerAttackRelease([pitch], releaseTime);
+    playPitchedSample(sampler: WrappedSampler, pitch: string, releaseTime: number) {
+        if (this.audioEnabled) {
+            // We can only play the sound if it's already loaded.
+            sampler.isLoadedPromise.then(() => {
+                sampler.triggerAttackRelease([pitch], releaseTime);
+            });
         }
     }
 
     playSoundForCharacterState(samplerKey: string, releaseTimeInMs: number, characterState: CharacterState) {
         if (this.audioEnabled) {
-            const releaseTime = releaseTimeInMs / 1000;
-            const noteName = getNoteForState(characterState);
+            this.startPromise.then(() => {
+                const releaseTime = releaseTimeInMs / 1000;
+                const noteName = getNoteForState(characterState);
 
-            const sampler: Sampler = this.samplers[samplerKey];
+                const sampler: WrappedSampler = this.samplers[samplerKey];
 
-            this.playPitchedSample(sampler, noteName, releaseTime);
+                this.playPitchedSample(sampler, noteName, releaseTime);
 
-            // Pan left/right to suggest the relative horizontal position.
-            // As we use a single Sampler grade, our best option for panning is
-            // to pan all sounds.  We can discuss adjusting this once we have
-            // multiple sound-producing elements in the environment.
-            const panningLevel = Math.min(1, Math.max(-1, (0.1 * characterState.xPos)));
+                // Pan left/right to suggest the relative horizontal position.
+                // As we use a single Sampler grade, our best option for panning is
+                // to pan all sounds.  We can discuss adjusting this once we have
+                // multiple sound-producing elements in the environment.
+                const panningLevel = Math.min(1, Math.max(-1, (0.1 * characterState.xPos)));
 
-            // TODO: Consider making the timing configurable or tying it to the movement timing.
-            this.panner.pan.rampTo(panningLevel, 0.5)
+                // TODO: Consider making the timing configurable or tying it to the movement timing.
+                this.panner.pan.rampTo(panningLevel, 0.5)
+            });
         }
     }
 
     setAudioEnabled(value: boolean) {
         this.audioEnabled = value;
     }
+
+    startTone = () => {
+        // Ensure that sound support is started on any user action.
+        if (!this.toneStartHasBeenCalled) {
+            ToneStart().then(this.createSoundInfrastructure, this.startReject);
+            this.toneStartHasBeenCalled = true;
+        }
+    }
 };