i thought this would be closer

musical-leptos/Cargo.toml

@@ -12,20 +12,18 @@ musical-lights-core = { path = "../musical-lights-core", features=["log"] }
 console_error_panic_hook = "0.1"
 console_log = "1"
 flume = "0.11.0"
+js-sys = "0.3.69"
 leptos = { version = "0.6", features = ["csr", "nightly"] }
 leptos_meta = { version = "0.6", features = ["csr", "nightly"] }
 leptos_router = { version = "0.6", features = ["csr", "nightly"] }
 log = "0.4"
+serde = { version = "1.0.198", default-features = false, features = ["derive"] }
+terrors = "0.3.0"
 wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4.42"
 web-sys = { version = "0.3", features = ["console", "AudioContext", "AudioContextOptions", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "BaseAudioContext", "BlobPropertyBag", "Document", "MediaDevices", "MediaStream", "MediaStreamConstraints", "MediaStreamAudioSourceNode", "Navigator", "TextDecoder", "Window"] }
-js-sys = "0.3.69"
-terrors = "0.3.0"
-
-# utils
-# strum = { version = "0.25", features = ["derive", "strum_macros"] }
-# strum_macros = "0.25"
 
+#leptos_workers = "0.2.1"
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3"

musical-leptos/src/components/dancing_lights.rs

@@ -46,7 +46,7 @@ pub fn DancingLights() -> impl IntoView {
 
     // TODO: this is wrong. this runs immediatly, not on first click. why?
     let start_listening = create_resource(listen, |x| async move {
-        if x == false {
+        if !x {
             return Ok(None);
         }
 
@@ -58,25 +58,17 @@ pub fn DancingLights() -> impl IntoView {
 
         info!("active media stream: {:?}", media_stream_id);
 
-        let audio_ctx = wasm_audio(
-            &media_stream,
-            Box::new(move |buf| {
-                // TODO: actually process it
-                let sum: f32 = buf.iter().sum();
-
-                info!("audio sum: {:?}", sum);
-
-                true
-            }),
-        )
-        .await
-        .map_err(|x| format!("audio_ctx error: {:?}", x))?;
+        let audio_ctx = wasm_audio(&media_stream)
+            .await
+            .map_err(|x| format!("audio_ctx error: {:?}", x))?;
 
         info!("audio context: {:?}", audio_ctx);
 
-        // TODO: do we need this?
-        let promise = audio_ctx.resume().unwrap();
-        let _ = wasm_bindgen_futures::JsFuture::from(promise).await.unwrap();
+        // // TODO: do we need this?
+        // let promise = audio_ctx.resume().unwrap();
+        // let _ = wasm_bindgen_futures::JsFuture::from(promise).await.unwrap();
+
+        // TODO: what do we do with the receiver?
 
         Ok::<_, String>(Some(media_stream_id))
     });

musical-leptos/src/dependent_module.rs

@@ -22,19 +22,12 @@ pub fn on_the_fly(code: &str) -> Result<String, JsValue> {
 
     // TODO: get the EncoderDecoderTogether from <https://github.com/anonyco/FastestSmallestTextEncoderDecoder>.
     // TODO: this doesn't seem to work. we still get TextDecoder not found
+    // TODO: do we even need this hack anymore?
     let header = format!(
-        "{}\n\nimport init, * as bindings from '{}';\n\n",
-        include_str!("./audio_polyfill.js"),
-        // wasm_bindgen::link_to!(module = "/src/audio_polyfill.js"),
+        "import init, * as bindings from '{}';\n\n",
         IMPORT_META.url(),
     );
 
-    // // TODO: why doesn't the above polyfill work?!
-    // let header = format!(
-    //     "import init, * as bindgen from '{}';\n\n",
-    //     IMPORT_META.url(),
-    // );
-
     Url::create_object_url_with_blob(&Blob::new_with_str_sequence_and_options(
         &Array::of2(&JsValue::from(header.as_str()), &JsValue::from(code)),
         BlobPropertyBag::new().type_("text/javascript"),

musical-leptos/src/my-wasm-processor.js

@@ -0,0 +1,37 @@
+
+class MyWasmProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+
+        console.log("options.processorOptions:", options.processorOptions);
+
+        let [module, foobar] = options.processorOptions;
+
+        // Fetch, compile, and instantiate the WebAssembly module
+        WebAssembly.instantiateStreaming(module)
+            .then(results => {
+                // `results` is an object with both the module and its instance
+                console.log('WebAssembly module instantiated successfully');
+
+                // Exported functions can now be called
+                const exports = results.instance.exports;
+                // const result = exports.yourExportedFunction();
+                // console.log('Result from your function:', result);
+
+                console.log("exports:", exports);
+            })
+            .catch(error => {
+                console.error('Error instantiating WebAssembly module:', error);
+            });
+    }
+    process(inputs, outputs, parameters) {
+        // TODO: send it over a channel to a regular worker? i'm stumped and just want it working even if it isn't real time
+
+        // TODO: this seems to be ignored
+        return true;
+
+        // return this.processor.process(inputs[0][0]);
+    }
+}
+
+registerProcessor("my-wasm-processor", MyWasmProcessor);

musical-leptos/src/wasm_audio.rs

@@ -1,40 +1,17 @@
 use crate::dependent_module;
 
 use log::debug;
-use log::info;
-use wasm_bindgen::prelude::*;
+use serde::{Deserialize, Serialize};
 use wasm_bindgen::JsValue;
 use wasm_bindgen_futures::JsFuture;
 use web_sys::MediaStream;
 use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions};
 
-type WasmAudioProcessorFn = Box<dyn FnMut(&mut [f32]) -> bool>;
-
-#[wasm_bindgen]
-pub struct WasmAudioProcessor(WasmAudioProcessorFn);
-
-#[wasm_bindgen]
-impl WasmAudioProcessor {
-    pub fn process(&mut self, buf: &mut [f32]) -> bool {
-        // TODO: instead of calling an arbitrary function, should we write a specific impl that does our audio processing?
-        self.0(buf)
-    }
-    pub fn pack(self) -> usize {
-        Box::into_raw(Box::new(self)) as usize
-    }
-    pub unsafe fn unpack(val: usize) -> Self {
-        *Box::from_raw(val as *mut _)
-    }
-}
-
 // Use wasm_audio if you have a single wasm audio processor in your application
 // whose samples should be played directly. Ideally, call wasm_audio based on
 // user interaction. Otherwise, resume the context on user interaction, so
 // playback starts reliably on all browsers.
-pub async fn wasm_audio(
-    media_stream: &MediaStream,
-    process: WasmAudioProcessorFn,
-) -> Result<AudioContext, JsValue> {
+pub async fn wasm_audio(media_stream: &MediaStream) -> Result<AudioContext, JsValue> {
     let ctx = AudioContext::new()?;
 
     prepare_wasm_audio(&ctx).await?;
@@ -43,12 +20,15 @@ pub async fn wasm_audio(
 
     let input = ctx.create_media_stream_source(media_stream).unwrap();
 
-    let worklet = wasm_audio_worklet(&ctx, process)?;
+    // TODO: pass a process callback to this somehow
+    let worklet = wasm_audio_worklet(&ctx)?;
 
     input.connect_with_audio_node(&worklet)?;
 
     worklet.connect_with_audio_node(&ctx.destination())?;
 
+    // TODO: i think we need to do something with worklet.port here to get the handled audio samples out
+
     debug!("audio input: {:?}", input);
     debug!("audio node: {:?}", worklet);
 
@@ -58,27 +38,24 @@ pub async fn wasm_audio(
 // wasm_audio_node creates an AudioWorkletNode running a wasm audio processor.
 // Remember to call prepare_wasm_audio once on your context before calling
 // this function.
-pub fn wasm_audio_worklet(
-    ctx: &AudioContext,
-    process: WasmAudioProcessorFn,
-) -> Result<AudioWorkletNode, JsValue> {
+pub fn wasm_audio_worklet(ctx: &AudioContext) -> Result<AudioWorkletNode, JsValue> {
     let mut audio_worklet_node = AudioWorkletNodeOptions::new();
 
     // TODO: one example passed wasm_bindgen::memory() here, but I don't think that is needed anymore
     let options = audio_worklet_node.processor_options(Some(&js_sys::Array::of2(
         &wasm_bindgen::module(),
-        &WasmAudioProcessor(process).pack().into(),
+        &"foobar".into(),
     )));
     debug!("options: {:?}", options);
 
-    let node = AudioWorkletNode::new_with_options(ctx, "WasmProcessor", options)?;
+    let node = AudioWorkletNode::new_with_options(ctx, "my-wasm-processor", options)?;
     debug!("node: {:?}", node);
 
     Ok(node)
 }
 
 pub async fn prepare_wasm_audio(ctx: &AudioContext) -> Result<(), JsValue> {
-    let mod_url = dependent_module!("audio_worklet.js")?;
+    let mod_url = dependent_module!("my-wasm-processor.js")?;
     JsFuture::from(ctx.audio_worklet()?.add_module(&mod_url)?).await?;
     Ok(())
 }