From f9a565cdcbb5defb0d5f1f41e66aa296c805cbb2 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Sat, 20 Apr 2024 13:09:32 -0700 Subject: [PATCH] i thought this would be closer --- musical-leptos/Cargo.lock | 1 + musical-leptos/Cargo.toml | 10 ++--- musical-leptos/src/audio_polyfill.js | 21 --------- musical-leptos/src/audio_worklet.js | 29 ------------- .../src/components/dancing_lights.rs | 26 ++++------- musical-leptos/src/dependent_module.rs | 11 +---- musical-leptos/src/my-wasm-processor.js | 37 ++++++++++++++++ musical-leptos/src/wasm_audio.rs | 43 +++++-------------- 8 files changed, 63 insertions(+), 115 deletions(-) delete mode 100644 musical-leptos/src/audio_polyfill.js delete mode 100644 musical-leptos/src/audio_worklet.js create mode 100644 musical-leptos/src/my-wasm-processor.js diff --git a/musical-leptos/Cargo.lock b/musical-leptos/Cargo.lock index eb436fc..5988d98 100644 --- a/musical-leptos/Cargo.lock +++ b/musical-leptos/Cargo.lock @@ -872,6 +872,7 @@ dependencies = [ "leptos_router", "log", "musical-lights-core", + "serde", "terrors", "wasm-bindgen", "wasm-bindgen-futures", diff --git a/musical-leptos/Cargo.toml b/musical-leptos/Cargo.toml index eb337c9..bdc7dee 100644 --- a/musical-leptos/Cargo.toml +++ b/musical-leptos/Cargo.toml @@ -12,20 +12,18 @@ musical-lights-core = { path = "../musical-lights-core", features=["log"] } console_error_panic_hook = "0.1" console_log = "1" flume = "0.11.0" +js-sys = "0.3.69" leptos = { version = "0.6", features = ["csr", "nightly"] } leptos_meta = { version = "0.6", features = ["csr", "nightly"] } leptos_router = { version = "0.6", features = ["csr", "nightly"] } log = "0.4" +serde = { version = "1.0.198", default-features = false, features = ["derive"] } +terrors = "0.3.0" wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4.42" web-sys = { version = "0.3", features = ["console", "AudioContext", "AudioContextOptions", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "BaseAudioContext", "BlobPropertyBag", "Document", "MediaDevices", "MediaStream", "MediaStreamConstraints", "MediaStreamAudioSourceNode", "Navigator", "TextDecoder", "Window"] } -js-sys = "0.3.69" -terrors = "0.3.0" - -# utils -# strum = { version = "0.25", features = ["derive", "strum_macros"] } -# strum_macros = "0.25" +#leptos_workers = "0.2.1" [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/musical-leptos/src/audio_polyfill.js b/musical-leptos/src/audio_polyfill.js deleted file mode 100644 index 69e2e1f..0000000 --- a/musical-leptos/src/audio_polyfill.js +++ /dev/null @@ -1,21 +0,0 @@ -// TODO: actual polyfill here. i was just hoping to see these stubs called, but we don't even get that far. i think because of "strict" - -globalThis.TextDecoder = class TextDecoder { - decode(arg) { - if (typeof arg !== 'undefined') { - throw Error('TextDecoder stub called'); - } else { - return ''; - } - } -}; - -globalThis.TextEncoder = class TextEncoder { - encode(arg) { - if (typeof arg !== 'undefined') { - throw Error('TextEncoder stub called'); - } else { - return new Uint8Array(0); - } - } -}; diff --git a/musical-leptos/src/audio_worklet.js b/musical-leptos/src/audio_worklet.js deleted file mode 100644 index 08fb9b7..0000000 --- a/musical-leptos/src/audio_worklet.js +++ /dev/null @@ -1,29 +0,0 @@ -class WasmProcessor extends AudioWorkletProcessor { - constructor(options) { - super(); - - let [module, handle] = options.processorOptions; - - console.log("module:", module); - console.log("handle:", handle); - - // // TODO: we aren't allowed to do this because they are read-only - // bindings.TextDecoder = globalThis.TextDecoder; - // bindings.TextEncoder = globalThis.TextEncoder; - - // // TODO: this currently fails because TextDecoder is not available inside an AudioWorklet - // // TODO: some examples use message passing for the wasm setup - // bindings.initSync(module); - // this.processor = bindings.WasmAudioProcessor.unpack(handle); - } - process(inputs, outputs) { - let sum = inputs[0][0].reduce((accumulator, currentValue) => accumulator + currentValue, 0); - console.log("sum:", sum); - - return true; - - // return this.processor.process(inputs[0][0]); - } -} - -registerProcessor("WasmProcessor", WasmProcessor); diff --git a/musical-leptos/src/components/dancing_lights.rs b/musical-leptos/src/components/dancing_lights.rs index 2fd1d86..5b98d1a 100644 --- a/musical-leptos/src/components/dancing_lights.rs +++ b/musical-leptos/src/components/dancing_lights.rs @@ -46,7 +46,7 @@ pub fn DancingLights() -> impl IntoView { // TODO: this is wrong. this runs immediatly, not on first click. why? let start_listening = create_resource(listen, |x| async move { - if x == false { + if !x { return Ok(None); } @@ -58,25 +58,17 @@ pub fn DancingLights() -> impl IntoView { info!("active media stream: {:?}", media_stream_id); - let audio_ctx = wasm_audio( - &media_stream, - Box::new(move |buf| { - // TODO: actually process it - let sum: f32 = buf.iter().sum(); - - info!("audio sum: {:?}", sum); - - true - }), - ) - .await - .map_err(|x| format!("audio_ctx error: {:?}", x))?; + let audio_ctx = wasm_audio(&media_stream) + .await + .map_err(|x| format!("audio_ctx error: {:?}", x))?; info!("audio context: {:?}", audio_ctx); - // TODO: do we need this? - let promise = audio_ctx.resume().unwrap(); - let _ = wasm_bindgen_futures::JsFuture::from(promise).await.unwrap(); + // // TODO: do we need this? + // let promise = audio_ctx.resume().unwrap(); + // let _ = wasm_bindgen_futures::JsFuture::from(promise).await.unwrap(); + + // TODO: what do we do with the receiver? Ok::<_, String>(Some(media_stream_id)) }); diff --git a/musical-leptos/src/dependent_module.rs b/musical-leptos/src/dependent_module.rs index 6cb2e9f..e3a30b1 100644 --- a/musical-leptos/src/dependent_module.rs +++ b/musical-leptos/src/dependent_module.rs @@ -22,19 +22,12 @@ pub fn on_the_fly(code: &str) -> Result { // TODO: get the EncoderDecoderTogether from . // TODO: this doesn't seem to work. we still get TextDecoder not found + // TODO: do we even need this hack anymore? let header = format!( - "{}\n\nimport init, * as bindings from '{}';\n\n", - include_str!("./audio_polyfill.js"), - // wasm_bindgen::link_to!(module = "/src/audio_polyfill.js"), + "import init, * as bindings from '{}';\n\n", IMPORT_META.url(), ); - // // TODO: why doesn't the above polyfill work?! - // let header = format!( - // "import init, * as bindgen from '{}';\n\n", - // IMPORT_META.url(), - // ); - Url::create_object_url_with_blob(&Blob::new_with_str_sequence_and_options( &Array::of2(&JsValue::from(header.as_str()), &JsValue::from(code)), BlobPropertyBag::new().type_("text/javascript"), diff --git a/musical-leptos/src/my-wasm-processor.js b/musical-leptos/src/my-wasm-processor.js new file mode 100644 index 0000000..66fa577 --- /dev/null +++ b/musical-leptos/src/my-wasm-processor.js @@ -0,0 +1,37 @@ + +class MyWasmProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + + console.log("options.processorOptions:", options.processorOptions); + + let [module, foobar] = options.processorOptions; + + // Fetch, compile, and instantiate the WebAssembly module + WebAssembly.instantiateStreaming(module) + .then(results => { + // `results` is an object with both the module and its instance + console.log('WebAssembly module instantiated successfully'); + + // Exported functions can now be called + const exports = results.instance.exports; + // const result = exports.yourExportedFunction(); + // console.log('Result from your function:', result); + + console.log("exports:", exports); + }) + .catch(error => { + console.error('Error instantiating WebAssembly module:', error); + }); + } + process(inputs, outputs, parameters) { + // TODO: send it over a channel to a regular worker? i'm stumped and just want it working even if it isn't real time + + // TODO: this seems to be ignored + return true; + + // return this.processor.process(inputs[0][0]); + } +} + +registerProcessor("my-wasm-processor", MyWasmProcessor); diff --git a/musical-leptos/src/wasm_audio.rs b/musical-leptos/src/wasm_audio.rs index 6f9f812..3627769 100644 --- a/musical-leptos/src/wasm_audio.rs +++ b/musical-leptos/src/wasm_audio.rs @@ -1,40 +1,17 @@ use crate::dependent_module; use log::debug; -use log::info; -use wasm_bindgen::prelude::*; +use serde::{Deserialize, Serialize}; use wasm_bindgen::JsValue; use wasm_bindgen_futures::JsFuture; use web_sys::MediaStream; use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions}; -type WasmAudioProcessorFn = Box bool>; - -#[wasm_bindgen] -pub struct WasmAudioProcessor(WasmAudioProcessorFn); - -#[wasm_bindgen] -impl WasmAudioProcessor { - pub fn process(&mut self, buf: &mut [f32]) -> bool { - // TODO: instead of calling an arbitrary function, should we write a specific impl that does our audio processing? - self.0(buf) - } - pub fn pack(self) -> usize { - Box::into_raw(Box::new(self)) as usize - } - pub unsafe fn unpack(val: usize) -> Self { - *Box::from_raw(val as *mut _) - } -} - // Use wasm_audio if you have a single wasm audio processor in your application // whose samples should be played directly. Ideally, call wasm_audio based on // user interaction. Otherwise, resume the context on user interaction, so // playback starts reliably on all browsers. -pub async fn wasm_audio( - media_stream: &MediaStream, - process: WasmAudioProcessorFn, -) -> Result { +pub async fn wasm_audio(media_stream: &MediaStream) -> Result { let ctx = AudioContext::new()?; prepare_wasm_audio(&ctx).await?; @@ -43,12 +20,15 @@ pub async fn wasm_audio( let input = ctx.create_media_stream_source(media_stream).unwrap(); - let worklet = wasm_audio_worklet(&ctx, process)?; + // TODO: pass a process callback to this somehow + let worklet = wasm_audio_worklet(&ctx)?; input.connect_with_audio_node(&worklet)?; worklet.connect_with_audio_node(&ctx.destination())?; + // TODO: i think we need to do something with worklet.port here to get the handled audio samples out + debug!("audio input: {:?}", input); debug!("audio node: {:?}", worklet); @@ -58,27 +38,24 @@ pub async fn wasm_audio( // wasm_audio_node creates an AudioWorkletNode running a wasm audio processor. // Remember to call prepare_wasm_audio once on your context before calling // this function. -pub fn wasm_audio_worklet( - ctx: &AudioContext, - process: WasmAudioProcessorFn, -) -> Result { +pub fn wasm_audio_worklet(ctx: &AudioContext) -> Result { let mut audio_worklet_node = AudioWorkletNodeOptions::new(); // TODO: one example passed wasm_bindgen::memory() here, but I don't think that is needed anymore let options = audio_worklet_node.processor_options(Some(&js_sys::Array::of2( &wasm_bindgen::module(), - &WasmAudioProcessor(process).pack().into(), + &"foobar".into(), ))); debug!("options: {:?}", options); - let node = AudioWorkletNode::new_with_options(ctx, "WasmProcessor", options)?; + let node = AudioWorkletNode::new_with_options(ctx, "my-wasm-processor", options)?; debug!("node: {:?}", node); Ok(node) } pub async fn prepare_wasm_audio(ctx: &AudioContext) -> Result<(), JsValue> { - let mod_url = dependent_module!("audio_worklet.js")?; + let mod_url = dependent_module!("my-wasm-processor.js")?; JsFuture::from(ctx.audio_worklet()?.add_module(&mod_url)?).await?; Ok(()) }