From acc106564998dfc119a8b732da085041f32323c9 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Mon, 6 Jun 2022 21:42:32 +0000 Subject: [PATCH 1/9] framework --- .cargo/config.toml | 2 -- .vscode/launch.json | 14 ++++++++++++++ .vscode/tasks.json | 13 +++++++++++++ Cargo.toml | 2 +- debugger/Cargo.toml | 8 ++++++++ debugger/src/main.rs | 3 +++ 6 files changed, 39 insertions(+), 3 deletions(-) delete mode 100644 .cargo/config.toml create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json create mode 100644 debugger/Cargo.toml create mode 100644 debugger/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 435ed75..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -target = "wasm32-unknown-unknown" \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4d48e0b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "preLaunchTask": "build:debugger", + "type": "lldb", + "request": "launch", + "name": "Debug", + "program": "${workspaceFolder}/target/debug/debugger", + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..4882838 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,13 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "cargo", + "group": "build", + "label": "build:debugger", + "command": "build", + "args": ["--package", "debugger"], + "problemMatcher": ["$rustc"] + } + ] +} diff --git a/Cargo.toml b/Cargo.toml index b9c07b0..3701a64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["examples/rust/*"] +members = ["examples/rust/*", "debugger"] diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml new file mode 100644 index 0000000..e008a99 --- /dev/null +++ b/debugger/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "debugger" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/debugger/src/main.rs b/debugger/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/debugger/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 83ea219b66d830b7f0628dd4eac7846e38337853 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Tue, 7 Jun 2022 01:13:56 +0000 Subject: [PATCH 2/9] working foundation --- .devcontainer/devcontainer.json | 3 + .vscode/launch.json | 4 +- .vscode/tasks.json | 21 ++++++- debugger/Cargo.toml | 5 ++ debugger/debugger.wit | 1 + debugger/src/main.rs | 105 +++++++++++++++++++++++++++++++- examples/rust/power/Cargo.toml | 4 ++ examples/rust/power/src/lib.rs | 11 +++- 8 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 debugger/debugger.wit diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5ad1370..cab08cc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,6 +15,9 @@ } }, + // needed for debugging + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + // Set *default* container specific settings.json values on container create. "settings": { "python.defaultInterpreterPath": "/usr/local/bin/python", diff --git a/.vscode/launch.json b/.vscode/launch.json index 4d48e0b..b99f0c9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,12 +2,12 @@ "version": "0.2.0", "configurations": [ { - "preLaunchTask": "build:debugger", + "preLaunchTask": "build: all", "type": "lldb", "request": "launch", "name": "Debug", "program": "${workspaceFolder}/target/debug/debugger", - "args": [], + "args": ["${relativeFileDirname}"], "cwd": "${workspaceFolder}" } ] diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4882838..e0b673c 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,10 +4,27 @@ { "type": "cargo", "group": "build", - "label": "build:debugger", + "label": "build: debugger", "command": "build", - "args": ["--package", "debugger"], "problemMatcher": ["$rustc"] + }, + { + "type": "cargo", + "group": "build", + "label": "build: rust wasm modules", + "command": "build", + "args": [ + "--target=wasm32-unknown-unknown", + "--workspace", + "--exclude", + "debugger" + ], + "problemMatcher": ["$rustc"] + }, + { + "group": "build", + "label": "build: all", + "dependsOn": ["build: debugger", "build: rust wasm modules"] } ] } diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index e008a99..969bf89 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml @@ -6,3 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +regex = "1" +anyhow = "1.0" +wasmtime = "0.35.3" +wasmtime-wasi = "0.35.3" +wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } diff --git a/debugger/debugger.wit b/debugger/debugger.wit new file mode 100644 index 0000000..d8b90b4 --- /dev/null +++ b/debugger/debugger.wit @@ -0,0 +1 @@ +handle-json: function(name: string, json: list) -> list \ No newline at end of file diff --git a/debugger/src/main.rs b/debugger/src/main.rs index e7a11a9..c60e44d 100644 --- a/debugger/src/main.rs +++ b/debugger/src/main.rs @@ -1,3 +1,104 @@ -fn main() { - println!("Hello, world!"); +use anyhow::Result; +use regex; +use std::path::Path; + +use wasmtime::{Config, Engine, Instance, Linker, Module, Store}; +use wasmtime_wasi; +use wit_bindgen_wasmtime; + +fn default_config() -> Result { + let mut config = Config::new(); + config.debug_info(true); + config.cache_config_load_default()?; + config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); + Ok(config) +} + +fn default_wasi() -> wasmtime_wasi::WasiCtx { + wasmtime_wasi::sync::WasiCtxBuilder::new() + .inherit_stdio() + .build() +} + +struct Context { + wasi: wasmtime_wasi::WasiCtx, + exports: E, +} + +fn instantiate( + wasm: &str, + mk_exports: impl FnOnce( + &mut Store>, + &Module, + &mut Linker>, + ) -> Result<(T, Instance)>, +) -> Result<(T, Store>)> { + let engine = Engine::new(&default_config()?)?; + let module = Module::from_file(&engine, wasm)?; + + let mut linker = Linker::new(&engine); + wasmtime_wasi::add_to_linker(&mut linker, |cx: &mut Context| &mut cx.wasi)?; + + let mut store = Store::new( + &engine, + Context { + wasi: default_wasi(), + exports: E::default(), + }, + ); + let (exports, _instance) = mk_exports(&mut store, &module, &mut linker)?; + Ok((exports, store)) +} + +wit_bindgen_wasmtime::import!("debugger.wit"); + +fn run(wasm: &str) -> Result<()> { + let (exports, mut store) = crate::instantiate(wasm, |store, module, linker| { + debugger::Debugger::instantiate(store, module, linker, |cx| &mut cx.exports) + })?; + + exports.handle_json(&mut store, "hello".into(), "bob".as_bytes())?; + + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + + // print usage if no args + if args.len() < 2 { + println!("Usage: {} ", args[0]); + std::process::exit(1); + } + + let target_path = args[1].clone(); + let target_package_regex = regex::Regex::new(r"examples/rust/([^/]+)").unwrap(); + let target_package = target_package_regex + .captures(&target_path) + .and_then(|cap| cap.get(1)) + .map(|m| m.as_str()) + .ok_or(format!( + "Could not find wasm rust package at path {}", + target_path + ))?; + + let target_wasm_path = format!( + "target/wasm32-unknown-unknown/debug/{}.wasm", + target_package + ); + let target_wit_path = format!("examples/rust/{}/{}.wit", target_package, target_package); + + if !Path::new(&target_wasm_path).exists() { + return Err(format!("Could not find wasm file at path {}", target_wasm_path).into()); + } + let wit_exists = Path::new(&target_wit_path).exists(); + + println!("debugging: {}", target_wasm_path); + if wit_exists { + println!("with wit: {}", target_wit_path); + } + + run(&target_wasm_path)?; + + Ok(()) } diff --git a/examples/rust/power/Cargo.toml b/examples/rust/power/Cargo.toml index 51d1c84..5e036f6 100644 --- a/examples/rust/power/Cargo.toml +++ b/examples/rust/power/Cargo.toml @@ -8,3 +8,7 @@ wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" [lib] crate-type = ["cdylib"] + +[profile.dev] +strip = "none" +lto = "off" diff --git a/examples/rust/power/src/lib.rs b/examples/rust/power/src/lib.rs index ae40500..50c36e8 100644 --- a/examples/rust/power/src/lib.rs +++ b/examples/rust/power/src/lib.rs @@ -1,5 +1,5 @@ -wit_bindgen_rust::export!("power.wit"); struct Power; +wit_bindgen_rust::export!("power.wit"); impl power::Power for Power { fn power_of(base: i32, exp: i32) -> i32 { let mut res = 1; @@ -9,3 +9,12 @@ impl power::Power for Power { res } } + +struct Debugger; +wit_bindgen_rust::export!("../../../debugger/debugger.wit"); +impl debugger::Debugger for Debugger { + fn handle_json(name: String, _json: Vec) -> Vec { + let foo = 123; + format!("{} {}", name, foo).into_bytes() + } +} From d2edabb539bf1535bb87370a2328cf1e76b53d64 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 8 Jun 2022 00:25:50 +0000 Subject: [PATCH 3/9] almost finished - just need to finalize the proc macro and document the create function stuff --- .devcontainer/Dockerfile | 8 +- .vscode/launch.json | 7 +- .vscode/tasks.json | 47 ++++++++-- Cargo.toml | 2 +- crates/debugger-macro/Cargo.toml | 17 ++++ crates/debugger-macro/src/lib.rs | 53 +++++++++++ {debugger => crates/debugger}/Cargo.toml | 5 + {debugger => crates/debugger}/debugger.wit | 0 crates/debugger/src/handle.rs | 80 ++++++++++++++++ crates/debugger/src/main.rs | 28 ++++++ crates/debugger/src/server.rs | 58 ++++++++++++ debugger/src/main.rs | 104 --------------------- examples/rust/power/Cargo.toml | 7 +- examples/rust/power/src/lib.rs | 16 ++-- scripts/debug | 14 +++ 15 files changed, 315 insertions(+), 131 deletions(-) create mode 100644 crates/debugger-macro/Cargo.toml create mode 100644 crates/debugger-macro/src/lib.rs rename {debugger => crates/debugger}/Cargo.toml (69%) rename {debugger => crates/debugger}/debugger.wit (100%) create mode 100644 crates/debugger/src/handle.rs create mode 100644 crates/debugger/src/main.rs create mode 100644 crates/debugger/src/server.rs delete mode 100644 debugger/src/main.rs create mode 100755 scripts/debug diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fce5813..2713180 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -45,5 +45,11 @@ RUN pip3 --disable-pip-version-check --no-cache-dir install mypy wasmtime \ # configure rust RUN rustup target add wasm32-unknown-unknown wasm32-wasi -RUN cargo install twiggy cargo-wasi cargo-expand mdbook +RUN cargo install \ + twiggy \ + cargo-wasi \ + cargo-expand \ + mdbook \ + cargo-get \ + cargo-workspaces RUN cargo install --git https://github.com/bytecodealliance/wit-bindgen wit-bindgen-cli \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index b99f0c9..4505bb6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,13 +2,12 @@ "version": "0.2.0", "configurations": [ { - "preLaunchTask": "build: all", "type": "lldb", - "request": "launch", + "request": "attach", "name": "Debug", "program": "${workspaceFolder}/target/debug/debugger", - "args": ["${relativeFileDirname}"], - "cwd": "${workspaceFolder}" + "preLaunchTask": "setup: debugger", + "postDebugTask": "Terminate All Tasks" } ] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e0b673c..80c7ad5 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,6 +6,7 @@ "group": "build", "label": "build: debugger", "command": "build", + "args": ["-p", "debugger"], "problemMatcher": ["$rustc"] }, { @@ -13,18 +14,48 @@ "group": "build", "label": "build: rust wasm modules", "command": "build", - "args": [ - "--target=wasm32-unknown-unknown", - "--workspace", - "--exclude", - "debugger" - ], + "args": ["--target=wasm32-wasi", "--workspace", "--exclude", "debugger"], "problemMatcher": ["$rustc"] }, { "group": "build", - "label": "build: all", - "dependsOn": ["build: debugger", "build: rust wasm modules"] + "label": "setup: debugger", + "dependsOn": ["build: debugger", "build: rust wasm modules"], + "type": "process", + "command": "scripts/debug", + "args": ["${fileDirname}"], + "isBackground": true, + "problemMatcher": [ + { + "pattern": [ + { + "regexp": ".", + "file": 1, + "location": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": true, + "beginsPattern": ".", + "endsPattern": "." + } + } + ] + }, + { + "label": "Terminate All Tasks", + "command": "echo ${input:terminate}", + "type": "shell", + "problemMatcher": [] + } + ], + "inputs": [ + { + "id": "terminate", + "type": "command", + "command": "workbench.action.tasks.terminate", + "args": "terminateAll" } ] } diff --git a/Cargo.toml b/Cargo.toml index 3701a64..9955e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["examples/rust/*", "debugger"] +members = ["examples/rust/*", "crates/*"] diff --git a/crates/debugger-macro/Cargo.toml b/crates/debugger-macro/Cargo.toml new file mode 100644 index 0000000..c78c9d5 --- /dev/null +++ b/crates/debugger-macro/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "debugger-macro" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0", features = [ + "full", + "printing", + "visit", + "extra-traits", + "parsing", +] } +quote = "1.0" diff --git a/crates/debugger-macro/src/lib.rs b/crates/debugger-macro/src/lib.rs new file mode 100644 index 0000000..8cdb471 --- /dev/null +++ b/crates/debugger-macro/src/lib.rs @@ -0,0 +1,53 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, parse_quote, visit::Visit}; + +struct Visitor {} + +impl Visit<'_> for Visitor { + fn visit_item_fn(&mut self, node: &'_ syn::ItemFn) { + let sig = &node.sig; + let name = &sig.ident; + /* + sig.inputs.iter().for_each(|input| { + if let syn::FnArg::Typed(x) = input { + self.source.push_str(quote! {#x}.to_string().as_str()); + } + }); + + self.source.push_str(") -> "); + + if let syn::ReturnType::Type(_, ref ty) = sig.output { + let type_name = quote! {#ty}.to_string(); + let type_name = type_name.replace("Vec", "list"); + self.source.push_str(type_name.as_str()); + self.source.push('\n'); + } + */ + } +} + +#[proc_macro_attribute] +pub fn debugger(_attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as syn::Item); + let mut visitor = Visitor {}; + visitor.visit_item(&input); + + let x = quote! { + #input + + #[derive(Serialize, Deserialize)] + struct Payload(i32, i32); + + struct Debugger; + wit_bindgen_rust::export!("../../../crates/debugger/debugger.wit"); + impl debugger::Debugger for Debugger { + fn handle_json(_name: String, json: Vec) -> Vec { + let payload: Payload = serde_json::from_slice(&json).unwrap(); + let result = ::power_of(payload.0, payload.1); + serde_json::to_vec(&vec![result]).unwrap() + } + } + }; + x.into() +} diff --git a/debugger/Cargo.toml b/crates/debugger/Cargo.toml similarity index 69% rename from debugger/Cargo.toml rename to crates/debugger/Cargo.toml index 969bf89..0ff0f70 100644 --- a/debugger/Cargo.toml +++ b/crates/debugger/Cargo.toml @@ -11,3 +11,8 @@ anyhow = "1.0" wasmtime = "0.35.3" wasmtime-wasi = "0.35.3" wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } + +tide = "0.16.0" +async-std = { version = "1.8.0", features = ["attributes"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" diff --git a/debugger/debugger.wit b/crates/debugger/debugger.wit similarity index 100% rename from debugger/debugger.wit rename to crates/debugger/debugger.wit diff --git a/crates/debugger/src/handle.rs b/crates/debugger/src/handle.rs new file mode 100644 index 0000000..573976c --- /dev/null +++ b/crates/debugger/src/handle.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use wasmtime::{Config, Engine, Linker, Module, Store}; +use wasmtime_wasi; +use wit_bindgen_wasmtime; + +wit_bindgen_wasmtime::import!("debugger.wit"); + +struct Context { + wasi: wasmtime_wasi::WasiCtx, + debugger_state: debugger::DebuggerData, +} + +#[derive(Clone)] +pub struct HandleFactory { + engine: Engine, + linker: Linker, + module: Module, +} + +impl HandleFactory { + fn default_config() -> Result { + let mut config = Config::new(); + config.debug_info(true); + config.cache_config_load_default()?; + config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); + Ok(config) + } + + fn default_wasi() -> wasmtime_wasi::WasiCtx { + wasmtime_wasi::sync::WasiCtxBuilder::new() + .inherit_stdout() + .inherit_stderr() + .build() + } + + pub fn new(wasm_path: &str) -> Result { + let engine = Engine::new(&Self::default_config()?)?; + let module = Module::from_file(&engine, wasm_path)?; + + let mut linker = Linker::new(&engine); + wasmtime_wasi::add_to_linker(&mut linker, |cx: &mut Context| &mut cx.wasi)?; + debugger::Debugger::add_to_linker(&mut linker, |cx: &mut Context| &mut cx.debugger_state)?; + + Ok(Self { + engine, + linker, + module, + }) + } + + pub fn make_handler(&self) -> Result { + let mut store = Store::new( + &self.engine, + Context { + wasi: Self::default_wasi(), + debugger_state: debugger::DebuggerData::default(), + }, + ); + let linked = self.linker.instantiate(&mut store, &self.module)?; + let instance = debugger::Debugger::new(&mut store, &linked, |cx: &mut Context| { + &mut cx.debugger_state + })?; + + Ok(Handler { store, instance }) + } +} + +pub struct Handler { + store: Store, + instance: debugger::Debugger, +} + +impl Handler { + pub fn handle_json(&mut self, name: String, json: Vec) -> Result> { + match self.instance.handle_json(&mut self.store, &name, &json) { + Ok(res) => Ok(res), + Err(err) => Err(err.into()), + } + } +} diff --git a/crates/debugger/src/main.rs b/crates/debugger/src/main.rs new file mode 100644 index 0000000..0940d3b --- /dev/null +++ b/crates/debugger/src/main.rs @@ -0,0 +1,28 @@ +use anyhow::Result; + +mod handle; +mod server; + +#[async_std::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + + // print usage if no args + if args.len() < 3 { + println!("Usage: {} ", args[0]); + std::process::exit(1); + } + + let port_number = args[1].parse::().unwrap(); + let wasm_path = args[2].clone(); + let wit_path = args.get(3); + + println!("debugging: {}", wasm_path); + if wit_path.is_some() { + println!("with wit: {}", wit_path.unwrap()); + } + + let factory = handle::HandleFactory::new(&wasm_path)?; + + server::listen_and_serve(port_number, factory).await +} diff --git a/crates/debugger/src/server.rs b/crates/debugger/src/server.rs new file mode 100644 index 0000000..642375a --- /dev/null +++ b/crates/debugger/src/server.rs @@ -0,0 +1,58 @@ +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; +use serde_json; +use std::iter; +use tide::{Body, Request, Response}; + +use crate::handle; + +#[derive(Clone)] +struct State { + factory: handle::HandleFactory, +} + +pub async fn listen_and_serve(port: u16, factory: handle::HandleFactory) -> Result<()> { + tide::log::start(); + + let state = State { factory }; + let mut app = tide::with_state(state); + app.with(tide::log::LogMiddleware::new()); + + app.at("/foo").post(handle_json); + app.listen(format!("0.0.0.0:{}", port)).await?; + + Ok(()) +} + +type Row = Vec; + +#[derive(Serialize, Deserialize)] +struct Payload { + data: Vec, +} + +async fn handle_json(mut req: Request) -> tide::Result { + let state = req.state(); + let mut handler = state.factory.make_handler()?; + + let payload: Payload = req.body_json().await?; + let path = req.url().path(); + + let mut result = Vec::new(); + for row in payload.data { + if row.len() == 0 { + return Err(anyhow!("Empty row").into()); + } + + let row_id = row[0].clone(); + let row_input = row[1..].to_vec(); + + let row_output_raw = handler.handle_json(path.into(), serde_json::to_vec(&row_input)?)?; + let row_output: Row = serde_json::from_slice(&row_output_raw)?; + + let row_final: Row = iter::once(row_id).chain(row_output.into_iter()).collect(); + result.push(row_final); + } + + Ok(Response::from(Body::from_json(&Payload { data: result })?)) +} diff --git a/debugger/src/main.rs b/debugger/src/main.rs deleted file mode 100644 index c60e44d..0000000 --- a/debugger/src/main.rs +++ /dev/null @@ -1,104 +0,0 @@ -use anyhow::Result; -use regex; -use std::path::Path; - -use wasmtime::{Config, Engine, Instance, Linker, Module, Store}; -use wasmtime_wasi; -use wit_bindgen_wasmtime; - -fn default_config() -> Result { - let mut config = Config::new(); - config.debug_info(true); - config.cache_config_load_default()?; - config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); - Ok(config) -} - -fn default_wasi() -> wasmtime_wasi::WasiCtx { - wasmtime_wasi::sync::WasiCtxBuilder::new() - .inherit_stdio() - .build() -} - -struct Context { - wasi: wasmtime_wasi::WasiCtx, - exports: E, -} - -fn instantiate( - wasm: &str, - mk_exports: impl FnOnce( - &mut Store>, - &Module, - &mut Linker>, - ) -> Result<(T, Instance)>, -) -> Result<(T, Store>)> { - let engine = Engine::new(&default_config()?)?; - let module = Module::from_file(&engine, wasm)?; - - let mut linker = Linker::new(&engine); - wasmtime_wasi::add_to_linker(&mut linker, |cx: &mut Context| &mut cx.wasi)?; - - let mut store = Store::new( - &engine, - Context { - wasi: default_wasi(), - exports: E::default(), - }, - ); - let (exports, _instance) = mk_exports(&mut store, &module, &mut linker)?; - Ok((exports, store)) -} - -wit_bindgen_wasmtime::import!("debugger.wit"); - -fn run(wasm: &str) -> Result<()> { - let (exports, mut store) = crate::instantiate(wasm, |store, module, linker| { - debugger::Debugger::instantiate(store, module, linker, |cx| &mut cx.exports) - })?; - - exports.handle_json(&mut store, "hello".into(), "bob".as_bytes())?; - - Ok(()) -} - -fn main() -> Result<(), Box> { - let args: Vec = std::env::args().collect(); - - // print usage if no args - if args.len() < 2 { - println!("Usage: {} ", args[0]); - std::process::exit(1); - } - - let target_path = args[1].clone(); - let target_package_regex = regex::Regex::new(r"examples/rust/([^/]+)").unwrap(); - let target_package = target_package_regex - .captures(&target_path) - .and_then(|cap| cap.get(1)) - .map(|m| m.as_str()) - .ok_or(format!( - "Could not find wasm rust package at path {}", - target_path - ))?; - - let target_wasm_path = format!( - "target/wasm32-unknown-unknown/debug/{}.wasm", - target_package - ); - let target_wit_path = format!("examples/rust/{}/{}.wit", target_package, target_package); - - if !Path::new(&target_wasm_path).exists() { - return Err(format!("Could not find wasm file at path {}", target_wasm_path).into()); - } - let wit_exists = Path::new(&target_wit_path).exists(); - - println!("debugging: {}", target_wasm_path); - if wit_exists { - println!("with wit: {}", target_wit_path); - } - - run(&target_wasm_path)?; - - Ok(()) -} diff --git a/examples/rust/power/Cargo.toml b/examples/rust/power/Cargo.toml index 5e036f6..d95cd0d 100644 --- a/examples/rust/power/Cargo.toml +++ b/examples/rust/power/Cargo.toml @@ -5,10 +5,9 @@ edition = "2018" [dependencies] wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +debugger-macro = { path = "../../../crates/debugger-macro" } [lib] crate-type = ["cdylib"] - -[profile.dev] -strip = "none" -lto = "off" diff --git a/examples/rust/power/src/lib.rs b/examples/rust/power/src/lib.rs index 50c36e8..2e7c794 100644 --- a/examples/rust/power/src/lib.rs +++ b/examples/rust/power/src/lib.rs @@ -1,5 +1,12 @@ +use debugger_macro::debugger; +use serde::{Deserialize, Serialize}; +use serde_json; +use std::str; + struct Power; wit_bindgen_rust::export!("power.wit"); + +#[debugger] impl power::Power for Power { fn power_of(base: i32, exp: i32) -> i32 { let mut res = 1; @@ -9,12 +16,3 @@ impl power::Power for Power { res } } - -struct Debugger; -wit_bindgen_rust::export!("../../../debugger/debugger.wit"); -impl debugger::Debugger for Debugger { - fn handle_json(name: String, _json: Vec) -> Vec { - let foo = 123; - format!("{} {}", name, foo).into_bytes() - } -} diff --git a/scripts/debug b/scripts/debug new file mode 100755 index 0000000..906be0b --- /dev/null +++ b/scripts/debug @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -eu + +FILE_TO_DEBUG="${1}" + +TARGET_DIR=$(cargo metadata --format-version 1 | jq -r '.target_directory') +CRATE_NAME=$(cargo get --name --root "${FILE_TO_DEBUG}") +CRATE_DIR=$(cargo workspaces list --json | jq -r ".[] | select(.name == \"${CRATE_NAME}\") | .location") + +WASM_PATH="${TARGET_DIR}/wasm32-wasi/debug/${CRATE_NAME}.wasm" +# currently unused +WIT_PATH="${CRATE_DIR}/${CRATE_NAME}.wit" + +${TARGET_DIR}/debug/debugger 3000 "${WASM_PATH}" \ No newline at end of file From b7422f156ce149d20b3636bf748bd3a90b4134b5 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Wed, 8 Jun 2022 23:10:26 +0000 Subject: [PATCH 4/9] working interactive debugger for wasm modules that only use simple types --- .vscode/tasks.json | 9 ++- crates/debugger-macro-impl/Cargo.toml | 18 ++++++ crates/debugger-macro-impl/src/lib.rs | 93 +++++++++++++++++++++++++++ crates/debugger-macro/Cargo.toml | 14 +--- crates/debugger-macro/src/lib.rs | 55 +--------------- crates/debugger/Cargo.toml | 1 - crates/debugger/src/server.rs | 6 +- examples/rust/power/Cargo.toml | 2 - examples/rust/power/src/lib.rs | 8 +-- 9 files changed, 129 insertions(+), 77 deletions(-) create mode 100644 crates/debugger-macro-impl/Cargo.toml create mode 100644 crates/debugger-macro-impl/src/lib.rs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 80c7ad5..bbe7d47 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -14,7 +14,14 @@ "group": "build", "label": "build: rust wasm modules", "command": "build", - "args": ["--target=wasm32-wasi", "--workspace", "--exclude", "debugger"], + "args": [ + "--target=wasm32-wasi", + "--workspace", + "--exclude", + "debugger", + "--exclude", + "debugger-macro" + ], "problemMatcher": ["$rustc"] }, { diff --git a/crates/debugger-macro-impl/Cargo.toml b/crates/debugger-macro-impl/Cargo.toml new file mode 100644 index 0000000..5585e2f --- /dev/null +++ b/crates/debugger-macro-impl/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "debugger-macro-impl" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0", features = [ + "full", + "printing", + "visit", + "extra-traits", + "parsing", +] } +quote = "1.0" +proc-macro2 = "1.0" diff --git a/crates/debugger-macro-impl/src/lib.rs b/crates/debugger-macro-impl/src/lib.rs new file mode 100644 index 0000000..380ed03 --- /dev/null +++ b/crates/debugger-macro-impl/src/lib.rs @@ -0,0 +1,93 @@ +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{parse_macro_input, visit::Visit}; + +struct Handler { + name: String, + src: TokenStream2, +} + +struct HandleVisitor { + impl_trait: syn::Path, + impl_type: syn::Type, + handlers: Vec, +} + +impl<'ast> Visit<'ast> for HandleVisitor { + fn visit_impl_item_method(&mut self, node: &'ast syn::ImplItemMethod) { + let sig = &node.sig; + let name = &sig.ident; + + let typed_args = sig.inputs.iter().filter_map(|input| match input { + syn::FnArg::Typed(x) => Some((*x.ty).clone()), + _ => None, + }); + let indexes = (0..typed_args.clone().count()).map(syn::Index::from); + + let args = quote! { (#(#typed_args,)*) }; + let args_splat = quote! { #(args.#indexes),* }; + + // XXX REMOVE? + // TODO: the idea here is to inspect the return_type + // and see if a json encoder is available. + + let return_type = match &sig.output { + syn::ReturnType::Type(_, ty) => (**ty).clone(), + _ => panic!("expected return type"), + }; + + let impl_trait = &self.impl_trait; + let impl_type = &self.impl_type; + + self.handlers.push(Handler { + name: name.to_string(), + src: quote! { + let args: #args = serde_json::from_slice(&json).unwrap(); + let result: #return_type = <#impl_type as #impl_trait>::#name(#args_splat); + serde_json::to_vec(&vec![result]).unwrap() + }, + }); + + syn::visit::visit_impl_item_method(self, node); + } +} + +#[proc_macro_attribute] +pub fn export_debug_handler(_attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as syn::ItemImpl); + + let (_, path, _) = input.trait_.as_ref().expect("expected trait"); + + let mut visitor = HandleVisitor { + impl_trait: path.clone(), + impl_type: (*input.self_ty).clone(), + handlers: vec![], + }; + visitor.visit_item_impl(&input); + + let handle_names = visitor.handlers.iter().map(|h| &h.name); + let handle_srcs = visitor.handlers.iter().map(|h| &h.src); + + let debugger_wit = include_str!("../../debugger/debugger.wit"); + + quote! { + #input + + use ::debugger_macro::serde_json; + + struct DebuggerImpl; + wit_bindgen_rust::export!({ + src["debugger_impl"]: #debugger_wit + }); + impl debugger_impl::DebuggerImpl for DebuggerImpl { + fn handle_json(name: String, json: Vec) -> Vec { + match name.as_str() { + #(#handle_names => {#handle_srcs})* + _ => panic!("unknown handler") + } + } + } + } + .into() +} diff --git a/crates/debugger-macro/Cargo.toml b/crates/debugger-macro/Cargo.toml index c78c9d5..785f86e 100644 --- a/crates/debugger-macro/Cargo.toml +++ b/crates/debugger-macro/Cargo.toml @@ -3,15 +3,7 @@ name = "debugger-macro" version = "0.1.0" edition = "2021" -[lib] -proc-macro = true - [dependencies] -syn = { version = "1.0", features = [ - "full", - "printing", - "visit", - "extra-traits", - "parsing", -] } -quote = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +debugger-macro-impl = { path = "../debugger-macro-impl" } diff --git a/crates/debugger-macro/src/lib.rs b/crates/debugger-macro/src/lib.rs index 8cdb471..32dd0ae 100644 --- a/crates/debugger-macro/src/lib.rs +++ b/crates/debugger-macro/src/lib.rs @@ -1,53 +1,2 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{parse_macro_input, parse_quote, visit::Visit}; - -struct Visitor {} - -impl Visit<'_> for Visitor { - fn visit_item_fn(&mut self, node: &'_ syn::ItemFn) { - let sig = &node.sig; - let name = &sig.ident; - /* - sig.inputs.iter().for_each(|input| { - if let syn::FnArg::Typed(x) = input { - self.source.push_str(quote! {#x}.to_string().as_str()); - } - }); - - self.source.push_str(") -> "); - - if let syn::ReturnType::Type(_, ref ty) = sig.output { - let type_name = quote! {#ty}.to_string(); - let type_name = type_name.replace("Vec", "list"); - self.source.push_str(type_name.as_str()); - self.source.push('\n'); - } - */ - } -} - -#[proc_macro_attribute] -pub fn debugger(_attr: TokenStream, item: TokenStream) -> TokenStream { - let input = parse_macro_input!(item as syn::Item); - let mut visitor = Visitor {}; - visitor.visit_item(&input); - - let x = quote! { - #input - - #[derive(Serialize, Deserialize)] - struct Payload(i32, i32); - - struct Debugger; - wit_bindgen_rust::export!("../../../crates/debugger/debugger.wit"); - impl debugger::Debugger for Debugger { - fn handle_json(_name: String, json: Vec) -> Vec { - let payload: Payload = serde_json::from_slice(&json).unwrap(); - let result = ::power_of(payload.0, payload.1); - serde_json::to_vec(&vec![result]).unwrap() - } - } - }; - x.into() -} +pub use ::serde_json; +pub use debugger_macro_impl::export_debug_handler; diff --git a/crates/debugger/Cargo.toml b/crates/debugger/Cargo.toml index 0ff0f70..cb3f540 100644 --- a/crates/debugger/Cargo.toml +++ b/crates/debugger/Cargo.toml @@ -11,7 +11,6 @@ anyhow = "1.0" wasmtime = "0.35.3" wasmtime-wasi = "0.35.3" wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } - tide = "0.16.0" async-std = { version = "1.8.0", features = ["attributes"] } serde = { version = "1.0", features = ["derive"] } diff --git a/crates/debugger/src/server.rs b/crates/debugger/src/server.rs index 642375a..2df1748 100644 --- a/crates/debugger/src/server.rs +++ b/crates/debugger/src/server.rs @@ -18,7 +18,7 @@ pub async fn listen_and_serve(port: u16, factory: handle::HandleFactory) -> Resu let mut app = tide::with_state(state); app.with(tide::log::LogMiddleware::new()); - app.at("/foo").post(handle_json); + app.at("/:name").post(handle_json); app.listen(format!("0.0.0.0:{}", port)).await?; Ok(()) @@ -36,7 +36,7 @@ async fn handle_json(mut req: Request) -> tide::Result { let mut handler = state.factory.make_handler()?; let payload: Payload = req.body_json().await?; - let path = req.url().path(); + let name = req.param("name")?; let mut result = Vec::new(); for row in payload.data { @@ -47,7 +47,7 @@ async fn handle_json(mut req: Request) -> tide::Result { let row_id = row[0].clone(); let row_input = row[1..].to_vec(); - let row_output_raw = handler.handle_json(path.into(), serde_json::to_vec(&row_input)?)?; + let row_output_raw = handler.handle_json(name.into(), serde_json::to_vec(&row_input)?)?; let row_output: Row = serde_json::from_slice(&row_output_raw)?; let row_final: Row = iter::once(row_id).chain(row_output.into_iter()).collect(); diff --git a/examples/rust/power/Cargo.toml b/examples/rust/power/Cargo.toml index d95cd0d..7ff8e98 100644 --- a/examples/rust/power/Cargo.toml +++ b/examples/rust/power/Cargo.toml @@ -5,8 +5,6 @@ edition = "2018" [dependencies] wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" debugger-macro = { path = "../../../crates/debugger-macro" } [lib] diff --git a/examples/rust/power/src/lib.rs b/examples/rust/power/src/lib.rs index 2e7c794..f03ce79 100644 --- a/examples/rust/power/src/lib.rs +++ b/examples/rust/power/src/lib.rs @@ -1,12 +1,8 @@ -use debugger_macro::debugger; -use serde::{Deserialize, Serialize}; -use serde_json; -use std::str; +wit_bindgen_rust::export!("power.wit"); struct Power; -wit_bindgen_rust::export!("power.wit"); -#[debugger] +#[debugger_macro::export_debug_handler] impl power::Power for Power { fn power_of(base: i32, exp: i32) -> i32 { let mut res = 1; From e1c57822b65c136c90f2ad966ac27e5b08d0ff5a Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Thu, 9 Jun 2022 00:01:05 +0000 Subject: [PATCH 5/9] implement remote debugger in all rust examples --- crates/debugger-macro-impl/src/lib.rs | 11 +--- examples/rust/sentiment/Cargo.toml | 30 ++++----- examples/rust/sentiment/src/lib.rs | 55 +++++++++------- examples/rust/split/Cargo.toml | 3 +- examples/rust/split/src/lib.rs | 18 +++++- examples/rust/usergenerator/Cargo.toml | 4 +- examples/rust/usergenerator/src/lib.rs | 62 +++++++++++-------- examples/rust/usergenerator/usergenerator.wit | 2 +- 8 files changed, 109 insertions(+), 76 deletions(-) diff --git a/crates/debugger-macro-impl/src/lib.rs b/crates/debugger-macro-impl/src/lib.rs index 380ed03..c839763 100644 --- a/crates/debugger-macro-impl/src/lib.rs +++ b/crates/debugger-macro-impl/src/lib.rs @@ -28,15 +28,6 @@ impl<'ast> Visit<'ast> for HandleVisitor { let args = quote! { (#(#typed_args,)*) }; let args_splat = quote! { #(args.#indexes),* }; - // XXX REMOVE? - // TODO: the idea here is to inspect the return_type - // and see if a json encoder is available. - - let return_type = match &sig.output { - syn::ReturnType::Type(_, ty) => (**ty).clone(), - _ => panic!("expected return type"), - }; - let impl_trait = &self.impl_trait; let impl_type = &self.impl_type; @@ -44,7 +35,7 @@ impl<'ast> Visit<'ast> for HandleVisitor { name: name.to_string(), src: quote! { let args: #args = serde_json::from_slice(&json).unwrap(); - let result: #return_type = <#impl_type as #impl_trait>::#name(#args_splat); + let result = <#impl_type as #impl_trait>::#name(#args_splat); serde_json::to_vec(&vec![result]).unwrap() }, }); diff --git a/examples/rust/sentiment/Cargo.toml b/examples/rust/sentiment/Cargo.toml index 6f55dae..e06c76f 100644 --- a/examples/rust/sentiment/Cargo.toml +++ b/examples/rust/sentiment/Cargo.toml @@ -1,14 +1,16 @@ -[package] -name = "sentiment-rust" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } -vader_sentiment = { git = "https://github.com/ckw017/vader-sentiment-rust" } -lazy_static = "1.4.0" - -[lib] -crate-type = ["cdylib"] \ No newline at end of file +[package] +name = "sentiment" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } +vader_sentiment = { git = "https://github.com/ckw017/vader-sentiment-rust" } +lazy_static = "1.4.0" +serde = { version = "1.0", features = ["derive"] } +debugger-macro = { path = "../../../crates/debugger-macro" } + +[lib] +crate-type = ["cdylib"] diff --git a/examples/rust/sentiment/src/lib.rs b/examples/rust/sentiment/src/lib.rs index ac0a399..19bd6dc 100644 --- a/examples/rust/sentiment/src/lib.rs +++ b/examples/rust/sentiment/src/lib.rs @@ -1,21 +1,34 @@ -wit_bindgen_rust::export!("sentiment.wit"); - -struct Sentiment(); - -impl sentiment::Sentiment for Sentiment { - - fn sentiment(input: String) -> sentiment::PolarityScores { - lazy_static::lazy_static! { - static ref ANALYZER: vader_sentiment::SentimentIntensityAnalyzer<'static> = - vader_sentiment::SentimentIntensityAnalyzer::new(); - } - - let scores = ANALYZER.polarity_scores(input.as_str()); - sentiment::PolarityScores { - compound: scores["compound"], - positive: scores["pos"], - negative: scores["neg"], - neutral: scores["neu"], - } - } -} +use serde::{ser::SerializeMap, Serialize, Serializer}; + +wit_bindgen_rust::export!("sentiment.wit"); + +impl Serialize for sentiment::PolarityScores { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("compound", &self.compound)?; + map.serialize_entry("pos", &self.positive)?; + map.serialize_entry("neg", &self.negative)?; + map.serialize_entry("neu", &self.neutral)?; + map.end() + } +} + +struct Sentiment; + +#[debugger_macro::export_debug_handler] +impl sentiment::Sentiment for Sentiment { + fn sentiment(input: String) -> sentiment::PolarityScores { + lazy_static::lazy_static! { + static ref ANALYZER: vader_sentiment::SentimentIntensityAnalyzer<'static> = + vader_sentiment::SentimentIntensityAnalyzer::new(); + } + + let scores = ANALYZER.polarity_scores(input.as_str()); + sentiment::PolarityScores { + compound: scores["compound"], + positive: scores["pos"], + negative: scores["neg"], + neutral: scores["neu"], + } + } +} diff --git a/examples/rust/split/Cargo.toml b/examples/rust/split/Cargo.toml index 38d2c82..3383717 100644 --- a/examples/rust/split/Cargo.toml +++ b/examples/rust/split/Cargo.toml @@ -5,7 +5,8 @@ edition = "2018" [dependencies] wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } +serde = { version = "1.0", features = ["derive"] } +debugger-macro = { path = "../../../crates/debugger-macro" } [lib] crate-type = ["cdylib"] - diff --git a/examples/rust/split/src/lib.rs b/examples/rust/split/src/lib.rs index 8944e22..0350062 100644 --- a/examples/rust/split/src/lib.rs +++ b/examples/rust/split/src/lib.rs @@ -1,16 +1,28 @@ +use serde::{ser::SerializeMap, Serialize, Serializer}; + wit_bindgen_rust::export!("split.wit"); -struct Split; + use crate::split::Subphrase; +impl Serialize for Subphrase { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("str", &self.str)?; + map.serialize_entry("idx", &self.idx)?; + map.end() + } +} -impl split::Split for Split { +struct Split; +#[debugger_macro::export_debug_handler] +impl split::Split for Split { fn split_str(phrase: String, delim: String) -> Vec { phrase .split(&delim) .scan(0, |idx, s| { let current = Subphrase { str: s.to_string(), - idx: *idx as i32 + idx: *idx as i32, }; *idx += (s.len() + delim.len()) as i32; Some(current) diff --git a/examples/rust/usergenerator/Cargo.toml b/examples/rust/usergenerator/Cargo.toml index 2868837..d942663 100644 --- a/examples/rust/usergenerator/Cargo.toml +++ b/examples/rust/usergenerator/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" [dependencies] fakeit = "1.1.1" wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } +serde = { version = "1.0", features = ["derive"] } +debugger-macro = { path = "../../../crates/debugger-macro" } [lib] -crate-type = ["cdylib"] \ No newline at end of file +crate-type = ["cdylib"] diff --git a/examples/rust/usergenerator/src/lib.rs b/examples/rust/usergenerator/src/lib.rs index fb3323b..9594947 100644 --- a/examples/rust/usergenerator/src/lib.rs +++ b/examples/rust/usergenerator/src/lib.rs @@ -1,46 +1,58 @@ -// wit_bindgen_rust::export!("fprofile.wit"); is used to export the "bindings" for the following rust example +// export! will load usergenerator.wit and generate Wasi compatible bindings wit_bindgen_rust::export!("usergenerator.wit"); -// From the bindings of the wit file, pull the User structure. This will be used later to generate multiple user profiles +// From the bindings of the wit file, pull the user structure. This will be used +// later to generate multiple user profiles use crate::usergenerator::User; -// Pull in the crate package "fakeit" and the modules for name, contact, password, unique and datetime -extern crate fakeit; -use fakeit::name; -use fakeit::contact; -use fakeit::datetime; -use fakeit::unique; -use fakeit::password; -// Another way to do this would be use fakeit::{name,contact,datetime,unique,password}; +// implement json serialization on user for the remote debugger +use serde::{ser::SerializeMap, Serialize, Serializer}; +impl Serialize for User { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("uid", &self.uid)?; + map.serialize_entry("created", &self.created)?; + map.serialize_entry("first-name", &self.first_name)?; + map.serialize_entry("last-name", &self.last_name)?; + map.serialize_entry("email", &self.email)?; + map.serialize_entry("passwd", &self.passwd)?; + map.end() + } +} + +// Pull in fakeit modules for name, contact, password, unique and datetime +use fakeit::{contact, datetime, name, password, unique}; -// We create the "Usergenerator" structure as a module pointer +// define a struct to be the concrete implementation of the wit interface struct Usergenerator; -// "Impl" is used to generate the "implementation types" of a module. Using the struct we created above, were going to iterate and implement each of these functions for our "fprofile" see more here - https://doc.rust-lang.org/std/keyword.impl.html +// implement the gen_users method of our wit interface (usergenerator.wit) +#[debugger_macro::export_debug_handler] impl usergenerator::Usergenerator for Usergenerator { - - // Our first function of our package! gen_users takes in a integer (a u32 in this case) and will output a vector with the User type we pulled from the wit file above - fn gen_users(pcount: u32) -> Vec { + // Our first function of our package! gen_users takes in a integer (a u32 in + // this case) and will output a vector with the User type we pulled from the + // wit file above + fn gen_users(count: u32) -> Vec { // We instantiate the vector to be returned - let mut pfs = Vec::new(); + let mut users = Vec::new(); - // Iterate from 0 to pcount for each profile we want to create - for _x in 0..pcount { - let data = User { + // Iterate from 0 to count for each user we want to create + for _x in 0..count { + let user = User { // See https://crates.io/crates/fakeit for all the fakeit types uid: unique::uuid_v4().to_string(), created: datetime::date().to_string(), first_name: name::first().to_string(), last_name: name::last().to_string(), email: contact::email().to_string(), - passwd: password::generate(true, true, true, 52).to_string() + passwd: password::generate(true, true, true, 52).to_string(), }; - // For each of these items, we push the data to the pfs vector - pfs.push(data); + // push the user into the vector + users.push(user); } - // Return the pfs data structure after completing - return pfs; + // Return the list of users + return users; } -} \ No newline at end of file +} diff --git a/examples/rust/usergenerator/usergenerator.wit b/examples/rust/usergenerator/usergenerator.wit index 0994fd2..5740157 100644 --- a/examples/rust/usergenerator/usergenerator.wit +++ b/examples/rust/usergenerator/usergenerator.wit @@ -7,4 +7,4 @@ record user { passwd: string } -gen-users: function(pcount: u32) -> list \ No newline at end of file +gen-users: function(count: u32) -> list \ No newline at end of file From eba29d50539df68f9eca42fe2627228109a4bcb7 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Thu, 9 Jun 2022 18:11:09 +0000 Subject: [PATCH 6/9] works for multi-returns --- crates/debugger-macro-impl/src/lib.rs | 2 +- crates/debugger/src/server.rs | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/debugger-macro-impl/src/lib.rs b/crates/debugger-macro-impl/src/lib.rs index c839763..6216535 100644 --- a/crates/debugger-macro-impl/src/lib.rs +++ b/crates/debugger-macro-impl/src/lib.rs @@ -36,7 +36,7 @@ impl<'ast> Visit<'ast> for HandleVisitor { src: quote! { let args: #args = serde_json::from_slice(&json).unwrap(); let result = <#impl_type as #impl_trait>::#name(#args_splat); - serde_json::to_vec(&vec![result]).unwrap() + serde_json::to_vec(&result).unwrap() }, }); diff --git a/crates/debugger/src/server.rs b/crates/debugger/src/server.rs index 2df1748..1c9909d 100644 --- a/crates/debugger/src/server.rs +++ b/crates/debugger/src/server.rs @@ -1,7 +1,6 @@ use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; -use serde_json; -use std::iter; +use serde_json::{self, Value}; use tide::{Body, Request, Response}; use crate::handle; @@ -24,7 +23,7 @@ pub async fn listen_and_serve(port: u16, factory: handle::HandleFactory) -> Resu Ok(()) } -type Row = Vec; +type Row = Vec; #[derive(Serialize, Deserialize)] struct Payload { @@ -47,11 +46,22 @@ async fn handle_json(mut req: Request) -> tide::Result { let row_id = row[0].clone(); let row_input = row[1..].to_vec(); - let row_output_raw = handler.handle_json(name.into(), serde_json::to_vec(&row_input)?)?; - let row_output: Row = serde_json::from_slice(&row_output_raw)?; + let output_raw = handler.handle_json(name.into(), serde_json::to_vec(&row_input)?)?; + let output: Value = serde_json::from_slice(&output_raw)?; - let row_final: Row = iter::once(row_id).chain(row_output.into_iter()).collect(); - result.push(row_final); + let encode_value = |v: &Value| match v { + Value::Array(_) | Value::Object(_) => Value::String(serde_json::to_string(v).unwrap()), + _ => v.clone(), + }; + + // if output is an array, then expand into multiple rows + if output.is_array() { + for row in output.as_array().unwrap() { + result.push(vec![row_id.clone(), encode_value(row)]); + } + } else { + result.push(vec![row_id.clone(), encode_value(&output)]); + } } Ok(Response::from(Body::from_json(&Payload { data: result })?)) From 082a91a7c6f95db47248525ab1177b3492997715 Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Thu, 9 Jun 2022 19:06:08 +0000 Subject: [PATCH 7/9] readme, echo example --- crates/debugger/README.md | 203 ++++++++++++++++++++++++++++++++++ examples/rust/README.md | 24 ++++ examples/rust/echo/Cargo.toml | 13 +++ examples/rust/echo/echo.wit | 1 + examples/rust/echo/src/lib.rs | 10 ++ 5 files changed, 251 insertions(+) create mode 100644 crates/debugger/README.md create mode 100644 examples/rust/echo/Cargo.toml create mode 100644 examples/rust/echo/echo.wit create mode 100644 examples/rust/echo/src/lib.rs diff --git a/crates/debugger/README.md b/crates/debugger/README.md new file mode 100644 index 0000000..031172d --- /dev/null +++ b/crates/debugger/README.md @@ -0,0 +1,203 @@ +# Wasm Remote Debugger Service + +This crate implements an [external functions][extfns] compatible http service which hosts wasm functions at specific endpoints. + +## Usage from VSCode + +This repo is already setup to ensure a seamless debugging experience. + +First, you need to make sure your wasm module is annotated with the debugger macro. We will use the following rust code as a starting point for adding debugger support: + +**Cargo.toml** +```toml +[package] +name = "echo" +version = "0.1.0" +edition = "2018" + +[dependencies] +wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } + +[lib] +crate-type = ["cdylib"] +``` + +**echo.wit** +```wit +echo: function(phrase: string) -> string +``` + +**lib.rs** +```rust +wit_bindgen_rust::export!("echo.wit"); +struct Echo; +impl echo::Echo for Echo { + fn echo(phrase: String) -> String { + format!("{} {}", phrase, phrase) + } +} +``` + +**In order to enable the debugger you need to make 2 changes.** + +The first is to add a dependency on the debugger-macro crate. You need to add this line to the `[dependencies]` section of `Cargo.toml`: + +```toml +debugger-macro = { path = "../../../crates/debugger-macro" } +``` + +Make sure the relative path is correct based on where your `Cargo.toml` is located. + +The second is to use the debugger macro in your code. This macro needs to annotate the `impl echo::Echo for Echo` like so: + +**lib.rs** +```rust +wit_bindgen_rust::export!("echo.wit"); +struct Echo; + +#[debugger_macro::export_debug_handler] +impl echo::Echo for Echo { + fn echo(phrase: String) -> String { + format!("{} {}", phrase, phrase) + } +} +``` + +Once you have done this simply press `F5` while you have your lib.rs open. The debugger will automatically build your crate targetting wasm32-wasi, and then start hosting it at `localhost:3000/echo`. + +**Note:** The path in the url needs to match the name of the function you want to call. For example, if the function in the code above was called "tell_joke" then the web service would host the function at `localhost:3000/tell_joke`. This does allow you to host multiple functions from the same Wasm module. + +## Calling your remote function from the command line + +For easy testing, you can now use curl (or any other http client) to test your Wasm code. Here is how I would use curl to test the function above: + +```bash +$ curl -s -XPOST localhost:3000/echo -d '{"data":[[1,"hello"]]}' | jq -r '.' +{ + "data": [ + [ + 1, + "hello hello" + ] + ] +} +``` + +For documentation on the input/output format please see the [external function][extfns] documentation. + +## Calling your remote function from SingleStore + +Now that you have your Wasm code hosted behind an external functions compatible web service, you can easily call your code from SingleStore by defining an external udf or tvf. Full [documentation on doing this is here][extfns]. + +As an example, lets test the echo function we defined above. Note, I am running the VSCode dev container in this repository, and my dev container has ip address 172.17.0.3. I also have SingleStore running on my machine in another docker container. So, in order to call the external function from SingleStore I will need to tell SingleStore how to connect to 172.17.0.3:3000. I can do this like so: + +```sql +MySQL [x]> create or replace external function echo (phrase text) returns text as remote service '172.17.0.3:3000/echo' format json; +Query OK, 1 row affected (0.014 sec) + +MySQL [x]> select echo("hi"); ++------------+ +| echo("hi") | ++------------+ +| hi hi | ++------------+ +1 row in set (0.049 sec) +``` + +## Using breakpoints + +Now that you have your wasm code hosted in the remote debugger, you can do some pretty magical things with it. The first thing you can do is use breakpoints. To continue with the example from above, let's open `lib.rs` and put a breakpoint at the line containing `format!("{} {}", phrase, phrase)`. + +Once the breakpoint is set make sure the debugger is running (press `F5` if it's not) and trigger your function from an HTTP client or SingleStore. As soon as you do, the breakpoint you set should be hit. + +Note - currently debugger support for Wasm is a bit thin. You will be able to step through your code and get nice back traces on failure, however you won't be able to inspect local variables yet. Hopefully that will be resolved in the future as debugger support increases for Wasm modules. + +## Logging + +Since you can't inspect variables in the debugger, how can you see what is going on? For now, the best answer is good ol `print` style debugging. Using the example above, let's add some logs to our echo function: + +**lib.rs** +```rust +#[debugger_macro::export_debug_handler] +impl echo::Echo for Echo { + fn echo(phrase: String) -> String { + dbg!("hello from wasm!", &phrase); + format!("{} {}", phrase, phrase) + } +} +``` + +When you run the debugger, you will see it's output in one of the VSCode terminals. Look for the phrase `tide::server Server listening on http://0.0.0.0:3000`. That's where your logs will go! + +Now send another request to the debugger from an HTTP client or SingleStore, and you should see something like the following output: + +``` +tide::log::middleware <-- Request received + method POST + path /echo +[examples/rust/echo/src/lib.rs:7] "hello from wasm!" = "hello from wasm!" +[examples/rust/echo/src/lib.rs:7] &phrase = "hi" +tide::log::middleware --> Response sent + method POST + path /echo + status 200 - OK + duration 8.712664ms +``` + +Pretty cool right? Well, hopefully this gets you started! See the FAQ below if you run into any issues. Otherwise, enjoy! + +# FAQ + +## A panic occurred! VSCode opened up some weird assembly code + +If VSCode opens a file that looks something like this: +```asm +; No Symbol Info +; Source location: /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/panic_abort/src/lib.rs:84 +7F30BF4235F0: 0F 0B ud2 +7F30BF4235F2: 55 pushq %rbp +7F30BF4235F3: 48 89 E5 movq %rsp, %rbp +7F30BF4235F6: 8B F2 movl %edx, %esi +7F30BF4235F8: 48 8B 87 88 00 00 00 movq 0x88(%rdi), %rax +7F30BF4235FF: 48 0F B7 74 30 00 movzwq (%rax,%rsi), %rsi +7F30BF423605: 48 89 F0 movq %rsi, %rax +``` + +That means you have most likely panicked. Don't fret! + +Check the call stack first. If you see your wasm function somewhere in the call stack click that to see where in your code the panic happened. Hopefully you can determine why and fix the issue. + +If you **don't see your wasm function in the call stack** then you most likely hit the wrong endpoint on the debugger. Check that the url you are requesting ends with the precise name of the function defined in your code. For example, in the `examples/rust/power` example project, the function in the code is called `power_of` so the debugger endpoint needs to be `IP_ADDRESS:3000/power_of` for it to work. + +## the trait bound `XXX: Serialize` is not satisfied + +If you get a Rust compilation error that looks something like this, it means you are using custom types in your Wasm code. + +``` +error[E0277]: the trait bound `PolarityScores: Serialize` is not satisfied + --> examples/rust/sentiment/src/lib.rs:18:1 + | +18 | #[debugger_macro::export_debug_handler] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Serialize` is not implemented for `PolarityScores` +``` + +Since external functions uses JSON, you will need to provide an implementation of the `serde::Serialize` trait for each of your custom types. For example, here is the implementation of `Serialize` for the PolarityScores type in the `examples/rust/sentiment` crate: + +```rust +impl Serialize for sentiment::PolarityScores { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("compound", &self.compound)?; + map.serialize_entry("pos", &self.positive)?; + map.serialize_entry("neg", &self.negative)?; + map.serialize_entry("neu", &self.neutral)?; + map.end() + } +} +``` + +Once you add a valid Serialize implementation for your custom type(s) the compilation error should go away. + + + +[extfns]: https://docs.singlestore.com/managed-service/en/reference/sql-reference/procedural-sql-reference/create--or-replace--external-function.html \ No newline at end of file diff --git a/examples/rust/README.md b/examples/rust/README.md index ece4412..1137a4f 100644 --- a/examples/rust/README.md +++ b/examples/rust/README.md @@ -5,6 +5,30 @@ See [Quickstart Build](./Quickstart-Build.md) for info on how to build Rust WASM ## Catalog ### Sentiment (VADER) +Perform sentiment analysis on a string of english text. [code](./sentiment/) +### Power + +Calculate X to the power of Y. + +[code](./power/) + +### echo + +Echo an input phrase twice. + +[code](./echo/) + +### split + +Split a string of text on a delimiter. + +[code](./split/) + +### usergenerator + +Use the Faker library to generate some fake user profiles. + +[code](./usergenerator/) \ No newline at end of file diff --git a/examples/rust/echo/Cargo.toml b/examples/rust/echo/Cargo.toml new file mode 100644 index 0000000..80fe6aa --- /dev/null +++ b/examples/rust/echo/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "echo" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen.git" } +debugger-macro = { path = "../../../crates/debugger-macro" } + +[lib] +crate-type = ["cdylib"] \ No newline at end of file diff --git a/examples/rust/echo/echo.wit b/examples/rust/echo/echo.wit new file mode 100644 index 0000000..123bc4a --- /dev/null +++ b/examples/rust/echo/echo.wit @@ -0,0 +1 @@ +echo: function(phrase: string) -> string \ No newline at end of file diff --git a/examples/rust/echo/src/lib.rs b/examples/rust/echo/src/lib.rs new file mode 100644 index 0000000..fc09704 --- /dev/null +++ b/examples/rust/echo/src/lib.rs @@ -0,0 +1,10 @@ +wit_bindgen_rust::export!("echo.wit"); +struct Echo; + +#[debugger_macro::export_debug_handler] +impl echo::Echo for Echo { + fn echo(phrase: String) -> String { + dbg!("hello from wasm!", &phrase); + format!("{} {}", phrase, phrase) + } +} From 537c2d1846269601a92b433c00cbf6e6ee79da3b Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Thu, 9 Jun 2022 20:05:21 +0000 Subject: [PATCH 8/9] use new wit syntax for functions --- crates/debugger/debugger.wit | 2 +- examples/cpp/power/power.wit | 2 +- examples/cpp/split/split.wit | 2 +- examples/rust/echo/echo.wit | 2 +- examples/rust/power/power.wit | 2 +- examples/rust/sentiment/sentiment.wit | 2 +- examples/rust/split/split.wit | 2 +- examples/rust/usergenerator/usergenerator.wit | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/debugger/debugger.wit b/crates/debugger/debugger.wit index d8b90b4..bb601a6 100644 --- a/crates/debugger/debugger.wit +++ b/crates/debugger/debugger.wit @@ -1 +1 @@ -handle-json: function(name: string, json: list) -> list \ No newline at end of file +handle-json: func(name: string, json: list) -> list \ No newline at end of file diff --git a/examples/cpp/power/power.wit b/examples/cpp/power/power.wit index 03f0590..ead59de 100644 --- a/examples/cpp/power/power.wit +++ b/examples/cpp/power/power.wit @@ -1 +1 @@ -power-of: function(base: s32, exp: s32) -> s32 +power-of: func(base: s32, exp: s32) -> s32 diff --git a/examples/cpp/split/split.wit b/examples/cpp/split/split.wit index a00104e..a33a908 100644 --- a/examples/cpp/split/split.wit +++ b/examples/cpp/split/split.wit @@ -2,5 +2,5 @@ record subphrase { str: string, idx: s32 } -split-str: function(phrase: string, delim: string) -> list +split-str: func(phrase: string, delim: string) -> list diff --git a/examples/rust/echo/echo.wit b/examples/rust/echo/echo.wit index 123bc4a..d53a43a 100644 --- a/examples/rust/echo/echo.wit +++ b/examples/rust/echo/echo.wit @@ -1 +1 @@ -echo: function(phrase: string) -> string \ No newline at end of file +echo: func(phrase: string) -> string \ No newline at end of file diff --git a/examples/rust/power/power.wit b/examples/rust/power/power.wit index 03f0590..ead59de 100644 --- a/examples/rust/power/power.wit +++ b/examples/rust/power/power.wit @@ -1 +1 @@ -power-of: function(base: s32, exp: s32) -> s32 +power-of: func(base: s32, exp: s32) -> s32 diff --git a/examples/rust/sentiment/sentiment.wit b/examples/rust/sentiment/sentiment.wit index 0fc047c..00e9a4e 100644 --- a/examples/rust/sentiment/sentiment.wit +++ b/examples/rust/sentiment/sentiment.wit @@ -5,4 +5,4 @@ record polarity-scores { neutral: float64, } -sentiment: function(input: string) -> polarity-scores \ No newline at end of file +sentiment: func(input: string) -> polarity-scores \ No newline at end of file diff --git a/examples/rust/split/split.wit b/examples/rust/split/split.wit index a00104e..a33a908 100644 --- a/examples/rust/split/split.wit +++ b/examples/rust/split/split.wit @@ -2,5 +2,5 @@ record subphrase { str: string, idx: s32 } -split-str: function(phrase: string, delim: string) -> list +split-str: func(phrase: string, delim: string) -> list diff --git a/examples/rust/usergenerator/usergenerator.wit b/examples/rust/usergenerator/usergenerator.wit index 5740157..fd85f6e 100644 --- a/examples/rust/usergenerator/usergenerator.wit +++ b/examples/rust/usergenerator/usergenerator.wit @@ -7,4 +7,4 @@ record user { passwd: string } -gen-users: function(count: u32) -> list \ No newline at end of file +gen-users: func(count: u32) -> list \ No newline at end of file From 826432417b55ba1cad1bef757c5025746317acae Mon Sep 17 00:00:00 2001 From: Carl Sverre Date: Thu, 9 Jun 2022 20:15:13 +0000 Subject: [PATCH 9/9] s/SingleStore/SingleStoreDB/ --- crates/debugger/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/debugger/README.md b/crates/debugger/README.md index 031172d..e0bd8f9 100644 --- a/crates/debugger/README.md +++ b/crates/debugger/README.md @@ -85,11 +85,11 @@ $ curl -s -XPOST localhost:3000/echo -d '{"data":[[1,"hello"]]}' | jq -r '.' For documentation on the input/output format please see the [external function][extfns] documentation. -## Calling your remote function from SingleStore +## Calling your remote function from SingleStoreDB -Now that you have your Wasm code hosted behind an external functions compatible web service, you can easily call your code from SingleStore by defining an external udf or tvf. Full [documentation on doing this is here][extfns]. +Now that you have your Wasm code hosted behind an external functions compatible web service, you can easily call your code from SingleStoreDB by defining an external udf or tvf. Full [documentation on doing this is here][extfns]. -As an example, lets test the echo function we defined above. Note, I am running the VSCode dev container in this repository, and my dev container has ip address 172.17.0.3. I also have SingleStore running on my machine in another docker container. So, in order to call the external function from SingleStore I will need to tell SingleStore how to connect to 172.17.0.3:3000. I can do this like so: +As an example, lets test the echo function we defined above. Note, I am running the VSCode dev container in this repository, and my dev container has ip address 172.17.0.3. I also have SingleStoreDB running on my machine in another docker container. So, in order to call the external function from SingleStoreDB I will need to tell SingleStoreDB how to connect to 172.17.0.3:3000. I can do this like so: ```sql MySQL [x]> create or replace external function echo (phrase text) returns text as remote service '172.17.0.3:3000/echo' format json; @@ -108,7 +108,7 @@ MySQL [x]> select echo("hi"); Now that you have your wasm code hosted in the remote debugger, you can do some pretty magical things with it. The first thing you can do is use breakpoints. To continue with the example from above, let's open `lib.rs` and put a breakpoint at the line containing `format!("{} {}", phrase, phrase)`. -Once the breakpoint is set make sure the debugger is running (press `F5` if it's not) and trigger your function from an HTTP client or SingleStore. As soon as you do, the breakpoint you set should be hit. +Once the breakpoint is set make sure the debugger is running (press `F5` if it's not) and trigger your function from an HTTP client or SingleStoreDB. As soon as you do, the breakpoint you set should be hit. Note - currently debugger support for Wasm is a bit thin. You will be able to step through your code and get nice back traces on failure, however you won't be able to inspect local variables yet. Hopefully that will be resolved in the future as debugger support increases for Wasm modules. @@ -129,7 +129,7 @@ impl echo::Echo for Echo { When you run the debugger, you will see it's output in one of the VSCode terminals. Look for the phrase `tide::server Server listening on http://0.0.0.0:3000`. That's where your logs will go! -Now send another request to the debugger from an HTTP client or SingleStore, and you should see something like the following output: +Now send another request to the debugger from an HTTP client or SingleStoreDB, and you should see something like the following output: ``` tide::log::middleware <-- Request received