(re)add --stub-wasi option

Early versions of this project had such an option, but I had to drop as part of
the refactoring required to support native extensions.  The implementation is a
bit tricky, since we need to import WASI during pre-initialization, but then
stub out the imports in the final component.  Normally, that would require
significant surgery on the pre-initialized component, but I've "cheated" by
applying the state snapshot to a stubbed component instead of the original one.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.12.0"
+version = "0.13.0"
 edition = "2021"
 exclude = ["cpython"]
 
@@ -15,19 +15,26 @@ tar = "0.4.40"
 tempfile = "3.8.1"
 zstd = "0.13.0"
 componentize-py-shared = { path = "shared" }
-wasmparser = "0.118.0"
-wasm-encoder = "0.38.1"
-wit-parser = "0.13.1"
-wit-component = "0.20.0"
+# TODO: switch to upstream release versions of these deps once
+# https://github.com/bytecodealliance/wasm-tools/pull/1459 is merged and
+# released:
+wasm-encoder = { git = "https://github.com/dicej/wasm-tools", branch = "adapt-world-imports" }
+wit-parser = { git = "https://github.com/dicej/wasm-tools", branch = "adapt-world-imports" }
+wit-component = { git = "https://github.com/dicej/wasm-tools", branch = "adapt-world-imports" }
+wasmparser = { git = "https://github.com/dicej/wasm-tools", branch = "adapt-world-imports" }
 indexmap = "2.1.0"
 bincode = "1.3.3"
 heck = "0.4.1"
 pyo3 = { version = "0.20.0", features = ["abi3-py37", "extension-module"], optional = true }
-wasmtime = { path = "../wasmtime/crates/wasmtime", features = ["component-model"] }
-wasmtime-wasi = { path = "../wasmtime/crates/wasi" }
-wasi-common = { path = "../wasmtime/crates/wasi-common" }
+# TODO: switch to upstream release versions of these deps once
+# https://github.com/bytecodealliance/wasm-tools/pull/1459 is merged and
+# released, and Wasmtime has adopted those releases:
+wasmtime = { git = "https://github.com/dicej/wasmtime", branch = "isyswasfa", features = ["component-model"] }
+wasmtime-wasi = { git = "https://github.com/dicej/wasmtime", branch = "isyswasfa" }
+wasi-common = { git = "https://github.com/dicej/wasmtime", branch = "isyswasfa" }
 once_cell = "1.18.0"
 component-init = { git = "https://github.com/dicej/component-init" }
+wasm-convert = { git = "https://github.com/dicej/wasm-convert" }
 async-trait = "0.1.74"
 futures = "0.3.29"
 tokio = { version = "1.34.0", features = ["macros", "rt", "rt-multi-thread", "fs"] }
@@ -37,7 +44,7 @@ cap-std = "2.0.0"
 im-rc = "15.1.0"
 serde = { version = "1.0.193", features = ["derive"] }
 toml = "0.8.8"
-isyswasfa-transform = { path = "../transform" }
+isyswasfa-transform = { git = "https://github.com/dicej/isyswasfa-transform" }
 semver = "1.0.22"
 
 [dev-dependencies]

examples/cli/README.md

@@ -10,15 +10,15 @@ run a Python-based component targetting the [wasi-cli] `command` world.
 ## Prerequisites
 
 * `Wasmtime` 18.0.0 or later
-* `componentize-py` 0.12.0
+* `componentize-py` 0.13.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
 cargo install --version 18.0.0 wasmtime-cli
-pip install componentize-py==0.12.0
+pip install componentize-py==0.13.0
 ```
 
 ## Running the demo

examples/http/README.md

@@ -10,15 +10,15 @@ run a Python-based component targetting the [wasi-http] `proxy` world.
 ## Prerequisites
 
 * `Wasmtime` 18.0.0 or later
-* `componentize-py` 0.12.0
+* `componentize-py` 0.13.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
 cargo install --version 18.0.0 wasmtime-cli
-pip install componentize-py==0.12.0
+pip install componentize-py==0.13.0
 ```
 
 ## Running the demo

examples/matrix-math/README.md

@@ -11,7 +11,7 @@ within a guest component.
 ## Prerequisites
 
 * `wasmtime` 18.0.0 or later
-* `componentize-py` 0.12.0
+* `componentize-py` 0.13.0
 * `NumPy`, built for WASI
 
 Note that we use an unofficial build of NumPy since the upstream project does
@@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
 cargo install --version 18.0.0 wasmtime-cli
-pip install componentize-py==0.12.0
+pip install componentize-py==0.13.0
 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz
 tar xf numpy-wasi.tar.gz
 ```

examples/sandbox/README.md

@@ -0,0 +1,65 @@
+# Example: `sandbox`
+
+This is an example of how to use
+[`wasmtime-py`](https://github.com/bytecodealliance/wasmtime-py) and
+[`componentize-py`](https://github.com/dicej/componentize-py) to execute
+sandboxed Python code snippets from within a Python app.
+
+## Prerequisites
+
+* `wasmtime-py` 18.0.0 or later
+* `componentize-py` 0.13.0
+
+```
+pip install componentize-py==0.13.0 wasmtime==18.0.2
+```
+
+## Running the demo
+
+```
+componentize-py -d wit -w sandbox componentize guest -o sandbox.wasm
+python3 -m wasmtime.bindgen sandbox.wasm --out-dir sandbox
+python3 host.py "2 + 2"
+```
+
+## Examples
+
+`host.py` accepts zero or more `exec` strings (e.g. newline-delimited
+statements) followed by a final `eval` string (i.e. an expression).  Note that
+any symbols you declare in an `exec` string must be explicitly added to the
+global scope using `global`.  This ensures they are visible to subsequent `exec`
+and `eval` strings.
+
+```shell-session
+ $ python3 host.py "2 + 2"
+result: 4
+ $ python3 host.py 'global foo
+def foo(): return 42' 'foo()'
+result: 42
+```
+
+### Time limit
+
+`host.py` enforces a twenty second timeout on guest execution.  If and when the
+timeout is reached, `wasmtime` will raise a `Trap` error.
+
+```shell-session
+ $ python3 host.py 'while True: pass' '1'
+timeout!
+Traceback (most recent call last):
+  File "/Users/dicej/p/component-sandbox-demo/host.py", line 31, in <module>
+    result = sandbox.exec(store, arg)
+             ^^^^^^^^^^^^^^^^^^^^^^^^
+...
+```
+
+### Memory limit
+
+`host.py` limits guest memory usage to 20MB.  Any attempt to allocate beyond
+that limit will fail.
+
+```shell-session
+ $ python3 host.py 'global foo
+foo = bytes(100 * 1024 * 1024)' 'foo[42]'
+exec error: MemoryError
+```

examples/sandbox/guest.py

@@ -0,0 +1,23 @@
+import sandbox
+from sandbox.types import Err
+import json
+
+def handle(e: Exception):
+    message = str(e)
+    if message == '':
+        raise Err(f"{type(e).__name__}")
+    else:
+        raise Err(f"{type(e).__name__}: {message}")
+
+class Sandbox(sandbox.Sandbox):
+    def eval(self, expression: str) -> str:
+        try:
+            return json.dumps(eval(expression))
+        except Exception as e:
+            handle(e)
+
+    def exec(self, statements: str):
+        try:
+            exec(statements)
+        except Exception as e:
+            handle(e)

examples/sandbox/host.py

@@ -0,0 +1,48 @@
+from sandbox import Root
+from sandbox.types import Ok, Err
+from wasmtime import Config, Engine, Store
+import json
+import sys
+from threading import Timer
+from typing import List, Tuple
+
+TIMEOUT_SECONDS = 20
+MEMORY_LIMIT_BYTES = 20 * 1024 * 1024
+
+args = sys.argv[1:]
+if len(args) == 0:
+    print("usage: python3 host.py [<statement>...] <expression>", file=sys.stderr)
+    exit(-1)
+
+config = Config()
+config.epoch_interruption = True
+
+def on_timeout(engine):
+    print("timeout!")
+    engine.increment_epoch()
+
+engine = Engine(config)
+timer = Timer(TIMEOUT_SECONDS, on_timeout, args=(engine,))
+timer.start()
+
+try:
+    store = Store(engine)
+    store.set_epoch_deadline(1)
+    store.set_limits(memory_size=MEMORY_LIMIT_BYTES)
+
+    sandbox = Root(store)
+    for arg in args[:-1]:
+        result = sandbox.exec(store, arg)
+        if isinstance(result, Err):
+            print(f"exec error: {result.value}")
+            exit(-1)
+
+    result = sandbox.eval(store, args[-1])
+    if isinstance(result, Ok):
+        result = json.loads(result.value)
+        print(f"result: {result}")
+    else:
+        print(f"eval error: {result.value}")
+
+finally:
+    timer.cancel()

examples/sandbox/sandbox.wit

@@ -0,0 +1,6 @@
+package local:sandbox;
+
+world sandbox {
+  export eval: func(expression: string) -> result<string, string>;
+  export exec: func(statements: string) -> result<_, string>;
+}

examples/tcp/README.md

@@ -11,15 +11,15 @@ making an outbound TCP request using `wasi-sockets`.
 ## Prerequisites
 
 * `Wasmtime` 18.0.0 or later
-* `componentize-py` 0.12.0
+* `componentize-py` 0.13.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
 cargo install --version 18.0.0 wasmtime-cli
-pip install componentize-py==0.12.0
+pip install componentize-py==0.13.0
 ```
 
 ## Running the demo

pyproject.toml

@@ -7,7 +7,7 @@ features = ["pyo3/extension-module"]
 
 [project]
 name = "componentize-py"
-version = "0.12.0"
+version = "0.13.0"
 description = "Tool to package Python applications as WebAssembly components"
 readme = "README.md"
 license = { file = "LICENSE" }

runtime/src/lib.rs

@@ -33,6 +33,7 @@ wit_bindgen::generate!({
     }
 });
 
+static STUB_WASI: OnceCell<bool> = OnceCell::new();
 static EXPORTS: OnceCell<Vec<Export>> = OnceCell::new();
 static TYPES: OnceCell<Vec<Type>> = OnceCell::new();
 static ENVIRON: OnceCell<Py<PyMapping>> = OnceCell::new();
@@ -169,7 +170,7 @@ fn componentize_py_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> {
     module.add_function(pyo3::wrap_pyfunction!(drop_resource, module)?)
 }
 
-fn do_init(app_name: String, symbols: Symbols) -> Result<()> {
+fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<()> {
     pyo3::append_to_inittab!(componentize_py_module);
 
     pyo3::prepare_freethreaded_python();
@@ -183,6 +184,8 @@ fn do_init(app_name: String, symbols: Symbols) -> Result<()> {
             }
         };
 
+        STUB_WASI.set(stub_wasi).unwrap();
+
         EXPORTS
             .set(
                 symbols
@@ -368,8 +371,8 @@ fn do_init(app_name: String, symbols: Symbols) -> Result<()> {
 struct MyExports;
 
 impl Guest for MyExports {
-    fn init(app_name: String, symbols: Symbols) -> Result<(), String> {
-        let result = do_init(app_name, symbols).map_err(|e| format!("{e:?}"));
+    fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> {
+        let result = do_init(app_name, symbols, stub_wasi).map_err(|e| format!("{e:?}"));
 
         // This tells the WASI Preview 1 component adapter to reset its state.  In particular, we want it to forget
         // about any open handles and re-request the stdio handles at runtime since we'll be running under a brand
@@ -413,24 +416,26 @@ pub unsafe extern "C" fn componentize_py_dispatch(
         // todo: is this sound, or do we need to `.into_iter().map(MaybeUninit::assume_init).collect()` instead?
         let params_py = mem::transmute::<Vec<MaybeUninit<&PyAny>>, Vec<&PyAny>>(params_py);
 
-        static ONCE: Once = Once::new();
-        ONCE.call_once(|| {
-            // We must call directly into the host to get the runtime environment since libc's version will only
-            // contain the build-time pre-init snapshot.
-            let environ = ENVIRON.get().unwrap().as_ref(py);
-            for (k, v) in environment::get_environment() {
-                environ.set_item(k, v).unwrap();
-            }
+        if !*STUB_WASI.get().unwrap() {
+            static ONCE: Once = Once::new();
+            ONCE.call_once(|| {
+                // We must call directly into the host to get the runtime environment since libc's version will only
+                // contain the build-time pre-init snapshot.
+                let environ = ENVIRON.get().unwrap().as_ref(py);
+                for (k, v) in environment::get_environment() {
+                    environ.set_item(k, v).unwrap();
+                }
 
-            // Likewise for CLI arguments.
-            for arg in environment::get_arguments() {
-                ARGV.get().unwrap().as_ref(py).append(arg).unwrap();
-            }
+                // Likewise for CLI arguments.
+                for arg in environment::get_arguments() {
+                    ARGV.get().unwrap().as_ref(py).append(arg).unwrap();
+                }
 
-            // Call `random.seed()` to ensure we get a fresh seed rather than the one that got baked in during
-            // pre-init.
-            SEED.get().unwrap().call0(py).unwrap();
-        });
+                // Call `random.seed()` to ensure we get a fresh seed rather than the one that got baked in during
+                // pre-init.
+                SEED.get().unwrap().call0(py).unwrap();
+            });
+        }
 
         let export = &EXPORTS.get().unwrap()[export];
         let result = match export {

src/command.rs

@@ -82,6 +82,14 @@ pub struct Componentize {
     /// Output file to which to write the resulting component
     #[arg(short = 'o', long, default_value = "index.wasm")]
     pub output: PathBuf,
+
+    /// If set, replace all WASI imports with trapping stubs.
+    ///
+    /// PLEASE NOTE: This has the effect of baking whatever PRNG seed is generated at build time into the
+    /// component, meaning Python's `random` module will return the exact same sequence each time the component is
+    /// run.  Do *not* use this option in situations where a secure source of randomness is required.
+    #[arg(short = 's', long)]
+    pub stub_wasi: bool,
 }
 
 #[derive(clap::Args, Debug)]
@@ -150,6 +158,7 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> {
         &componentize.output,
         None,
         common.isyswasfa.as_deref(),
+        componentize.stub_wasi,
     ))?;
 
     if !common.quiet {