various updates to support wasi:http@0.2.0-rc-2023-10-18

This fixes various issues:

- Broken generated code indentation for resources in some cases
- Type annotations that refer to non-yet-declared types confuse CPython, so we disable them
  - However, MyPy has no trouble with them, so we enable them by default for the `bindings` subcommand
- Support WIT version annotations (i.e. pass them through to the generated component)
  - This partially addresses #19, but doesn't support importing or exporting multiple versions of the same interface
- Update the `http` example to match `wasi:http@0.2.0-rc-2023-10-18`
- Update to Wasmtime 14 and the latest `wit-parser`, `wit-component`, etc.
  - and update the WASI preview 1 adapter to match

This also bumps the version to 0.6.0.

Note that I've had to remove the `matrix-math` example since `wasmtime-py` does
not yet support resources.  Although the example itself doesn't use them, the
new WASI Preview 1 adapter pulls them in as WASI Preview 2 imports, and there's
no feasible way to work around that.  Ideally, we'd provide the option to allow
users to supply their own adapter, in which case we could use a pre-resource
version of the adapter.  However, that won't work given that pre-initialization
is central to how `componentize-py` works.  Hopefully we can bring back this
example in the future, e.g. when `wasmtime-py` adds support for resources.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 exclude = ["cpython"]
 
@@ -17,17 +17,15 @@ zstd = "0.11.1"
 componentize-py-shared = { path = "shared" }
 wasmparser = "0.107.0"
 wasm-encoder = "0.29.0"
-# TODO: switch to release once https://github.com/bytecodealliance/wasm-tools/pull/1226 is merged and released:
-wit-parser = { git = "https://github.com/dicej/wasm-tools", branch = "adapter-export-resources" }
-wit-component = { git = "https://github.com/dicej/wasm-tools", branch = "adapter-export-resources" }
+wit-parser = "0.12.2"
+wit-component = "0.17.0"
 indexmap = "2.0.0"
 bincode = "1.3.3"
 heck = "0.4.1"
 pyo3 = { version = "0.18.3", features = ["abi3-py37", "extension-module"], optional = true }
-# TODO: switch to Wasmtime 14 when released:
-wasmtime-wasi = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966" }
-wasi-common = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966" }
-wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966", features = [ "component-model" ] }
+wasmtime-wasi = "14.0.3"
+wasi-common = "14.0.3"
+wasmtime = { version = "14.0.3", features = [ "component-model" ] }
 once_cell = "1.17.1"
 component-init = { git = "https://github.com/dicej/component-init" }
 async-trait = "0.1.68"

adapters/README.md

@@ -0,0 +1,8 @@
+The subdirectory of this directory contains a build of the WASI Preview 1
+component adapter.  It was built from commit `e8766e49` of
+https://github.com/dicej/wasmtime using the
+`ci/build-wasi-preview1-component-adapter.sh` script.
+
+TODO: Switch back to upstream once
+https://github.com/bytecodealliance/wasmtime/pull/7444 has been merged and
+released.

build.rs

@@ -185,7 +185,7 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> {
         bail!("no such directory: {}", path.display())
     }
     compress(
-        &repo_dir.join("adapters/40c1f9b8"),
+        &repo_dir.join("adapters/e8766e49"),
         "wasi_snapshot_preview1.reactor.wasm",
         out_dir,
         false,

examples/http/README.md

@@ -1,24 +1,27 @@
 # Example: `http`
 
-This is an example of how to use [componentize-py] and [Spin] to build and run a
-Python-based component targetting the [wasi-http] `proxy` world.
+This is an example of how to use [componentize-py] and [Wasmtime] to build and
+run a Python-based component targetting the [wasi-http] `proxy` world.
 
 Note that, as of this writing, neither `wasi-http` nor the portions of
 `wasi-cli` on which it is based have stabilized.  Here we use a snapshot of both,
 which may differ from later revisions.
 
 [componentize-py]: https://github.com/bytecodealliance/componentize-py
-[Spin]: https://github.com/fermyon/spin
+[Wasmtime]: https://github.com/bytecodealliance/wasmtime
 [wasi-http]: https://github.com/WebAssembly/wasi-http
 
 ## Prerequisites
 
-* `dicej/spin` branch `wasi-http-wasmtime-2ad057d7`
-* `componentize-py` 0.5.0
-* `Rust`, for installing `Spin`
+* `Wasmtime` 14.0.3 (later versions may use a different, incompatible `wasi-http` snapshot)
+* `componentize-py` 0.6.0
+
+Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
+you don't have `cargo`, you can download and install from
+https://github.com/bytecodealliance/wasmtime/releases/tag/v14.0.3.
 
 ```
-cargo install --locked --git https://github.com/dicej/spin --branch wasi-http-wasmtime-2ad057d7 spin-cli
+cargo install --version 14.0.3 wasmtime-cli
 pip install componentize-py
 ```
 
@@ -27,13 +30,14 @@ pip install componentize-py
 First, build the app and run it:
 
 ```
-spin build --up
+componentize-py -d wit -w proxy componentize app -o http.wasm
+wasmtime serve http.wasm
 ```
 
 Then, in another terminal, use cURL to send a request to the app:
 
 ```
-curl -i -H 'content-type: text/plain' --data-binary @- http://127.0.0.1:3000/echo <<EOF
+curl -i -H 'content-type: text/plain' --data-binary @- http://127.0.0.1:8080/echo <<EOF
 ’Twas brillig, and the slithy toves
       Did gyre and gimble in the wabe:
 All mimsy were the borogoves,
@@ -52,7 +56,7 @@ curl -i \
     -H 'url: https://webassembly.github.io/spec/core/' \
     -H 'url: https://www.w3.org/groups/wg/wasm/' \
     -H 'url: https://bytecodealliance.org/' \
-    http://127.0.0.1:3000/hash-all
+    http://127.0.0.1:8080/hash-all
 ```
 
 If you run into any problems, please file an issue!

examples/http/app.py

@@ -5,31 +5,25 @@
 to thread I/O through coroutines.
 """
 
-# Note that we've temporarily renamed various `wasi-http` and `wasi-cli`
-# interfaces (appending a 2 to each name) to avoid conflicting with the
-# implementations in `wasmtime-wasi` which are still under development.
-#
-# Also note that `wasi-http` currently uses pseudo-resources (represented as
-# integers) to model requests, responses, etc.  As of this writing, proper WIT
-# resource support is still under development; we'll update this example to use
-# them once they're ready.
-
 import asyncio
 import hashlib
 import poll_loop
 
 from proxy import exports
 from proxy.types import Ok
-from proxy.imports import types2 as types, outgoing_handler2 as outgoing_handler
-from proxy.imports.types2 import MethodGet, MethodPost, Scheme, SchemeHttp, SchemeHttps, SchemeOther
+from proxy.imports import types
+from proxy.imports.types import (
+    MethodGet, MethodPost, Scheme, SchemeHttp, SchemeHttps, SchemeOther, IncomingRequest, ResponseOutparam,
+    OutgoingResponse, Fields, OutgoingBody, OutgoingRequest
+)
 from poll_loop import Stream, Sink, PollLoop
 from typing import Tuple, cast
 from urllib import parse
 
-class IncomingHandler2(exports.IncomingHandler2):
+class IncomingHandler(exports.IncomingHandler):
     """Implements the `export`ed portion of the `wasi-http` `proxy` world."""
 
-    def handle(self, request: int, response_out: int):
+    def handle(self, request: IncomingRequest, response_out: ResponseOutparam):
         """Handle the specified `request` (represented as a pseudo-resource), sending
         the response to `response_out`.
         """
@@ -39,13 +33,13 @@ def handle(self, request: int, response_out: int):
         asyncio.set_event_loop(loop)
         loop.run_until_complete(handle_async(request, response_out))
 
-async def handle_async(request: int, response_out: int):
+async def handle_async(request: IncomingRequest, response_out: ResponseOutparam):
     """Handle the specified `request` (represented as a pseudo-resource), sending
     the response to `response_out`."""
 
-    method = types.incoming_request_method(request)
-    path = types.incoming_request_path_with_query(request)
-    headers = types.fields_entries(types.incoming_request_headers(request))
+    method = request.method()
+    path = request.path_with_query()
+    headers = request.headers().entries()
 
     if isinstance(method, MethodGet) and path == "/hash-all":
         # Collect one or more "url" headers, download their contents
@@ -55,29 +49,33 @@ async def handle_async(request: int, response_out: int):
 
         urls = map(lambda pair: str(pair[1], "utf-8"), filter(lambda pair: pair[0] == "url", headers))
 
-        response = types.new_outgoing_response(200, types.new_fields([("content-type", b"text/plain")]))
+        response = OutgoingResponse(200, Fields([("content-type", b"text/plain")]))
 
-        types.set_response_outparam(response_out, Ok(response))
+        response_body = response.write()
 
-        sink = Sink(types.outgoing_response_write(response))
-
+        ResponseOutparam.set(response_out, Ok(response))
+
+        sink = Sink(response_body)
         for result in asyncio.as_completed(map(sha256, urls)):
             url, sha = await result
             await sink.send(bytes(f"{url}: {sha}\n", "utf-8"))
 
         sink.close()
+
     elif isinstance(method, MethodPost) and path == "/echo":
         # Echo the request body back to the client without buffering.
-        
-        response = types.new_outgoing_response(
+
+        response = OutgoingResponse(
             200,
-            types.new_fields(list(filter(lambda pair: pair[0] == "content-type", headers)))
+            Fields(list(filter(lambda pair: pair[0] == "content-type", headers)))
         )
-        types.set_response_outparam(response_out, Ok(response))
 
-        stream = Stream(types.incoming_request_consume(request))
-        sink = Sink(types.outgoing_response_write(response))
+        response_body = response.write()
+
+        ResponseOutparam.set(response_out, Ok(response))
 
+        stream = Stream(request.consume())
+        sink = Sink(response_body)
         while True:
             chunk = await stream.next()
             if chunk is None:
@@ -87,9 +85,9 @@ async def handle_async(request: int, response_out: int):
 
         sink.close()
     else:
-        response = types.new_outgoing_response(400, types.new_fields([]))
-        types.set_response_outparam(response_out, Ok(response))
-        types.finish_outgoing_stream(types.outgoing_response_write(response))
+        response = OutgoingResponse(400, Fields([]))
+        ResponseOutparam.set(response_out, Ok(response))
+        OutgoingBody.finish(response.write(), None)
 
 async def sha256(url: str) -> Tuple[str, str]:
     """Download the contents of the specified URL, computing the SHA-256
@@ -109,43 +107,24 @@ async def sha256(url: str) -> Tuple[str, str]:
         case _:
             scheme = SchemeOther(url_parsed.scheme)
 
-    request = types.new_outgoing_request(
+    request = OutgoingRequest(
         MethodGet(),
         url_parsed.path,
         scheme,
         url_parsed.netloc,
-        types.new_fields([])
+        Fields([])
     )
 
-    response = await outgoing_request_send(request)
-
-    status = types.incoming_response_status(response)
+    response = await poll_loop.send(request)
+    status = response.status()
     if status < 200 or status > 299:
         return url, f"unexpected status: {status}"
 
-    stream = Stream(types.incoming_response_consume(response))
-
+    stream = Stream(response.consume())
     hasher = hashlib.sha256()
     while True:
         chunk = await stream.next()
         if chunk is None:
             return url, hasher.hexdigest()
         else:
             hasher.update(chunk)
-
-async def outgoing_request_send(request: int) -> int:
-    """Send the specified request and wait asynchronously for the response."""
-
-    future = outgoing_handler.handle(request, None)
-    pollable = types.listen_to_future_incoming_response(future)
-
-    while True:
-        response = types.future_incoming_response_get(future)
-        if response is None:
-            await poll_loop.register(cast(PollLoop, asyncio.get_event_loop()), pollable)
-        else:
-            if isinstance(response, Ok):
-                return response.value
-            else:
-                raise response
-