wit-component: implement a tool to create WebAssembly components. (#183)

* wit-component: initial implementation.

This commit provides the initial implementation for the `wit-component` tool.

The tool is responsible for converting a core module implementing the canonical
ABI into a component based on the WebAssembly component model proposal.

Users provide the tool with the interfaces to be exported or imported by the
component in `wit` format. The output of the tool is a component file.

The tool validates that the core module is satisfied with the given imports and
also exports the provided interfaces.

The output component is also verified with the validator from `wasmparser`.

* wit-component: add more tests.

This commit adds a test to encode an interface, decode the bytes using
`wasmprinter`, and then compare the output with a baseline wat file.

It also adds a `simple` test case for encoding components where a component
exports a single default interface.

* wit-component: add more component tests.

This commit adds more tests to `wit-component` for component encoding.

* wit-component: add error handling test cases.

This commit adds several test cases for validation of the module that will be
encoded as a component.

* wit-component: be smarter about canonical option encoding.

This commit only adds an encoding option if the function signatures contain
strings.

It also only adds the `into` option only if the function signature requires it.

Added tests for lifting and lowering of various function signatures to ensure
the canonical options emitted are correct.

* wit-component: update and better doc test cases.

This commit updates the documentation for the tests in `wit-component` and
restructures the interface encoding tests directory layout to match that of the
component encoding tests.

* wit-component: update test baseline for new wasmprinter.

This commit updates wasmprinter and the test baselines now that it correctly
prints outer aliases.

* wit-component: update assert_eq! for better diff printing.

The `pretty_assertions` crate prints a nice diff for `assert_eq!` failures, but
the tests were reversing left and right to make it more confusing when
conceptually seeing changes from the baseline.

This commit swaps the two arguments to `assert_eq!` so that left is the
baseline and right is the output of the test.

* wit-component: give names to one of the test core modules.

This commit adds some names to one of the test module's canonical definitions.

This helps to test that the names survive encoding and are also present when
printed.

* wit-component: add an import and export test case.

This commit adds a test to cover both importing and exporting interfaces from
the same component.

Fixes a bug where importing functions wasn't contributing to the function type
list.

* wit-component: use type information from wasmparser for validation.

This commit uses the type information returned from wasmparser for the
validation of the core module.

It also reduces the complexity of parsing the core module by using methods
provided by wasmparser.

* wit-component: code review feedback changes.

Various code review feedback changes.

* wit-component: be smarter about import lowerings.

This commit changes how imports are lowered.

Previously, the tool used indirect lowerings for all imports, even if the
import didn't need the `into` option. Now such lowerings are lowered before the
inner core module is instantiated and directly given as an argument.

Additionally, if there are indirect lowerings to perform, at most two
additional modules are encoded: a module storing a table for only the indirect
lowerings and another that initializes the table once the core module has been
instantiated.

crates/parser/src/ast/lex.rs

@@ -87,6 +87,7 @@ pub enum Token {
 }
 
 #[derive(Eq, PartialEq, Debug)]
+#[allow(dead_code)]
 pub enum Error {
     InvalidCharInString(usize, char),
     InvalidCharInId(usize, char),

crates/wit-component/Cargo.toml

@@ -4,6 +4,11 @@ version = "0.1.0"
 authors = ["Peter Huene <peter@huene.dev>"]
 edition = "2021"
 
+[[bin]]
+name = "wit-component"
+path = "src/bin/wit-component.rs"
+required-features = ["cli"]
+
 [[bin]]
 name = "wit2wasm"
 path = "src/bin/wit2wasm.rs"
@@ -20,10 +25,16 @@ wasm-encoder = { git = "https://github.com/bytecodealliance/wasm-tools" }
 wat = { git = "https://github.com/bytecodealliance/wasm-tools" }
 wit-parser = { path = "../parser" }
 anyhow = "1.0.55"
+indexmap = "1.8.0"
 clap = { version = "3.1.0", features = ["derive"], optional = true }
 env_logger = { version = "0.9.0", optional = true }
 log = { version = "0.4.14", optional = true }
 
+[dev-dependencies]
+wasmprinter = { git = "https://github.com/bytecodealliance/wasm-tools" }
+glob = "0.3.0"
+pretty_assertions = "1.2.0"
+
 [features]
 default = ["cli"]
 cli = ["clap", "env_logger", "log"]

crates/wit-component/README.md

@@ -19,6 +19,9 @@
 
 ## Tools
 
+* `wit-component` - creates a WebAssembly component from a core WebAssembly module and a set of
+  `.wit` files representing the component's imported and exported interfaces.
+
 * `wit2wasm` - encodes an interface definition (in `wit`) as an "interface-only" WebAssembly component.
   A `.wasm` component file will be generated that stores a full description of the original interface.
 

crates/wit-component/src/bin/wit-component.rs

@@ -0,0 +1,13 @@
+use clap::Parser;
+use wit_component::cli::WitComponentApp;
+
+fn main() {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
+        .format_target(false)
+        .init();
+
+    if let Err(e) = WitComponentApp::parse().execute() {
+        log::error!("{:?}", e);
+        std::process::exit(1);
+    }
+}

crates/wit-component/src/cli.rs

@@ -2,21 +2,122 @@
 
 #![deny(missing_docs)]
 
-use crate::{decode_interface_component, encode_interface_component, InterfacePrinter};
+use crate::{
+    decode_interface_component, encoding::ComponentEncoder, InterfacePrinter, StringEncoding,
+};
 use anyhow::{bail, Context, Result};
 use clap::Parser;
 use std::path::{Path, PathBuf};
 use wit_parser::Interface;
 
-fn read_interface(path: impl AsRef<Path>) -> Result<Interface> {
-    let path = path.as_ref();
+fn parse_named_interface(s: &str) -> Result<Interface> {
+    let (name, path) = s
+        .split_once('=')
+        .ok_or_else(|| anyhow::anyhow!("expected a value with format `NAME=INTERFACE`"))?;
 
+    parse_interface(Some(name.to_string()), Path::new(path))
+}
+
+fn parse_unnamed_interface(s: &str) -> Result<Interface> {
+    parse_interface(None, Path::new(s))
+}
+
+fn parse_interface(name: Option<String>, path: &Path) -> Result<Interface> {
     if !path.is_file() {
         bail!("interface file `{}` does not exist", path.display(),);
     }
 
-    Interface::parse_file(&path)
-        .with_context(|| format!("failed to parse interface file `{}`", path.display()))
+    let mut interface = Interface::parse_file(&path)
+        .with_context(|| format!("failed to parse interface file `{}`", path.display()))?;
+
+    interface.name = name.unwrap_or_else(|| "".to_string());
+
+    Ok(interface)
+}
+
+/// WebAssembly component encoder.
+///
+/// Encodes a WebAssembly component from a core WebAssembly module.
+#[derive(Debug, Parser)]
+#[clap(name = "component-encoder", version = env!("CARGO_PKG_VERSION"))]
+pub struct WitComponentApp {
+    /// The path to an interface definition file the component imports.
+    #[clap(long = "import", value_name = "NAME=INTERFACE", parse(try_from_str = parse_named_interface))]
+    pub imports: Vec<Interface>,
+
+    /// The path to an interface definition file the component exports.
+    #[clap(long = "export", value_name = "NAME=INTERFACE", parse(try_from_str = parse_named_interface))]
+    pub exports: Vec<Interface>,
+
+    /// The path of the output WebAssembly component.
+    #[clap(long, short = 'o', value_name = "OUTPUT")]
+    pub output: Option<PathBuf>,
+
+    /// The default interface of the component.
+    #[clap(long, short = 'i', value_name = "INTERFACE", parse(try_from_str = parse_unnamed_interface))]
+    pub interface: Option<Interface>,
+
+    /// Skip validation of the output component.
+    #[clap(long)]
+    pub skip_validation: bool,
+
+    /// The expected string encoding format for the component.
+    /// Supported values are: `utf8` (default), `utf16`, and `compact-utf16`.
+    #[clap(long, value_name = "ENCODING")]
+    pub encoding: Option<StringEncoding>,
+
+    /// Path to the WebAssembly module to encode.
+    #[clap(index = 1, value_name = "MODULE")]
+    pub module: PathBuf,
+}
+
+impl WitComponentApp {
+    /// Executes the application.
+    pub fn execute(self) -> Result<()> {
+        if !self.module.is_file() {
+            bail!(
+                "module `{}` does not exist as a file",
+                self.module.display()
+            );
+        }
+
+        let output = self.output.unwrap_or_else(|| {
+            let mut stem: PathBuf = self.module.file_stem().unwrap().into();
+            stem.set_extension("wasm");
+            stem
+        });
+
+        let module = wat::parse_file(&self.module)
+            .with_context(|| format!("failed to parse module `{}`", self.module.display()))?;
+
+        let mut encoder = ComponentEncoder::default()
+            .module(&module)
+            .imports(&self.imports)
+            .exports(&self.exports)
+            .validate(!self.skip_validation);
+
+        if let Some(interface) = &self.interface {
+            encoder = encoder.interface(interface);
+        }
+
+        if let Some(encoding) = &self.encoding {
+            encoder = encoder.encoding(*encoding);
+        }
+
+        let bytes = encoder.encode().with_context(|| {
+            format!(
+                "failed to encode a component from module `{}`",
+                self.module.display()
+            )
+        })?;
+
+        std::fs::write(&output, bytes)
+            .with_context(|| format!("failed to write output file `{}`", output.display()))?;
+
+        println!("encoded component `{}`", output.display());
+
+        Ok(())
+    }
 }
 
 /// WebAssembly interface encoder.
@@ -26,11 +127,11 @@ fn read_interface(path: impl AsRef<Path>) -> Result<Interface> {
 #[clap(name = "wit2wasm", version = env!("CARGO_PKG_VERSION"))]
 pub struct WitToWasmApp {
     /// The path of the output WebAssembly component.
-    #[clap(long, short = 'o', value_name = "OUTPUT", parse(from_os_str))]
+    #[clap(long, short = 'o', value_name = "OUTPUT")]
     pub output: Option<PathBuf>,
 
     /// The path to the WebAssembly interface file to encode.
-    #[clap(index = 1, value_name = "INTERFACE", parse(from_os_str))]
+    #[clap(index = 1, value_name = "INTERFACE")]
     pub interface: PathBuf,
 }
 
@@ -43,8 +144,18 @@ impl WitToWasmApp {
             stem
         });
 
-        let interface = read_interface(self.interface)?;
-        let bytes = encode_interface_component(&interface)?;
+        let interface = parse_interface(None, &self.interface)?;
+
+        let encoder = ComponentEncoder::default()
+            .interface(&interface)
+            .types_only(true);
+
+        let bytes = encoder.encode().with_context(|| {
+            format!(
+                "failed to encode a component from interface `{}`",
+                self.interface.display()
+            )
+        })?;
 
         std::fs::write(&output, bytes)
             .with_context(|| format!("failed to write output file `{}`", output.display()))?;
@@ -62,11 +173,11 @@ impl WitToWasmApp {
 #[clap(name = "wit2wasm", version = env!("CARGO_PKG_VERSION"))]
 pub struct WasmToWitApp {
     /// The path of the output WebAssembly interface file.
-    #[clap(long, short = 'o', value_name = "OUTPUT", parse(from_os_str))]
+    #[clap(long, short = 'o', value_name = "OUTPUT")]
     pub output: Option<PathBuf>,
 
     /// The path to the WebAssembly component to decode.
-    #[clap(index = 1, value_name = "COMPONENT", parse(from_os_str))]
+    #[clap(index = 1, value_name = "COMPONENT")]
     pub component: PathBuf,
 }
 

crates/wit-component/src/decoding.rs

@@ -24,14 +24,12 @@ impl<'a> ComponentInfo<'a> {
     pub fn new(mut bytes: &'a [u8]) -> Result<Self> {
         let mut parser = Parser::new(0);
         let mut parsers = Vec::new();
-        let mut validator = Validator::new();
-        let mut exported_types = Vec::new();
-        let mut exported_functions = Vec::new();
-
-        validator.wasm_features(WasmFeatures {
+        let mut validator = Validator::new_with_features(WasmFeatures {
             component_model: true,
             ..Default::default()
         });
+        let mut exported_types = Vec::new();
+        let mut exported_functions = Vec::new();
 
         loop {
             match parser.parse(bytes, true)? {