Auto merge of #47102 - Diggsey:wasm-syscall, r=alexcrichton

Implement extensible syscall interface for wasm

Currently it's possible to run tests with the native wasm target, but it's not possible to tell whether they pass or to capture the output, because libstd throws away stdout, stderr and the exit code. While advanced libstd features should probably require more specific targets (eg. wasm-unknown-web) I think even the unknown target should at least support basic I/O.

Any solution is constrained by these factors:
- It must not be javascript specific
- There must not be too strong coupling between libstd and the host environment (because it's an "unknown" target)
- WebAssembly does not allow "optional" imports - all imports *must* be resolved.
- WebAssembly does not support calling the host environment through any channel *other* than imports.

The best solution I could find to these constraints was to give libstd a single required import, and implement a syscall-style interface through that import. Each syscall is designed such that a no-op implementation gives the most reasonable fallback behaviour. This means that the following import table would be perfectly valid:
```javascript
imports.env = { rust_wasm_syscall: function(index, data) {} }
```

Currently I have implemented these system calls:
- Read from stdin
- Write to stdout/stderr
- Set the exit code
- Get command line arguments
- Get environment variable
- Set environment variable
- Get time

It need not be extended beyond this set if being able to run tests for this target is the only goal.

edit:
As part of this PR I had to make a further change. Previously, the rust entry point would be automatically called when the webassembly module was instantiated. This was problematic because from the javascript side it was impossible to call exported functions, access program memory or get a reference to the instance.

To solve this, ~I changed the default behaviour to not automatically call the entry point, and added a crate-level attribute to regain the old behaviour. (`#![wasm_auto_run]`)~ I disabled this behaviour when building tests.

config.toml.example

@@ -312,6 +312,11 @@
 # bootstrap)
 #codegen-backends = ["llvm"]
 
+# Flag indicating whether `libstd` calls an imported function to hande basic IO
+# when targetting WebAssembly. Enable this to debug tests for the `wasm32-unknown-unknown`
+# target, as without this option the test output will not be captured.
+#wasm-syscall = false
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/config.rs

@@ -107,6 +107,7 @@ pub struct Config {
     pub debug_jemalloc: bool,
     pub use_jemalloc: bool,
     pub backtrace: bool, // support for RUST_BACKTRACE
+    pub wasm_syscall: bool,
 
     // misc
     pub low_priority: bool,
@@ -282,6 +283,7 @@ struct Rust {
     test_miri: Option<bool>,
     save_toolstates: Option<String>,
     codegen_backends: Option<Vec<String>>,
+    wasm_syscall: Option<bool>,
 }
 
 /// TOML representation of how each build target is configured.
@@ -463,6 +465,7 @@ impl Config {
             set(&mut config.rust_dist_src, rust.dist_src);
             set(&mut config.quiet_tests, rust.quiet_tests);
             set(&mut config.test_miri, rust.test_miri);
+            set(&mut config.wasm_syscall, rust.wasm_syscall);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);

src/bootstrap/lib.rs

@@ -423,6 +423,9 @@ impl Build {
         if self.config.profiler {
             features.push_str(" profiler");
         }
+        if self.config.wasm_syscall {
+            features.push_str(" wasm_syscall");
+        }
         features
     }
 

src/bootstrap/test.rs

@@ -1286,6 +1286,14 @@ impl Step for Crate {
             cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
                       build.config.nodejs.as_ref().expect("nodejs not configured"));
         } else if target.starts_with("wasm32") {
+            // Warn about running tests without the `wasm_syscall` feature enabled.
+            // The javascript shim implements the syscall interface so that test
+            // output can be correctly reported.
+            if !build.config.wasm_syscall {
+                println!("Libstd was built without `wasm_syscall` feature enabled: \
+                          test output may not be visible.");
+            }
+
             // On the wasm32-unknown-unknown target we're using LTO which is
             // incompatible with `-C prefer-dynamic`, so disable that here
             cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");

src/etc/wasm32-shim.js

@@ -28,14 +28,76 @@ let m = new WebAssembly.Module(buffer);
 
 let memory = null;
 
+function viewstruct(data, fields) {
+  return new Uint32Array(memory.buffer).subarray(data/4, data/4 + fields);
+}
+
 function copystr(a, b) {
-  if (memory === null) {
-    return null
-  }
-  let view = new Uint8Array(memory.buffer).slice(a, a + b);
+  let view = new Uint8Array(memory.buffer).subarray(a, a + b);
   return String.fromCharCode.apply(null, view);
 }
 
+function syscall_write([fd, ptr, len]) {
+  let s = copystr(ptr, len);
+  switch (fd) {
+    case 1: process.stdout.write(s); break;
+    case 2: process.stderr.write(s); break;
+  }
+}
+
+function syscall_exit([code]) {
+  process.exit(code);
+}
+
+function syscall_args(params) {
+  let [ptr, len] = params;
+
+  // Calculate total required buffer size
+  let totalLen = -1;
+  for (let i = 2; i < process.argv.length; ++i) {
+    totalLen += Buffer.byteLength(process.argv[i]) + 1;
+  }
+  if (totalLen < 0) { totalLen = 0; }
+  params[2] = totalLen;
+
+  // If buffer is large enough, copy data
+  if (len >= totalLen) {
+    let view = new Uint8Array(memory.buffer);
+    for (let i = 2; i < process.argv.length; ++i) {
+      let value = process.argv[i];
+      Buffer.from(value).copy(view, ptr);
+      ptr += Buffer.byteLength(process.argv[i]) + 1;
+    }
+  }
+}
+
+function syscall_getenv(params) {
+  let [keyPtr, keyLen, valuePtr, valueLen] = params;
+
+  let key = copystr(keyPtr, keyLen);
+  let value = process.env[key];
+
+  if (value == null) {
+    params[4] = 0xFFFFFFFF;
+  } else {
+    let view = new Uint8Array(memory.buffer);
+    let totalLen = Buffer.byteLength(value);
+    params[4] = totalLen;
+    if (valueLen >= totalLen) {
+      Buffer.from(value).copy(view, valuePtr);
+    }
+  }
+}
+
+function syscall_time(params) {
+  let t = Date.now();
+  let secs = Math.floor(t / 1000);
+  let millis = t % 1000;
+  params[1] = Math.floor(secs / 0x100000000);
+  params[2] = secs % 0x100000000;
+  params[3] = Math.floor(millis * 1000000);
+}
+
 let imports = {};
 imports.env = {
   // These are generated by LLVM itself for various intrinsic calls. Hopefully
@@ -48,68 +110,25 @@ imports.env = {
   log10: Math.log10,
   log10f: Math.log10,
 
-  // These are called in src/libstd/sys/wasm/stdio.rs and are used when
-  // debugging is enabled.
-  rust_wasm_write_stdout: function(a, b) {
-    let s = copystr(a, b);
-    if (s !== null) {
-      process.stdout.write(s);
-    }
-  },
-  rust_wasm_write_stderr: function(a, b) {
-    let s = copystr(a, b);
-    if (s !== null) {
-      process.stderr.write(s);
-    }
-  },
-
-  // These are called in src/libstd/sys/wasm/args.rs and are used when
-  // debugging is enabled.
-  rust_wasm_args_count: function() {
-    if (memory === null)
-      return 0;
-    return process.argv.length - 2;
-  },
-  rust_wasm_args_arg_size: function(i) {
-    return Buffer.byteLength(process.argv[i + 2]);
-  },
-  rust_wasm_args_arg_fill: function(idx, ptr) {
-    let arg = process.argv[idx + 2];
-    let view = new Uint8Array(memory.buffer);
-    Buffer.from(arg).copy(view, ptr);
-  },
-
-  // These are called in src/libstd/sys/wasm/os.rs and are used when
-  // debugging is enabled.
-  rust_wasm_getenv_len: function(a, b) {
-    let key = copystr(a, b);
-    if (key === null) {
-      return -1;
+  rust_wasm_syscall: function(index, data) {
+    switch (index) {
+      case 1: syscall_write(viewstruct(data, 3)); return true;
+      case 2: syscall_exit(viewstruct(data, 1)); return true;
+      case 3: syscall_args(viewstruct(data, 3)); return true;
+      case 4: syscall_getenv(viewstruct(data, 5)); return true;
+      case 6: syscall_time(viewstruct(data, 4)); return true;
+      default:
+        console.log("Unsupported syscall: " + index);
+        return false;
     }
-    if (!(key in process.env)) {
-      return -1;
-    }
-    return Buffer.byteLength(process.env[key]);
-  },
-  rust_wasm_getenv_data: function(a, b, ptr) {
-    let key = copystr(a, b);
-    let value = process.env[key];
-    let view = new Uint8Array(memory.buffer);
-    Buffer.from(value).copy(view, ptr);
-  },
-};
-
-let module_imports = WebAssembly.Module.imports(m);
-
-for (var i = 0; i < module_imports.length; i++) {
-  let imp = module_imports[i];
-  if (imp.module != 'env') {
-    continue
   }
-  if (imp.name == 'memory' && imp.kind == 'memory') {
-    memory = new WebAssembly.Memory({initial: 20});
-    imports.env.memory = memory;
-  }
-}
+};
 
 let instance = new WebAssembly.Instance(m, imports);
+memory = instance.exports.memory;
+try {
+  instance.exports.main();
+} catch (e) {
+  console.error(e);
+  process.exit(101);
+}

src/librustc_trans/back/write.rs

@@ -824,9 +824,7 @@ fn binaryen_assemble(cgcx: &CodegenContext,
     if cgcx.debuginfo != config::NoDebugInfo {
         options.debuginfo(true);
     }
-    if cgcx.crate_types.contains(&config::CrateTypeExecutable) {
-        options.start("main");
-    }
+
     options.stack(1024 * 1024);
     options.import_memory(cgcx.wasm_import_memory);
     let assembled = input.and_then(|input| {
@@ -1452,7 +1450,7 @@ fn start_executing_work(tcx: TyCtxt,
         target_pointer_width: tcx.sess.target.target.target_pointer_width.clone(),
         binaryen_linker: tcx.sess.linker_flavor() == LinkerFlavor::Binaryen,
         debuginfo: tcx.sess.opts.debuginfo,
-        wasm_import_memory: wasm_import_memory,
+        wasm_import_memory,
         assembler_cmd,
     };
 

src/libstd/Cargo.toml

@@ -48,3 +48,4 @@ jemalloc = ["alloc_jemalloc"]
 force_alloc_system = []
 panic-unwind = ["panic_unwind"]
 profiler = ["profiler_builtins"]
+wasm_syscall = []

src/libstd/sys/wasm/args.rs

@@ -10,8 +10,8 @@
 
 use ffi::OsString;
 use marker::PhantomData;
-use mem;
 use vec;
+use sys::ArgsSysCall;
 
 pub unsafe fn init(_argc: isize, _argv: *const *const u8) {
     // On wasm these should always be null, so there's nothing for us to do here
@@ -21,38 +21,10 @@ pub unsafe fn cleanup() {
 }
 
 pub fn args() -> Args {
-    // When the runtime debugging is enabled we'll link to some extra runtime
-    // functions to actually implement this. These are for now just implemented
-    // in a node.js script but they're off by default as they're sort of weird
-    // in a web-wasm world.
-    if !super::DEBUG {
-        return Args {
-            iter: Vec::new().into_iter(),
-            _dont_send_or_sync_me: PhantomData,
-        }
-    }
-
-    // You'll find the definitions of these in `src/etc/wasm32-shim.js`. These
-    // are just meant for debugging and should not be relied on.
-    extern {
-        fn rust_wasm_args_count() -> usize;
-        fn rust_wasm_args_arg_size(a: usize) -> usize;
-        fn rust_wasm_args_arg_fill(a: usize, ptr: *mut u8);
-    }
-
-    unsafe {
-        let cnt = rust_wasm_args_count();
-        let mut v = Vec::with_capacity(cnt);
-        for i in 0..cnt {
-            let n = rust_wasm_args_arg_size(i);
-            let mut data = vec![0; n];
-            rust_wasm_args_arg_fill(i, data.as_mut_ptr());
-            v.push(mem::transmute::<Vec<u8>, OsString>(data));
-        }
-        Args {
-            iter: v.into_iter(),
-            _dont_send_or_sync_me: PhantomData,
-        }
+    let v = ArgsSysCall::perform();
+    Args {
+        iter: v.into_iter(),
+        _dont_send_or_sync_me: PhantomData,
     }
 }