Make JS harness optional and compatible with async

[rossberg]

This allows for use of the JS-converted files with alternative test harnesses.

[bnjbvr]

convesion => conversion

[bnjbvr]

LGTM.

Could there a way to also wrap up all the function calls with arbitrary JS, or some predefined string? For instance, transforming instance(byteArray); function calls into test(() => instance(byteArray), "instance call") would make the whole web harness more resilient to errors.

[rossberg]

Hm, what prevents the instance function you provide from doing such wrapping itself? That way it's factored out cleanly into the harness with no complicated interplay.

[bnjbvr]

It's because the harness test function is asynchronous, and we want to be able to instantiate and make use of the returned instance.

For example: https://github.com/bnjbvr/wasm-spec/blob/testing/test/out/js/call.wast.js#L7 one error could occur in the instance() function if the VM can't compile it for some reason, and then the getprop/callprop on the exports object will make the test harness itself fail. If we could wrap the whole line with a test() call, we wouldn't make the test harness itself fail:

That is, I'd like to turn this:

instance("\x00\x61\x73\x6d\x0d\x00\x00\x00\x01\x88\x80\x80\x80\x00\x02\x60\x00\x00\x60\x00\x01\x7e\x02\x8f\x80\x80\x80\x00\x01\x02\x24\x24\x08\x74\x79\x70\x65\x2d\x69\x36\x34\x00\x01\x03\x82\x80\x80\x80\x00\x01\x00\x07\x87\x80\x80\x80\x00\x01\x03\x72\x75\x6e\x00\x01\x0a\x98\x80\x80\x80\x00\x01\x92\x80\x80\x80\x00\x00\x02\x40\x10\x00\x01\x42\xe4\x02\x01\x51\x45\x0d\x00\x0f\x0b\x00\x0b", {$$: $$.exports}).exports.run(); // assert_return(() => $$.exports["type-i64"](), int64("356"))

into that:

test(() => instance("\x00\x61\x73\x6d\x0d\x00\x00\x00\x01\x88\x80\x80\x80\x00\x02\x60\x00\x00\x60\x00\x01\x7e\x02\x8f\x80\x80\x80\x00\x01\x02\x24\x24\x08\x74\x79\x70\x65\x2d\x69\x36\x34\x00\x01\x03\x82\x80\x80\x80\x00\x01\x00\x07\x87\x80\x80\x80\x00\x01\x03\x72\x75\x6e\x00\x01\x0a\x98\x80\x80\x80\x00\x01\x92\x80\x80\x80\x00\x00\x02\x40\x10\x00\x01\x42\xe4\x02\x01\x51\x45\x0d\x00\x0f\x0b\x00\x0b", {$$: $$.exports}).exports.run());

(out of curiosity, for this particular test: does this create a wasm instance that emulates the behavior of assert_return in wasm, since JS can't handle returned i64? otherwise, we'd need to think about how to test i64 and non-canonical NaNs.)

Alternatively, the compile() and instance() functions (from the spec test harness) could return Promises (or even have callbacks) and the generated JS code would use these.

[rossberg]

I see. Isn't the bigger problem that you would need to make the lets generated for module definitions async? AFAICS, no amount of extra function call around instance can make

let $M = instance(...)
assert_return(() => $M.exports["f"]())

work asynchronously. You would need to have an await on the RHS of the let -- which isn't allowed on the JS toplevel, though.

If only JS had real monads instead of its hacky async/promise mess. :)

The only way I can see this being solved in a generic manner is by not generating uses of exports directly but instead introducing one more harness abstraction, say call(module, name, args). The above would then become

let $M = instance(...)
assert_return(() => call($M, "f", []))

In an async harness, $M would be a promise and call an async function that first awaits it. The test function would not be needed, AFAICS. Does that make sense?

Yes, the auxiliary modules whose run export is called immediately are wrappers implementing i64 and float assertions. They implement the pseudo JS code shown in the trailing comment.

[bnjbvr]

Yes, this makes sense and would be great to have indeed. Thanks!

[rossberg]

I added harness abstractions for call, get, and exports. With that, I was able to run tests async with the following modified harness that uses W.{compile,instantiate} instead of the constructors (the main other change is that all harness function are now async and wrapped by the wrap function that observes async exceptions):

'use strict';

let soft_validate = true;

let spectest = {
  print: print || ((...xs) => console.log(...xs)),
  global: 666,
  table: new WebAssembly.Table({initial: 10, maximum: 20, element: 'anyfunc'}),  memory: new WebAssembly.Memory({initial: 1, maximum: 2}),};

let registry = {spectest};
let $$;

function wrap(f) {
  return (...args) => f(...args).catch(e => { print(e.stack); quit(1) })
}

let register = wrap(async function(name, instance) {
  registry[name] = (await instance).exports;
})

function binary(bytes) {
  let buffer = new ArrayBuffer(bytes.length);
  let view = new Uint8Array(buffer);
  for (let i = 0; i < bytes.length; ++i) {
    view[i] = bytes.charCodeAt(i);
  }
  return buffer;
}

function module(bytes, valid = true) {
  let buffer = binary(bytes);
  let validated;
  try {
    validated = WebAssembly.validate(buffer);
  } catch (e) {
    throw new Error("Wasm validate throws");
  }
  if (validated !== valid) {
    throw new Error("Wasm validate failure" + (valid ? "" : " expected"));
  }
  return WebAssembly.compile(buffer);
}

let instance = wrap(async function(bytes, imports = registry) {
  return WebAssembly.instantiate(binary(bytes), await imports);
})

let call = wrap(async function(instance, name, args) {
  return (await instance).exports[name](...args);
})

let get = wrap(async function(instance, name) {
  return (await instance).exports[name];
})

let exports = wrap(async function(name, instance) {
  return {[name]: (await instance).exports};
})

let assert_malformed = wrap(async function(bytes) {
  try { await (module(bytes, false)) } catch (e) {
    if (e instanceof WebAssembly.CompileError) return;
  }
  throw new Error("Wasm decoding failure expected");
})

let assert_invalid = wrap(async function(bytes) {
  try { await (module(bytes, false)) } catch (e) {
    if (e instanceof WebAssembly.CompileError) return;
  }
  throw new Error("Wasm validation failure expected");
})

let assert_soft_invalid = wrap(async function(bytes) {
  try { await (module(bytes, soft_validate)) } catch (e) {
    if (e instanceof WebAssembly.CompileError) return;
    throw new Error("Wasm validation failure expected");
  }
  if (soft_validate)
    throw new Error("Wasm validation failure expected");
})

let assert_unlinkable = wrap(function(bytes) {
  try { await (WebAssembly.instantiate(binary(bytes), registry)) } catch (e) {
    if (e instanceof WebAssembly.LinkError) return;
  }
  throw new Error("Wasm linking failure expected");
})

let assert_uninstantiable = wrap(async function(bytes) {
  try { await (WebAssembly.instantiate(binary(bytes), registry)) } catch (e) {
    if (e instanceof WebAssembly.RuntimeError) return;
  }
  throw new Error("Wasm trap expected");
})

let assert_trap = wrap(async function(action) {
  try { await (action()) } catch (e) {
    if (e instanceof WebAssembly.RuntimeError) return;
  }
  throw new Error("Wasm trap expected");
})

let StackOverflow;
try { (function f() { 1 + f() })() } catch (e) { StackOverflow = e.constructor }

let assert_exhaustion = wrap(async function(action) {
  try { await (action()) } catch (e) {
    if (e instanceof StackOverflow) return;
  }
  throw new Error("Wasm resource exhaustion expected");
})

let assert_return = wrap(async function(action, expected) {
  let actual = await (action());
  if (!Object.is(actual, expected)) {
    throw new Error("Wasm return value " + expected + " expected, got " + actual);
  };
})

let assert_return_nan = wrap(async function(action) {
  let actual = await action();
  if (!Number.isNaN(actual)) {
    throw new Error("Wasm return value NaN expected, got " + actual);
  };
})