From 19c05d9785c2ab7eafc25f4e204948b2394320dc Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 24 Aug 2021 22:31:57 -0700 Subject: [PATCH] Add cpp-to-wasm test (#968) * Add cpp-to-wasm test * Run wasm test on CI * Fix CI * Add host test to test-ffi * pin emsdk * Pin emscripten test to older nightly too --- .github/workflows/build-test.yml | 18 ++++- Cargo.lock | 12 ++-- Makefile.toml | 6 +- ffi/capi/Cargo.toml | 21 +++++- ffi/capi/src/crate_type_hack.rs | 10 +++ ffi/cpp/examples/fixeddecimal/Makefile | 2 +- ffi/cpp/examples/fixeddecimal/test.cpp | 1 - ffi/cpp/examples/fixeddecimal_wasm/.gitignore | 8 +++ ffi/cpp/examples/fixeddecimal_wasm/Makefile | 53 +++++++++++++++ ffi/cpp/examples/fixeddecimal_wasm/README | 7 ++ .../examples/fixeddecimal_wasm/node-test.js | 6 ++ .../examples/fixeddecimal_wasm/package.json | 6 ++ ffi/cpp/examples/fixeddecimal_wasm/test.cpp | 65 +++++++++++++++++++ tools/scripts/ffi.toml | 2 + tools/scripts/wasm.toml | 28 ++++++-- 15 files changed, 225 insertions(+), 20 deletions(-) create mode 100644 ffi/capi/src/crate_type_hack.rs create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/.gitignore create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/Makefile create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/README create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/node-test.js create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/package.json create mode 100644 ffi/cpp/examples/fixeddecimal_wasm/test.cpp diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9d382a6d47a..9a8e7dd2640 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -229,7 +229,9 @@ jobs: - name: Load nightly Rust toolchain for WASM. run: | rustup install nightly-2021-02-28 - rustup target add wasm32-unknown-unknown --toolchain nightly-2021-02-28-x86_64-unknown-linux-gnu + rustup target add wasm32-unknown-unknown --toolchain nightly-2021-02-28 + rustup target add wasm32-unknown-emscripten --toolchain nightly-2021-02-28 + rustup component add rust-src --toolchain nightly-2021-02-28 - name: Install WASM tools run: | sudo apt-get install wabt binaryen @@ -273,7 +275,13 @@ jobs: with: crate: cargo-make version: latest - + - name: Install emsdk + run: | + cd ~ + git clone https://github.com/emscripten-core/emsdk.git --branch 2.0.27 + cd emsdk + ./emsdk install latest + ./emsdk activate latest - name: Get Diplomat version id: diplomat-version run: | @@ -310,6 +318,12 @@ jobs: with: command: make args: wasm-test-release + # This has to be a separate test since the emscripten sdk + # will otherwise interfere with other node-using tests + - name: Run emscripten test + run: | + . ~/emsdk/emsdk_env.sh + cargo make wasm-cpp-emscripten # Fmt job - runs cargo fmt fmt: diff --git a/Cargo.lock b/Cargo.lock index e0169a64b0e..1cd2c68f4ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,9 +1740,9 @@ checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" [[package]] name = "postcard" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9083a4e478606d952014ad10e2fad31c43030942cec1f2bf8f12d4cc7089e62d" +checksum = "c8863e251332eb18520388099b8b0acc4810ed6e602e3b6f674e8a46ba20e15c" dependencies = [ "heapless", "postcard-cobs", @@ -2110,9 +2110,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.125" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +checksum = "1056a0db1978e9dbf0f6e4fca677f6f9143dc1c19de346f22cac23e422196834" dependencies = [ "serde_derive", ] @@ -2149,9 +2149,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.125" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +checksum = "13af2fbb8b60a8950d6c72a56d2095c28870367cc8e10c55e9745bac4995a2c4" dependencies = [ "proc-macro2", "quote", diff --git a/Makefile.toml b/Makefile.toml index b7000348317..d76bc53d706 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -87,7 +87,11 @@ dependencies = [ description = "Run all tests for the CI 'wasm' job" category = "CI" dependencies = [ - "wasm-test-release", + # note: CI does not call `cargo make ci-job-wasm` since + # we have to set up the environment for the emscripten job separately + # Instead, each of these is called individually. + "wasm-release", + "wasm-cpp-emscripten", ] [tasks.ci-all] diff --git a/ffi/capi/Cargo.toml b/ffi/capi/Cargo.toml index 4ceb92cb745..c07af5ca978 100644 --- a/ffi/capi/Cargo.toml +++ b/ffi/capi/Cargo.toml @@ -31,11 +31,11 @@ all-features = true skip_optional_dependencies = true # Bench feature gets tested separately and is only relevant for CI. # wearos/freertos/x86tiny are not relevant in normal environments, -# and smaller_static gets tested on the FFI job anyway +# smaller_static gets tested on the FFI job anyway denylist = ["bench", "wearos", "freertos", "x86tiny", "smaller_static"] [lib] -crate-type = ["staticlib", "rlib", "cdylib"] +crate-type = ["staticlib", "rlib"] path = "src/lib.rs" [features] @@ -86,3 +86,20 @@ icu_provider_fs = { path = "../../provider/fs/", optional = true } [target.'cfg(target_os = "none")'.dependencies] freertos-rust = { version = "0.1.2", optional = true } cortex-m = { version = "0.7.3", optional = true } + +# Unfortunately, --crate-type cannot be set per-target +# (https://github.com/rust-lang/cargo/issues/4881) +# and emscripten has link errors when compiling icu_capi due to +# symbols like log_js being undefined. There is no way to ask Cargo +# to only build a particular crate type for an invocation +# +# As a workaround, we define an example crate that just reexports icu_capi, +# but is built as a cdylib. Due to how Cargo invocations work around examples, +# `--features` is still passed down to `icu_capi`, but the end result is an +# `icu_capi_cdylib.wasm`/`icu_capi_cdylib.so`/etc file that is for all intents +# and purposes identical to the file one would get from adding "cdylib" to +# `crate-type` above. +[[example]] +name = "icu_capi_cdylib" +path = "src/crate_type_hack.rs" +crate-type = ["cdylib"] diff --git a/ffi/capi/src/crate_type_hack.rs b/ffi/capi/src/crate_type_hack.rs new file mode 100644 index 00000000000..77e8eba94b8 --- /dev/null +++ b/ffi/capi/src/crate_type_hack.rs @@ -0,0 +1,10 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +// See comment in icu_capi's Cargo.toml +// +// This is essentially a hack that allows icu_capi to be compiled +// to arbitrary `--crate-type`s without having to add a `--crate-type` +// to the list in Cargo.toml +extern crate icu_capi; diff --git a/ffi/cpp/examples/fixeddecimal/Makefile b/ffi/cpp/examples/fixeddecimal/Makefile index 07725ba397f..09529d249ab 100644 --- a/ffi/cpp/examples/fixeddecimal/Makefile +++ b/ffi/cpp/examples/fixeddecimal/Makefile @@ -22,4 +22,4 @@ a.out: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.cpp build: a.out test: build - ./a.out \ No newline at end of file + ./a.out diff --git a/ffi/cpp/examples/fixeddecimal/test.cpp b/ffi/cpp/examples/fixeddecimal/test.cpp index 691377690c8..5ffe9fde933 100644 --- a/ffi/cpp/examples/fixeddecimal/test.cpp +++ b/ffi/cpp/examples/fixeddecimal/test.cpp @@ -12,7 +12,6 @@ int main() { ICU4XLocale locale = ICU4XLocale::create("bn").value(); std::cout << "Running test for locale " << locale.tostring().ok().value() << std::endl; ICU4XDataProvider dp = ICU4XDataProvider::create_fs(path).provider.value(); - ICU4XFixedDecimalFormatOptions opts = {ICU4XFixedDecimalGroupingStrategy::Auto, ICU4XFixedDecimalSignDisplay::Auto}; ICU4XFixedDecimalFormat fdf = ICU4XFixedDecimalFormat::try_new(locale, dp, opts).fdf.value(); diff --git a/ffi/cpp/examples/fixeddecimal_wasm/.gitignore b/ffi/cpp/examples/fixeddecimal_wasm/.gitignore new file mode 100644 index 00000000000..d5ff1562df2 --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/.gitignore @@ -0,0 +1,8 @@ +web-version.html +web-version.wasm +web-version.js +node-version.js +node-version.wasm +package-lock.json +node_modules +a.out \ No newline at end of file diff --git a/ffi/cpp/examples/fixeddecimal_wasm/Makefile b/ffi/cpp/examples/fixeddecimal_wasm/Makefile new file mode 100644 index 00000000000..9d0d2be4c7a --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/Makefile @@ -0,0 +1,53 @@ +# This file is part of ICU4X. For terms of use, please see the file +# called LICENSE at the top level of the ICU4X source tree +# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +.DEFAULT_GOAL := build +.PHONY: build test clean serve build-host test-host + +ALL_HEADERS := $(wildcard ../../include/*.hpp) $(wildcard ../../../capi/include/*.h) +ALL_RUST := $(wildcard ../../../capi//src/*.rs) + +$(ALL_RUST): + +$(ALL_HEADERS): + +../../../../target/debug/libicu_capi.a: $(ALL_RUST) + cargo build -p icu_capi + +a.out: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.cpp + g++ -std=c++17 test.cpp ../../../../target/debug/libicu_capi.a -ldl -lpthread -lm -g + +../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a: $(ALL_RUST) + RUSTFLAGS="-Cpanic=abort" cargo +nightly-2021-02-28 build --release -p icu_capi --target wasm32-unknown-emscripten -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort + +web-version.html: ../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a $(ALL_HEADERS) test.cpp + emcc -std=c++17 test.cpp ../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a -ldl -lpthread -lm -g -o web-version.html --bind --emrun -sENVIRONMENT=web -sWASM=1 -sEXPORT_ES6=1 -sMODULARIZE=1 + +node-version.js: ../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a $(ALL_HEADERS) test.cpp + emcc -std=c++17 test.cpp ../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a -ldl -lpthread -lm -g -o node-version.js --bind -sWASM=1 -sENVIRONMENT=node -sWASM_ASYNC_COMPILATION=0 -DNOMAIN + +build: web-version.html node-version.js + +test: node-version.js + exec node ./node-test.js + +serve: web-version.html + emrun web-version.html + +# These make it possible to ensure that the C++ code is up to date with the bindings +# without needing to set up emsdk. This way `make test-ffi` works without emsdk. +build-host: a.out + +test-host: build-host + ./a.out + +clean: + rm -f web-version.html + rm -f web-version.wasm + rm -f web-version.js + rm -f node-version.js + rm -f node-version.wasm + rm -f ../../../../target/wasm32-unknown-emscripten/release/libicu_capi.a + rm -f ../../../../target/debug/libicu_capi.a + rm -f a.out diff --git a/ffi/cpp/examples/fixeddecimal_wasm/README b/ffi/cpp/examples/fixeddecimal_wasm/README new file mode 100644 index 00000000000..1299d40bf02 --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/README @@ -0,0 +1,7 @@ +This folder contains a test for calling ICU4X from C++ compiled to WASM (via emscripten). + +You need the [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) downloaded and sourced into your environment to run this. + +There are two ways to run the test. Firstly, you can call `make test`, which runs `node node-test.js` after building the appropriate WASM files. This runs a CLI test with the fixed decimal example in test.cpp. + +The other way is to run `make serve`, which will open a web page running test.cpp in your browser. \ No newline at end of file diff --git a/ffi/cpp/examples/fixeddecimal_wasm/node-test.js b/ffi/cpp/examples/fixeddecimal_wasm/node-test.js new file mode 100644 index 00000000000..d4a7a99d419 --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/node-test.js @@ -0,0 +1,6 @@ +wasm = require("./node-version.js"); + +const exitCode = wasm.runFixedDecimal(); +if (exitCode !== 0) { + throw new Error(`Test failed with exit code ${exitCode}`) +} diff --git a/ffi/cpp/examples/fixeddecimal_wasm/package.json b/ffi/cpp/examples/fixeddecimal_wasm/package.json new file mode 100644 index 00000000000..5567954da21 --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/package.json @@ -0,0 +1,6 @@ +{ + "type": "commonjs", + "scripts": { + "test": "node node-test.js" + } +} diff --git a/ffi/cpp/examples/fixeddecimal_wasm/test.cpp b/ffi/cpp/examples/fixeddecimal_wasm/test.cpp new file mode 100644 index 00000000000..fd6b900eaf6 --- /dev/null +++ b/ffi/cpp/examples/fixeddecimal_wasm/test.cpp @@ -0,0 +1,65 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +#ifdef __EMSCRIPTEN__ +#include +#endif + +#include "../../include/ICU4XFixedDecimalFormat.hpp" + +#include + +extern "C" void diplomat_init(); +extern "C" void log_js(char* s) { + std::cout<<"LOG: " <