Separate RTS Builds (Incremental and Non-Incremental GC)

bin/wrapper.sh

@@ -18,7 +18,7 @@ declare -A envs # list of expected environment variables with paths to products
 # Define build products here
 real[moc]=src/moc
 hint[moc]="make -C $rel_root/src moc"
-envs[moc]="MOC_RTS MOC_DEBUG_RTS"
+envs[moc]="MOC_NON_INCREMENTAL_RELEASE_RTS MOC_NON_INCREMENTAL_DEBUG_RTS MOC_INCREMENTAL_RELEASE_RTS MOC_INCREMENTAL_DEBUG_RTS"
 real[mo-ld]=src/mo-ld
 hint[mo-ld]="make -C $rel_root/src mo-ld"
 real[mo-doc]=src/mo-doc
@@ -30,11 +30,16 @@ hint[deser]="make -C $rel_root/src deser"
 real[candid-tests]=src/candid-tests
 hint[candid-tests]="make -C $rel_root/src candid-tests"
 
-real[MOC_RTS]=rts/mo-rts.wasm
-hint[MOC_RTS]="make -C $rel_root/rts"
+rts_hint="make -C $rel_root/rts"
 
-real[MOC_DEBUG_RTS]=rts/mo-rts-debug.wasm
-hint[MOC_DEBUG_RTS]="make -C $rel_root/rts"
+real[MOC_NON_INCREMENTAL_RELEASE_RTS]=rts/mo-rts.wasm
+real[MOC_NON_INCREMENTAL_DEBUG_RTS]=rts/mo-rts-debug.wasm
+real[MOC_INCREMENTAL_RELEASE_RTS]=rts/mo-rts-incremental.wasm
+real[MOC_INCREMENTAL_DEBUG_RTS]=rts/mo-rts-incremental-debug.wasm
+
+for var in ${envs[moc]}; do
+  hint[$var]=$rts_hint
+done
 
 # This is the command we want to run
 exe=$(basename "$0")

default.nix

@@ -259,6 +259,8 @@ rec {
         mkdir -p $out/rts
         cp mo-rts.wasm $out/rts
         cp mo-rts-debug.wasm $out/rts
+        cp mo-rts-incremental.wasm $out/rts
+        cp mo-rts-incremental-debug.wasm $out/rts
       '';
 
       # This needs to be self-contained. Remove mention of nix path in debug
@@ -269,6 +271,11 @@ rec {
           -t ${rtsDeps} \
           -t ${rustStdDeps} \
           $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm
+        remove-references-to \
+          -t ${nixpkgs.rustc-nightly} \
+          -t ${rtsDeps} \
+          -t ${rustStdDeps} \
+          $out/rts/mo-rts-incremental.wasm $out/rts/mo-rts-incremental-debug.wasm
       '';
 
       allowedRequisites = [];

rts/.gitignore

@@ -1,3 +1,5 @@
 mo-rts.wasm
 mo-rts-debug.wasm
+mo-rts-incremental.wasm
+mo-rts-incremental-debug.wasm
 _build

rts/Makefile

@@ -111,7 +111,7 @@ CLANG_FLAGS = \
 
 .PHONY: all
 
-all: mo-rts.wasm mo-rts-debug.wasm
+all: mo-rts.wasm mo-rts-debug.wasm mo-rts-incremental.wasm mo-rts-incremental-debug.wasm
 
 _build:
 	mkdir -p $@
@@ -172,9 +172,6 @@ $(MUSL_WASM_A): $(MUSL_WASM_O)
 # The rust code code of the RTS
 #
 
-RTS_RUST_WASM_A=_build/wasm/libmotoko_rts.a
-RTS_RUST_DEBUG_WASM_A=_build/wasm/libmotoko_rts_debug.a
-
 # This relies on bash and globstar, see https://stackoverflow.com/questions/2483182/recursive-wildcards-in-gnu-make
 RTS_RUST_FILES=$(shell ls **/*.rs)
 RTS_CARGO_FILES=$(shell ls **/Cargo.toml)
@@ -221,23 +218,54 @@ $(TOMMATH_BINDINGS_RS): | _build
 	# Note that bindgen can't generate Rust macros or functions for CPP macros, so
 	# macros like `mp_get_u32` and `mp_isneg` need to be manually implemented.
 
+RTS_DEPENDENCIES=$(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
+RTS_BUILD=cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
+RTS_DEBUG_BUILD=$(RTS_BUILD)
+RTS_RELEASE_BUILD=$(RTS_BUILD) --release
+RTS_DEBUG_TARGET=motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a
+RTS_RELEASE_TARGET=motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a
+
+RTS_RUST_NON_INCREMENTAL_WASM_A=_build/wasm/libmotoko_rts.a
+RTS_RUST_NON_INCREMENTAL_DEBUG_WASM_A=_build/wasm/libmotoko_rts_debug.a
+
+RTS_RUST_INCREMENTAL_WASM_A=_build/wasm/libmotoko_rts_incremental.a
+RTS_RUST_INCREMENTAL_DEBUG_WASM_A=_build/wasm/libmotoko_rts_incremental_debug.a
+
+$(RTS_RUST_NON_INCREMENTAL_WASM_A): $(RTS_DEPENDENCIES)
+	$(RTS_RELEASE_BUILD)
+	cp $(RTS_RELEASE_TARGET) $@
 
-$(RTS_RUST_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && cargo build --release --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
-	cp motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a $@
+$(RTS_RUST_NON_INCREMENTAL_DEBUG_WASM_A): $(RTS_DEPENDENCIES)
+	$(RTS_DEBUG_BUILD)
+	cp $(RTS_DEBUG_TARGET) $@
 
-$(RTS_RUST_DEBUG_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
-	cp motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a $@
+$(RTS_RUST_INCREMENTAL_WASM_A): $(RTS_DEPENDENCIES)
+	$(RTS_RELEASE_BUILD) --features incremental_gc
+	cp $(RTS_RELEASE_TARGET) $@
+
+$(RTS_RUST_INCREMENTAL_DEBUG_WASM_A): $(RTS_DEPENDENCIES)
+	$(RTS_DEBUG_BUILD) --features incremental_gc
+	cp $(RTS_DEBUG_TARGET) $@
 
 #
 # The test suite
 #
 
+TEST_DEPENDENCIES=$(TOMMATH_WASM_A) $(TOMMATH_BINDINGS_RS)
+TEST_BUILD=cd motoko-rts-tests && cargo build --target=wasm32-wasi
+TEST_RUN=wasmtime --disable-cache motoko-rts-tests/target/wasm32-wasi/debug/motoko-rts-tests.wasm
+
 .PHONY: test
-test: $(TOMMATH_WASM_A) $(TOMMATH_BINDINGS_RS)
-	cd motoko-rts-tests && cargo build --target=wasm32-wasi
-	wasmtime --disable-cache motoko-rts-tests/target/wasm32-wasi/debug/motoko-rts-tests.wasm
+
+test: test-non-incremental test-incremental
+
+test-non-incremental: $(TEST_DEPENDENCIES)
+	$(TEST_BUILD)
+	$(TEST_RUN)
+
+test-incremental: $(TEST_DEPENDENCIES)
+	$(TEST_BUILD) --features incremental_gc
+	$(TEST_RUN)
 
 #
 # Putting it all together
@@ -260,30 +288,45 @@ EXPORTED_SYMBOLS=\
   fmod \
   log \
 
-mo-rts.wasm: $(TOMMATH_WASM_A) $(MUSL_WASM_A) $(RTS_RUST_WASM_A)
+WASM_A_DEPENDENCIES=$(TOMMATH_WASM_A) $(MUSL_WASM_A)
+LINKER_OPTIONS=\
+  --import-memory --shared --no-entry --gc-sections \
+  $(EXPORTED_SYMBOLS:%=--export=%) \
+  --whole-archive
+
+mo-rts.wasm: $(RTS_RUST_NON_INCREMENTAL_WASM_A) $(WASM_A_DEPENDENCIES)
 	$(WASM_LD) -o $@ \
-		--import-memory --shared --no-entry --gc-sections \
-                $(EXPORTED_SYMBOLS:%=--export=%) \
-		--whole-archive \
+		$(LINKER_OPTIONS) \
 		$+
 
-mo-rts-debug.wasm: $(RTS_RUST_DEBUG_WASM_A) $(TOMMATH_WASM_A) $(MUSL_WASM_A)
+mo-rts-debug.wasm: $(RTS_RUST_NON_INCREMENTAL_DEBUG_WASM_A) $(WASM_A_DEPENDENCIES)
 	$(WASM_LD) -o $@ \
-		--import-memory --shared --no-entry --gc-sections \
-                $(EXPORTED_SYMBOLS:%=--export=%) \
-		--whole-archive \
+		$(LINKER_OPTIONS) \
 		$+
 
+mo-rts-incremental.wasm: $(RTS_RUST_INCREMENTAL_WASM_A) $(WASM_A_DEPENDENCIES)
+	$(WASM_LD) -o $@ \
+		$(LINKER_OPTIONS) \
+		$+
+
+mo-rts-incremental-debug.wasm: $(RTS_RUST_INCREMENTAL_DEBUG_WASM_A) $(WASM_A_DEPENDENCIES)
+	$(WASM_LD) -o $@ \
+		$(LINKER_OPTIONS) \
+		$+
 
 format:
 	cargo fmt --verbose --manifest-path motoko-rts/Cargo.toml
 	cargo fmt --verbose --manifest-path motoko-rts-tests/Cargo.toml
+	cargo fmt --verbose --manifest-path motoko-rts-macros/Cargo.toml
 
 clean:
 	rm -rf \
 	  _build \
 	  mo-rts.wasm \
 	  mo-rts-debug.wasm \
+	  mo-rts-incremental.wasm \
+	  mo-rts-incremental-debug.wasm \
 	  motoko-rts/target \
 	  motoko-rts-tests/target \
+	  motoko-rts-macros/target \
 	  motoko-rts/cargo-home

rts/motoko-rts-macros/src/lib.rs

@@ -136,3 +136,34 @@ pub fn ic_mem_fn(attr: TokenStream, input: TokenStream) -> TokenStream {
     )
     .into()
 }
+
+/// Feature macro for incremental GC features, in particular forwarding pointers.
+/// Equivalent to using the attribute `#[cfg(feature = "incremental_gc")]`.
+#[proc_macro_attribute]
+pub fn incremental_gc(attr: TokenStream, input: TokenStream) -> TokenStream {
+    assert!(attr.is_empty());
+    let block = syn::parse_macro_input!(input as syn::Item);
+    quote!(
+        #[cfg(feature = "incremental_gc")]
+        #block
+    )
+    .into()
+}
+
+/// Feature macro for non-incremental GC features, in particular forwarding pointers.
+/// Equivalent to using the attribute `#[cfg(not(feature = "incremental_gc"))]`.
+#[proc_macro_attribute]
+pub fn non_incremental_gc(attr: TokenStream, input: TokenStream) -> TokenStream {
+    assert!(attr.is_empty());
+    let block = syn::parse_macro_input!(input as syn::Item);
+    quote!(
+        #[cfg(not(feature = "incremental_gc"))]
+        #block
+    )
+    .into()
+}
+
+#[proc_macro]
+pub fn is_incremental_gc(_item: TokenStream) -> TokenStream {
+    "cfg!(feature = \"incremental_gc\")".parse().unwrap()
+}

rts/motoko-rts-tests/Cargo.toml

@@ -4,10 +4,14 @@ version = "0.1.0"
 authors = ["dfinity <sdk@dfinity.org"]
 edition = "2018"
 
+[features]
+incremental_gc = ["motoko-rts/incremental_gc"]
+
 [dependencies]
 byteorder = "1.4.3"
 fxhash = "0.2.1"
 libc = { version = "0.2.139", default_features = false }
 motoko-rts = { path = "../motoko-rts/native" }
 oorandom = "11.1.3"
 proptest = { version = "1.0.0", default-features = false, features = ["std"] }
+motoko-rts-macros = { path = "../motoko-rts-macros" }