wasm: add a webassembly compiler backend, based on using binaryen (#17368)

Author: l1mey112

.github/workflows/native_backend_tests_ci.yml

@@ -10,7 +10,6 @@ on:
       - 'vlib/builtin/**.v'
       - 'vlib/v/ast/**.v'
       - 'vlib/v/scanner/**.v'
-      - 'vlib/v/scanner/**.v'
       - 'vlib/v/parser/**.v'
       - 'vlib/v/checker/**.v'
       - 'vlib/v/gen/c/**.v'
@@ -31,7 +30,6 @@ on:
       - 'vlib/builtin/**.v'
       - 'vlib/v/ast/**.v'
       - 'vlib/v/scanner/**.v'
-      - 'vlib/v/scanner/**.v'
       - 'vlib/v/parser/**.v'
       - 'vlib/v/checker/**.v'
       - 'vlib/v/gen/c/**.v'

.github/workflows/wasm_backend_tests_ci.yml

@@ -0,0 +1,95 @@
+name: wasm backend CI
+
+on:
+  push:
+    paths:
+      - '!**'
+      - '!**.md'
+      - 'cmd/tools/builders/**.v'
+      - 'vlib/builtin/**.v'
+      - 'vlib/v/ast/**.v'
+      - 'vlib/v/scanner/**.v'
+      - 'vlib/v/parser/**.v'
+      - 'vlib/v/checker/**.v'
+      - 'vlib/v/gen/c/**.v'
+      - 'vlib/v/builder/**.v'
+      - 'vlib/v/cflag/**.v'
+      - 'vlib/v/live/**.v'
+      - 'vlib/v/util/**.v'
+      - 'vlib/v/markused/**.v'
+      - 'vlib/v/preludes/**.v'
+      - 'vlib/v/gen/wasm/**.v'
+      - 'vlib/v/gen/wasm/tests/**.v'
+  pull_request:
+    paths:
+      - '!**'
+      - '!**.md'
+      - 'cmd/tools/builders/**.v'
+      - 'vlib/builtin/**.v'
+      - 'vlib/v/ast/**.v'
+      - 'vlib/v/scanner/**.v'
+      - 'vlib/v/parser/**.v'
+      - 'vlib/v/checker/**.v'
+      - 'vlib/v/gen/c/**.v'
+      - 'vlib/v/builder/**.v'
+      - 'vlib/v/cflag/**.v'
+      - 'vlib/v/live/**.v'
+      - 'vlib/v/util/**.v'
+      - 'vlib/v/markused/**.v'
+      - 'vlib/v/preludes/**.v'
+      - 'vlib/v/gen/wasm/**.v'
+      - 'vlib/v/gen/wasm/tests/**.v'
+
+concurrency:
+  group: wasm-backend-ci-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  wasm-backend-ubuntu:
+    runs-on: ubuntu-22.04
+    if: github.event_name != 'push' || github.event.ref == 'refs/heads/master' || github.event.repository.full_name != 'vlang/v'
+    timeout-minutes: 121
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --quiet -y clang gcc
+
+      - name: Build V
+        run: make -j4 && ./v symlink -githubci
+
+      - name: Install binaryen as build dependency for the V WASM backend
+        run: ./v cmd/tools/install_binaryen.vsh
+
+      - name: Build the V WASM backend
+        run: ./v -cc clang -showcc -v cmd/tools/builders/wasm_builder.v
+
+      - name: Test the WASM backend
+        run: ./v test vlib/v/gen/wasm/tests/
+
+      - name: Build examples
+        run: VTEST_ONLY=wasm ./v build-examples
+
+##  wasm-backend-macos:
+##    runs-on: macOS-12
+##    if: github.event_name != 'push' || github.event.ref == 'refs/heads/master' || github.event.repository.full_name != 'vlang/v'
+##    timeout-minutes: 121
+##    steps:
+##      - uses: actions/checkout@v3
+##
+##      - name: Build V
+##        run: make -j4 && ./v symlink -githubci
+##
+##      - name: Install binaryen as build dependency for the V WASM backend
+##        run: ./v cmd/tools/install_binaryen.vsh
+##
+##      - name: Build the V WASM backend
+##        run: ./v -cc clang -showcc -v cmd/tools/builders/wasm_builder.v
+##
+##      - name: Test the WASM backend
+##        run: ./v test vlib/v/gen/wasm/tests/
+##
+##      - name: Build examples
+##        run: VTEST_ONLY=wasm ./v build-examples

.gitignore

@@ -108,6 +108,7 @@ flake.nix
 .envrc
 
 thirdparty/stdatomic/nix/cpp/*.h
+thirdparty/binaryen*
 
 # ignore VLS log
 vls.log
@@ -120,3 +121,4 @@ vls.log
 # ignore Intellij files
 .idea/
 /*.iml
+wasm.v

cmd/tools/builders/wasm_builder.v

@@ -0,0 +1,7 @@
+module main
+
+import v.builder.wasmbuilder
+
+fn main() {
+	wasmbuilder.start()
+}

cmd/tools/install_binaryen.vsh

@@ -0,0 +1,68 @@
+#!/usr/bin/env -S v -raw-vsh-tmp-prefix tmp
+
+import net.http
+import json
+import os
+
+struct JQ {
+	tag_name string
+}
+
+fn main() {
+	root := os.real_path(os.dir(os.getenv_opt('VEXE') or { @VEXE }))
+	os.chdir(root)! // make sure that the workfolder is stable
+
+	tloc := os.join_path(root, 'thirdparty')
+	loc := os.join_path(tloc, 'binaryen')
+
+	if os.exists(loc) {
+		eprintln('thirdparty/binaryen exists, will not overwrite')
+		eprintln('delete the folder, and execute again')
+		exit(1)
+	}
+
+	jq := http.get_text('https://api.github.com/repos/WebAssembly/binaryen/releases/latest')
+	tag := json.decode(JQ, jq)!.tag_name
+
+	name := $if windows {
+		'x86_64-windows'
+	} $else $if macos {
+		$if arm64 {
+			'arm64-macos'
+		} $else {
+			'x86_64-macos'
+		}
+	} $else $if linux {
+		'x86_64-linux'
+	} $else {
+		eprintln('A premade binary library is not available for your system.')
+		eprintln('Build it from source, following the documentation here: https://github.com/WebAssembly/binaryen/#building')
+		exit(1)
+	}
+
+	fname := 'binaryen-${tag}'
+	url := 'https://github.com/WebAssembly/binaryen/releases/download/${tag}/${fname}-${name}.tar.gz'
+
+	saveloc := os.join_path(tloc, '${fname}.tar.gz')
+	if !os.exists(saveloc) {
+		println('Downloading archive: ${saveloc}, from url: ${url} ...')
+		http.download_file(url, saveloc)!
+		// defer { os.rm(saveloc) or {}! }
+	}
+
+	mkdir_all(loc)!
+	println(loc)
+
+	println('Extracting `${tloc}/${fname}` to `${tloc}/binaryen` ...')
+	cmd := 'tar -xvf ${saveloc} --directory ${tloc}'
+	if os.system(cmd) != 0 {
+		eprintln('`${cmd}` exited with a non zero exit code')
+		exit(1)
+	}
+
+	println(cmd)
+	println('Moving `${tloc}/${fname}` to `${tloc}/binaryen` ...')
+
+	os.rename_dir('${tloc}/${fname}', loc)!
+	println('Done. You can now use `v -b wasm file.v` .')
+}

cmd/tools/modules/testing/common.v

@@ -252,6 +252,18 @@ pub fn new_test_session(_vargs string, will_compile bool) TestSession {
 		$if !macos {
 			skip_files << 'examples/macos_tray/tray.v'
 		}
+		// examples/wasm/mandelbrot/mandelbrot.v requires special compilation flags: `-b wasm -os browser`, skip it for now:
+		skip_files << 'examples/wasm/mandelbrot/mandelbrot.v'
+
+		// TODO: always build the wasm_builder in the future, not just when it was build manually before:
+		wasm_builder_executable := $if !windows {
+			'cmd/tools/builders/wasm_builder'
+		} $else {
+			'cmd/tools/builders/wasm_builder.exe'
+		}
+		if !os.exists(wasm_builder_executable) {
+			skip_files << os.join_path('cmd/tools/builders/wasm_builder.v')
+		}
 	}
 	vargs := _vargs.replace('-progress', '')
 	vexe := pref.vexe_path()

cmd/tools/vbuild-examples.v

@@ -3,7 +3,7 @@ module main
 import os
 import testing
 
-const vroot = @VMODROOT
+const vroot = os.dir(os.real_path(os.getenv_opt('VEXE') or { @VEXE }))
 
 // build as a project folder
 const efolders = [
@@ -12,11 +12,14 @@ const efolders = [
 	'examples/vweb_fullstack',
 ]
 
+pub fn normalised_vroot_path(path string) string {
+	return os.real_path(os.join_path_single(vroot, path)).replace('\\', '/')
+}
+
 fn main() {
 	args_string := os.args[1..].join(' ')
 	params := args_string.all_before('build-examples')
-	skip_prefixes := efolders.map(os.real_path(os.join_path_single(vroot, it)).replace('\\',
-		'/'))
+	mut skip_prefixes := efolders.map(normalised_vroot_path(it))
 	res := testing.v_build_failing_skipped(params, 'examples', skip_prefixes, fn (mut session testing.TestSession) {
 		for x in efolders {
 			pathsegments := x.split_any('/')

cmd/tools/vtest-self.v

@@ -329,6 +329,10 @@ fn main() {
 		tsession.skip_files << 'vlib/net/udp_test.v'
 	}
 
+	if !os.exists('cmd/tools/builders/wasm_builder') {
+		tsession.skip_files << 'vlib/v/gen/wasm/tests/wasm_test.v'
+	}
+
 	mut werror := false
 	mut sanitize_memory := false
 	mut sanitize_address := false

cmd/v/v.v

@@ -192,5 +192,19 @@ fn rebuild(prefs &pref.Preferences) {
 			println('using Go WIP backend...')
 			util.launch_tool(prefs.is_verbose, 'builders/golang_builder', os.args[1..])
 		}
+		.wasm {
+			assert_wasm_backend_thirdparty()
+			util.launch_tool(prefs.is_verbose, 'builders/wasm_builder', os.args[1..])
+		}
+	}
+}
+
+fn assert_wasm_backend_thirdparty() {
+	vroot := os.dir(pref.vexe_path())
+	if !os.exists('${vroot}/cmd/tools/builders/wasm_builder')
+		&& !os.exists('${vroot}/thirdparty/binaryen') {
+		eprintln('The WebAssembly backend requires `binaryen`, an external library dependency')
+		eprintln('This can be installed with `./cmd/tools/install_binaryen.vsh`, to download prebuilt libraries for your platform')
+		exit(1)
 	}
 }

examples/wasm/.gitignore

@@ -0,0 +1 @@
+*.wasm