Add WASI based WebAssembly support

.github/workflows/wasm.yml

@@ -0,0 +1,106 @@
+name: WebAssembly
+on:
+  push:
+    paths-ignore:
+      - 'doc/**'
+      - '**.md'
+      - '**.rdoc'
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '**.md'
+      - '**.rdoc'
+
+concurrency:
+  group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }}
+  cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }}
+
+jobs:
+  make:
+    strategy:
+      matrix:
+        entry:
+#         # wasmtime can't compile non-optimized Asyncified binary due to locals explosion
+#         - { name: O0-debuginfo, optflags: "-O0", debugflags: "-g", wasmoptflags: "-O1" }
+#         - { name: O1,           optflags: "-O1", debugflags: ""  , wasmoptflags: "-O1" }
+          - { name: O2,           optflags: "-O2", debugflags: ""  , wasmoptflags: "-O2" }
+#         - { name: O3,           optflags: "-O3", debugflags: ""  , wasmoptflags: "-O3" }
+#         # -O4 is equivalent to -O3 in clang, but it's different in wasm-opt
+#         - { name: O4,           optflags: "-O3", debugflags: ""  , wasmoptflags: "-O4" }
+#         - { name: Oz,           optflags: "-Oz", debugflags: ""  , wasmoptflags: "-Oz" }
+      fail-fast: false
+    env:
+      RUBY_TESTOPTS: '-q --tty=no'
+      GITPULLOPTIONS: --no-tags origin ${{github.ref}}
+      WASI_SDK_VERSION_MAJOR: 14
+      WASI_SDK_VERSION_MINOR: 0
+      # Use older version, which uses glibc instead of musl, to avoid https://github.com/WebAssembly/binaryen/issues/4401
+      BINARYEN_VERSION: 91
+      WASMTIME_VERSION: v0.33.0
+    runs-on: ubuntu-20.04
+    if: ${{ !startsWith(github.event.head_commit.message, '[DOC]') }}
+    steps:
+      - run: mkdir build
+        working-directory:
+      - name: git config
+        run: |
+          git config --global advice.detachedHead 0
+          git config --global init.defaultBranch garbage
+      - uses: actions/checkout@v2
+        with:
+          path: src
+      - name: Install libraries
+        run: |
+          set -ex
+          sudo apt-get update -q || :
+          sudo apt-get install --no-install-recommends -q -y ruby bison make autoconf git wget
+
+          wasi_sdk_deb="wasi-sdk_${WASI_SDK_VERSION_MAJOR}.${WASI_SDK_VERSION_MINOR}_amd64.deb"
+          wget "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION_MAJOR}/${wasi_sdk_deb}"
+          sudo dpkg -i "$wasi_sdk_deb"
+          rm -f "$wasi_sdk_deb"
+
+          mkdir build-sdk
+          pushd build-sdk
+
+          wasmtime_url="https://github.com/bytecodealliance/wasmtime/releases/download/${WASMTIME_VERSION}/wasmtime-${WASMTIME_VERSION}-x86_64-linux.tar.xz"
+          wget -O - "$wasmtime_url" | tar xJf -
+          sudo ln -fs "$PWD/wasmtime-${WASMTIME_VERSION}-x86_64-linux/wasmtime" /usr/local/bin/wasmtime
+
+          binaryen_tarball="binaryen-version_${BINARYEN_VERSION}-x86-linux.tar.gz"
+          binaryen_url="https://github.com/WebAssembly/binaryen/releases/download/version_${BINARYEN_VERSION}/${binaryen_tarball}"
+          wget -O - "$binaryen_url" | tar xfz -
+          sudo ln -fs "$PWD/binaryen-version_${BINARYEN_VERSION}/wasm-opt" /usr/local/bin/wasm-opt
+        working-directory: src
+      - name: Set ENV
+        run: |
+          echo "MAKEFLAGS=-j$((1 + $(sysctl -n hw.activecpu)))" >> $GITHUB_ENV
+          echo "WASI_SDK_PATH=/opt/wasi-sdk" >> $GITHUB_ENV
+      - run: ./autogen.sh
+        working-directory: src
+      - name: Run configure
+        run: |
+          ../src/configure \
+            --host wasm32-unknown-wasi \
+            --with-static-linked-ext \
+            LDFLAGS=" \
+              -Xlinker --stack-first \
+              -Xlinker -z -Xlinker stack-size=16777216 \
+            " \
+            optflags="${{ matrix.entry.optflags }}" \
+            debugflags="${{ matrix.entry.debugflags }}" \
+            wasmoptflags="${{ matrix.entry.wasmoptflags }} ${{ matrix.entry.debugflags }}"
+
+      - run: make ruby
+      - name: Run basictest
+        run: wasmtime run ./../build/miniruby --mapdir /::./ -- basictest/test.rb
+        working-directory: src
+      - name: Run bootstraptest (no thread)
+        run: |
+          NO_THREAD_TESTS="$(grep -L Thread -R ./bootstraptest | awk -F/ '{ print $NF }' | uniq | sed -n 's/test_\(.*\).rb/\1/p' | paste -s -d, -)"
+          ruby ./bootstraptest/runner.rb --ruby="$(which wasmtime) run $PWD/../build/ruby --mapdir /::./ -- " --verbose "--sets=$NO_THREAD_TESTS"
+        working-directory: src
+
+defaults:
+  run:
+    working-directory: build

.gitignore

@@ -232,3 +232,6 @@ lcov*.info
 /rb_mjit_header.h
 /mjit_config.h
 /include/ruby-*/*/rb_mjit_min_header-*.h
+
+# /wasm/
+/wasm/tests/*.wasm

basictest/test.rb

@@ -1960,6 +1960,8 @@ module EvTest
 test_ok(i7 == nil)
 end
 
+# WASI doesn't support spawning a new process for now.
+unless /wasi/ =~ RUBY_PLATFORM
 test_check "system"
 test_ok(`echo foobar` == "foobar\n")
 test_ok(`./miniruby -e 'print "foobar"'` == 'foobar')
@@ -2010,6 +2012,7 @@ module EvTest
 
 File.unlink script_tmp or `/bin/rm -f "#{script_tmp}"`
 File.unlink "#{script_tmp}.bak" or `/bin/rm -f "#{script_tmp}.bak"`
+end # not /wasi/ =~ RUBY_PLATFORM
 
 test_check "const"
 TEST1 = 1

bootstraptest/runner.rb

@@ -271,6 +271,14 @@ def show_progress(message = '')
   end
 end
 
+def target_platform
+  if @ruby
+    `#{@ruby} --disable-gems -e 'print RUBY_PLATFORM'`
+  else
+    RUBY_PLATFORM
+  end
+end
+
 def show_limit(testsrc, opt = '', **argh)
   result = get_result_string(testsrc, opt, **argh)
   if @tty and @verbose

bootstraptest/test_literal.rb

@@ -65,8 +65,11 @@
 assert_equal 'Symbol',          ':"a#{1+2}c".class'
 
 # xstring
-assert_equal "foo\n",           %q(`echo foo`)
-assert_equal "foo\n",           %q(s = "foo"; `echo #{s}`)
+# WASI doesn't support spawning a new process for now.
+if /wasi/ !~ target_platform
+  assert_equal "foo\n",           %q(`echo foo`)
+  assert_equal "foo\n",           %q(s = "foo"; `echo #{s}`)
+end
 
 # regexp
 assert_equal '',                '//.source'

configure.ac

@@ -42,6 +42,7 @@ m4_include([tool/m4/ruby_try_cflags.m4])dnl
 m4_include([tool/m4/ruby_try_cxxflags.m4])dnl
 m4_include([tool/m4/ruby_try_ldflags.m4])dnl
 m4_include([tool/m4/ruby_universal_arch.m4])dnl
+m4_include([tool/m4/ruby_wasm_tools.m4])dnl
 m4_include([tool/m4/ruby_werror_flag.m4])dnl
 
 AC_ARG_VAR([cflags], [additional CFLAGS (ignored when CFLAGS is given)])dnl
@@ -136,6 +137,9 @@ AS_IF([test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC"], [
   AC_MSG_ERROR(cached CC is different -- throw away $cache_file
 (it is also a good idea to do 'make clean' before compiling))
 ])
+
+RUBY_WASM_TOOLS
+
 AS_CASE(["${build_os}"],
 [linux*|cygwin*|msys*], [
     # Naruse prefers GCC on Linux
@@ -402,6 +406,15 @@ AS_CASE(["$build_os"],
     ])
     rm -fr conftest*
   ])
+AS_CASE(["$target_os"],
+  [wasi*], [
+    # Clang linker automatically uses wasm-opt with -O if it found.
+    # https://github.com/llvm/llvm-project/blob/812828984c10857a4cd260eb638c52a4411f9143/clang/lib/Driver/ToolChains/WebAssembly.cpp#L95-L118
+    # However optimization before asyncify causes misoptimization,
+    # so wrap clang to insert our fake wasm-opt, which does nothing, in PATH.
+    CC_WRAPPER=`cd -P "${tooldir}" && pwd`/wasm-clangw
+    CC="$CC_WRAPPER $CC"
+  ])
 
 cc_version=
 for option in --version -v -V -qversion; do
@@ -739,7 +752,7 @@ AS_IF([test "$GCC" = yes], [
 
     # -fstack-protector
     AS_CASE(["$target_os"],
-    [mingw*|emscripten*], [
+    [mingw*|emscripten*|wasi*], [
 	stack_protector=no
     ])
     AS_IF([test -z "${stack_protector+set}"], [
@@ -1215,6 +1228,14 @@ main()
 		RUBY_APPEND_OPTIONS(LDFLAGS, "-sFORCE_FILESYSTEM=1")
 		ac_cv_func_shutdown=no
 		],
+[wasi*],[	LIBS="-lm -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-getpid -lwasi-emulated-process-clocks $LIBS"
+		RUBY_APPEND_OPTIONS(CFLAGS, -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID -D_WASI_EMULATED_PROCESS_CLOCKS)
+		RUBY_APPEND_OPTIONS(CPPFLAGS, -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID -D_WASI_EMULATED_PROCESS_CLOCKS)
+		POSTLINK="\$(WASMOPT) --asyncify \$(wasmoptflags) --pass-arg=asyncify-ignore-imports -o \$@ \$@${POSTLINK:+; $POSTLINK}"
+		# wasi-libc's sys/socket.h is not compatible with -std=gnu99,
+		# so re-declare shutdown in include/ruby/missing.h
+		ac_cv_func_shutdown=no
+		]
 [	LIBS="-lm $LIBS"])
 : ${ORIG_LIBS=$LIBS}
 
@@ -1276,6 +1297,7 @@ AC_CHECK_HEADERS(sanitizer/asan_interface.h)
 AC_CHECK_HEADERS(sanitizer/msan_interface.h)
 AC_CHECK_HEADERS(setjmpex.h)
 AC_CHECK_HEADERS(stdalign.h)
+AC_CHECK_HEADERS(stdio.h)
 AC_CHECK_HEADERS(sys/attr.h)
 AC_CHECK_HEADERS(sys/eventfd.h)
 AC_CHECK_HEADERS(sys/fcntl.h)
@@ -1908,7 +1930,14 @@ AC_DEFUN([RUBY_REQUIRE_FUNC], [
     AS_IF([test "$ac_cv_func_[]AS_TR_SH($1)" = yes], [],
           [AC_MSG_ERROR($1[() must be supported])])
 ])
-m4_map_args_w([dup dup2], [RUBY_REQUIRE_FUNC(], [)])
+
+# dup and dup2 are always available except for WASI
+AS_CASE(["$target_os"],
+    [wasi*], [],
+    [
+        m4_map_args_w([dup dup2], [RUBY_REQUIRE_FUNC(], [)])
+    ]
+)
 
 AC_REPLACE_FUNCS(acosh)
 AC_REPLACE_FUNCS(cbrt)
@@ -1949,6 +1978,8 @@ AC_CHECK_FUNCS(_longjmp)		# used for AC_ARG_WITH(setjmp-type)
 test x$ac_cv_func__longjmp = xno && ac_cv_func__setjmp=no
 AC_CHECK_FUNCS(arc4random_buf)
 AC_CHECK_FUNCS(atan2l atan2f)
+AC_CHECK_FUNCS(chmod)
+AC_CHECK_FUNCS(chown)
 AC_CHECK_FUNCS(chroot)
 AC_CHECK_FUNCS(chsize)
 AC_CHECK_FUNCS(clock_gettime)
@@ -1964,6 +1995,9 @@ AC_CHECK_FUNCS(dup3)
 AC_CHECK_FUNCS(eaccess)
 AC_CHECK_FUNCS(endgrent)
 AC_CHECK_FUNCS(eventfd)
+AC_CHECK_FUNCS(execl)
+AC_CHECK_FUNCS(execle)
+AC_CHECK_FUNCS(execve)
 AC_CHECK_FUNCS(explicit_memset)
 AC_CHECK_FUNCS(fcopyfile)
 AC_CHECK_FUNCS(fchmod)
@@ -1979,7 +2013,10 @@ AC_CHECK_FUNCS(ftruncate)
 AC_CHECK_FUNCS(ftruncate64)		# used for Win32 platform
 AC_CHECK_FUNCS(getattrlist)
 AC_CHECK_FUNCS(getcwd)
+AC_CHECK_FUNCS(getegid)
 AC_CHECK_FUNCS(getentropy)
+AC_CHECK_FUNCS(geteuid)
+AC_CHECK_FUNCS(getgid)
 AC_CHECK_FUNCS(getgidx)
 AC_CHECK_FUNCS(getgrnam)
 AC_CHECK_FUNCS(getgrnam_r)
@@ -1988,6 +2025,7 @@ AC_CHECK_FUNCS(getlogin)
 AC_CHECK_FUNCS(getlogin_r)
 AC_CHECK_FUNCS(getpgid)
 AC_CHECK_FUNCS(getpgrp)
+AC_CHECK_FUNCS(getppid)
 AC_CHECK_FUNCS(getpriority)
 AC_CHECK_FUNCS(getpwnam)
 AC_CHECK_FUNCS(getpwnam_r)
@@ -1999,13 +2037,15 @@ AC_CHECK_FUNCS(getresuid)
 AC_CHECK_FUNCS(getrlimit)
 AC_CHECK_FUNCS(getsid)
 AC_CHECK_FUNCS(gettimeofday)		# for making ac_cv_func_gettimeofday
+AC_CHECK_FUNCS(getuid)
 AC_CHECK_FUNCS(getuidx)
 AC_CHECK_FUNCS(gmtime_r)
 AC_CHECK_FUNCS(grantpt)
 AC_CHECK_FUNCS(initgroups)
 AC_CHECK_FUNCS(ioctl)
 AC_CHECK_FUNCS(isfinite)
 AC_CHECK_FUNCS(issetugid)
+AC_CHECK_FUNCS(kill)
 AC_CHECK_FUNCS(killpg)
 AC_CHECK_FUNCS(lchmod)
 AC_CHECK_FUNCS(lchown)
@@ -2029,9 +2069,13 @@ AC_CHECK_FUNCS(mktime)
 AC_CHECK_FUNCS(mmap)
 AC_CHECK_FUNCS(mremap)
 AC_CHECK_FUNCS(openat)
+AC_CHECK_FUNCS(pclose)
+AC_CHECK_FUNCS(pipe)
 AC_CHECK_FUNCS(pipe2)
 AC_CHECK_FUNCS(poll)
+AC_CHECK_FUNCS(popen)
 AC_CHECK_FUNCS(posix_fadvise)
+AC_CHECK_FUNCS(posix_madvise)
 AC_CHECK_FUNCS(posix_memalign)
 AC_CHECK_FUNCS(ppoll)
 AC_CHECK_FUNCS(pread)
@@ -2070,12 +2114,15 @@ AC_CHECK_FUNCS(spawnv)
 AC_CHECK_FUNCS(symlink)
 AC_CHECK_FUNCS(syscall)
 AC_CHECK_FUNCS(sysconf)
+AC_CHECK_FUNCS(system)
 AC_CHECK_FUNCS(tanh)
 AC_CHECK_FUNCS(telldir)
 AC_CHECK_FUNCS(timegm)
 AC_CHECK_FUNCS(times)
 AC_CHECK_FUNCS(truncate)
 AC_CHECK_FUNCS(truncate64)		# used for Win32
+AC_CHECK_FUNCS(tzset)
+AC_CHECK_FUNCS(umask)
 AC_CHECK_FUNCS(unsetenv)
 AC_CHECK_FUNCS(utimensat)
 AC_CHECK_FUNCS(utimes)
@@ -2584,6 +2631,9 @@ AS_CASE([$coroutine_type], [yes|''], [
         [*-emscripten*], [
             coroutine_type=emscripten
         ],
+        [*-wasi*], [
+            coroutine_type=asyncify
+        ],
         [
             AC_CHECK_FUNCS([getcontext swapcontext makecontext],
                 [coroutine_type=ucontext],
@@ -3682,12 +3732,15 @@ AC_SUBST(INSTALLDOC)
 
 AC_ARG_ENABLE(jit-support,
         AS_HELP_STRING([--disable-jit-support], [disable JIT features]),
-        [MJIT_SUPPORT=$enableval
-         AS_IF([test x"$enable_jit_support" = "xyes"],
-                 [AC_DEFINE(USE_MJIT, 1)],
-                 [AC_DEFINE(USE_MJIT, 0)])],
-        [MJIT_SUPPORT=yes
-         AC_DEFINE(USE_MJIT, 1)])
+        [MJIT_SUPPORT=$enableval],
+        # Enable mjit by default except for WASI
+        [AS_IF([test x"$target_os" != "xwasi"],
+		[MJIT_SUPPORT=yes],
+		[MJIT_SUPPORT=no ])])
+
+AS_IF([test x"$MJIT_SUPPORT" = "xyes"],
+	[AC_DEFINE(USE_MJIT, 1)],
+	[AC_DEFINE(USE_MJIT, 0)])
 
 AC_SUBST(MJIT_SUPPORT)
 
@@ -3786,6 +3839,17 @@ AS_CASE(["$target_os"],
 	    LIBRUBY='lib$(RUBY_SO_NAME).a'
 	    LIBRUBYARG='-l$(RUBY_SO_NAME)'
 	])
+	],
+    [wasi*], [
+        FIRSTMAKEFILE=GNUmakefile:wasm/GNUmakefile.in
+        AC_LIBOBJ([wasm/missing])
+        AC_LIBOBJ([wasm/runtime])
+        AC_LIBOBJ([wasm/fiber])
+        AC_LIBOBJ([wasm/machine])
+        AC_LIBOBJ([wasm/setjmp])
+        AC_LIBOBJ([wasm/machine_core])
+        AC_LIBOBJ([wasm/setjmp_core])
+        PLATFORM_DIR=wasm
 ])
 
 MINIOBJS="$MINIDLNOBJ"
@@ -4216,8 +4280,7 @@ AS_MESSAGE([$PACKAGE library version = $ruby_version])
 AS_IF([test x"$CC_WRAPPER" != x], [
     CC='$(CC_WRAPPER) '"${CC@%:@$CC_WRAPPER }"
     CPP='$(CC_WRAPPER) '"${CPP@%:@$CC_WRAPPER }"
-    CC_WRAPPER='$(rubyarchdir)/darwin-cc'
-    XCC_WRAPPER='$(top_srcdir)/tool/darwin-cc'
+    XCC_WRAPPER="$CC_WRAPPER"
 ])
 AC_SUBST(CC_WRAPPER, '')
 AC_SUBST(XCC_WRAPPER)

coroutine/asyncify/Context.c

@@ -0,0 +1,10 @@
+#include "Context.h"
+
+void coroutine_trampoline(void * _start, void * _context)
+{
+    coroutine_start start = (coroutine_start)_start;
+    struct coroutine_context * context = _context;
+    rb_wasm_set_stack_pointer(context->current_sp);
+
+    start(context->from, context);
+}