Latest wasi-libc (#719)

* gh: fix merge conflicts

* wasi: add mising symbols in latest wasi-libc

* wamr: fix bug in argc/argv processing

* wasi-libc: new wasi-libc implements getcwd

* wamr: make sure to de-instantiate the function

* gha: bump cpython version

* wamr: not mprotect

* tests: fix two failing tests

* docker: bump cpython version in base dockerfile

* gh: bump code version

* codegen: add command line parsing to properly pass the clean flag everywhere

* nit: no space between namespaces

* cpp: more wasi-libc updates

* python: bump submodule

* tests: temporarily disable calloc test

* tests: comment out more python tests

* tests: fix python and calloc tests

* tests: comment out microbench test

* tests: use a different python shared library

* cpp: fix mmap/mmap_big tests. wasi-libc's mmap implementation does not support PROT_EXEC

* dist-tests: temporarily disable failing test

* wamr: don't change config in this pr

* cpp: bump submodule for formatting changes

* wamr: comment out exit usage

* wamr: add link to upstream issue for failing test

* wasm: optionally compile our faasm-managed wasm memory allocation

* cleanup + enable faasm-managed memory see if this is the root of the errors

* bump submodules to revert back to faasm-based mman

* tests: fix microbench runner by patching faabric

* tests: fix failing wasm tests

* faabric: fix for distributed tests

* tests: uncomment failing dist test

* memory: revert the commit to introduce faasm_wasm_mman, out of scope for this pr

* nits: self-review cleanup

* clients: bump all submodules after re-tagging

* gh: fix after rebase

* nits: run clang-format

* gh: bump cpp and python again

* gh: bump submodules after merging and re-tagging

.env

@@ -1,12 +1,12 @@
-FAASM_VERSION=0.9.1
-FAASM_CLI_IMAGE=faasm/cli:0.9.1
-FAASM_WORKER_IMAGE=faasm/worker:0.9.1
+FAASM_VERSION=0.9.2
+FAASM_CLI_IMAGE=faasm/cli:0.9.2
+FAASM_WORKER_IMAGE=faasm/worker:0.9.2
 
-CPP_VERSION=0.2.1
-CPP_CLI_IMAGE=faasm/cpp-sysroot:0.2.1
+CPP_VERSION=0.2.2
+CPP_CLI_IMAGE=faasm/cpp-sysroot:0.2.2
 
-PYTHON_VERSION=0.2.2
-PYTHON_CLI_IMAGE=faasm/cpython:0.2.2
+PYTHON_VERSION=0.2.3
+PYTHON_CLI_IMAGE=faasm/cpython:0.2.3
 
 COMPOSE_PROJECT_NAME=faasm-dev
 

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cli:0.9.1
+      image: faasm/cli:0.9.2
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v3
@@ -49,7 +49,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cli:0.9.1
+      image: faasm/cli:0.9.2
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v3
@@ -62,7 +62,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cpp-sysroot:0.2.1
+      image: faasm/cpp-sysroot:0.2.2
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v3
@@ -92,7 +92,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cpython:0.2.2
+      image: faasm/cpython:0.2.3
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v3
@@ -119,7 +119,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm/cli:0.9.1
+      image: faasm/cli:0.9.2
     steps:
       - name: "Conan cache"
         uses: faasm/conan-cache-action@v1
@@ -153,12 +153,12 @@ jobs:
       TSAN_OPTIONS: "history_size=0 halt_on_error=1 suppressions=./thread-sanitizer-ignorelist.txt flush_memory_ms=5000"
       UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=./ub-sanitizer-ignorelist.txt"
     container:
-      image: faasm/cli:0.9.1
+      image: faasm/cli:0.9.2
     services:
       redis:
-        image: faasm/redis:0.9.1
+        image: faasm/redis:0.9.2
       minio:
-        image: faasm/minio:0.9.1
+        image: faasm/minio:0.9.2
         env:
           MINIO_ROOT_USER: minio
           MINIO_ROOT_PASSWORD: minio123
@@ -233,12 +233,12 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: faasm/cli-sgx-sim:0.9.1
+      image: faasm/cli-sgx-sim:0.9.2
     services:
       redis:
-        image: faasm/redis:0.9.1
+        image: faasm/redis:0.9.2
       minio:
-        image: faasm/minio:0.9.1
+        image: faasm/minio:0.9.2
         env:
           MINIO_ROOT_USER: minio
           MINIO_ROOT_PASSWORD: minio123

VERSION

@@ -1 +1 @@
-0.9.1
+0.9.2

deploy/k8s-common/minio.yml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: minio-main
-      image: faasm/minio:0.9.1
+      image: faasm/minio:0.9.2
       env:
         - name: MINIO_ROOT_USER
           value: "minio"

deploy/k8s-common/redis.yml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: master
-      image: faasm/redis:0.9.1
+      image: faasm/redis:0.9.2
       ports:
         - containerPort: 6379
 
@@ -28,7 +28,7 @@ metadata:
 spec:
   containers:
     - name: master
-      image: faasm/redis:0.9.1
+      image: faasm/redis:0.9.2
       ports:
         - containerPort: 6379
 

deploy/k8s-sgx/upload.yml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: upload
-      image: faasm/upload:0.9.1
+      image: faasm/upload:0.9.2
       ports:
         - containerPort: 8002
         - containerPort: 5000

deploy/k8s-sgx/worker.yml

@@ -31,7 +31,7 @@ spec:
               weight: 100
 
       containers:
-        - image: faasm/worker-sgx:0.9.1
+        - image: faasm/worker-sgx:0.9.2
           name: faasm-worker
           ports:
             - containerPort: 8080

deploy/k8s/upload.yml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: upload
-      image: faasm/upload:0.9.1
+      image: faasm/upload:0.9.2
       ports:
         - containerPort: 8002
         - containerPort: 5000

deploy/k8s/worker.yml

@@ -31,7 +31,7 @@ spec:
               weight: 100
 
       containers:
-        - image: faasm/worker:0.9.1
+        - image: faasm/worker:0.9.2
           name: faasm-worker
           ports:
             - containerPort: 8080

docker/base.dockerfile

@@ -1,5 +1,5 @@
 # Stage to extract Python runtime files
-FROM faasm/cpython:0.2.2 as python
+FROM faasm/cpython:0.2.3 as python
 
 # Note - we don't often rebuild cpp-root so this dep may be behind
 FROM faasm/cpp-root:0.9.1

include/wamr/WAMRModuleMixin.h

@@ -64,6 +64,7 @@ struct WAMRModuleMixin
             validateNativePointer(nextBuffer, thisStr.size() + 1);
 
             std::copy(thisStr.begin(), thisStr.end(), nextBuffer);
+            nextBuffer[thisStr.size()] = '\0';
             strOffsets[i] = nativePointerToWasmOffset(nextBuffer);
 
             nextBuffer += thisStr.size() + 1;

include/wasm/WasmEnvironment.h

@@ -4,10 +4,12 @@
 #include <unordered_map>
 #include <vector>
 
+// Defined in wasi-libc/libc-bottom-half/sources/getcwd.c
+#define WASI_LIBC_CWD "/"
+
 #define FAKE_NAME "faasm"
 #define FAKE_PASSWORD "foobar123"
 #define FAKE_HOME "/home/faasm/"
-#define FAKE_WORKING_DIR "/bin"
 #define FAKE_HOSTNAME "faasm"
 
 #define FAKE_PID 23

src/enclave/inside/filesystem.cpp

@@ -21,11 +21,20 @@ static int wasi_fd_fdstat_get(wasm_exec_env_t execEnv, int a, int b)
     return 0;
 }
 
+static int32_t wasi_fd_fdstat_set_rights(wasm_exec_env_t exec_env,
+                                         int32_t a,
+                                         int64_t b,
+                                         int64_t c)
+{
+    return 0;
+}
+
 static NativeSymbol wasiNs[] = {
     REG_WASI_NATIVE_FUNC(fd_close, "(i)i"),
     REG_WASI_NATIVE_FUNC(fd_seek, "(iIii)i"),
     REG_WASI_NATIVE_FUNC(fd_write, "(iiii)i"),
     REG_WASI_NATIVE_FUNC(fd_fdstat_get, "(ii)i"),
+    REG_WASI_NATIVE_FUNC(fd_fdstat_set_rights, "(iII)i"),
 };
 
 uint32_t getFaasmWasiFilesystemApi(NativeSymbol** nativeSymbols)

src/wamr/WAMRWasmModule.cpp

@@ -89,6 +89,7 @@ void WAMRWasmModule::reset(faabric::Message& msg,
     std::string funcStr = faabric::util::funcToString(msg, true);
     SPDLOG_DEBUG("WAMR resetting after {} (snap key {})", funcStr, snapshotKey);
 
+    wasm_runtime_deinstantiate(moduleInstance);
     bindInternal(msg);
 }
 

src/wamr/codegen.cpp

@@ -61,7 +61,7 @@ std::vector<uint8_t> wamrCodegen(std::vector<uint8_t>& wasmBytes, bool isSgx)
     option.opt_level = 3;
     option.size_level = 3;
     option.output_format = AOT_FORMAT_FILE;
-    option.bounds_checks = 2;
+    option.bounds_checks = 0;
     option.enable_bulk_memory = false;
     option.enable_ref_types = true;
     option.is_jit_mode = false;

src/wamr/filesystem.cpp

@@ -37,18 +37,6 @@ static uint32_t dup_wrapper(wasm_exec_env_t exec_env, uint32_t fd)
     return doWasiDup(fd);
 }
 
-static uint32_t getcwd_wrapper(wasm_exec_env_t exec_env,
-                               char* buf,
-                               uint32_t bufLen)
-{
-    SPDLOG_DEBUG("S - getcwd");
-
-    // Fake working directory
-    std::strcpy(buf, FAKE_WORKING_DIR);
-
-    return __WASI_ESUCCESS;
-}
-
 static uint32_t getpwnam_wrapper(wasm_exec_env_t exec_env, uint32_t a)
 {
     SPDLOG_DEBUG("S - getpwnam");
@@ -71,9 +59,11 @@ static int32_t tempnam_wrapper(wasm_exec_env_t exec_env, int32_t a, int32_t b)
 }
 
 static NativeSymbol ns[] = {
-    REG_NATIVE_FUNC(__wasi_fd_dup, "(i*)i"), REG_NATIVE_FUNC(dup, "(i)i"),
-    REG_NATIVE_FUNC(getcwd, "(*~)i"),        REG_NATIVE_FUNC(getpwnam, "(i)i"),
-    REG_NATIVE_FUNC(sendfile, "(iiii)i"),    REG_NATIVE_FUNC(tempnam, "(ii)i"),
+    REG_NATIVE_FUNC(__wasi_fd_dup, "(i*)i"),
+    REG_NATIVE_FUNC(dup, "(i)i"),
+    REG_NATIVE_FUNC(getpwnam, "(i)i"),
+    REG_NATIVE_FUNC(sendfile, "(iiii)i"),
+    REG_NATIVE_FUNC(tempnam, "(ii)i"),
 };
 
 uint32_t getFaasmFilesystemApi(NativeSymbol** nativeSymbols)
@@ -143,6 +133,15 @@ static int32_t wasi_fd_fdstat_set_flags(wasm_exec_env_t exec_env,
     throw std::runtime_error("fd_fdstat_set_flags not implemented");
 }
 
+static int32_t wasi_fd_fdstat_set_rights(wasm_exec_env_t exec_env,
+                                         int32_t a,
+                                         int64_t b,
+                                         int64_t c)
+{
+    SPDLOG_DEBUG("S - fd_fdstat_set_rights");
+    throw std::runtime_error("fd_fdstat_set_rights not implemented");
+}
+
 static int32_t doFileStat(uint32_t fd,
                           const std::string& relativePath,
                           __wasi_filestat_t* statWasm)
@@ -558,6 +557,7 @@ static NativeSymbol wasiNs[] = {
     REG_WASI_NATIVE_FUNC(fd_close, "(i)i"),
     REG_WASI_NATIVE_FUNC(fd_fdstat_get, "(i*)i"),
     REG_WASI_NATIVE_FUNC(fd_fdstat_set_flags, "(ii)i"),
+    REG_WASI_NATIVE_FUNC(fd_fdstat_set_rights, "(iII)i"),
     REG_WASI_NATIVE_FUNC(fd_filestat_get, "(i*)i"),
     REG_WASI_NATIVE_FUNC(fd_pread, "(i*iI*)i"),
     REG_WASI_NATIVE_FUNC(fd_prestat_dir_name, "(i*~)i"),

src/wavm/env.cpp

@@ -105,7 +105,7 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, "getpwuid", I32, getpwuid, I32 uid)
 
     // Set up strings to place in passwd
     std::string fakeName = std::string(FAKE_NAME);
-    std::string fakeDir = std::string(FAKE_WORKING_DIR);
+    std::string fakeDir = std::string(WASI_LIBC_CWD);
 
     // Create enough memory
     size_t nameOffset = sizeof(wasm_passwd);
@@ -150,12 +150,6 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, "getgid", I32, getgid)
     return FAKE_GID;
 }
 
-WAVM_DEFINE_INTRINSIC_FUNCTION(env, "getpid", I32, getpid)
-{
-    SPDLOG_DEBUG("S - getpid");
-    return FAKE_PID;
-}
-
 WAVM_DEFINE_INTRINSIC_FUNCTION(env, "getppid", I32, getppid)
 {
     throwException(Runtime::ExceptionTypes::calledUnimplementedIntrinsic);
@@ -331,26 +325,6 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(wasi,
     return __WASI_ESUCCESS;
 }
 
-WAVM_DEFINE_INTRINSIC_FUNCTION(env,
-                               "getcwd",
-                               I32,
-                               getcwd,
-                               I32 bufPtr,
-                               I32 bufLen)
-{
-    SPDLOG_DEBUG("S - getcwd - {} {}", bufPtr, bufLen);
-
-    Runtime::Memory* memoryPtr = getExecutingWAVMModule()->defaultMemory;
-    char* hostBuf =
-      Runtime::memoryArrayPtr<char>(memoryPtr, (Uptr)bufPtr, (Uptr)bufLen);
-
-    // Fake working directory
-    std::strcpy(hostBuf, FAKE_WORKING_DIR);
-
-    // Note, this needs to return the buffer on success, NOT zero
-    return bufPtr;
-}
-
 // --------------------------
 // Unsupported
 // --------------------------

src/wavm/io.cpp

@@ -592,6 +592,19 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(wasi,
     return 0;
 }
 
+WAVM_DEFINE_INTRINSIC_FUNCTION(wasi,
+                               "fd_fdstat_set_rights",
+                               I32,
+                               wasi_fd_fdstat_set_rights,
+                               I32 a,
+                               I64 b,
+                               I64 c)
+{
+    SPDLOG_DEBUG("S - fd_fdstat_set_rights - {} {} {}", a, b, c);
+
+    throwException(Runtime::ExceptionTypes::calledUnimplementedIntrinsic);
+}
+
 I32 doFileStat(int fd, const std::string& relativePath, I32 statPtr)
 {
     WAVMWasmModule* module = getExecutingWAVMModule();

src/wavm/network.cpp

@@ -371,6 +371,17 @@ WAVM_DEFINE_INTRINSIC_FUNCTION(env, "socket", I32, socket, I32 a, I32 b, I32 c)
     throwException(Runtime::ExceptionTypes::calledUnimplementedIntrinsic);
 }
 
+WAVM_DEFINE_INTRINSIC_FUNCTION(wasi,
+                               "sock_accept",
+                               I32,
+                               sock_accept,
+                               I32 a,
+                               I32 b,
+                               I32 c)
+{
+    throwException(Runtime::ExceptionTypes::calledUnimplementedIntrinsic);
+}
+
 WAVM_DEFINE_INTRINSIC_FUNCTION(wasi,
                                "sock_send",
                                I32,