Squashed commit of the following:

commit 0d9fb0e
Author: hydai <hydai@secondstate.io>
Date:   Tue Oct 17 06:46:01 2023 +0800

    [WASI-NN] Enable OpenVINO target again (#2940)

    Signed-off-by: hydai <z54981220@gmail.com>

commit 278904b
Author: Saikat Dey <57017288+notfathomless@users.noreply.github.com>
Date:   Fri Oct 13 20:19:59 2023 +0530

    [Plugin] zlib: Changed a few naming style issues for the zlib plugin tests. (#2947)

    Signed-off-by: Saikat Dey <saikatdey2100@gmail.com>

commit a4e9457
Author: dm4 <dm4@secondstate.io>
Date:   Thu Oct 12 19:53:12 2023 +0800

    [WASI-NN] Add STREAM_TO_STDOUT support in ggml backend

    Signed-off-by: dm4 <dm4@secondstate.io>

commit 3260157
Author: dm4 <dm4@secondstate.io>
Date:   Thu Oct 12 14:10:08 2023 +0800

    [CI] Temporary disable OpenCVMini testing

    Signed-off-by: dm4 <dm4@secondstate.io>

commit 7a36709
Author: dm4 <dm4@secondstate.io>
Date:   Thu Oct 5 10:43:51 2023 +0800

    [WASI-NN] Add AUTO device support in ggml backend

    Signed-off-by: dm4 <dm4@secondstate.io>

commit db51a07
Author: dm4 <dm4@secondstate.io>
Date:   Wed Oct 4 17:15:18 2023 +0800

    [WASI-NN] Unified preload options with case-insensitive matching

    Signed-off-by: dm4 <dm4@secondstate.io>

commit 630fb80
Author: Wck-iipi <110763795+Wck-iipi@users.noreply.github.com>
Date:   Fri Oct 6 15:54:18 2023 +0530

    [Plugin] Added imgproc 9 functions to opencvmini plugin. (#2794)

    * Added imgproc's 9 functions.

    Added BilateralFilter, BoxFilter, MiniDilate, MiniErode, GaussianBlur, MiniLaplacian, MedianBlur, PyrDown, PyrUp

    Kernel is now added through function.

    Added WasmEdgeOpenCVMiniEmptyMat

    Signed-off-by: wck-iipi <21dcs006@nith.ac.in>

    * Changed Expect in opencvmini_func.h

    Signed-off-by: wck-iipi <21dcs006@nith.ac.in>

    * Changed OpenCVMiniEmptyMat in opencvmini_func.cpp

    Signed-off-by: wck-iipi <21dcs006@nith.ac.in>

    * Updated tests for opencvmini

    Signed-off-by: wck-iipi <21dcs006@nith.ac.in>

    ---------

    Signed-off-by: wck-iipi <21dcs006@nith.ac.in>

Signed-off-by: Lîm Tsú-thuàn <dannypsnl@secondstate.io>

.github/workflows/build-extensions.yml

@@ -78,10 +78,12 @@ jobs:
     env:
       output_dir: build/plugins/wasi_nn
       test_dir: build/test/plugins/wasi_nn
-      build_options: -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=PyTorch -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=TensorFlowLite -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=GGML
-      tar_names: wasi_nn-pytorch wasi_nn-tensorflowlite wasi_nn-ggml
+      build_options: -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=PyTorch -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=OpenVINO -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=TensorFlowLite -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=GGML
+      tar_names: wasi_nn-pytorch wasi_nn-openvino wasi_nn-tensorflowlite wasi_nn-ggml
       test_bin: wasiNNTests
       output_bin: libwasmedgePluginWasiNN.so
+      OPENVINO_VERSION: "2023.0.2"
+      OPENVINO_YEAR: "2023"
       PYTORCH_VERSION: "1.8.2"
       PYTORCH_INSTALL_TO: "."
     needs: [get_version]
@@ -99,6 +101,7 @@ jobs:
         run: |
           apt update
           apt install -y unzip libopenblas-dev pkg-config
+          bash utils/wasi-nn/install-openvino.sh
           bash utils/wasi-nn/install-pytorch.sh
       - name: Build and test WASI-NN using ${{ matrix.compiler }} with ${{ matrix.build_type }} mode
         shell: bash
@@ -128,6 +131,11 @@ jobs:
         with:
           name: WasmEdge-plugin-wasi_nn-pytorch-${{ needs.get_version.outputs.version }}-ubuntu22.04-${{ matrix.compiler }}.tar.gz
           path: plugin_wasi_nn-pytorch.tar.gz
+      - name: Upload artifact - wasi_nn-openvino
+        uses: actions/upload-artifact@v3
+        with:
+          name: WasmEdge-plugin-wasi_nn-openvino-${{ needs.get_version.outputs.version }}-ubuntu22.04-${{ matrix.compiler }}.tar.gz
+          path: plugin_wasi_nn-openvino.tar.gz
       - name: Upload artifact - wasi_nn-tensorflowlite
         uses: actions/upload-artifact@v3
         with:
@@ -538,6 +546,10 @@ jobs:
           testbin_array=(${test_bins})
           for (( i=0; i<${#plugin_array[@]}; i++ ));
           do
+            if [[ ${plugin_array[$i]} == "wasmedge_opencvmini" ]]; then
+              echo "Temporary disable ${plugin_array[$i]} testing"
+              continue
+            fi
             echo "Testing ${plugin_array[$i]} :"
             cd ${test_prefix}/${plugin_array[$i]}
             ./${testbin_array[$i]}

.github/workflows/release.yml

@@ -123,9 +123,11 @@ jobs:
     runs-on: ubuntu-latest
     env:
       output_dir: build/plugins/wasi_nn
-      build_options: -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=PyTorch -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=TensorFlowLite -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=GGML
-      tar_names: wasi_nn-pytorch wasi_nn-tensorflowlite wasi_nn-ggml
+      build_options: -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=PyTorch -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=OpenVINO -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=TensorFlowLite -DWASMEDGE_PLUGIN_WASI_NN_BACKEND=GGML
+      tar_names: wasi_nn-pytorch wasi_nn-openvino wasi_nn-tensorflowlite wasi_nn-ggml
       output_bin: libwasmedgePluginWasiNN.so
+      OPENVINO_VERSION: "2023.0.2"
+      OPENVINO_YEAR: "2023"
       PYTORCH_VERSION: "1.8.2"
       PYTORCH_INSTALL_TO: "."
     needs: create_release
@@ -144,6 +146,7 @@ jobs:
         run: |
           apt update
           apt install unzip libopenblas-dev pkg-config
+          bash utils/wasi-nn/install-openvino.sh
           bash utils/wasi-nn/install-pytorch.sh
       - name: Build WASI-NN plugin
         shell: bash
@@ -179,6 +182,12 @@ jobs:
         run: |
           mv plugin_wasi_nn-pytorch.tar.gz WasmEdge-plugin-wasi_nn-pytorch-${{ needs.create_release.outputs.version }}-ubuntu20.04_x86_64.tar.gz
           gh release upload ${{ needs.create_release.outputs.version }} WasmEdge-plugin-wasi_nn-pytorch-${{ needs.create_release.outputs.version }}-ubuntu20.04_x86_64.tar.gz --clobber
+      - name: Upload wasi_nn-openvino plugin tar.gz package
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv plugin_wasi_nn-openvino.tar.gz WasmEdge-plugin-wasi_nn-openvino-${{ needs.create_release.outputs.version }}-ubuntu20.04_x86_64.tar.gz
+          gh release upload ${{ needs.create_release.outputs.version }} WasmEdge-plugin-wasi_nn-openvino-${{ needs.create_release.outputs.version }}-ubuntu20.04_x86_64.tar.gz --clobber
       - name: Upload wasi_nn-tensorflowlite plugin tar.gz package
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

plugins/wasi_nn/ggml.cpp

@@ -239,10 +239,18 @@ Expect<ErrNo> compute(WasiNNEnvironment &Env, uint32_t ContextId) noexcept {
       break;
     }
 
-    // Append the new token.
-    CxtRef.LlamaOutputs +=
+    std::string NextToken =
         llama_token_to_piece(GraphRef.LlamaContext, NewTokenId);
 
+    // When setting STREAM_TO_STDOUT, we print the output to stdout.
+    const char *StreamOutput = std::getenv("STREAM_TO_STDOUT");
+    if (StreamOutput != nullptr) {
+      std::cout << NextToken << std::flush;
+    }
+
+    // Append the new token.
+    CxtRef.LlamaOutputs += NextToken;
+
     // Prepare the next batch
     LlamaBatch.n_tokens = 0;
 

plugins/wasi_nn/types.h

@@ -22,7 +22,7 @@ enum class ErrNo : uint32_t {
 
 enum class TensorType : uint8_t { F16 = 0, F32 = 1, U8 = 2, I32 = 3 };
 
-enum class Device : uint32_t { CPU = 0, GPU = 1, TPU = 2 };
+enum class Device : uint32_t { CPU = 0, GPU = 1, TPU = 2, AUTO = 3 };
 
 enum class Backend : uint8_t {
   OpenVINO = 0,

plugins/wasi_nn/wasinnenv.cpp

@@ -17,16 +17,18 @@ create(const Plugin::PluginModule::ModuleDescriptor *) noexcept {
 }
 
 std::map<std::string_view, Backend> BackendMap = {
-    {"OpenVINO"sv, Backend::OpenVINO},
-    {"ONNX"sv, Backend::ONNX},
-    {"Tensorflow"sv, Backend::Tensorflow},
-    {"PyTorch"sv, Backend::PyTorch},
-    {"TensorflowLite"sv, Backend::TensorflowLite},
-    {"Autodetect"sv, Backend::Autodetect},
-    {"GGML"sv, Backend::GGML}};
+    {"openvino"sv, Backend::OpenVINO},
+    {"onnx"sv, Backend::ONNX},
+    {"tensorflow"sv, Backend::Tensorflow},
+    {"pytorch"sv, Backend::PyTorch},
+    {"tensorflowlite"sv, Backend::TensorflowLite},
+    {"autodetect"sv, Backend::Autodetect},
+    {"ggml"sv, Backend::GGML}};
 
-std::map<std::string_view, Device> DeviceMap = {
-    {"CPU"sv, Device::CPU}, {"GPU"sv, Device::GPU}, {"TPU"sv, Device::TPU}};
+std::map<std::string_view, Device> DeviceMap = {{"cpu"sv, Device::CPU},
+                                                {"gpu"sv, Device::GPU},
+                                                {"tpu"sv, Device::TPU},
+                                                {"auto"sv, Device::AUTO}};
 
 bool load(const std::filesystem::path &Path, std::vector<uint8_t> &Data) {
   std::ifstream File(Path, std::ios::binary);
@@ -61,6 +63,10 @@ WasiNNEnvironment::WasiNNEnvironment() noexcept {
     }
     std::vector<std::vector<uint8_t>> Models;
     Models.reserve(Paths.size());
+    std::transform(Encode.begin(), Encode.end(), Encode.begin(),
+                   [](unsigned char C) { return std::tolower(C); });
+    std::transform(Target.begin(), Target.end(), Target.begin(),
+                   [](unsigned char C) { return std::tolower(C); });
     auto Backend = BackendMap.find(Encode);
     auto Device = DeviceMap.find(Target);
     if (Backend != BackendMap.end() && Device != DeviceMap.end()) {

plugins/wasmedge_opencvmini/opencvmini_func.cpp

@@ -3,7 +3,11 @@
 
 #include "opencvmini_func.h"
 #include "common/defines.h"
+#include "common/errcode.h"
 
+#include <cstdint>
+#include <opencv2/core/mat.hpp>
+#include <opencv2/imgproc.hpp>
 #include <opencv2/opencv.hpp>
 #include <string>
 #include <vector>
@@ -67,6 +71,109 @@ Expect<uint32_t> WasmEdgeOpenCVMiniBlur::body(const Runtime::CallingFrame &,
   return Env.insertMat(Dst);
 }
 
+Expect<uint32_t>
+WasmEdgeOpenCVMiniBilateralFilter::body(const Runtime::CallingFrame &,
+                                        uint32_t SrcMatKey, uint32_t D,
+                                        double SigmaColor, double SigmaSpace) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::bilateralFilter(*Src, Dst, D, SigmaColor, SigmaSpace);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t>
+WasmEdgeOpenCVMiniBoxFilter::body(const Runtime::CallingFrame &,
+                                  uint32_t SrcMatKey, uint32_t Ddepth,
+                                  uint32_t KernelWidth, uint32_t KernelHeight) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::boxFilter(*Src, Dst, Ddepth, cv::Size(KernelWidth, KernelHeight));
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t>
+WasmEdgeOpenCVMiniEmptyMat::body(const Runtime::CallingFrame &) {
+  cv::Mat Kernel;
+  return Env.insertMat(Kernel);
+}
+
+Expect<uint32_t> WasmEdgeOpenCVMiniDilate::body(const Runtime::CallingFrame &,
+                                                uint32_t SrcMatKey,
+                                                uint32_t KernelMatKey) {
+  cv::Mat Dst;
+  auto Kernel = Env.getMat(KernelMatKey);
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::dilate(*Src, Dst, *Kernel);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t> WasmEdgeOpenCVMiniErode::body(const Runtime::CallingFrame &,
+                                               uint32_t SrcMatKey,
+                                               uint32_t KernelMatKey) {
+  cv::Mat Dst;
+  auto Kernel = Env.getMat(KernelMatKey);
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::erode(*Src, Dst, *Kernel);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t>
+WasmEdgeOpenCVMiniGaussianBlur::body(const Runtime::CallingFrame &,
+                                     uint32_t SrcMatKey, uint32_t KernelWidth,
+                                     uint32_t KernelHeight, double SigmaX) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::GaussianBlur(*Src, Dst, cv::Size(KernelWidth, KernelHeight), SigmaX);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t>
+WasmEdgeOpenCVMiniLaplacian::body(const Runtime::CallingFrame &,
+                                  uint32_t SrcMatKey, uint32_t Ddepth) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::Laplacian(*Src, Dst, Ddepth);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t>
+WasmEdgeOpenCVMiniMedianBlur::body(const Runtime::CallingFrame &,
+                                   uint32_t SrcMatKey, uint32_t Ksize) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::medianBlur(*Src, Dst, Ksize);
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t> WasmEdgeOpenCVMiniPyrDown::body(const Runtime::CallingFrame &,
+                                                 uint32_t SrcMatKey,
+                                                 uint32_t KernelWidth,
+                                                 uint32_t KernelHeight) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::pyrDown(*Src, Dst, cv::Size(KernelWidth, KernelHeight));
+  }
+  return Env.insertMat(Dst);
+}
+
+Expect<uint32_t> WasmEdgeOpenCVMiniPyrUp::body(const Runtime::CallingFrame &,
+                                               uint32_t SrcMatKey,
+                                               uint32_t KernelWidth,
+                                               uint32_t KernelHeight) {
+  cv::Mat Dst;
+  if (auto Src = Env.getMat(SrcMatKey); Src) {
+    cv::pyrUp(*Src, Dst, cv::Size(KernelWidth, KernelHeight));
+  }
+  return Env.insertMat(Dst);
+}
+
 Expect<void> WasmEdgeOpenCVMiniImwrite::body(const Runtime::CallingFrame &Frame,
                                              uint32_t TargetFileNamePtr,
                                              uint32_t TargetFileNameLen,

plugins/wasmedge_opencvmini/opencvmini_func.h

@@ -59,6 +59,16 @@ class WasmEdgeOpenCVMiniBlur
                         uint32_t KernelWidth, uint32_t KernelHeight);
 };
 
+class WasmEdgeOpenCVMiniBilateralFilter
+    : public WasmEdgeOpenCVMini<WasmEdgeOpenCVMiniBilateralFilter> {
+public:
+  WasmEdgeOpenCVMiniBilateralFilter(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+
+  Expect<uint32_t> body(const Runtime::CallingFrame &Frame, uint32_t SrcMatKey,
+                        uint32_t D, double SigmaColor, double SigmaSpace);
+};
+
 class WasmEdgeOpenCVMiniImwrite
     : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniImwrite> {
 public:
@@ -91,6 +101,88 @@ class WasmEdgeOpenCVMiniBilinearSampling
                         uint32_t OutImgW, uint32_t OutImgH);
 };
 
+class WasmEdgeOpenCVMiniBoxFilter
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniBoxFilter> {
+public:
+  WasmEdgeOpenCVMiniBoxFilter(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t Ddepth, uint32_t KernelWidth,
+                        uint32_t KernelHeight);
+};
+
+class WasmEdgeOpenCVMiniEmptyMat
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniEmptyMat> {
+public:
+  WasmEdgeOpenCVMiniEmptyMat(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &);
+};
+
+class WasmEdgeOpenCVMiniDilate
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniDilate> {
+public:
+  WasmEdgeOpenCVMiniDilate(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t KernelMatKey);
+};
+
+class WasmEdgeOpenCVMiniErode
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniErode> {
+public:
+  WasmEdgeOpenCVMiniErode(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t Kernel);
+};
+
+class WasmEdgeOpenCVMiniGaussianBlur
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniGaussianBlur> {
+public:
+  WasmEdgeOpenCVMiniGaussianBlur(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t KernelWidth, uint32_t KernelHeight,
+                        double SigmaX);
+};
+
+class WasmEdgeOpenCVMiniLaplacian
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniLaplacian> {
+public:
+  WasmEdgeOpenCVMiniLaplacian(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t Ddepth);
+};
+
+class WasmEdgeOpenCVMiniMedianBlur
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniMedianBlur> {
+public:
+  WasmEdgeOpenCVMiniMedianBlur(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t Ksize);
+};
+
+class WasmEdgeOpenCVMiniPyrDown
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniPyrDown> {
+public:
+  WasmEdgeOpenCVMiniPyrDown(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t KernelWidth, uint32_t KernelHeight);
+};
+
+class WasmEdgeOpenCVMiniPyrUp
+    : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniPyrUp> {
+public:
+  WasmEdgeOpenCVMiniPyrUp(WasmEdgeOpenCVMiniEnvironment &HostEnv)
+      : WasmEdgeOpenCVMini(HostEnv) {}
+  Expect<uint32_t> body(const Runtime::CallingFrame &, uint32_t SrcMatKey,
+                        uint32_t KernelWidth, uint32_t KernelHeight);
+};
+
 class WasmEdgeOpenCVMiniRectangle
     : public WasmEdgeOpenCVMini<class WasmEdgeOpenCVMiniRectangle> {
 public: