ncnn importer

.github/workflows/code-format.yml

@@ -2,11 +2,20 @@ name: code-format
 
 on: [pull_request]
 
+concurrency:
+  group: code-format-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
 jobs:
   code-format:
+    permissions:
+      contents: write  # for stefanzweifel/git-auto-commit-action to push code in repo
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Setup .NET
       uses: actions/setup-dotnet@v1

.github/workflows/compiler-build.yml

@@ -261,6 +261,7 @@ jobs:
           dotnet-coverage collect -s tools/dotnet_coverage.settings.xml -f cobertura -o coverage/tflite_basic.xml pytest tests/importer/tflite_/basic/ --doctest-modules --junitxml=test_results/tflite_basic.xml
           dotnet-coverage collect -s tools/dotnet_coverage.settings.xml -f cobertura -o coverage/tflite_combine.xml pytest tests/importer/tflite_/combine/ --doctest-modules --junitxml=test_results/tflite_combine.xml
           dotnet-coverage collect -s tools/dotnet_coverage.settings.xml -f cobertura -o coverage/tflite_model.xml pytest tests/importer/tflite_/model/ --doctest-modules --junitxml=test_results/tflite_model.xml
+          dotnet-coverage collect -s tools/dotnet_coverage.settings.xml -f cobertura -o coverage/ncnn_basic.xml pytest tests/importer/ncnn_/basic/ --doctest-modules --junitxml=test_results/ncnn_basic.xml
           dotnet-coverage merge -o coverage.integration.xml -f cobertura -r coverage/*.xml
 
       - name: Upload Coverage

.gitignore

@@ -262,6 +262,9 @@ __pycache__/
 # vscode
 .vscode/
 
+# clangd
+.cache/
+
 # pip
 /python/*/dist
 *egg-info

Directory.Packages.props

@@ -49,7 +49,7 @@
     <PackageVersion Include="OrtKISharp" Version="0.0.2" />
     <PackageVersion Include="RazorLight" Version="2.3.0" />
     <PackageVersion Include="Singulink.Collections.Weak" Version="1.0.2" />
-    <PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.435" />
+    <PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.507" />
     <PackageVersion Include="System.CommandLine.Hosting" Version="0.3.0-alpha.21216.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Reactive" Version="5.0.0" />

docs/ncnn_ops.md

@@ -0,0 +1,19 @@
+## Supported ncnn ops
+
+## 支持的 ncnn 算子
+
+| Operator | Is Supported |
+|-------|------------------ |
+| Input | ✅ |
+| Clip | ✅ |
+| Concat | ✅ |
+| Convolution | ✅ |
+| ConvolutionDepthWise | ✅ |
+| HardSigmoid | ✅ |
+| HardSwish | ✅ |
+| Innerproduct | ✅ |
+| Pooling | ✅ |
+| ReLU | ✅ |
+| Sigmoid | ✅ |
+| Softmax | ✅ |
+| Split | ✅ |

examples/user_guide/test.param

@@ -0,0 +1,33 @@
+7767517
+31 31
+Input                    data                     0 1 data -23330=4,3,224,224,3 0=224 1=224 2=3
+Convolution              conv1                    1 1 data conv1_relu1 -23330=4,3,112,112,32 0=32 1=3 3=2 4=1 5=1 6=864 9=1
+ConvolutionDepthWise     conv2_1/dw               1 1 conv1_relu1 conv2_1/dw_relu2_1/dw -23330=4,3,112,112,32 0=32 1=3 4=1 5=1 6=288 7=32 9=1
+Convolution              conv2_1/sep              1 1 conv2_1/dw_relu2_1/dw conv2_1/sep_relu2_1/sep -23330=4,3,112,112,64 0=64 1=1 5=1 6=2048 9=1
+ConvolutionDepthWise     conv2_2/dw               1 1 conv2_1/sep_relu2_1/sep conv2_2/dw_relu2_2/dw -23330=4,3,56,56,64 0=64 1=3 3=2 4=1 5=1 6=576 7=64 9=1
+Convolution              conv2_2/sep              1 1 conv2_2/dw_relu2_2/dw conv2_2/sep_relu2_2/sep -23330=4,3,56,56,128 0=128 1=1 5=1 6=8192 9=1
+ConvolutionDepthWise     conv3_1/dw               1 1 conv2_2/sep_relu2_2/sep conv3_1/dw_relu3_1/dw -23330=4,3,56,56,128 0=128 1=3 4=1 5=1 6=1152 7=128 9=1
+Convolution              conv3_1/sep              1 1 conv3_1/dw_relu3_1/dw conv3_1/sep_relu3_1/sep -23330=4,3,56,56,128 0=128 1=1 5=1 6=16384 9=1
+ConvolutionDepthWise     conv3_2/dw               1 1 conv3_1/sep_relu3_1/sep conv3_2/dw_relu3_2/dw -23330=4,3,28,28,128 0=128 1=3 3=2 4=1 5=1 6=1152 7=128 9=1
+Convolution              conv3_2/sep              1 1 conv3_2/dw_relu3_2/dw conv3_2/sep_relu3_2/sep -23330=4,3,28,28,256 0=256 1=1 5=1 6=32768 9=1
+ConvolutionDepthWise     conv4_1/dw               1 1 conv3_2/sep_relu3_2/sep conv4_1/dw_relu4_1/dw -23330=4,3,28,28,256 0=256 1=3 4=1 5=1 6=2304 7=256 9=1
+Convolution              conv4_1/sep              1 1 conv4_1/dw_relu4_1/dw conv4_1/sep_relu4_1/sep -23330=4,3,28,28,256 0=256 1=1 5=1 6=65536 9=1
+ConvolutionDepthWise     conv4_2/dw               1 1 conv4_1/sep_relu4_1/sep conv4_2/dw_relu4_2/dw -23330=4,3,14,14,256 0=256 1=3 3=2 4=1 5=1 6=2304 7=256 9=1
+Convolution              conv4_2/sep              1 1 conv4_2/dw_relu4_2/dw conv4_2/sep_relu4_2/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=131072 9=1
+ConvolutionDepthWise     conv5_1/dw               1 1 conv4_2/sep_relu4_2/sep conv5_1/dw_relu5_1/dw -23330=4,3,14,14,512 0=512 1=3 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_1/sep              1 1 conv5_1/dw_relu5_1/dw conv5_1/sep_relu5_1/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=262144 9=1
+ConvolutionDepthWise     conv5_2/dw               1 1 conv5_1/sep_relu5_1/sep conv5_2/dw_relu5_2/dw -23330=4,3,14,14,512 0=512 1=3 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_2/sep              1 1 conv5_2/dw_relu5_2/dw conv5_2/sep_relu5_2/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=262144 9=1
+ConvolutionDepthWise     conv5_3/dw               1 1 conv5_2/sep_relu5_2/sep conv5_3/dw_relu5_3/dw -23330=4,3,14,14,512 0=512 1=3 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_3/sep              1 1 conv5_3/dw_relu5_3/dw conv5_3/sep_relu5_3/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=262144 9=1
+ConvolutionDepthWise     conv5_4/dw               1 1 conv5_3/sep_relu5_3/sep conv5_4/dw_relu5_4/dw -23330=4,3,14,14,512 0=512 1=3 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_4/sep              1 1 conv5_4/dw_relu5_4/dw conv5_4/sep_relu5_4/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=262144 9=1
+ConvolutionDepthWise     conv5_5/dw               1 1 conv5_4/sep_relu5_4/sep conv5_5/dw_relu5_5/dw -23330=4,3,14,14,512 0=512 1=3 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_5/sep              1 1 conv5_5/dw_relu5_5/dw conv5_5/sep_relu5_5/sep -23330=4,3,14,14,512 0=512 1=1 5=1 6=262144 9=1
+ConvolutionDepthWise     conv5_6/dw               1 1 conv5_5/sep_relu5_5/sep conv5_6/dw_relu5_6/dw -23330=4,3,7,7,512 0=512 1=3 3=2 4=1 5=1 6=4608 7=512 9=1
+Convolution              conv5_6/sep              1 1 conv5_6/dw_relu5_6/dw conv5_6/sep_relu5_6/sep -23330=4,3,7,7,1024 0=1024 1=1 5=1 6=524288 9=1
+ConvolutionDepthWise     conv6/dw                 1 1 conv5_6/sep_relu5_6/sep conv6/dw_relu6/dw -23330=4,3,7,7,1024 0=1024 1=3 4=1 5=1 6=9216 7=1024 9=1
+Convolution              conv6/sep                1 1 conv6/dw_relu6/dw conv6/sep_relu6/sep -23330=4,3,7,7,1024 0=1024 1=1 5=1 6=1048576 9=1
+Pooling                  pool6                    1 1 conv6/sep_relu6/sep pool6 -23330=4,1,1024,1,1 0=1 4=1
+InnerProduct             fc7                      1 1 pool6 fc7 -23330=4,1,1000,1,1 0=1000 1=1 2=1024000
+Softmax                  prob                     1 1 fc7 output -23330=4,1,1000,1,1

modules/Nncase.Modules.StackVM/packages.lock.json

@@ -4,11 +4,11 @@
     "net7.0": {
       "StyleCop.Analyzers": {
         "type": "Direct",
-        "requested": "[1.2.0-beta.435, )",
-        "resolved": "1.2.0-beta.435",
-        "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==",
+        "requested": "[1.2.0-beta.507, )",
+        "resolved": "1.2.0-beta.507",
+        "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==",
         "dependencies": {
-          "StyleCop.Analyzers.Unstable": "1.2.0.435"
+          "StyleCop.Analyzers.Unstable": "1.2.0.507"
         }
       },
       "Google.OrTools.runtime.linux-arm64": {
@@ -103,8 +103,8 @@
       },
       "StyleCop.Analyzers.Unstable": {
         "type": "Transitive",
-        "resolved": "1.2.0.435",
-        "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg=="
+        "resolved": "1.2.0.507",
+        "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw=="
       },
       "System.Buffers": {
         "type": "Transitive",

python/nncase/__init__.py

@@ -186,11 +186,15 @@ def import_caffe(self, model: bytes, prototxt: bytes) -> None:
 
     def import_onnx(self, model_content: bytes, options: ImportOptions) -> None:
         self._compile_options.input_format = "onnx"
-        self._import_module(model_content)
+        self._import_onnx_module(model_content)
 
     def import_tflite(self, model_content: bytes, options: ImportOptions) -> None:
         self._compile_options.input_format = "tflite"
-        self._import_module(model_content)
+        self._import_tflite_module(model_content)
+
+    def import_ncnn(self, model_param: bytes, model_bin : bytes, options: ImportOptions) -> None:
+        self._compile_options.input_format = "ncnn"
+        self._import_ncnn_module(model_param, model_bin)
 
     def use_ptq(self, ptq_dataset_options: PTQTensorOptions) -> None:
         dataset = [_nncase.RTValue.from_runtime_tensor(
@@ -275,9 +279,18 @@ def __process_compile_options(self, compile_options: CompileOptions) -> ClCompil
         self._compile_options.dump_flags = dump_flags
         self._compile_options.dump_dir = compile_options.dump_dir
 
-    def _import_module(self, model_content: bytes | io.RawIOBase) -> None:
+    def _import_onnx_module(self, model_content: bytes | io.RawIOBase) -> None:
+        stream = io.BytesIO(model_content) if isinstance(model_content, bytes) else model_content
+        self._module = IRModule(self._compiler.import_onnx_module(stream))
+
+    def _import_tflite_module(self, model_content: bytes | io.RawIOBase) -> None:
         stream = io.BytesIO(model_content) if isinstance(model_content, bytes) else model_content
-        self._module = IRModule(self._compiler.import_module(stream))
+        self._module = IRModule(self._compiler.import_tflite_module(stream))
+
+    def _import_ncnn_module(self, model_param: bytes | io.RawIOBase, model_bin: bytes | io.RawIOBase) -> None:
+        param_stream = io.BytesIO(model_param) if isinstance(model_param, bytes) else model_param
+        bin_stream = io.BytesIO(model_bin) if isinstance(model_bin, bytes) else model_bin
+        self._module = IRModule(self._compiler.import_ncnn_module(param_stream, bin_stream))
 
 
 def check_target(target: str):

python/nncase/native/ffi.cpp

@@ -299,7 +299,9 @@ PYBIND11_MODULE(_nncase, m) {
         .def_property_readonly("entry", &ir_module::entry);
 
     py::class_<compiler>(m, "Compiler")
-        .def("import_module", &compiler::import_module)
+        .def("import_tflite_module", &compiler::import_tflite_module)
+        .def("import_onnx_module", &compiler::import_onnx_module)
+        .def("import_ncnn_module", &compiler::import_ncnn_module)
         .def("compile", &compiler::compile)
         .def("gencode", &compiler::gencode);
 

requirements.test.txt

@@ -6,6 +6,7 @@ onnx==1.12.0
 onnx-simplifier==0.3.6
 onnxoptimizer==0.2.6
 onnxruntime==1.12.0
+ncnn==1.0.20230816
 numpy==1.21.0
 torch==1.9.0
 torchvision==0.10.0
@@ -15,8 +16,6 @@ kendryte-caffe
 pytest
 pytest-xdist
 pyyaml
-pythonnet==3.0.1
-clr_loader==0.2.4
 toml==0.10.2
 pandas
 tabulate

src/Native/include/nncase/compiler.h

@@ -159,8 +159,13 @@ typedef struct {
     clr_object_handle_t (*compile_session_get_compiler)(
         clr_object_handle_t compile_session);
     void (*compiler_initialize)();
-    clr_object_handle_t (*compiler_import_module)(clr_object_handle_t compiler,
-                                                  clr_object_handle_t stream);
+    clr_object_handle_t (*compiler_import_tflite_module)(
+        clr_object_handle_t compiler, clr_object_handle_t stream);
+    clr_object_handle_t (*compiler_import_onnx_module)(
+        clr_object_handle_t compiler, clr_object_handle_t stream);
+    clr_object_handle_t (*compiler_import_ncnn_module)(
+        clr_object_handle_t compiler, clr_object_handle_t param_stream,
+        clr_object_handle_t bin_stream);
     void (*compiler_compile)(clr_object_handle_t compiler);
     void (*compiler_gencode)(clr_object_handle_t compiler,
                              clr_object_handle_t stream);
@@ -642,9 +647,20 @@ class compiler : public clr_object_base {
   public:
     using clr_object_base::clr_object_base;
 
-    ir_module import_module(cstream &stream) {
+    ir_module import_tflite_module(cstream &stream) {
+        return {std::in_place, nncase_clr_api()->compiler_import_tflite_module(
+                                   get(), stream.get())};
+    }
+
+    ir_module import_onnx_module(cstream &stream) {
+        return {std::in_place, nncase_clr_api()->compiler_import_onnx_module(
+                                   get(), stream.get())};
+    }
+
+    ir_module import_ncnn_module(cstream &param_stream, cstream &bin_stream) {
         return {std::in_place,
-                nncase_clr_api()->compiler_import_module(get(), stream.get())};
+                nncase_clr_api()->compiler_import_ncnn_module(
+                    get(), param_stream.get(), bin_stream.get())};
     }
 
     void compile() { nncase_clr_api()->compiler_compile(obj_.get()); }

src/Nncase.Cli/Commands/Compile.cs

@@ -86,6 +86,10 @@ public Compile()
           alias: "--calib-method",
           description: $"model quant options, default is {Quantization.CalibMethod.Kld}",
           getDefaultValue: () => Quantization.CalibMethod.Kld));
+        AddOption(new Option<bool>(
+          alias: "--benchmark-only",
+          description: $"benchmark only",
+          getDefaultValue: () => false));
 
         Handler = CommandHandler.Create<CliCompileOptions, IHost>(RunAsync);
     }
@@ -138,17 +142,14 @@ private async Task RunAsync(CliCompileOptions cliOptions, IHost host)
                 },
                 ModelQuantMode = cliOptions.ModelQuantMode,
             },
+            IsBenchmarkOnly = cliOptions.BenchmarkOnly,
         };
 
         // 2. import the model
         var target = CompilerServices.GetTarget(cliOptions.Target);
         using var compileSession = CompileSession.Create(target, compileOptions);
         var compiler = compileSession.Compiler;
-        IRModule module;
-        using (var model_stream = File.OpenRead(compileOptions.InputFile))
-        {
-            module = await compiler.ImportModuleAsync(model_stream);
-        }
+        var module = await compiler.ImportModuleAsync(compileOptions.InputFormat, compileOptions.InputFile, compileOptions.IsBenchmarkOnly);
 
         // 3. create the calib dataset
         if (compileOptions.QuantizeOptions.ModelQuantMode == Quantization.ModelQuantMode.UsePTQ)
@@ -206,6 +207,8 @@ internal sealed class CliCompileOptions
     public string Dataset { get; set; }
 
     public DatasetFormat DatasetFormat { get; set; }
+
+    public bool BenchmarkOnly { get; set; }
 }
 
 #pragma warning restore CS8618

src/Nncase.Cli/packages.lock.json

@@ -33,11 +33,11 @@
       },
       "StyleCop.Analyzers": {
         "type": "Direct",
-        "requested": "[1.2.0-beta.435, )",
-        "resolved": "1.2.0-beta.435",
-        "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==",
+        "requested": "[1.2.0-beta.507, )",
+        "resolved": "1.2.0-beta.507",
+        "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==",
         "dependencies": {
-          "StyleCop.Analyzers.Unstable": "1.2.0.435"
+          "StyleCop.Analyzers.Unstable": "1.2.0.507"
         }
       },
       "System.CommandLine.Hosting": {
@@ -344,8 +344,8 @@
       },
       "StyleCop.Analyzers.Unstable": {
         "type": "Transitive",
-        "resolved": "1.2.0.435",
-        "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg=="
+        "resolved": "1.2.0.507",
+        "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw=="
       },
       "System.Buffers": {
         "type": "Transitive",

src/Nncase.CodeGen/packages.lock.json

@@ -10,11 +10,11 @@
       },
       "StyleCop.Analyzers": {
         "type": "Direct",
-        "requested": "[1.2.0-beta.435, )",
-        "resolved": "1.2.0-beta.435",
-        "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==",
+        "requested": "[1.2.0-beta.507, )",
+        "resolved": "1.2.0-beta.507",
+        "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==",
         "dependencies": {
-          "StyleCop.Analyzers.Unstable": "1.2.0.435"
+          "StyleCop.Analyzers.Unstable": "1.2.0.507"
         }
       },
       "Microsoft.Extensions.Configuration.Abstractions": {
@@ -53,8 +53,8 @@
       },
       "StyleCop.Analyzers.Unstable": {
         "type": "Transitive",
-        "resolved": "1.2.0.435",
-        "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg=="
+        "resolved": "1.2.0.507",
+        "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw=="
       },
       "System.Buffers": {
         "type": "Transitive",