Onnxifi backend

.gitmodules

@@ -8,3 +8,6 @@
 [submodule "test/lib/filesystem"]
 	path = test/lib/filesystem
 	url = https://github.com/wjakob/filesystem.git
+[submodule "external/fmt"]
+	path = external/fmt
+	url = https://github.com/fmtlib/fmt.git

CMakeLists.txt

@@ -16,6 +16,12 @@ option(ENABLE_EXAMPLE "Build example" ON)
 
 option(BUILD_SHARED_LIBS "Build shared libs" ON)
 
+option(ENABLE_ONNXIFI_BACKEND_DEBUG "" OFF)
+if(ENABLE_ONNXIFI_BACKEND_DEBUG)
+    message(STATUS "Set ENABLE_ONNIFI_BACKEND_DEBUG ON")
+    add_definitions(-DMENOH_ONNXIFI_BACKEND_DEBUG)
+endif()
+
 option(SHOW_ALL_VARIABLES "Debug: show all variables" OFF)
 
 ## C++ setup
@@ -99,9 +105,18 @@ configure_file(${EXTERNAL_DIR}/onnx-v1.3.0-patch_CMakeLists.txt ${EXTERNAL_DIR}/
 
 set(BUILD_SHARED_LIBS_SAVED "${BUILD_SHARED_LIBS}")
 set(BUILD_SHARED_LIBS OFF)
+if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+    add_definitions(-DONNXIFI_LOADER_LOGGING)
+endif()
 add_subdirectory(${EXTERNAL_DIR}/onnx EXCLUDE_FROM_ALL) # Note: BUILD_SHARED_LIBS must be OFF in this place
 set(BUILD_SHARED_LIBS "${BUILD_SHARED_LIBS_SAVED}")
 
+## Retrieve fmt
+set(FMT_SRC_DIR ${EXTERNAL_DIR}/fmt)
+execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init -- ${FMT_SRC_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+add_definitions(-DFMT_HEADER_ONLY=1)
+set(FMT_INCLUDE_DIR ${FMT_SRC_DIR}/include)
+
 if(${ENABLE_TEST})
     message(STATUS "Adding test")
     enable_testing()

benchmark/vgg16_benchmark.cpp

@@ -11,16 +11,22 @@
 int main(int argc, char** argv) {
     using clock = std::chrono::high_resolution_clock;
     cmdline::parser a;
-    a.add<std::string>("input", '\0', "input_data");
+    a.add<std::string>("backend", '\0', "backend name", false,
+                       "mkldnn_with_generic_fallback");
+    a.add<std::string>("config", '\0', "backend config", false, "");
+    a.add<std::string>("input", '\0', "input_data", false,
+                       "../data/random_input_1_3_224_224.txt");
     a.add<std::string>("model", '\0', "onnx model path", false,
                        "../data/vgg16.onnx");
     a.add<int>("iteration", '\0', "number of iterations", false, 1);
     a.parse_check(argc, argv);
 
     constexpr auto category_num = 1000;
-    auto input_data = menoh_impl::load_np_array_as_array(a.get<std::string>("input"));
+    auto input_data =
+      menoh_impl::load_np_array_as_array(a.get<std::string>("input"));
     auto batch_size = input_data.dims().at(0);
-    menoh_impl::array output_data(menoh_impl::dtype_t::float_, {batch_size, category_num});
+    menoh_impl::array output_data(menoh_impl::dtype_t::float_,
+                                  {batch_size, category_num});
 
     auto input_image_path = a.get<std::string>("input");
     auto onnx_model_path = a.get<std::string>("model");
@@ -41,7 +47,8 @@ int main(int argc, char** argv) {
 
     // Build model
     menoh::model_builder model_builder(vpt);
-    auto model = model_builder.build_model(model_data, "mkldnn_with_generic_fallback");
+    auto model = model_builder.build_model(
+      model_data, a.get<std::string>("backend"), a.get<std::string>("config"));
     model_data
       .reset(); // you can delete model_data explicitly after model building
 
@@ -53,16 +60,19 @@ int main(int argc, char** argv) {
 
     std::vector<std::chrono::microseconds> usecs;
 
-    for (int i = 0; i < a.get<int>("iteration"); ++i) {
+    for(int i = 0; i < a.get<int>("iteration"); ++i) {
         auto start = clock::now();
 
         // Run inference
         model.run();
 
         auto end = clock::now();
-        usecs.push_back(std::chrono::duration_cast<std::chrono::microseconds>(end - start));
+        usecs.push_back(
+          std::chrono::duration_cast<std::chrono::microseconds>(end - start));
     }
 
-    auto total_usec = std::accumulate(usecs.begin(), usecs.end(), std::chrono::microseconds::zero());
-    std::cout << total_usec.count() / usecs.size() * 0.001 << " msec" << std::endl;
+    auto total_usec = std::accumulate(usecs.begin(), usecs.end(),
+                                      std::chrono::microseconds::zero());
+    std::cout << total_usec.count() / usecs.size() * 0.001 << " msec"
+              << std::endl;
 }

menoh/CMakeLists.txt

@@ -35,6 +35,7 @@ add_library(menoh_objlib OBJECT
     mkldnn/operator/fc.cpp
     mkldnn/operator/batch_norm.cpp
     mkldnn/model_core.cpp
+    onnxifi/model_core.hpp
     menoh.cpp
     model_core.cpp
 )
@@ -44,7 +45,8 @@ target_include_directories(menoh_objlib PUBLIC
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
     $<BUILD_INTERFACE:${PROTOBUF_INCLUDE_DIRS}>
-    $<BUILD_INTERFACE:${ONNX_INCLUDE_DIRS}>
+    "$<BUILD_INTERFACE:${ONNX_INCLUDE_DIRS}>"
+    $<BUILD_INTERFACE:${FMT_INCLUDE_DIR}>
     $<BUILD_INTERFACE:${MKLDNN_INCLUDE_DIR}>
 )
 
@@ -56,7 +58,7 @@ if(NOT APPLE AND NOT MSVC)
         TARGET menoh APPEND_STRING PROPERTY
             LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/menoh.map")
 endif()
-target_link_libraries(menoh PRIVATE ${MKLDNN_LIBRARIES} onnx)
+target_link_libraries(menoh PRIVATE ${MKLDNN_LIBRARIES} onnx onnxifi_loader)
 if(LINK_STATIC_LIBGCC)
     target_link_libraries(menoh PRIVATE -static-libgcc)
 endif()
@@ -66,7 +68,7 @@ endif()
 
 # menoh_test_target: only used in `test` subdirectory
 add_library(menoh_test_target $<TARGET_OBJECTS:menoh_objlib>)
-target_link_libraries(menoh_test_target PRIVATE ${MKLDNN_LIBRARIES} onnx)
+target_link_libraries(menoh_test_target PRIVATE ${MKLDNN_LIBRARIES} onnx onnxifi_loader)
 
 install(TARGETS menoh
     RUNTIME DESTINATION "bin"

menoh/model_core_factory.cpp

@@ -3,6 +3,7 @@
 
 #include <menoh/composite_backend/model_core.hpp>
 #include <menoh/mkldnn/model_core.hpp>
+#include <menoh/onnxifi/model_core.hpp>
 
 #include <menoh/json.hpp>
 
@@ -32,6 +33,11 @@ namespace menoh_impl {
               composite_backend::make_model_core(
                 input_table, required_output_table, output_profile_table,
                 model_data, config));
+        } else if(backend_name == "onnxifi") {
+            return std::make_unique<onnxifi_backend::model_core>(
+              onnxifi_backend::make_model_core(
+                input_table, required_output_table, output_profile_table,
+                model_data, config));
         }
 
         throw invalid_backend_name(backend_name);

menoh/model_data.hpp

@@ -13,6 +13,8 @@
 namespace menoh_impl {
 
     struct model_data {
+        int32_t ir_version;
+        int64_t opset_version;
         std::vector<node> node_list;
         std::vector<std::pair<std::string, array>>
           parameter_name_and_array_list;

menoh/onnx.cpp

@@ -219,7 +219,9 @@ namespace menoh_impl {
         auto parameter_table =
           extract_parameter_name_and_array_list_from_onnx_graph(
             *onnx_model.mutable_graph(), model_parameter_name_list);
-        return model_data{node_list, parameter_table};
+        return model_data{static_cast<int32_t>(onnx_model.ir_version()),
+                          onnx_model.opset_import(0).version(), node_list,
+                          parameter_table};
     }
 
     model_data make_model_data_from_onnx_file(std::string const& filename) {