From f5060600e38fb41c4eeaa2089efc2f9a28fde198 Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 Date: Mon, 15 Sep 2025 00:34:40 -0700 Subject: [PATCH 1/2] [OVEP] Updated CXX sample --- c_cxx/OpenVINO_EP/Windows/CMakeLists.txt | 8 +- c_cxx/OpenVINO_EP/Windows/README.md | 80 +++++++--------- .../Windows/model-explorer/model-explorer.cpp | 93 ++++++++++++++----- .../squeezenet_classification/CMakeLists.txt | 11 ++- .../squeezenet_cpp_app.cpp | 14 ++- 5 files changed, 118 insertions(+), 88 deletions(-) diff --git a/c_cxx/OpenVINO_EP/Windows/CMakeLists.txt b/c_cxx/OpenVINO_EP/Windows/CMakeLists.txt index 5d63b54c6..23391cec5 100644 --- a/c_cxx/OpenVINO_EP/Windows/CMakeLists.txt +++ b/c_cxx/OpenVINO_EP/Windows/CMakeLists.txt @@ -24,8 +24,8 @@ if(OPENCV_ROOTDIR) set(OPENCV_FOUND true) set(OPENCV_INCLUDE_DIRS "${OPENCV_ROOTDIR}/include") set(OPENCV_LIBDIR "${OPENCV_ROOTDIR}/x64/vc16/lib") - file(GLOB OPENCV_DEBUG_LIBRARIES ${OPENCV_LIBDIR}/opencv_world470d.lib) - file(GLOB OPENCV_RELEASE_LIBRARIES ${OPENCV_LIBDIR}/opencv_world470.lib) + file(GLOB OPENCV_DEBUG_LIBRARIES "${OPENCV_LIBDIR}/opencv_world*d.lib") + file(GLOB OPENCV_RELEASE_LIBRARIES "${OPENCV_LIBDIR}/opencv_world*.lib") list(FILTER OPENCV_RELEASE_LIBRARIES EXCLUDE REGEX ".*d\\.lib") endif() @@ -41,8 +41,4 @@ if(OPENCV_FOUND) add_subdirectory(squeezenet_classification) endif() -if(OPENCL_FOUND) - add_subdirectory(squeezenet_classification_io_buffer) -endif() - add_subdirectory(model-explorer) diff --git a/c_cxx/OpenVINO_EP/Windows/README.md b/c_cxx/OpenVINO_EP/Windows/README.md index b36e7bb61..50c923d07 100644 --- a/c_cxx/OpenVINO_EP/Windows/README.md +++ b/c_cxx/OpenVINO_EP/Windows/README.md @@ -2,34 +2,43 @@ 1. model-explorer - This sample application demonstrates how to use components of the experimental C++ API to query for model inputs/outputs and how to run inferrence using OpenVINO Execution Provider for ONNXRT on a model. The source code for this sample is available [here](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx/OpenVINO_EP/Windows/model-explorer). + This sample application demonstrates how to use the **ONNX Runtime C++ API** with the OpenVINO Execution Provider (OVEP). + It loads an ONNX model, inspects the input/output node names and shapes, creates random input data, and runs inference. + The sample is useful for exploring model structure and verifying end-to-end execution with OVEP. [here](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx/OpenVINO_EP/Windows/model-explorer). 2. Squeezenet classification sample - The sample involves presenting an image to the ONNX Runtime (RT), which uses the OpenVINO Execution Provider for ONNXRT to run inference on various Intel hardware devices like Intel CPU, GPU, VPU and more. The sample uses OpenCV for image processing and ONNX Runtime OpenVINO EP for inference. After the sample image is inferred, the terminal will output the predicted label classes in order of their confidence. The source code for this sample is available [here](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx/OpenVINO_EP/Windows/squeezenet_classification). - -3. Squeezenet classification sample with IO Buffer feature - - This sample is also doing the same process but with IO Buffer optimization enabled. With IO Buffer interfaces we can avoid any memory copy overhead when plugging OpenVINO™ inference into an existing GPU pipeline. It also enables OpenCL kernels to participate in the pipeline to become native buffer consumers or producers of the OpenVINO™ inference. Refer [here](https://docs.openvino.ai/latest/openvino_docs_OV_UG_supported_plugins_GPU_RemoteTensor_API.html) for more details. This sample is for GPUs only. The source code for this sample is available [here](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx/OpenVINO_EP/Windows/squeezenet_classification_io_buffer). + The sample involves presenting an image to the ONNX Runtime (RT), which uses the OpenVINO Execution Provider for ONNXRT to run inference on various Intel hardware devices like Intel CPU, GPU and NPU. The sample uses OpenCV for image processing and ONNX Runtime OpenVINO EP for inference. After the sample image is inferred, the terminal will output the predicted label classes in order of their confidence. The source code for this sample is available [here](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx/OpenVINO_EP/Windows/squeezenet_classification). ## How to build ## Prerequisites -1. [The Intel® Distribution of OpenVINO toolkit](https://docs.openvinotoolkit.org/latest/index.html) -2. Use opencv -3. Use opencl for IO buffer sample (squeezenet_cpp_app_io.cpp). -4. Use any sample image as input to the sample. -5. Download the latest Squeezenet model from the ONNX Model Zoo. - This example was adapted from [ONNX Model Zoo](https://github.com/onnx/models).Download the latest version of the [Squeezenet](https://github.com/onnx/models/tree/master/validated/vision/classification/squeezenet) model from here. +1. [The Intel® Distribution of OpenVINO toolkit](https://docs.openvino.ai/2025/get-started/install-openvino.html) +2. Use opencv [OpenCV](https://opencv.org/releases/) +3. Use any sample image as input to the sample. +4. Download the latest Squeezenet model from the ONNX Model Zoo. + This example was adapted from [ONNX Model Zoo](https://github.com/onnx/models). Download the latest version of the [Squeezenet](https://github.com/onnx/models/tree/master/vision/classification/squeezenet) model from here. -#### Build ONNX Runtime -Open x64 Native Tools Command Prompt for VS 2019. -For running the sample with IO Buffer optimization feature, make sure you set the OpenCL paths. For example if you are setting the path from openvino source build folder, the paths will be like: +## Build ONNX Runtime with OpenVINO on Windows +Make sure you open **x64 Native Tools Command Prompt for VS 2019** before running the following steps. + +### 1. Download OpenVINO package +Download the OpenVINO archive package from the official repository: +[OpenVINO Archive Packages](https://storage.openvinotoolkit.org/repositories/openvino/packages) + +Extract the downloaded archive to a directory (e.g., `C:\openvino`). + +--- + +### 2. Set up OpenVINO environment +After extracting, run the following command to set up environment variables: + +```cmd +"C:\openvino\setupvars.bat" ``` -set OPENCL_LIBS=\path\to\openvino\folder\bin\intel64\Release\OpenCL.lib -set OPENCL_INCS=\path\to\openvino\folder\thirdparty\ocl\clhpp_headers\include -``` + +### 3. Build ONNX Runtime ``` build.bat --config RelWithDebInfo --use_openvino CPU --build_shared_lib --parallel --cmake_extra_defines CMAKE_INSTALL_PREFIX=c:\dev\ort_install --skip_tests @@ -43,44 +52,23 @@ cd build\Windows\RelWithDebInfo msbuild INSTALL.vcxproj /p:Configuration=RelWithDebInfo ``` -#### Build the samples +### Build the samples -Open x64 Native Tools Command Prompt for VS 2019, Git clone the sample repo. +Open x64 Native Tools Command Prompt for VS 2022, Git clone the sample repo. ``` git clone https://github.com/microsoft/onnxruntime-inference-examples.git ``` Change your current directory to c_cxx\OpenVINO_EP\Windows, then run + ```bat mkdir build && cd build cmake .. -A x64 -T host=x64 -Donnxruntime_USE_OPENVINO=ON -DONNXRUNTIME_ROOTDIR=c:\dev\ort_install -DOPENCV_ROOTDIR="path\to\opencv" ``` Choose required opencv path. Skip the opencv flag if you don't want to build squeezenet sample. -To get the squeezenet sample with IO buffer feature enabled, pass opencl paths as well: -```bat -mkdir build && cd build -cmake .. -A x64 -T host=x64 -Donnxruntime_USE_OPENVINO=ON -DONNXRUNTIME_ROOTDIR=c:\dev\ort_install -DOPENCV_ROOTDIR="path\to\opencv" -DOPENCL_LIB=path\to\openvino\folder\bin\intel64\Release\ -DOPENCL_INCLUDE="path\to\openvino\folder\thirdparty\ocl\clhpp_headers\include;path\to\openvino\folder\thirdparty\ocl\cl_headers" -``` - -**Note:** -If you are using the opencv from openvino package, below are the paths: -* For openvino version 2022.1.0, run download_opencv.ps1 in \path\to\openvino\extras\script and the opencv folder will be downloaded at \path\to\openvino\extras. -* For older openvino version, opencv folder is available at openvino directory itself. -* The current cmake files are adjusted with the opencv folders coming along with openvino packages. Plase make sure you are updating the opencv paths according to your custom builds. - -For the squeezenet IO buffer sample: -Make sure you are creating the opencl context for the right GPU device in a multi-GPU environment. - -Build samples using msbuild for Debug configuration. For Release configuration replace Debug with Release. - -```bat -msbuild onnxruntime_samples.sln /p:Configuration=Debug -``` - +### Note To run the samples make sure you source openvino variables using setupvars.bat. -To run the samples download and install(extract) OpenCV from: [download OpenCV](https://github.com/opencv/opencv/releases/download/4.7.0/opencv-4.7.0-windows.exe). Also copy OpenCV dll (opencv_world470.dll which is located at: "path\to\opencv\build\x64\vc16\bin") to the location of the application exe file(Release dll for release build) and (opencv_world470d.dll which is located at:"path\to\opencv\build\x64\vc16\bin") to the location of the application exe file (debug dll for debug build). - #### Run the sample - To Run the general sample @@ -96,13 +84,9 @@ To run the samples download and install(extract) OpenCV from: [download OpenCV]( ``` run_squeezenet.exe --use_cpu ``` - - To Run the sample for IO Buffer Optimization feature - ``` - run_squeezenet.exe - ``` ## References: -[OpenVINO Execution Provider](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/faster-inferencing-with-one-line-of-code.html) +[OpenVINO Execution Provider](https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html) [Other ONNXRT Reference Samples](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/c_cxx) \ No newline at end of file diff --git a/c_cxx/OpenVINO_EP/Windows/model-explorer/model-explorer.cpp b/c_cxx/OpenVINO_EP/Windows/model-explorer/model-explorer.cpp index 3eb01ceee..c291b0ea1 100644 --- a/c_cxx/OpenVINO_EP/Windows/model-explorer/model-explorer.cpp +++ b/c_cxx/OpenVINO_EP/Windows/model-explorer/model-explorer.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include // pretty prints a shape dimension vector std::string print_shape(const std::vector& v) { @@ -64,59 +64,102 @@ int main(int argc, char** argv) { //Appending OpenVINO Execution Provider API #ifdef USE_OPENVINO // Using OPENVINO backend - OrtOpenVINOProviderOptions options; - options.device_type = "CPU"; - std::cout << "OpenVINO device type is set to: " << options.device_type << std::endl; - session_options.AppendExecutionProvider_OpenVINO(options); + std::unordered_map options; + options["device_type"] = "CPU"; + std::cout << "OpenVINO device type is set to: " << options["device_type"] << std::endl; + session_options.AppendExecutionProvider_OpenVINO_V2(options); #endif - Ort::Experimental::Session session = Ort::Experimental::Session(env, model_file, session_options); // access experimental components via the Experimental namespace - - // print name/shape of inputs - std::vector input_names = session.GetInputNames(); - std::vector > input_shapes = session.GetInputShapes(); - cout << "Input Node Name/Shape (" << input_names.size() << "):" << endl; - for (size_t i = 0; i < input_names.size(); i++) { + Ort::Session session(env, model_file.c_str(), session_options); + Ort::AllocatorWithDefaultOptions allocator; + + size_t num_input_nodes = session.GetInputCount(); + std::vector input_names; + std::vector> input_shapes; + + cout << "Input Node Name/Shape (" << num_input_nodes << "):" << endl; + for (size_t i = 0; i < num_input_nodes; i++) { + // Get input name + auto input_name = session.GetInputNameAllocated(i, allocator); + input_names.push_back(std::string(input_name.get())); + + // Get input shape + Ort::TypeInfo input_type_info = session.GetInputTypeInfo(i); + auto input_tensor_info = input_type_info.GetTensorTypeAndShapeInfo(); + std::vector input_dims = input_tensor_info.GetShape(); + input_shapes.push_back(input_dims); + cout << "\t" << input_names[i] << " : " << print_shape(input_shapes[i]) << endl; + } - // print name/shape of outputs - std::vector output_names = session.GetOutputNames(); - std::vector > output_shapes = session.GetOutputShapes(); - cout << "Output Node Name/Shape (" << output_names.size() << "):" << endl; - for (size_t i = 0; i < output_names.size(); i++) { + size_t num_output_nodes = session.GetOutputCount(); + std::vector output_names; + std::vector> output_shapes; + + cout << "Output Node Name/Shape (" << num_output_nodes << "):" << endl; + for (size_t i = 0; i < num_output_nodes; i++) { + // Get output name + auto output_name = session.GetOutputNameAllocated(i, allocator); + output_names.push_back(std::string(output_name.get())); + + // Get output shape + Ort::TypeInfo output_type_info = session.GetOutputTypeInfo(i); + auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo(); + std::vector output_dims = output_tensor_info.GetShape(); + output_shapes.push_back(output_dims); + cout << "\t" << output_names[i] << " : " << print_shape(output_shapes[i]) << endl; + } - - // Assume model has 1 input node and 1 output node. + assert(input_names.size() == 1 && output_names.size() == 1); // Create a single Ort tensor of random numbers auto input_shape = input_shapes[0]; int total_number_elements = calculate_product(input_shape); std::vector input_tensor_values(total_number_elements); - std::generate(input_tensor_values.begin(), input_tensor_values.end(), [&] { return rand() % 255; }); // generate random numbers in the range [0, 255] + std::generate(input_tensor_values.begin(), input_tensor_values.end(), [&] { return rand() % 255; }); + + // Create input tensor + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); std::vector input_tensors; - input_tensors.push_back(Ort::Experimental::Value::CreateTensor(input_tensor_values.data(), input_tensor_values.size(), input_shape)); + input_tensors.push_back(Ort::Value::CreateTensor(memory_info, input_tensor_values.data(), + input_tensor_values.size(), input_shape.data(), + input_shape.size())); // double-check the dimensions of the input tensor assert(input_tensors[0].IsTensor() && input_tensors[0].GetTensorTypeAndShapeInfo().GetShape() == input_shape); cout << "\ninput_tensor shape: " << print_shape(input_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; + // Create input/output name arrays for Run() + std::vector input_names_char(input_names.size(), nullptr); + std::vector output_names_char(output_names.size(), nullptr); + + for (size_t i = 0; i < input_names.size(); i++) { + input_names_char[i] = input_names[i].c_str(); + } + for (size_t i = 0; i < output_names.size(); i++) { + output_names_char[i] = output_names[i].c_str(); + } + // pass data through model cout << "Running model..."; try { - auto output_tensors = session.Run(session.GetInputNames(), input_tensors, session.GetOutputNames()); + auto output_tensors = session.Run(Ort::RunOptions{nullptr}, input_names_char.data(), + input_tensors.data(), input_names_char.size(), + output_names_char.data(), output_names_char.size()); cout << "done" << endl; // double-check the dimensions of the output tensors - // NOTE: the number of output tensors is equal to the number of output nodes specifed in the Run() call - assert(output_tensors.size() == session.GetOutputNames().size() && - output_tensors[0].IsTensor()); + assert(output_tensors.size() == output_names.size() && output_tensors[0].IsTensor()); cout << "output_tensor_shape: " << print_shape(output_tensors[0].GetTensorTypeAndShapeInfo().GetShape()) << endl; } catch (const Ort::Exception& exception) { cout << "ERROR running model inference: " << exception.what() << endl; exit(-1); } + + return 0; + } diff --git a/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/CMakeLists.txt b/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/CMakeLists.txt index 64aa6ed13..7e36469f9 100644 --- a/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/CMakeLists.txt +++ b/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/CMakeLists.txt @@ -13,13 +13,14 @@ if(OPENCV_LIBDIR) endif() #In onnxruntime deafault install path, the required dlls are in lib and bin folders -set(DLL_DIRS "${ONNXRUNTIME_ROOTDIR}/lib;${ONNXRUNTIME_ROOTDIR}/bin") +set(DLL_DIRS "${ONNXRUNTIME_ROOTDIR}/lib;${ONNXRUNTIME_ROOTDIR}/bin;${OPENCV_ROOTDIR}/x64/vc16/bin") + foreach(DLL_DIR IN LISTS DLL_DIRS) file(GLOB ALL_DLLS ${DLL_DIR}/*.dll) - foreach(ORTDll IN LISTS ALL_DLLS) + foreach(DLLFile IN LISTS ALL_DLLS) add_custom_command(TARGET run_squeezenet POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${ORTDll}" - $) + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${DLLFile}" + $) endforeach() endforeach() \ No newline at end of file diff --git a/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/squeezenet_cpp_app.cpp b/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/squeezenet_cpp_app.cpp index 779c4e654..113710e7e 100644 --- a/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/squeezenet_cpp_app.cpp +++ b/c_cxx/OpenVINO_EP/Windows/squeezenet_classification/squeezenet_cpp_app.cpp @@ -218,10 +218,16 @@ int main(int argc, char* argv[]) //Appending OpenVINO Execution Provider API if (useOPENVINO) { // Using OPENVINO backend - OrtOpenVINOProviderOptions options; - options.device_type = "CPU"; - std::cout << "OpenVINO device type is set to: " << options.device_type << std::endl; - sessionOptions.AppendExecutionProvider_OpenVINO(options); + std::unordered_map options; + options["device_type"] = "CPU"; + std::string config = R"({ + "CPU": { + "INFERENCE_NUM_THREADS": "1" + } + })"; + options["load_config"] = config; + std::cout << "OpenVINO device type is set to: " << options["device_type"] << std::endl; + sessionOptions.AppendExecutionProvider_OpenVINO_V2(options); } // Sets graph optimization level From 9b2ecae4e2f0b87f71f7b31ef0e0a693d65e98bf Mon Sep 17 00:00:00 2001 From: jatinwadhwa921 Date: Tue, 16 Sep 2025 04:04:35 -0700 Subject: [PATCH 2/2] [OVEP] Update python sample --- .../tiny_yolo_v2_object_detection/README.md | 2 +- ...OVEP_tiny_yolov2_obj_detection_sample.ipynb | 18 +++++++++--------- .../requirements.txt | 14 +++++++------- .../tiny_yolov2_obj_detection_sample.py | 8 ++++---- .../yolov4_object_detection/README.md | 14 ++++++-------- .../OVEP_yolov4_obj_detection_sample.ipynb | 18 +++++++++--------- .../yolov4_object_detection/requirements.txt | 14 +++++++------- .../yolov4_object_detection/yolov4.py | 17 ++++++++++++++--- 8 files changed, 57 insertions(+), 48 deletions(-) diff --git a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/README.md b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/README.md index 7e274b42f..3c7ea0422 100644 --- a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/README.md +++ b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/README.md @@ -45,7 +45,7 @@ python3 tiny_yolov2_obj_detection_sample.py --h ``` ## Running the ONNXRuntime OpenVINO™ Execution Provider sample ```bash -python3 tiny_yolov2_obj_detection_sample.py --video face-demographics-walking-and-pause.mp4 --model tinyyolov2.onnx --device CPU_FP32 +python3 tiny_yolov2_obj_detection_sample.py --video face-demographics-walking-and-pause.mp4 --model tinyyolov2.onnx --device CPU ``` ## To stop the sample from running diff --git a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/notebooks/OVEP_tiny_yolov2_obj_detection_sample.ipynb b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/notebooks/OVEP_tiny_yolov2_obj_detection_sample.ipynb index cf9d51f42..5d3bbb313 100644 --- a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/notebooks/OVEP_tiny_yolov2_obj_detection_sample.ipynb +++ b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/notebooks/OVEP_tiny_yolov2_obj_detection_sample.ipynb @@ -372,12 +372,12 @@ " \n", "1. Create a ONNX Runtime Session Option instance using `onnxruntime.SessionOptions()`\n", "2. Using the session options instance create a Inference Session object by passing the model and the execution provider as arguments.\n", - "Execution Providers are the hardware device options e.g. CPU, Myriad, GPU, etc. on which the session will be executed.\n", + "Execution Providers are the hardware device options e.g. CPU, GPU and NPU on which the session will be executed.\n", "\n", "The below `create_sess` function actually takes care of the above steps. All we need to do is pass the device arguement to it. It'll return the appropriate session according to the selected device along with the input name for the model.\n", "\n", "The device option should be chosen from any one of the below options: \n", - "- `cpu, CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VADM_FP16`" + "- `cpu, CPU, GPU, NPU`" ] }, { @@ -396,16 +396,16 @@ " print(\"Device type selected is 'cpu' which is the default CPU Execution Provider (MLAS)\")\n", " #Specify the path to the ONNX model on your machine and register the CPU EP\n", " sess = rt.InferenceSession(model, so, providers=['CPUExecutionProvider'])\n", - " elif (device == 'CPU_FP32' or device == 'GPU_FP32' or device == 'GPU_FP16' or device == 'MYRIAD_FP16' or device == 'VADM_FP16'):\n", + " elif (device == 'CPU' or device == 'GPU' or device == 'NPU'):\n", " #Specify the path to the ONNX model on your machine and register the OpenVINO EP\n", " sess = rt.InferenceSession(model, so, providers=['OpenVINOExecutionProvider'], provider_options=[{'device_type' : device}])\n", " print(\"Device type selected is: \" + device + \" using the OpenVINO Execution Provider\")\n", " '''\n", " other 'device_type' options are: (Any hardware target can be assigned if you have the access to it)\n", - " 'CPU_FP32', 'GPU_FP32', 'GPU_FP16', 'MYRIAD_FP16', 'VAD-M_FP16'\n", + " 'CPU', 'GPU', 'NPU'\n", " '''\n", " else:\n", - " raise Exception(\"Device type selected is not [cpu, CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VADM_FP16]\")\n", + " raise Exception(\"Device type selected is not [cpu, CPU, GPU, NPU]\")\n", "\n", " # Get the input name of the model\n", " input_name = sess.get_inputs()[0].name\n", @@ -504,7 +504,7 @@ "\n", "Now the `tinyyolov2-8.onnx` model will run inference on `cat.jpg` image using the below two execution providers:\n", "- `cpu`: default CPU Execution Provider (MLAS) \n", - "- `CPU_FP32`: Execution on CPU with OpenVino Execution Provider\n", + "- `CPU`: Execution on CPU with OpenVino Execution Provider\n", "\n", "The below code block performs the following operations:\n", "\n", @@ -567,7 +567,7 @@ }, "source": [ "### Run the inference with OpenVINO Execution Provider\n", - "The below code block performs the same opertions as [before](#cpu_exec) with `CPU_FP32` as device, that runs on OpenVINO Execution Provider for ONNX Runtime." + "The below code block performs the same opertions as [before](#cpu_exec) with `CPU` as device, that runs on OpenVINO Execution Provider for ONNX Runtime." ] }, { @@ -580,7 +580,7 @@ "outputs": [], "source": [ "### create a session with CPU_FP32 using the OpenVINO Execution Provider\n", - "sess, input_name = create_sess(\"CPU_FP32\")\n", + "sess, input_name = create_sess(\"CPU\")\n", "\n", "#capturing one frame at a time from the video feed and performing the inference\n", "frame = cap.copy()\n", @@ -607,7 +607,7 @@ "postprocess_output(out, frame, x_scale, y_scale)\n", "\n", "#Show the output\n", - "show_bbox(\"CPU_FP32\", frame, inference_time)\n", + "show_bbox(\"CPU\", frame, inference_time)\n", "\n", "#Write the frame with the detection boxes\n", "cv2.imwrite(output_file, frame.astype(np.uint8))\n", diff --git a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/requirements.txt b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/requirements.txt index 15dac4435..d60309fae 100644 --- a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/requirements.txt +++ b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/requirements.txt @@ -1,8 +1,8 @@ -certifi==2024.7.4 -flatbuffers==2.0 +certifi==2025.8.3 +flatbuffers==25.2.10 onnx -opencv-python==4.8.1.78 -Pillow==10.2.0 -protobuf==3.20.2 -scipy==1.11.4 -typing-extensions==4.2.0 +opencv-python==4.12.0.88 +Pillow==11.3.0 +protobuf==6.32.1 +scipy==1.16.2 +typing-extensions==4.15.0 \ No newline at end of file diff --git a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py index ce246c1e5..1ddacee3c 100644 --- a/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py +++ b/python/OpenVINO_EP/tiny_yolo_v2_object_detection/tiny_yolov2_obj_detection_sample.py @@ -29,7 +29,7 @@ def parse_arguments(): parser = argparse.ArgumentParser(description='Object Detection using YOLOv2 in OPENCV using OpenVINO Execution Provider for ONNXRuntime') - parser.add_argument('--device', default='CPU_FP32', help="Device to perform inference on 'cpu (MLAS)' or on devices supported by OpenVINO-EP [CPU_FP32, CPU_FP32, GPU_FP32, GPU_FP16].") + parser.add_argument('--device', default='CPU', help="Device to perform inference on 'cpu (MLAS)' or on devices supported by OpenVINO-EP [CPU, GPU, NPU].") parser.add_argument('--video', help='Path to video file.') parser.add_argument('--model', help='Path to model.') args = parser.parse_args() @@ -150,16 +150,16 @@ def main(): print("Device type selected is 'cpu' which is the default CPU Execution Provider (MLAS)") #Specify the path to the ONNX model on your machine and register the CPU EP sess = rt.InferenceSession(args.model, so, providers=['CPUExecutionProvider']) - elif (args.device == 'CPU_FP32', args.device == 'CPU_FP16' or args.device == 'GPU_FP32' or args.device == 'GPU_FP16'): + elif (args.device == 'CPU', args.device == 'GPU' or args.device == 'NPU'): #Specify the path to the ONNX model on your machine and register the OpenVINO EP sess = rt.InferenceSession(args.model, so, providers=['OpenVINOExecutionProvider'], provider_options=[{'device_type' : args.device}]) print("Device type selected is: " + args.device + " using the OpenVINO Execution Provider") ''' other 'device_type' options are: (Any hardware target can be assigned if you have the access to it) - 'CPU_FP32', 'CPU_FP16', 'GPU_FP32', 'GPU_FP16' + 'CPU', 'GPU', 'NPU' ''' else: - raise Exception("Device type selected is not [cpu, CPU_FP32, GPU_FP32, GPU_FP16]") + raise Exception("Device type selected is not [CPU, GPU, NPU]") # Get the input name of the model input_name = sess.get_inputs()[0].name diff --git a/python/OpenVINO_EP/yolov4_object_detection/README.md b/python/OpenVINO_EP/yolov4_object_detection/README.md index 17c2110e3..e9b633d6f 100644 --- a/python/OpenVINO_EP/yolov4_object_detection/README.md +++ b/python/OpenVINO_EP/yolov4_object_detection/README.md @@ -54,10 +54,10 @@ python3 yolov4.py --h ### Run the sample on OpenVINO™ Execution Provider ```bash -python3 yolov4.py --device CPU_FP32 --video classroom.mp4 --model yolov4.onnx +python3 yolov4.py --device CPU --video classroom.mp4 --model yolov4.onnx ``` Note: -* You can pick different device options to run on OpenVINO™ Execution Provider like GPU_FP32, GPU_FP16 and MYRIAD_FP16. +* You can pick different device options to run on OpenVINO™ Execution Provider like GPU and NPU. ### Run the sample on default CPU Execution Provider (MLAS) ```bash @@ -66,17 +66,17 @@ python3 yolov4.py --device cpu --video classroom.mp4 --model yolov4.onnx ### Run the sample with video as Input ```bash -python3 yolov4.py --device CPU_FP32 --video classroom.mp4 --model yolov4.onnx +python3 yolov4.py --device CPU --video classroom.mp4 --model yolov4.onnx ``` ### Run the sample with Image as Input ```bash -python3 yolov4.py --device CPU_FP32 --image cat.jpg --model yolov4.onnx +python3 yolov4.py --device CPU --image cat.jpg --model yolov4.onnx ``` ### Run the sample with Live Input stream Like webcam ```bash -python3 yolov4.py --device CPU_FP32 --model yolov4.onnx +python3 yolov4.py --device CPU --model yolov4.onnx ``` ## To stop the sample from running @@ -88,9 +88,7 @@ Just press the letter 'q' or Ctrl+C if on Windows [Download OpenVINO™ Execution Provider Latest pip wheels from here](https://pypi.org/project/onnxruntime-openvino/) -[OpenVINO™ Execution Provider](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/faster-inferencing-with-one-line-of-code.html) - -[Docker Containers](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/openvino-execution-provider-docker-container.html) +[OpenVINO™ Execution Provider](https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html) [Python Pip Wheel Packages](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/openvino-execution-provider-for-onnx-runtime.html) diff --git a/python/OpenVINO_EP/yolov4_object_detection/notebooks/OVEP_yolov4_obj_detection_sample.ipynb b/python/OpenVINO_EP/yolov4_object_detection/notebooks/OVEP_yolov4_obj_detection_sample.ipynb index c3d7432e5..bd1c55ecf 100644 --- a/python/OpenVINO_EP/yolov4_object_detection/notebooks/OVEP_yolov4_obj_detection_sample.ipynb +++ b/python/OpenVINO_EP/yolov4_object_detection/notebooks/OVEP_yolov4_obj_detection_sample.ipynb @@ -648,12 +648,12 @@ " \n", "1. Create a ONNX Runtime Session Option instance using `onnxruntime.SessionOptions()`\n", "2. Using the session options instance create a Inference Session object by passing the model and the execution provider as arguments.\n", - "Execution Providers are the hardware device options e.g. CPU, Myriad, GPU, etc. on which the session will be executed.\n", + "Execution Providers are the hardware device options e.g. CPU, GPU, NPU on which the session will be executed.\n", "\n", "The below `create_sess` function actually takes care of the above steps. All we need to do is pass the device arguement to it. It'll return the appropriate session according to the selected device along with the input name for the model.\n", "\n", "The device option should be chosen from any one of the below options: \n", - "- `cpu, CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VADM_FP16`" + "- `cpu, CPU, GPU, NPU`" ] }, { @@ -672,16 +672,16 @@ " print(\"Device type selected is 'cpu' which is the default CPU Execution Provider (MLAS)\")\n", " #Specify the path to the ONNX model on your machine and register the CPU EP\n", " sess = rt.InferenceSession(model, so, providers=['CPUExecutionProvider'])\n", - " elif (device == 'CPU_FP32' or device == 'GPU_FP32' or device == 'GPU_FP16' or device == 'MYRIAD_FP16' or device == 'VADM_FP16'):\n", + " elif (device == 'CPU' or device == 'GPU' or device == 'NPU'):\n", " #Specify the path to the ONNX model on your machine and register the OpenVINO EP\n", " sess = rt.InferenceSession(model, so, providers=['OpenVINOExecutionProvider'], provider_options=[{'device_type' : device}])\n", " print(\"Device type selected is: \" + device + \" using the OpenVINO Execution Provider\")\n", " '''\n", " other 'device_type' options are: (Any hardware target can be assigned if you have the access to it)\n", - " 'CPU_FP32', 'GPU_FP32', 'GPU_FP16', 'MYRIAD_FP16', 'VAD-M_FP16'\n", + " 'CPU', 'GPU', 'NPU'\n", " '''\n", " else:\n", - " raise Exception(\"Device type selected is not [cpu, CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VADM_FP16]\")\n", + " raise Exception(\"Device type selected is not [cpu, CPU, GPU, NPU]\")\n", "\n", " # Get the input name of the model\n", " input_name = sess.get_inputs()[0].name\n", @@ -807,7 +807,7 @@ "source": [ "Now the `yolov4.onnx` model will run inference on `cat.jpg` image using the below two execution providers:\n", "- `cpu`: default CPU Execution Provider (MLAS) \n", - "- `CPU_FP32`: Execution on CPU with OpenVino Execution Provider\n", + "- `CPU`: Execution on CPU with OpenVino Execution Provider\n", "\n", "The below code block performs the following operations:\n", "\n", @@ -887,7 +887,7 @@ "id": "8c04daa0" }, "source": [ - "The below code block performs the same opertions as [before](#cpu_exec) with `CPU_FP32` as device, that runs on OpenVINO Execution Provider for ONNX Runtime." + "The below code block performs the same opertions as [before](#cpu_exec) with `CPU` as device, that runs on OpenVINO Execution Provider for ONNX Runtime." ] }, { @@ -900,7 +900,7 @@ "outputs": [], "source": [ "### create a session with CPU_FP32 using the OpenVINO Execution Provider\n", - "sess, input_name = create_sess(\"CPU_FP32\")\n", + "sess, input_name = create_sess(\"CPU\")\n", "\n", "input_size = 416\n", "original_image = cap.copy()\n", @@ -931,7 +931,7 @@ "bboxes = nms(bboxes, 0.213, method='nms')\n", "image_out = draw_bbox(original_image, bboxes)\n", "\n", - "cv2.putText(image_out,\"CPU_FP32\",(10,20),cv2.FONT_HERSHEY_COMPLEX,0.5,(255,255,255),1)\n", + "cv2.putText(image_out,\"CPU\",(10,20),cv2.FONT_HERSHEY_COMPLEX,0.5,(255,255,255),1)\n", "\n", "image_out = cv2.cvtColor(image_out, cv2.COLOR_BGR2RGB)\n", "cv2_imshow(image_out)\n", diff --git a/python/OpenVINO_EP/yolov4_object_detection/requirements.txt b/python/OpenVINO_EP/yolov4_object_detection/requirements.txt index 2ce4e5e00..119a25ffb 100644 --- a/python/OpenVINO_EP/yolov4_object_detection/requirements.txt +++ b/python/OpenVINO_EP/yolov4_object_detection/requirements.txt @@ -1,8 +1,8 @@ -certifi==2023.7.22 -flatbuffers==2.0 +certifi==2025.8.3 +flatbuffers==25.2.10 onnx -opencv-python==4.8.1.78 -Pillow==10.3.0 -protobuf==3.20.2 -scipy==1.10.0 -typing-extensions==4.2.0 +opencv-python==4.12.0.88 +Pillow==11.3.0 +protobuf==6.32.1 +scipy==1.16.2 +typing-extensions==4.15.0 diff --git a/python/OpenVINO_EP/yolov4_object_detection/yolov4.py b/python/OpenVINO_EP/yolov4_object_detection/yolov4.py index b3e8a6c44..c818efb8f 100644 --- a/python/OpenVINO_EP/yolov4_object_detection/yolov4.py +++ b/python/OpenVINO_EP/yolov4_object_detection/yolov4.py @@ -19,6 +19,7 @@ import sys import time import platform +import json if platform.system() == "Windows": import onnxruntime.tools.add_openvino_win_libs as utils @@ -220,7 +221,7 @@ def get_anchors(anchors_path, tiny=False): def parse_arguments(): parser = argparse.ArgumentParser(description='Object Detection using YOLOv4 in OPENCV using OpenVINO Execution Provider for ONNXRuntime') - parser.add_argument('--device', default='CPU_FP32', help="Device to perform inference on 'cpu (MLAS)' or on devices supported by OpenVINO-EP [CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16].") + parser.add_argument('--device', default='CPU', help="Device to perform inference on 'cpu (MLAS)' or on devices supported by OpenVINO-EP [CPU, GPU, NPU].") parser.add_argument('--image', help='Path to image file.') parser.add_argument('--video', help='Path to video file.') parser.add_argument('--model', help='Path to model.') @@ -283,11 +284,21 @@ def main(): sess = rt.InferenceSession(args.model, so, providers=['CPUExecutionProvider']) else: #Specify the path to the ONNX model on your machine and register the OpenVINO EP - sess = rt.InferenceSession(args.model, so, providers=['OpenVINOExecutionProvider'], provider_options=[{'device_type' : device}]) + + config_dict = { + "CPU": { + "INFERENCE_NUM_THREADS": "5", + "CACHE_DIR": "C:\\" + } + } + + config_json = json.dumps(config_dict) + + sess = rt.InferenceSession(args.model, so, providers=['OpenVINOExecutionProvider'], provider_options=[{'device_type' : device, 'load_config' : config_json}]) print("Device type selected is: " + device + " using the OpenVINO Execution Provider") ''' other 'device_type' options are: (Any hardware target can be assigned if you have the access to it) - 'CPU_FP32', 'GPU_FP32', 'GPU_FP16', 'MYRIAD_FP16', 'VAD-M_FP16' + 'CPU', 'GPU', 'NPU' ''' input_name = sess.get_inputs()[0].name