[Accuracy tool] Update build and README with relevant changes for ONNX Runtime 1.18.0

c_cxx/accuracy_tool/CMakeLists.txt

@@ -32,7 +32,7 @@ if(QNN_SDK_ROOTDIR)
   endif()
 
   if(NOT QNN_HEXAGON_ARCH_VERSION)
-    set(QNN_HEXAGON_ARCH_VERSION "68")
+    set(QNN_HEXAGON_ARCH_VERSION "73")
     message(WARNING "Assuming QNN Hexagon version ${QNN_HEXAGON_ARCH_VERSION}. Set with -DQNN_HEXAGON_ARCH_VERSION")
   endif()
 
@@ -94,12 +94,13 @@ endfunction()
 # Windows might have an onnxruntime.dll in the system directory so it's more robust to manually copy the dlls to
 # the output dir.
 if(MSVC)
-  target_copy_artifacts(accuracy_test "${ONNXRUNTIME_ROOTDIR}/bin/*.dll")
+  target_copy_artifacts(accuracy_test "${ONNXRUNTIME_ROOTDIR}/bin/*")
 endif()
 
 if(EXISTS "${QNN_SDK_ROOTDIR}")
   set(QNN_SDK_HEX_DIR "${QNN_SDK_ROOTDIR}/lib/hexagon-v${QNN_HEXAGON_ARCH_VERSION}/unsigned")
   target_copy_artifacts(accuracy_test "${QNN_SDK_HEX_DIR}/*.so")
+  target_copy_artifacts(accuracy_test "${QNN_SDK_HEX_DIR}/*.cat")
 
   if(WIN32)
     set(QNN_WIN_ARCH_DIR "x86_64-windows-msvc")

c_cxx/accuracy_tool/README.md

@@ -2,10 +2,22 @@
 This tool measures the accuracy of a set of models on a given execution provider. The accuracy is computed by comparing with the expected results, which are either loaded from file or attained by running the model with the CPU execution provider.
 
 ## Build instructions on Windows
-Run the following commands in a terminal to generate a Visual Studio project and compile the tool. Make sure to specify the location of your ONNX Runtime installation. You can either [download an ONNX Runtime release package](https://github.com/microsoft/onnxruntime/releases/) or you can [build ONNX Runtime from source](https://www.onnxruntime.ai/docs/build/).
+### Using an ONNX Runtime NuGet package
+Download an ONNX Runtime NuGet package with the desired execution provider(s):
+- [Microsoft.ML.OnnxRuntime](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime)
+- [Microsoft.ML.OnnxRuntime.QNN](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.QNN)
+- [Microsoft.ML.OnnxRuntime.Gpu](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.Gpu)
+- Others: https://www.nuget.org/packages?q=Microsoft.ML.OnnxRuntime
+
+Clone this onnxruntime-inference-examples repository:
+```shell
+ git clone https://github.com/Microsoft/onnxruntime-inference-examples.git
+ cd onnxruntime-inference-examples\c_cxx\accuracy_tool
+```
 
+Run `build.bat` with the path to the ONNX Runtime NuGet package as the first argument.
 ```shell
-$ build.bat 'C:\onnxruntime'
+$ build.bat .\microsoft.ml.onnxruntime.1.18.0.nupkg
 ```
 
 Run the following command to open the solution file with Visual Studio.
@@ -20,20 +32,76 @@ Alternatively, you can directly run the executable from the terminal:
 .\build\Release\accuracy_test.exe --help
 ```
 
-### Building with QNN execution provider
-To test model accuracy with the QNN execution provider, provide the path to location of your Qualcomm AI Engine Direct SDK (QNN SDK) and, optionally, the hexagon architecture version (e.g., 68 or 73).
-The QNN SDK can be downloaded from https://qpm.qualcomm.com/main/tools/details/qualcomm_ai_engine_direct.
-Providing the QNN SDK path will ensure that the appropriate QNN SDK dynamic libraries (e.g., QnnHtp.dll) are automatically copied to the build directory.
+### Using an ONNX Runtime source build
+#### Build ONNX Runtime from source
+Refer to the documentation for [building ONNX Runtime from source](https://www.onnxruntime.ai/docs/build/) with the desired execution providers.
+
+The following commands build ONNX Runtime from source with the CPU EP.
+
+Clone the ONNX Runtime repository:
+```shell
+ git clone --recursive https://github.com/Microsoft/onnxruntime.git
+ cd onnxruntime
+```
 
+Build ONNX Runtime from source. Replace `<ORT_INSTALL_DIR>` with your desired installation directory.
 ```shell
-$ build.bat 'C:\onnxruntime' 'C:\Qualcomm\AIStack\QNN\2.17.0.231124' 68
+.\build.bat --config RelWithDebInfo --build_shared_lib --parallel --compile_no_warning_as_error --skip_submodule_sync --skip_tests --cmake_extra_defines CMAKE_INSTALL_PREFIX=<ORT_INSTALL_DIR>
 ```
 
+Install ONNX Runtime to `<ORT_INSTALL_DIR>`:
+```shell
+ cmake --install .\build\RelWithDebInfo --config RelWithDebInfo
+```
+
+#### Build accuracy tool
+Clone this onnxruntime-inference-examples repository:
+```shell
+ git clone https://github.com/Microsoft/onnxruntime-inference-examples.git
+ cd onnxruntime-inference-examples\c_cxx\accuracy_tool
+```
 
-Note that you can also provide a [NuGet package](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.QNN) as the first argument to `build.bat`:
+Run `build.bat` with the path to the ONNX Runtime installation directory as the first argument.
+```shell
+$ build.bat <ORT_INSTALL_DIR>
+```
+
+Run the following command to open the solution file with Visual Studio.
+
+```shell
+$ devenv .\build\onnxruntime_accuracy_test.sln
+```
+
+Alternatively, you can directly run the executable from the terminal:
 
 ```shell
-$ build.bat '.\microsoft.ml.onnxruntime.qnn.1.16.0.nupkg' 'C:\Qualcomm\AIStack\QNN\2.17.0.231124' 68
+.\build\Release\accuracy_test.exe --help
+```
+
+### Using an ONNX Runtime Github release package
+Download an ONNX Runtime release package from https://github.com/microsoft/onnxruntime/releases/ and extract it to your desired installation directory (`<ORT_INSTALL_DIR>`).
+
+Clone this onnxruntime-inference-examples repository:
+```shell
+ git clone https://github.com/Microsoft/onnxruntime-inference-examples.git
+ cd onnxruntime-inference-examples\c_cxx\accuracy_tool
+```
+
+Run `build.bat` with the path to the extracted ONNX Runtime installation directory as the first argument.
+```shell
+$ build.bat <ORT_INSTALL_DIR>
+```
+
+Run the following command to open the solution file with Visual Studio.
+
+```shell
+$ devenv .\build\onnxruntime_accuracy_test.sln
+```
+
+Alternatively, you can directly run the executable from the terminal:
+
+```shell
+.\build\Release\accuracy_test.exe --help
 ```
 
 ## Setup test models and inputs

c_cxx/accuracy_tool/build.bat

@@ -48,17 +48,14 @@ IF "%~x1" == ".nupkg" (
   mkdir _generated_onnxruntime_rootdir\bin 2>nul
   copy /y _ort_extracted_nuget\build\native\include\* _generated_onnxruntime_rootdir\include\
   IF "%PROCESSOR_ARCHITECTURE%" == "ARM64" (
-    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\*.lib _generated_onnxruntime_rootdir\lib\
-    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\*.dll _generated_onnxruntime_rootdir\lib\
-    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\*.dll _generated_onnxruntime_rootdir\bin\
-    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\*.exe _generated_onnxruntime_rootdir\bin\
+    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\* _generated_onnxruntime_rootdir\lib\
+    copy /y _ort_extracted_nuget\runtimes\win-arm64\native\* _generated_onnxruntime_rootdir\bin\
   ) ELSE (
-    copy /y _ort_extracted_nuget\runtimes\win-x64\native\*.lib _generated_onnxruntime_rootdir\lib\
-    copy /y _ort_extracted_nuget\runtimes\win-x64\native\*.dll _generated_onnxruntime_rootdir\lib\
-    copy /y _ort_extracted_nuget\runtimes\win-x64\native\*.dll _generated_onnxruntime_rootdir\bin\
-    copy /y _ort_extracted_nuget\runtimes\win-x64\native\*.exe _generated_onnxruntime_rootdir\bin\
+    copy /y _ort_extracted_nuget\runtimes\win-x64\native\* _generated_onnxruntime_rootdir\lib\
+    copy /y _ort_extracted_nuget\runtimes\win-x64\native\* _generated_onnxruntime_rootdir\bin\
   )
   copy /y _ort_extracted_nuget\*.txt _generated_onnxruntime_rootdir\
+  copy /y _ort_extracted_nuget\*.pdf _generated_onnxruntime_rootdir\
   copy /y _ort_extracted_nuget\LICENSE _generated_onnxruntime_rootdir\
   copy /y _ort_extracted_nuget\README.md _generated_onnxruntime_rootdir\
   copy /y _ort_extracted_nuget\Privacy.md _generated_onnxruntime_rootdir\