Skip to content

Minor cleanup of ONNX before threading refactor. #1382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mysticial merged 1 commit into from
Aug 12, 2026
Merged

Minor cleanup of ONNX before threading refactor. #1382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 43 additions & 20 deletions SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,14 @@ PaddleOCRPipeline::PaddleOCRPipeline(Language language)
PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, std::string dict_path)
: m_env{create_ORT_env()}
// , det_session(env, std::wstring(det_path.begin(), det_path.end()).c_str(), Ort::SessionOptions{})
, m_rec_session(create_session(m_env, rec_path, ML_MODEL_CACHE_PATH() + "PaddleOCRPipeline/", GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE0))
, m_rec_session(
create_session(
m_env,
rec_path,
ML_MODEL_CACHE_PATH() + "PaddleOCRPipeline/",
GlobalSettings::instance().USE_GPU_FOR_ML_INFERENCE0
)
)
// , memory_info(Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault))
, m_language(language)
, m_input_name(m_rec_session.GetInputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get())
Expand All @@ -69,7 +76,7 @@ PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, st
}

void PaddleOCRPipeline::run(const std::string& img_path){
#if 0
#if 0
cv::Mat img = cv::imread(img_path);
if (img.empty()) return;

Expand All @@ -82,7 +89,7 @@ void PaddleOCRPipeline::run(const std::string& img_path){
std::string text = recognize(cropped);
std::cout << "Detected Text: " << text << std::endl;
}
#endif
#endif
}


Expand Down Expand Up @@ -144,17 +151,19 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){

// 4b. Apply Mean/Std (Standard for PaddleOCR). except for Chinese
// Mean: [0.485, 0.456, 0.406], Std: [0.229, 0.224, 0.225]
if (!(m_language == Language::ChineseSimplified ||
m_language == Language::ChineseTraditional ||
m_language == Language::Japanese ||
m_language == Language::Korean))
{
#if 0
switch (m_language){
case Language::ChineseSimplified:
case Language::ChineseTraditional:
case Language::Japanese:
case Language::Korean:
break;
default:;
#if 0
cv::Scalar mean(0.485, 0.456, 0.406);
cv::Scalar std(0.229, 0.224, 0.225);
cv::subtract(resized, mean, resized);
cv::divide(resized, std, resized);
#endif
#endif
}


Expand All @@ -167,13 +176,17 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){
// 7. Create tensor with its own managed memory
Ort::AllocatorWithDefaultOptions allocator;
auto input_tensor = Ort::Value::CreateTensor<float>(
allocator, input_shape.data(), input_shape.size()
allocator,
input_shape.data(),
input_shape.size()
);

// Copy your processed data into that memory
std::memcpy(input_tensor.GetTensorMutableData<float>(),
input_tensor_values.data(),
input_tensor_values.size() * sizeof(float));
std::memcpy(
input_tensor.GetTensorMutableData<float>(),
input_tensor_values.data(),
input_tensor_values.size() * sizeof(float)
);

const char* input_names[] = {m_input_name.c_str()};
const char* output_names[] = {m_output_name.c_str()};
Expand All @@ -188,9 +201,17 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){
output_names, // char**
1 // output_count
);
return decode_CTC(outputs[0].GetTensorMutableData<float>(), outputs[0].GetTensorTypeAndShapeInfo().GetShape(), m_dictionary);
return decode_CTC(
outputs[0].GetTensorMutableData<float>(),
outputs[0].GetTensorTypeAndShapeInfo().GetShape(),
m_dictionary
);
}catch (Ort::Exception& e){
throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "PaddleOCRPipeline::recognize(): Failed." + std::string(e.what()));
throw InternalProgramError(
nullptr,
PA_CURRENT_FUNCTION,
"PaddleOCRPipeline::recognize(): Failed." + std::string(e.what())
);
}

}
Expand Down Expand Up @@ -369,10 +390,12 @@ cv::Mat imageviewrgb32_to_cv_mat_rgb(const ImageViewRGB32& image){
cv::Rect ImageFloatBox_to_cv_Rect(size_t width, size_t height, const ImageFloatBox& box){
ImagePixelBox pixelbox = floatbox_to_pixelbox(width, height, box);

return cv::Rect(safe_convert<int>(pixelbox.min_x),
safe_convert<int>(pixelbox.min_y),
safe_convert<int>(pixelbox.width()),
safe_convert<int>(pixelbox.height()));
return cv::Rect(
safe_convert<int>(pixelbox.min_x),
safe_convert<int>(pixelbox.min_y),
safe_convert<int>(pixelbox.width()),
safe_convert<int>(pixelbox.height())
);
}


Expand Down
7 changes: 2 additions & 5 deletions SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#ifndef PokemonAutomation_ML_PaddleOCRPipeline_H
#define PokemonAutomation_ML_PaddleOCRPipeline_H


#include <string>
#include <vector>
#include <onnxruntime_cxx_api.h>
Expand All @@ -17,13 +16,11 @@
#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
#include "CommonFramework/ImageTools/ImageBoxes.h"



namespace PokemonAutomation{
namespace ML{


class PaddleOCRPipeline {
class PaddleOCRPipeline{
public:
PaddleOCRPipeline(Language language);
PaddleOCRPipeline(Language language, std::string rec_path, std::string dict_path);
Expand Down Expand Up @@ -71,4 +68,4 @@ cv::Rect ImageFloatBox_to_cv_Rect(size_t width, size_t height, const ImageFloatB

}
}
#endif
#endif
136 changes: 68 additions & 68 deletions SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,85 +45,85 @@ Ort::SessionOptions create_session_options(const std::string& model_cache_path,
Ort::SessionOptions so;
std::cout << "Set potential model cache path in session options: " << model_cache_path << std::endl;

if (use_gpu){
if (use_gpu){
#if __APPLE__
// create session using Apple ML acceleration library CoreML
std::unordered_map<std::string, std::string> provider_options;
// See for provider options: https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html
// "NeuralNetwork" is a faster ModelFormat than "MLProgram".
provider_options["ModelFormat"] = std::string("NeuralNetwork");
provider_options["ModelCacheDirectory"] = model_cache_path;

// provider_options["MLComputeUnits"] = "ALL";
// provider_options["RequireStaticInputShapes"] = "0";
// provider_options["EnableOnSubgraphs"] = "0";
so.AppendExecutionProvider("CoreML", provider_options);
std::cout << "Using CoreML execution provider for GPU acceleration" << std::endl;
// create session using Apple ML acceleration library CoreML
std::unordered_map<std::string, std::string> provider_options;
// See for provider options: https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html
// "NeuralNetwork" is a faster ModelFormat than "MLProgram".
provider_options["ModelFormat"] = std::string("NeuralNetwork");
provider_options["ModelCacheDirectory"] = model_cache_path;

// provider_options["MLComputeUnits"] = "ALL";
// provider_options["RequireStaticInputShapes"] = "0";
// provider_options["EnableOnSubgraphs"] = "0";
so.AppendExecutionProvider("CoreML", provider_options);
std::cout << "Using CoreML execution provider for GPU acceleration" << std::endl;
#elif _WIN32
// Try CUDA first for NVIDIA GPUs (best performance)
// CUDA requires NVIDIA GPU and CUDA runtime installation
// See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html
bool cuda_available = false;
try{
OrtCUDAProviderOptions cuda_options{};
cuda_options.device_id = 0;
so.AppendExecutionProvider_CUDA(cuda_options);
std::cout << "Using CUDA execution provider for GPU acceleration" << std::endl;
cuda_available = true;
}catch (const Ort::Exception& e){
std::cout << "CUDA execution provider not available: " << e.what() << std::endl;
}

bool rocm_available = false;
if (!cuda_available){
// Try ROCm next for AMD GPUs
// See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html
// Try CUDA first for NVIDIA GPUs (best performance)
// CUDA requires NVIDIA GPU and CUDA runtime installation
// See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html
bool cuda_available = false;
try{
OrtROCMProviderOptions rocm_options{};
rocm_options.device_id = 0;
so.AppendExecutionProvider_ROCM(rocm_options);
std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl;
rocm_available = true;
OrtCUDAProviderOptions cuda_options{};
cuda_options.device_id = 0;
so.AppendExecutionProvider_CUDA(cuda_options);
std::cout << "Using CUDA execution provider for GPU acceleration" << std::endl;
cuda_available = true;
}catch (const Ort::Exception& e){
std::cout << "ROCm execution provider not available: " << e.what() << std::endl;
std::cout << "CUDA execution provider not available: " << e.what() << std::endl;
}
}

// Fallback to DirectML for all GPU vendors (NVIDIA, AMD, Intel)
// DirectML is built into Windows 10 1903+ and requires no additional runtime installation
// See: https://onnxruntime.ai/docs/execution-providers/DirectML-ExecutionProvider.html
if (!cuda_available and !rocm_available){
try{
so.AppendExecutionProvider("DML");
std::cout << "Using DirectML execution provider for GPU acceleration" << std::endl;
}catch (const Ort::Exception& e){
std::cout << "DirectML execution provider not available, falling back to CPU: " << e.what() << std::endl;
bool rocm_available = false;
if (!cuda_available){
// Try ROCm next for AMD GPUs
// See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html
try{
OrtROCMProviderOptions rocm_options{};
rocm_options.device_id = 0;
so.AppendExecutionProvider_ROCM(rocm_options);
std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl;
rocm_available = true;
}catch (const Ort::Exception& e){
std::cout << "ROCm execution provider not available: " << e.what() << std::endl;
}
}

// Fallback to DirectML for all GPU vendors (NVIDIA, AMD, Intel)
// DirectML is built into Windows 10 1903+ and requires no additional runtime installation
// See: https://onnxruntime.ai/docs/execution-providers/DirectML-ExecutionProvider.html
if (!cuda_available and !rocm_available){
try{
so.AppendExecutionProvider("DML");
std::cout << "Using DirectML execution provider for GPU acceleration" << std::endl;
}catch (const Ort::Exception& e){
std::cout << "DirectML execution provider not available, falling back to CPU: " << e.what() << std::endl;
}
}
}
#elif defined(__linux__)
// Try CUDA first for NVIDIA GPUs (best performance)
// See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html
try{
OrtCUDAProviderOptions cuda_options{};
cuda_options.device_id = 0;
so.AppendExecutionProvider_CUDA(cuda_options);
std::cout << "Using CUDA execution provider for GPU acceleration" << std::endl;
}catch (const Ort::Exception& e){
std::cout << "CUDA execution provider not available, trying ROCm: " << e.what() << std::endl;

// Try ROCm next for AMD GPUs
// See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html
// Try CUDA first for NVIDIA GPUs (best performance)
// See: https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html
try{
OrtROCMProviderOptions rocm_options{};
rocm_options.device_id = 0;
so.AppendExecutionProvider_ROCM(rocm_options);
std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl;
OrtCUDAProviderOptions cuda_options{};
cuda_options.device_id = 0;
so.AppendExecutionProvider_CUDA(cuda_options);
std::cout << "Using CUDA execution provider for GPU acceleration" << std::endl;
}catch (const Ort::Exception& e){
std::cout << "ROCm execution provider not available, falling back to CPU: " << e.what() << std::endl;
std::cout << "CUDA execution provider not available, trying ROCm: " << e.what() << std::endl;

// Try ROCm next for AMD GPUs
// See: https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html
try{
OrtROCMProviderOptions rocm_options{};
rocm_options.device_id = 0;
so.AppendExecutionProvider_ROCM(rocm_options);
std::cout << "Using ROCm execution provider for GPU acceleration" << std::endl;
}catch (const Ort::Exception& e){
std::cout << "ROCm execution provider not available, falling back to CPU: " << e.what() << std::endl;
}
}
}
#endif
}
}

// CPU fallback is always available
return so;
Expand Down Expand Up @@ -220,7 +220,7 @@ Ort::Session create_session(
try {
logger.log("Creating dedicated CPU-only session...");

Ort::SessionOptions cpu_options = create_session_options(model_cache_path, false);;
Ort::SessionOptions cpu_options = create_session_options(model_cache_path, false);

Ort::Session session{env, onnx_path.c_str(), cpu_options};
logger.log("Ort::Session created");
Expand Down
Loading