Skip to content

Commit

Permalink
Copy mpashchenkov's changes
Browse files Browse the repository at this point in the history
Minor refactoring

Partially address review comments

Move DX-related stuff from the sample to a default source

Simplify the default OneVPL config

Address minor review comments

Add class for the default VPL source

WIP: Add initial stub for tests with description

Removing default vpl source and minor refactoring

Refactor default files

Fix build and application crash

Address review comments

Add test on VPL + OCL interaction compared to CPU behavior

Fix test
  • Loading branch information
smirnov-alexey committed Nov 16, 2022
1 parent 2aad039 commit 4c74e6d
Show file tree
Hide file tree
Showing 16 changed files with 386 additions and 16 deletions.
13 changes: 7 additions & 6 deletions modules/gapi/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ set(gapi_srcs
src/streaming/onevpl/cfg_params.cpp
src/streaming/onevpl/cfg_params_parser.cpp
src/streaming/onevpl/utils.cpp
src/streaming/onevpl/default.cpp
src/streaming/onevpl/data_provider_interface_exception.cpp
src/streaming/onevpl/accelerators/surface/base_frame_adapter.cpp
src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp
Expand Down Expand Up @@ -367,20 +368,20 @@ ocv_add_samples()


# Required for sample with inference on host
if(TARGET example_gapi_onevpl_infer_single_roi)
if(TARGET example_gapi_onevpl_infer_with_advanced_device_selection)
if(TARGET ocv.3rdparty.openvino AND OPENCV_GAPI_WITH_OPENVINO)
ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ocv.3rdparty.openvino)
ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE ocv.3rdparty.openvino)
endif()
if(HAVE_DIRECTX AND HAVE_D3D11)
ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE d3d11 dxgi)
ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE d3d11 dxgi)
endif()
if(HAVE_D3D11 AND HAVE_OPENCL)
ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS})
ocv_target_include_directories(example_gapi_onevpl_infer_with_advanced_device_selection SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS})
endif()
if(UNIX AND HAVE_VA)
message ("GAPI VPL samples with VAAPI")
ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${VA_INCLUDE_DIR})
ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ${VA_LIBRARIES})
ocv_target_include_directories(example_gapi_onevpl_infer_with_advanced_device_selection SYSTEM PRIVATE ${VA_INCLUDE_DIR})
ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE ${VA_LIBRARIES})
endif()
endif()

Expand Down
4 changes: 4 additions & 0 deletions modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ template<typename U> struct ocl_get_in<cv::GArray<U> >
{
static const std::vector<U>& get(GOCLContext &ctx, int idx) { return ctx.inArg<VectorRef>(idx).rref<U>(); }
};
template<> struct ocl_get_in<cv::GFrame>
{
static cv::MediaFrame get(GOCLContext &ctx, int idx) { return ctx.inArg<cv::MediaFrame>(idx); }
};
template<typename U> struct ocl_get_in<cv::GOpaque<U> >
{
static const U& get(GOCLContext &ctx, int idx) { return ctx.inArg<OpaqueRef>(idx).rref<U>(); }
Expand Down
29 changes: 29 additions & 0 deletions modules/gapi/include/opencv2/gapi/streaming/onevpl/default.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2022 Intel Corporation

#ifndef OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP
#define OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP

#include <opencv2/gapi/own/exports.hpp> // GAPI_EXPORTS
#include <opencv2/gapi/streaming/onevpl/cfg_params.hpp>
#include <opencv2/gapi/streaming/onevpl/device_selector_interface.hpp>

namespace cv {
namespace gapi {
namespace wip {
namespace onevpl {

/**
* @brief Provides default device selector based on config.
*/
GAPI_EXPORTS std::shared_ptr<IDeviceSelector> getDefaultDeviceSelector(const std::vector<CfgParam>& cfg_params);

} // namespace onevpl
} // namespace wip
} // namespace gapi
} // namespace cv

#endif // OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP
106 changes: 106 additions & 0 deletions modules/gapi/samples/onevpl_source_to_bgr_conv.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#include <algorithm>
#include <fstream>
#include <iostream>
#include <cctype>
#include <tuple>
#include <memory>

#include <opencv2/imgproc.hpp>
#include <opencv2/gapi.hpp>
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/gpu/ggpukernel.hpp>
#include <opencv2/gapi/streaming/onevpl/source.hpp>
#include <opencv2/gapi/streaming/onevpl/data_provider_interface.hpp>
#include <opencv2/gapi/streaming/onevpl/default.hpp>
#include <opencv2/highgui.hpp> // CommandLineParser
#include <opencv2/gapi/ocl/core.hpp>

const std::string about =
"This is an example presents decoding on GPU using VPL Source and passing it to OpenCL backend";
const std::string keys =
"{ h help | | Print this help message }"
"{ input | | Path to the input video file. Use .avi extension }"
"{ accel_mode | mfxImplDescription.AccelerationMode:MFX_ACCEL_MODE_VIA_D3D11 | Acceleration mode for VPL }";

namespace {
namespace cfg {
// FIXME: Move OneVPL arguments parser to a single place
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line);
} // namespace cfg
} // anonymous namespace

int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
cmd.about(about);
if (cmd.has("help")) {
cmd.printMessage();
return 0;
}

// Get file name
const auto input = cmd.get<std::string>("input");
const auto accel_mode = cmd.get<std::string>("accel_mode");

// Create VPL config
std::vector<cv::gapi::wip::onevpl::CfgParam> source_cfgs;
source_cfgs.push_back(cfg::create_from_string(accel_mode));

// Create VPL-based source
std::shared_ptr<cv::gapi::wip::onevpl::IDeviceSelector> default_device_selector =
cv::gapi::wip::onevpl::getDefaultDeviceSelector(source_cfgs);

cv::gapi::wip::IStreamSource::Ptr source = cv::gapi::wip::make_onevpl_src(input, source_cfgs,
default_device_selector);

// Build the graph
cv::GFrame in; // input frame from VPL source
auto bgr_gmat = cv::gapi::streaming::BGR(in); // conversion from VPL source frame to BGR UMat
auto out = cv::gapi::blur(bgr_gmat, cv::Size(4,4)); // ocl kernel of blur operation

cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
.compileStreaming(cv::compile_args(cv::gapi::core::ocl::kernels()));
pipeline.setSource(std::move(source));

// The execution part
size_t frames = 0u;
cv::TickMeter tm;
cv::Mat outMat;

pipeline.start();
tm.start();

while (pipeline.pull(cv::gout(outMat))) {
cv::imshow("OutVideo", outMat);
cv::waitKey(1);
++frames;
}
tm.stop();
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;

return 0;
}

namespace {
namespace cfg {
typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) {
using namespace cv::gapi::wip;

if (line.empty()) {
throw std::runtime_error("Cannot parse CfgParam from emply line");
}

std::string::size_type name_endline_pos = line.find(':');
if (name_endline_pos == std::string::npos) {
throw std::runtime_error("Cannot parse CfgParam from: " + line +
"\nExpected separator \":\"");
}

std::string name = line.substr(0, name_endline_pos);
std::string value = line.substr(name_endline_pos + 1);

return cv::gapi::wip::onevpl::CfgParam::create(name, value,
/* vpp params strongly optional */
name.find("vpp.") == std::string::npos);
}
} // namespace cfg
} // anonymous namespace
4 changes: 3 additions & 1 deletion modules/gapi/src/backends/ocl/goclbackend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ cv::GArg cv::gimpl::GOCLExecutable::packArg(const GArg &arg)
GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY
&& arg.kind != cv::detail::ArgKind::GOPAQUE);
&& arg.kind != cv::detail::ArgKind::GOPAQUE
&& arg.kind != cv::detail::ArgKind::GFRAME);

if (arg.kind != cv::detail::ArgKind::GOBJREF)
{
Expand All @@ -136,6 +137,7 @@ cv::GArg cv::gimpl::GOCLExecutable::packArg(const GArg &arg)
// Note: .at() is intentional for GOpaque as object MUST be already there
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GOPAQUE: return GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;
Expand Down
95 changes: 95 additions & 0 deletions modules/gapi/src/backends/ocl/goclcore.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,32 @@


#include "precomp.hpp"
#include "logger.hpp"

#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/ocl/core.hpp>
#include <opencv2/gapi/util/throw.hpp>

#include "backends/ocl/goclcore.hpp"

#ifdef HAVE_DIRECTX
#ifdef HAVE_D3D11
#pragma comment(lib,"d3d11.lib")

// get rid of generate macro max/min/etc from DX side
#define D3D11_NO_HELPERS
#define NOMINMAX
#include <d3d11.h>
#pragma comment(lib, "dxgi")
#undef NOMINMAX
#undef D3D11_NO_HELPERS
#include <opencv2/core/directx.hpp>
#endif // HAVE_D3D11
#endif // HAVE_DIRECTX

#include <opencv2/core/ocl.hpp>
#include "streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp"

GAPI_OCL_KERNEL(GOCLAdd, cv::gapi::core::GAdd)
{
static void run(const cv::UMat& a, const cv::UMat& b, int dtype, cv::UMat& out)
Expand Down Expand Up @@ -523,6 +544,79 @@ GAPI_OCL_KERNEL(GOCLTranspose, cv::gapi::core::GTranspose)
}
};

GAPI_OCL_KERNEL(GOCLBGR, cv::gapi::streaming::GBGR)
{
static void run(const cv::MediaFrame& in, cv::UMat& out)
{
cv::util::suppress_unused_warning(in);
cv::util::suppress_unused_warning(out);
#ifdef HAVE_DIRECTX
#ifdef HAVE_D3D11
#ifdef HAVE_ONEVPL
auto d = in.desc();
if (d.fmt != cv::MediaFormat::NV12)
{
GAPI_LOG_FATAL(nullptr, "Unsupported format provided: " << static_cast<int>(d.fmt) <<
". Expected cv::MediaFormat::NV12.");
cv::util::throw_error(std::logic_error("Unsupported MediaFrame format provided"));
}

// FIXME: consider a better solution.
// Current approach cannot be easily extended for other adapters (getHandle).
auto adapterPtr = in.get<cv::gapi::wip::onevpl::VPLMediaFrameDX11Adapter>();
if (adapterPtr == nullptr)
{
GAPI_LOG_FATAL(nullptr, "Unsupported adapter type. Only VPLMediaFrameDX11Adapter is supported");
cv::util::throw_error(std::logic_error("Unsupported adapter type. Only VPLMediaFrameDX11Adapter is supported"));
}

auto params = adapterPtr->getHandle();
auto handle = cv::util::any_cast<mfxHDLPair>(params);
ID3D11Texture2D* texture = reinterpret_cast<ID3D11Texture2D*>(handle.first);
if (texture == nullptr)
{
GAPI_LOG_FATAL(nullptr, "mfxHDLPair contains ID3D11Texture2D that is nullptr. Handle address" <<
reinterpret_cast<uint64_t>(handle.first));
cv::util::throw_error(std::logic_error("mfxHDLPair contains ID3D11Texture2D that is nullptr"));
}

// FIXME: Assuming here that we only have 1 device
// TODO: Textures are reusable, so to improve the peroformance here
// consider creating a hash map texture <-> device/ctx
static thread_local ID3D11Device* pD3D11Device = nullptr;
if (pD3D11Device == nullptr)
{
texture->GetDevice(&pD3D11Device);
}
if (pD3D11Device == nullptr)
{
GAPI_LOG_FATAL(nullptr, "D3D11Texture2D::GetDevice returns pD3D11Device that is nullptr");
cv::util::throw_error(std::logic_error("D3D11Texture2D::GetDevice returns pD3D11Device that is nullptr"));
}

// FIXME: assuming here that the context is always the same
// TODO: Textures are reusable, so to improve the peroformance here
// consider creating a hash map texture <-> device/ctx
static thread_local cv::ocl::Context ctx = cv::directx::ocl::initializeContextFromD3D11Device(pD3D11Device);
if (ctx.ptr() == nullptr)
{
GAPI_LOG_FATAL(nullptr, "initializeContextFromD3D11Device returned null context");
cv::util::throw_error(std::logic_error("initializeContextFromD3D11Device returned null context"));
}

cv::directx::convertFromD3D11Texture2D(texture, out);
#else
GAPI_LOG_FATAL(nullptr, "HAVE_ONEVPL is not set. Please, check your cmake flags");
cv::util::throw_error(std::logic_error("HAVE_ONEVPL is not set. Please, check your cmake flags"));
#endif // HAVE_ONEVPL
#else
GAPI_LOG_FATAL(nullptr, "HAVE_D3D11 or HAVE_DIRECTX is not set. Please, check your cmake flags");
cv::util::throw_error(std::logic_error("HAVE_D3D11 or HAVE_DIRECTX is not set. Please, check your cmake flags"));
#endif // HAVE_D3D11
#endif // HAVE_DIRECTX
}
};

cv::GKernelPackage cv::gapi::core::ocl::kernels()
{
static auto pkg = cv::gapi::kernels
Expand Down Expand Up @@ -587,6 +681,7 @@ cv::GKernelPackage cv::gapi::core::ocl::kernels()
, GOCLLUT
, GOCLConvertTo
, GOCLTranspose
, GOCLBGR
>();
return pkg;
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) {

cv::util::any VPLMediaFrameCPUAdapter::blobParams() const {
throw std::runtime_error("VPLMediaFrameCPUAdapter::blobParams() is not implemented");
return {};
}

void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,24 @@ MediaFrame::View VPLMediaFrameDX11Adapter::access(MediaFrame::Access mode) {
}
}

mfxHDLPair VPLMediaFrameDX11Adapter::getHandle() const {
auto surface_ptr_copy = get_surface();
const Surface::data_t& data = surface_ptr_copy->get_data();
NativeHandleAdapter* native_handle_getter = reinterpret_cast<NativeHandleAdapter*>(data.MemId);

mfxHDLPair handle{};
native_handle_getter->get_handle(data.MemId, reinterpret_cast<mfxHDL&>(handle));
return handle;
}

cv::util::any VPLMediaFrameDX11Adapter::blobParams() const {
/*GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not fully integrated"
"in OpenVINO InferenceEngine and would be temporary disable.");*/
#ifdef HAVE_INF_ENGINE
mfxHDLPair handle = getHandle();

auto surface_ptr_copy = get_surface();
Surface::data_t& data = surface_ptr_copy->get_data();
const Surface::info_t& info = surface_ptr_copy->get_info();
NativeHandleAdapter* native_handle_getter = reinterpret_cast<NativeHandleAdapter*>(data.MemId);

mfxHDLPair handle{};
native_handle_getter->get_handle(data.MemId, reinterpret_cast<mfxHDL&>(handle));

GAPI_Assert(frame_desc.fmt == MediaFormat::NV12 &&
"blobParams() for VPLMediaFrameDX11Adapter supports NV12 only");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ class VPLMediaFrameDX11Adapter final: public BaseFrameAdapter,
GAPI_EXPORTS ~VPLMediaFrameDX11Adapter();
MediaFrame::View access(MediaFrame::Access) override;

// FIXME: Consider a better solution since this approach
// is not easily extendable for other adapters (oclcore.cpp)
// FIXME: Use with caution since the handle might become invalid
// due to reference counting
mfxHDLPair getHandle() const;
// The default implementation does nothing
cv::util::any blobParams() const override;
void serialize(cv::gapi::s11n::IOStream&) override;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ namespace gapi {
namespace wip {
namespace onevpl {

class PlatformSpecificParams;
struct PlatformSpecificParams;
std::vector<CfgParam> update_param_with_accel_type(std::vector<CfgParam> &&param_array, AccelType type);

struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector {
Expand Down

0 comments on commit 4c74e6d

Please sign in to comment.