From e59012a31942765de5b3c7a9e149a9945ba33c0b Mon Sep 17 00:00:00 2001 From: Dmitry Matveev Date: Tue, 29 Sep 2020 22:19:54 +0300 Subject: [PATCH] Merge pull request #18415 from dmatveev:dm/gframe_01_new_host_type * G-API: Introduce cv::MediaFrame, a host type for cv::GFrame * G-API: RMat -- address review comments --- modules/gapi/CMakeLists.txt | 1 + modules/gapi/include/opencv2/gapi.hpp | 3 + modules/gapi/include/opencv2/gapi/gframe.hpp | 8 ++ modules/gapi/include/opencv2/gapi/media.hpp | 72 ++++++++++ modules/gapi/src/api/media.cpp | 42 ++++++ modules/gapi/test/gapi_frame_tests.cpp | 135 ++++++++++++++++++- 6 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/media.hpp create mode 100644 modules/gapi/src/api/media.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 76d457777474..fa0b4708bc18 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -77,6 +77,7 @@ set(gapi_srcs src/api/render_ocv.cpp src/api/ginfer.cpp src/api/ft_render.cpp + src/api/media.cpp # Compiler part src/compiler/gmodel.cpp diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index 2c99c8650d28..c6ab3f13fdf2 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -24,6 +24,9 @@ #include #include +#include +#include +#include #include #include #include diff --git a/modules/gapi/include/opencv2/gapi/gframe.hpp b/modules/gapi/include/opencv2/gapi/gframe.hpp index 7e19dbf7f798..d0668730b6d4 100644 --- a/modules/gapi/include/opencv2/gapi/gframe.hpp +++ b/modules/gapi/include/opencv2/gapi/gframe.hpp @@ -42,12 +42,20 @@ class GAPI_EXPORTS_W_SIMPLE GFrame }; /** @} */ +enum class MediaFormat +{ + BGR = 0, + NV12, +}; + /** * \addtogroup gapi_meta_args * @{ */ struct GAPI_EXPORTS GFrameDesc { + MediaFormat fmt; + cv::Size size; }; static inline GFrameDesc empty_gframe_desc() { return GFrameDesc{}; } /** @} */ diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp new file mode 100644 index 000000000000..394db9b14c23 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -0,0 +1,72 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_MEDIA_HPP +#define OPENCV_GAPI_MEDIA_HPP + +#include // unique_ptr<>, shared_ptr<> +#include // array<> +#include // function<> +#include // forward<>() + +#include + +namespace cv { + +class GAPI_EXPORTS MediaFrame { +public: + enum class Access { R, W }; + class IAdapter; + class View; + using AdapterPtr = std::unique_ptr; + + MediaFrame(); + explicit MediaFrame(AdapterPtr &&); + template static cv::MediaFrame Create(Args&&...); + + View access(Access); + cv::GFrameDesc desc() const; + +private: + struct Priv; + std::shared_ptr m; +}; + +template +inline cv::MediaFrame cv::MediaFrame::Create(Args&&... args) { + std::unique_ptr ptr(new T(std::forward(args)...)); + return cv::MediaFrame(std::move(ptr)); +} + +class GAPI_EXPORTS MediaFrame::View final { +public: + static constexpr const size_t MAX_PLANES = 4; + using Ptrs = std::array; + using Strides = std::array; // in bytes + using Callback = std::function; + + View(Ptrs&& ptrs, Strides&& strs, Callback &&cb = [](){}); + View(const View&) = delete; + View(View&&) = default; + ~View(); + + Ptrs ptr; + Strides stride; + +private: + Callback m_cb; +}; + +class GAPI_EXPORTS MediaFrame::IAdapter { +public: + virtual ~IAdapter() = 0; + virtual cv::GFrameDesc meta() const = 0; + virtual MediaFrame::View access(MediaFrame::Access) = 0; +}; + +} //namespace cv + +#endif // OPENCV_GAPI_MEDIA_HPP diff --git a/modules/gapi/src/api/media.cpp b/modules/gapi/src/api/media.cpp new file mode 100644 index 000000000000..1edfb80e5016 --- /dev/null +++ b/modules/gapi/src/api/media.cpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" +#include + +struct cv::MediaFrame::Priv { + std::unique_ptr adapter; +}; + +cv::MediaFrame::MediaFrame() { +} + +cv::MediaFrame::MediaFrame(AdapterPtr &&ptr) + : m(new Priv{std::move(ptr)}) { +} + +cv::GFrameDesc cv::MediaFrame::desc() const { + return m->adapter->meta(); +} + +cv::MediaFrame::View cv::MediaFrame::access(Access code) { + return m->adapter->access(code); +} + +cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb) + : ptr (std::move(ptrs)) + , stride(std::move(strs)) + , m_cb (std::move(cb)) { +} + +cv::MediaFrame::View::~View() { + if (m_cb) { + m_cb(); + } +} + +cv::MediaFrame::IAdapter::~IAdapter() { +} diff --git a/modules/gapi/test/gapi_frame_tests.cpp b/modules/gapi/test/gapi_frame_tests.cpp index 04fa21ef5b6f..c0361bda1b8c 100644 --- a/modules/gapi/test/gapi_frame_tests.cpp +++ b/modules/gapi/test/gapi_frame_tests.cpp @@ -6,10 +6,12 @@ #include "test_precomp.hpp" -#include +#include -namespace opencv_test -{ +//////////////////////////////////////////////////////////////////////////////// +// cv::GFrame tests + +namespace opencv_test { G_API_OP(GBlurFrame, , "test.blur_frame") { static GMatDesc outMeta(GMatDesc in) { @@ -53,4 +55,131 @@ TEST_F(GFrameTest, Input) { check(); } +//////////////////////////////////////////////////////////////////////////////// +// cv::MediaFrame tests +namespace { +class TestMediaBGR final: public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + using Cb = cv::MediaFrame::View::Callback; + Cb m_cb; + +public: + explicit TestMediaBGR(cv::Mat m, Cb cb = [](){}) + : m_mat(m), m_cb(cb) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, cv::Size(m_mat.cols, m_mat.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb}); + } +}; + +class TestMediaNV12 final: public cv::MediaFrame::IAdapter { + cv::Mat m_y; + cv::Mat m_uv; +public: + TestMediaNV12(cv::Mat y, cv::Mat uv) : m_y(y), m_uv(uv) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size(m_y.cols, m_y.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { + m_y.ptr(), m_uv.ptr(), nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + m_y.step, m_uv.step, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } +}; +} // anonymous namespace + +struct MediaFrame_Test: public ::testing::Test { + using M = cv::Mat; + using MF = cv::MediaFrame; + MF frame; +}; + +struct MediaFrame_BGR: public MediaFrame_Test { + M bgr; + MediaFrame_BGR() + : bgr(M::eye(240, 320, CV_8UC3)) { + frame = MF::Create(bgr); + } +}; + +TEST_F(MediaFrame_BGR, Meta) { + auto meta = frame.desc(); + EXPECT_EQ(cv::MediaFormat::BGR, meta.fmt); + EXPECT_EQ(cv::Size(320,240), meta.size); +} + +TEST_F(MediaFrame_BGR, Access) { + cv::MediaFrame::View view1 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(bgr.ptr(), view1.ptr[0]); + EXPECT_EQ(bgr.step, view1.stride[0]); + + cv::MediaFrame::View view2 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(bgr.ptr(), view2.ptr[0]); + EXPECT_EQ(bgr.step, view2.stride[0]); +} + +struct MediaFrame_NV12: public MediaFrame_Test { + cv::Size sz; + cv::Mat buf, y, uv; + MediaFrame_NV12() + : sz {320, 240} + , buf(M::eye(sz.height*3/2, sz.width, CV_8UC1)) + , y (buf.rowRange(0, sz.height)) + , uv (buf.rowRange(sz.height, sz.height*3/2)) { + frame = MF::Create(y, uv); + } +}; + +TEST_F(MediaFrame_NV12, Meta) { + auto meta = frame.desc(); + EXPECT_EQ(cv::MediaFormat::NV12, meta.fmt); + EXPECT_EQ(cv::Size(320,240), meta.size); +} + +TEST_F(MediaFrame_NV12, Access) { + cv::MediaFrame::View view1 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(y. ptr(), view1.ptr [0]); + EXPECT_EQ(y. step, view1.stride[0]); + EXPECT_EQ(uv.ptr(), view1.ptr [1]); + EXPECT_EQ(uv.step, view1.stride[1]); + + cv::MediaFrame::View view2 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(y. ptr(), view2.ptr [0]); + EXPECT_EQ(y. step, view2.stride[0]); + EXPECT_EQ(uv.ptr(), view2.ptr [1]); + EXPECT_EQ(uv.step, view2.stride[1]); +} + +TEST(MediaFrame, Callback) { + int counter = 0; + cv::Mat bgr = cv::Mat::eye(240, 320, CV_8UC3); + cv::MediaFrame frame = cv::MediaFrame::Create(bgr, [&counter](){counter++;}); + + // Test that the callback (in this case, incrementing the counter) + // is called only on View destruction. + EXPECT_EQ(0, counter); + { + cv::MediaFrame::View v1 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(0, counter); + } + EXPECT_EQ(1, counter); + { + cv::MediaFrame::View v1 = frame.access(cv::MediaFrame::Access::R); + EXPECT_EQ(1, counter); + cv::MediaFrame::View v2 = frame.access(cv::MediaFrame::Access::W); + EXPECT_EQ(1, counter); + } + EXPECT_EQ(3, counter); +} + } // namespace opencv_test