G-API: Introduce cv::MediaFrame, a host type for cv::GFrame

modules/gapi/CMakeLists.txt

@@ -77,6 +77,7 @@ set(gapi_srcs
     src/api/render_ocv.cpp
     src/api/ginfer.cpp
     src/api/ft_render.cpp
+    src/api/media.cpp
 
     # Compiler part
     src/compiler/gmodel.cpp

modules/gapi/include/opencv2/gapi.hpp

@@ -24,6 +24,9 @@
 
 #include <opencv2/gapi/gmat.hpp>
 #include <opencv2/gapi/garray.hpp>
+#include <opencv2/gapi/gscalar.hpp>
+#include <opencv2/gapi/gopaque.hpp>
+#include <opencv2/gapi/gframe.hpp>
 #include <opencv2/gapi/gcomputation.hpp>
 #include <opencv2/gapi/gcompiled.hpp>
 #include <opencv2/gapi/gtyped.hpp>

modules/gapi/include/opencv2/gapi/gframe.hpp

@@ -42,12 +42,20 @@ class GAPI_EXPORTS_W_SIMPLE GFrame
 };
 /** @} */
 
+enum class MediaFormat
+{
+    BGR = 0,
+    NV12,
+};
+
 /**
  * \addtogroup gapi_meta_args
  * @{
  */
 struct GAPI_EXPORTS GFrameDesc
 {
+    MediaFormat fmt;
+    cv::Size size;
 };
 static inline GFrameDesc empty_gframe_desc() { return GFrameDesc{}; }
 /** @} */

modules/gapi/include/opencv2/gapi/media.hpp

@@ -0,0 +1,72 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#ifndef OPENCV_GAPI_MEDIA_HPP
+#define OPENCV_GAPI_MEDIA_HPP
+
+#include <memory>     // unique_ptr<>, shared_ptr<>
+#include <array>      // array<>
+#include <functional> // function<>
+#include <utility>    // forward<>()
+
+#include <opencv2/gapi/gframe.hpp>
+
+namespace cv {
+
+class GAPI_EXPORTS MediaFrame {
+public:
+    enum class Access { R, W };
+    class IAdapter;
+    class View;
+    using AdapterPtr = std::unique_ptr<IAdapter>;
+
+    MediaFrame();
+    explicit MediaFrame(AdapterPtr &&);
+    template<class T, class... Args> static cv::MediaFrame Create(Args&&...);
+
+    View access(Access);
+    cv::GFrameDesc desc() const;
+
+private:
+    struct Priv;
+    std::shared_ptr<Priv> m;
+};
+
+template<class T, class... Args>
+inline cv::MediaFrame cv::MediaFrame::Create(Args&&... args) {
+    std::unique_ptr<T> ptr(new T(std::forward<Args>(args)...));
+    return cv::MediaFrame(std::move(ptr));
+}
+
+class GAPI_EXPORTS MediaFrame::View final {
+public:
+    static constexpr const size_t MAX_PLANES = 4;
+    using Ptrs     = std::array<void*, MAX_PLANES>;
+    using Strides  = std::array<std::size_t, MAX_PLANES>; // in bytes
+    using Callback = std::function<void()>;
+
+    View(Ptrs&& ptrs, Strides&& strs, Callback &&cb = [](){});
+    View(const View&) = delete;
+    View(View&&) = default;
+    ~View();
+
+    Ptrs    ptr;
+    Strides stride;
+
+private:
+    Callback m_cb;
+};
+
+class GAPI_EXPORTS MediaFrame::IAdapter {
+public:
+    virtual ~IAdapter() = 0;
+    virtual cv::GFrameDesc meta() const = 0;
+    virtual MediaFrame::View access(MediaFrame::Access) = 0;
+};
+
+} //namespace cv
+
+#endif // OPENCV_GAPI_MEDIA_HPP

modules/gapi/src/api/media.cpp

@@ -0,0 +1,42 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#include "precomp.hpp"
+#include <opencv2/gapi/media.hpp>
+
+struct cv::MediaFrame::Priv {
+    std::unique_ptr<IAdapter> adapter;
+};
+
+cv::MediaFrame::MediaFrame() {
+}
+
+cv::MediaFrame::MediaFrame(AdapterPtr &&ptr)
+    : m(new Priv{std::move(ptr)}) {
+}
+
+cv::GFrameDesc cv::MediaFrame::desc() const {
+    return m->adapter->meta();
+}
+
+cv::MediaFrame::View cv::MediaFrame::access(Access code) {
+    return m->adapter->access(code);
+}
+
+cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb)
+    : ptr   (std::move(ptrs))
+    , stride(std::move(strs))
+    , m_cb  (std::move(cb)) {
+}
+
+cv::MediaFrame::View::~View() {
+    if (m_cb) {
+        m_cb();
+    }
+}
+
+cv::MediaFrame::IAdapter::~IAdapter() {
+}

modules/gapi/test/gapi_frame_tests.cpp

@@ -6,10 +6,12 @@
 
 #include "test_precomp.hpp"
 
-#include <opencv2/gapi/cpu/gcpukernel.hpp>
+#include <opencv2/gapi/media.hpp>
 
-namespace opencv_test
-{
+////////////////////////////////////////////////////////////////////////////////
+// cv::GFrame tests
+
+namespace opencv_test {
 
 G_API_OP(GBlurFrame, <GMat(GFrame)>, "test.blur_frame") {
     static GMatDesc outMeta(GMatDesc in) {
@@ -53,4 +55,131 @@ TEST_F(GFrameTest, Input) {
     check();
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// cv::MediaFrame tests
+namespace {
+class TestMediaBGR final: public cv::MediaFrame::IAdapter {
+    cv::Mat m_mat;
+    using Cb = cv::MediaFrame::View::Callback;
+    Cb m_cb;
+
+public:
+    explicit TestMediaBGR(cv::Mat m, Cb cb = [](){})
+        : m_mat(m), m_cb(cb) {
+    }
+    cv::GFrameDesc meta() const override {
+        return cv::GFrameDesc{cv::MediaFormat::BGR, cv::Size(m_mat.cols, m_mat.rows)};
+    }
+    cv::MediaFrame::View access(cv::MediaFrame::Access) override {
+        cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr };
+        cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u };
+        return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb});
+    }
+};
+
+class TestMediaNV12 final: public cv::MediaFrame::IAdapter {
+    cv::Mat m_y;
+    cv::Mat m_uv;
+public:
+    TestMediaNV12(cv::Mat y, cv::Mat uv) : m_y(y), m_uv(uv) {
+    }
+    cv::GFrameDesc meta() const override {
+        return cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size(m_y.cols, m_y.rows)};
+    }
+    cv::MediaFrame::View access(cv::MediaFrame::Access) override {
+        cv::MediaFrame::View::Ptrs pp = {
+            m_y.ptr(), m_uv.ptr(), nullptr, nullptr
+        };
+        cv::MediaFrame::View::Strides ss = {
+            m_y.step, m_uv.step, 0u, 0u
+        };
+        return cv::MediaFrame::View(std::move(pp), std::move(ss));
+    }
+};
+} // anonymous namespace
+
+struct MediaFrame_Test: public ::testing::Test {
+    using M = cv::Mat;
+    using MF = cv::MediaFrame;
+    MF frame;
+};
+
+struct MediaFrame_BGR: public MediaFrame_Test {
+    M bgr;
+    MediaFrame_BGR()
+        : bgr(M::eye(240, 320, CV_8UC3)) {
+        frame = MF::Create<TestMediaBGR>(bgr);
+    }
+};
+
+TEST_F(MediaFrame_BGR, Meta) {
+    auto meta = frame.desc();
+    EXPECT_EQ(cv::MediaFormat::BGR, meta.fmt);
+    EXPECT_EQ(cv::Size(320,240),    meta.size);
+}
+
+TEST_F(MediaFrame_BGR, Access) {
+    cv::MediaFrame::View view1 = frame.access(cv::MediaFrame::Access::R);
+    EXPECT_EQ(bgr.ptr(), view1.ptr[0]);
+    EXPECT_EQ(bgr.step,  view1.stride[0]);
+
+    cv::MediaFrame::View view2 = frame.access(cv::MediaFrame::Access::R);
+    EXPECT_EQ(bgr.ptr(), view2.ptr[0]);
+    EXPECT_EQ(bgr.step,  view2.stride[0]);
+}
+
+struct MediaFrame_NV12: public MediaFrame_Test {
+    cv::Size sz;
+    cv::Mat buf, y, uv;
+    MediaFrame_NV12()
+        : sz {320, 240}
+        , buf(M::eye(sz.height*3/2, sz.width, CV_8UC1))
+        , y  (buf.rowRange(0, sz.height))
+        , uv (buf.rowRange(sz.height, sz.height*3/2)) {
+        frame = MF::Create<TestMediaNV12>(y, uv);
+    }
+};
+
+TEST_F(MediaFrame_NV12, Meta) {
+    auto meta = frame.desc();
+    EXPECT_EQ(cv::MediaFormat::NV12, meta.fmt);
+    EXPECT_EQ(cv::Size(320,240),     meta.size);
+}
+
+TEST_F(MediaFrame_NV12, Access) {
+    cv::MediaFrame::View view1 = frame.access(cv::MediaFrame::Access::R);
+    EXPECT_EQ(y. ptr(), view1.ptr   [0]);
+    EXPECT_EQ(y. step,  view1.stride[0]);
+    EXPECT_EQ(uv.ptr(), view1.ptr   [1]);
+    EXPECT_EQ(uv.step,  view1.stride[1]);
+
+    cv::MediaFrame::View view2 = frame.access(cv::MediaFrame::Access::R);
+    EXPECT_EQ(y. ptr(), view2.ptr   [0]);
+    EXPECT_EQ(y. step,  view2.stride[0]);
+    EXPECT_EQ(uv.ptr(), view2.ptr   [1]);
+    EXPECT_EQ(uv.step,  view2.stride[1]);
+}
+
+TEST(MediaFrame, Callback) {
+    int counter = 0;
+    cv::Mat bgr = cv::Mat::eye(240, 320, CV_8UC3);
+    cv::MediaFrame frame = cv::MediaFrame::Create<TestMediaBGR>(bgr, [&counter](){counter++;});
+
+    // Test that the callback (in this case, incrementing the counter)
+    // is called only on View destruction.
+    EXPECT_EQ(0, counter);
+    {
+        cv::MediaFrame::View v1 = frame.access(cv::MediaFrame::Access::R);
+        EXPECT_EQ(0, counter);
+    }
+    EXPECT_EQ(1, counter);
+    {
+        cv::MediaFrame::View v1 = frame.access(cv::MediaFrame::Access::R);
+        EXPECT_EQ(1, counter);
+        cv::MediaFrame::View v2 = frame.access(cv::MediaFrame::Access::W);
+        EXPECT_EQ(1, counter);
+    }
+    EXPECT_EQ(3, counter);
+}
+
 } // namespace opencv_test