Ported VAS OT into OpenCV G-API

modules/gapi/CMakeLists.txt

@@ -88,6 +88,7 @@ set(gapi_srcs
     src/api/kernels_imgproc.cpp
     src/api/kernels_video.cpp
     src/api/kernels_nnparsers.cpp
+    src/api/kernels_ot.cpp
     src/api/kernels_streaming.cpp
     src/api/kernels_stereo.cpp
     src/api/render.cpp
@@ -130,6 +131,7 @@ set(gapi_srcs
     src/backends/cpu/gcpustereo.cpp
     src/backends/cpu/gcpuvideo.cpp
     src/backends/cpu/gcpucore.cpp
+    src/backends/cpu/gcpuot.cpp
     src/backends/cpu/gnnparsers.cpp
 
     # Fluid Backend (also built-in, FIXME:move away)
@@ -240,18 +242,25 @@ set(gapi_srcs
     src/utils/itt.cpp
     )
 
+file(GLOB_RECURSE gapi_3rdparty_srcs
+    "${CMAKE_CURRENT_LIST_DIR}/src/3rdparty/vasot/src/*.cpp"
+)
+
 ocv_add_dispatched_file(backends/fluid/gfluidimgproc_func SSE4_1 AVX2)
 ocv_add_dispatched_file(backends/fluid/gfluidcore_func SSE4_1 AVX2)
 
 ocv_list_add_prefix(gapi_srcs "${CMAKE_CURRENT_LIST_DIR}/")
 
 # For IDE users
-ocv_source_group("Src"     FILES ${gapi_srcs})
+ocv_source_group("Src"     FILES ${gapi_srcs} ${gapi_3rdparty_srcs})
 ocv_source_group("Include" FILES ${gapi_ext_hdrs})
 
-ocv_set_module_sources(HEADERS ${gapi_ext_hdrs} SOURCES ${gapi_srcs})
+ocv_set_module_sources(HEADERS ${gapi_ext_hdrs} SOURCES ${gapi_srcs} ${gapi_3rdparty_srcs})
 ocv_module_include_directories("${CMAKE_CURRENT_LIST_DIR}/src")
 
+# VAS Object Tracking includes
+ocv_module_include_directories(${CMAKE_CURRENT_LIST_DIR}/src/3rdparty/vasot/include)
+
 ocv_create_module()
 
 ocv_target_link_libraries(${the_module} PRIVATE ade)
@@ -386,7 +395,6 @@ endif()
 ocv_add_perf_tests()
 ocv_add_samples()
 
-
 # Required for sample with inference on host
 if(TARGET example_gapi_onevpl_infer_with_advanced_device_selection)
   if(TARGET ocv.3rdparty.openvino AND OPENCV_GAPI_WITH_OPENVINO)

modules/gapi/include/opencv2/gapi/cpu/ot.hpp

@@ -0,0 +1,29 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2018 Intel Corporation
+
+
+#ifndef OPENCV_GAPI_CPU_OT_API_HPP
+#define OPENCV_GAPI_CPU_OT_API_HPP
+
+#include <opencv2/core/cvdef.h>     // GAPI_EXPORTS
+#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
+
+namespace cv {
+namespace gapi {
+/**
+ * @brief This namespace contains G-API Operation Types for
+ * VAS Object Tracking module functionality.
+ */
+namespace ot {
+namespace cpu {
+GAPI_EXPORTS GKernelPackage kernels();
+} // namespace cpu
+} // namespace ot
+} // namespace gapi
+} // namespace cv
+
+
+#endif // OPENCV_GAPI_CPU_OT_API_HPP

modules/gapi/include/opencv2/gapi/ot.hpp

@@ -0,0 +1,190 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2023 Intel Corporation
+
+#ifndef OPENCV_GAPI_OT_HPP
+#define OPENCV_GAPI_OT_HPP
+
+#include <opencv2/gapi.hpp>
+#include <opencv2/gapi/s11n.hpp>
+#include <opencv2/gapi/gkernel.hpp>
+
+namespace cv {
+namespace gapi {
+/**
+ * @brief This namespace contains G-API Operation Types for
+ * VAS Object Tracking module functionality.
+ */
+namespace ot {
+
+/**
+ * @enum TrackingStatus
+ *
+ * Tracking status twin for vas::ot::TrackingStatus
+ */
+enum class TrackingStatus: int32_t
+{
+    NEW = 0,     /**< The object is newly added. */
+    TRACKED,     /**< The object is being tracked. */
+    LOST         /**< The object gets lost now. The object can be tracked again
+                      by specifying detected object manually. */
+};
+
+struct ObjectTrackerParams
+{
+    /**
+     * Maximum number of trackable objects in a frame.
+     * Valid range: 1 <= max_num_objects. Or it can be -1 if there is no limitation
+     * of maximum number in X86. KMB/TBH has limitation up to 1024.
+     * Default value is -1 which means there is no limitation in X86. KMB/TBH is -1 means 200.
+     */
+    int32_t max_num_objects = -1;
+
+    /**
+     * Input color format. Supports 0(BGR), 1(NV12), 2(BGRX) and 4(I420)
+     */
+    int32_t input_image_format = 0;
+
+    /**
+     * Specifies whether tracker to use detection class for keeping id of an object.
+     * If it is true, new detection will be associated from previous tracking only when
+     * those two have same class.
+     * class id of an object is fixed across video frames.
+     * If it is false, new detection can be associated across different-class objects.
+     * In this case, the class id of an object may change across video frames depending on the tracker input.
+     * It is recommended to turn this option off when it is likely that detector confuses the class of object.
+     * For example, when detector confuses bicycle and motorbike. Turning this option off will increase
+     * the tracking reliability as tracker will ignore the class label of detector.
+     * @n
+     * Default value is true.
+     */
+    bool tracking_per_class = true;
+
+    bool operator==(const ObjectTrackerParams& other) const
+    {
+        return max_num_objects == other.max_num_objects
+            && input_image_format == other.input_image_format
+            && tracking_per_class == other.tracking_per_class;
+    }
+};
+
+using GTrackedInfo = std::tuple<cv::GArray<cv::Rect>, cv::GArray<int32_t>, cv::GArray<uint64_t>, cv::GArray<TrackingStatus>>;
+
+G_API_OP(GTrackFromMat, <GTrackedInfo(cv::GMat, cv::GArray<cv::Rect>, cv::GArray<int32_t>, float)>, "com.intel.track_from_mat")
+{
+    static std::tuple<cv::GArrayDesc, cv::GArrayDesc,
+                      cv::GArrayDesc, cv::GArrayDesc> outMeta(cv::GMatDesc, cv::GArrayDesc, cv::GArrayDesc, float)
+    {
+        return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc(),
+                               cv::empty_array_desc(), cv::empty_array_desc());
+    }
+};
+
+G_API_OP(GTrackFromFrame, <GTrackedInfo(cv::GFrame, cv::GArray<cv::Rect>, cv::GArray<int32_t>, float)>, "com.intel.track_from_frame")
+{
+    static std::tuple<cv::GArrayDesc, cv::GArrayDesc,
+                      cv::GArrayDesc, cv::GArrayDesc> outMeta(cv::GFrameDesc, cv::GArrayDesc, cv::GArrayDesc, float)
+    {
+       return std::make_tuple(cv::empty_array_desc(), cv::empty_array_desc(),
+                              cv::empty_array_desc(), cv::empty_array_desc());
+    }
+};
+
+/**
+ * @brief   Tracks objects with video frames.
+ *          If a detected object is overlapped enough with one of tracked object, the tracked object's
+ *          informationis updated with the input detected object.
+ *          On the other hand, if a detected object is overlapped with none of tracked objects,
+ *          the detected object is newly added and ObjectTracker starts to track the object.
+ *          In zero term tracking type, ObjectTracker clears tracked objects in case that empty
+ *          list of detected objects is passed in.
+ *
+ * @param mat                       Input frame.
+ * @param detected_rects            Detected objects rectangles in the input frame.
+ * @param detected_class_labels     Detected objects class labels in the input frame.
+ * @param delta                     Frame_delta_t Delta time between two consecutive tracking in seconds.
+ *                                  The valid range is [0.005 ~ 0.5].
+ * @return                          Tracking results of target objects.
+ *                                  cv::GArray<cv::Rect>          Array of rectangles for tracked objects.
+ *                                  cv::GArray<int32_t>           Array of detected objects labels.
+ *                                  cv::GArray<uint64_t>          Array of tracking IDs for objects.
+ *                                                                Numbering sequence starts from 1.
+ *                                                                The value 0 means the tracking ID of this object has
+ *                                                                not been assigned.
+ *                                  cv::GArray<TrackingStatus>    Array of tracking statuses for objects.
+ */
+GAPI_EXPORTS std::tuple<cv::GArray<cv::Rect>,
+                        cv::GArray<int32_t>,
+                        cv::GArray<uint64_t>,
+                        cv::GArray<TrackingStatus>> track(const cv::GMat& mat,
+                                                          const cv::GArray<cv::Rect>& detected_rects,
+                                                          const cv::GArray<int>& detected_class_labels,
+                                                          float delta);
+
+/**
+ * @brief   Tracks objects with video frames. Overload of track(...) for frame as GFrame.
+ *
+ * @param frame                     Input frame.
+ * @param detected_rects            Detected objects rectangles in the input frame.
+ * @param detected_class_labels     Detected objects class labels in the input frame.
+ * @param delta                     Frame_delta_t Delta time between two consecutive tracking in seconds.
+ *                                  The valid range is [0.005 ~ 0.5].
+ * @return                          Tracking results of target objects.
+ * @return                          Tracking results of target objects.
+ *                                  cv::GArray<cv::Rect>          Array of rectangles for tracked objects.
+ *                                  cv::GArray<int32_t>           Array of detected objects labels.
+ *                                  cv::GArray<uint64_t>          Array of tracking IDs for objects.
+ *                                                                Numbering sequence starts from 1.
+ *                                                                The value 0 means the tracking ID of this object has
+ *                                                                not been assigned.
+ *                                  cv::GArray<TrackingStatus>    Array of tracking statuses for objects.
+ */
+GAPI_EXPORTS std::tuple<cv::GArray<cv::Rect>,
+                        cv::GArray<int32_t>,
+                        cv::GArray<uint64_t>,
+                        cv::GArray<TrackingStatus>> track(const cv::GFrame& frame,
+                                                          const cv::GArray<cv::Rect>& detected_rects,
+                                                          const cv::GArray<int>& detected_class_labels,
+                                                          float delta);
+} // namespace ot
+} // namespace gapi
+} // namespace cv
+
+// FIXME: move to a separate file?
+namespace cv
+{
+namespace detail
+{
+template<> struct CompileArgTag<cv::gapi::ot::ObjectTrackerParams>
+{
+    static const char* tag()
+    {
+        return "cv.gapi.ot.object_tracker_params";
+    }
+};
+} // namespace detail
+
+namespace gapi
+{
+namespace s11n
+{
+namespace detail
+{
+template<> struct S11N<cv::gapi::ot::ObjectTrackerParams> {
+    static void serialize(IOStream &os, const cv::gapi::ot::ObjectTrackerParams &p) {
+        os << p. max_num_objects << p.input_image_format << p.tracking_per_class;
+    }
+    static cv::gapi::ot::ObjectTrackerParams deserialize(IIStream &is) {
+        cv::gapi::ot::ObjectTrackerParams p;
+        is >> p. max_num_objects >> p.input_image_format >> p.tracking_per_class;
+        return p;
+    }
+};
+} // namespace detail
+} // namespace s11n
+} // namespace gapi
+} // namespace cv
+
+#endif // OPENCV_GAPI_OT_HPP

modules/gapi/src/3rdparty/vasot/include/vas/common.hpp

@@ -0,0 +1,83 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2023 Intel Corporation
+
+#ifndef VAS_COMMON_HPP
+#define VAS_COMMON_HPP
+
+#include <cstdint>
+
+#define OT_VERSION_MAJOR 1
+#define OT_VERSION_MINOR 0
+#define OT_VERSION_PATCH 0
+
+#define VAS_EXPORT //__attribute__((visibility("default")))
+
+namespace vas {
+
+/**
+ * @class Version
+ *
+ * Contains version information.
+ */
+class Version {
+  public:
+    /**
+     * Constructor.
+     *
+     * @param[in] major Major version.
+     * @param[in] minor Minor version.
+     * @param[in] patch Patch version.
+     */
+    explicit Version(uint32_t major, uint32_t minor, uint32_t patch) : major_(major), minor_(minor), patch_(patch) {
+    }
+
+    /**
+     * Returns major version.
+     */
+    uint32_t GetMajor() const noexcept {
+        return major_;
+    }
+
+    /**
+     * Returns minor version.
+     */
+    uint32_t GetMinor() const noexcept {
+        return minor_;
+    }
+
+    /**
+     * Returns patch version.
+     */
+    uint32_t GetPatch() const noexcept {
+        return patch_;
+    }
+
+  private:
+    uint32_t major_;
+    uint32_t minor_;
+    uint32_t patch_;
+};
+
+/**
+ * @enum BackendType
+ *
+ * Represents HW backend types.
+ */
+enum class BackendType {
+    CPU,  /**< CPU */
+    GPU  /**< GPU */
+};
+
+/**
+ * @enum ColorFormat
+ *
+ * Represents Color formats.
+ */
+enum class ColorFormat { BGR, NV12, BGRX, GRAY, I420 };
+
+}; // namespace vas
+
+#endif // VAS_COMMON_HPP