Merge pull request #23264 from AleksandrPanov:add_detect_qr_with_aruco

Add detect qr with aruco #23264

Using Aruco to detect finder patterns to search QR codes.

TODO (in next PR):
- add single QR detect (update `detect()` and `detectAndDecode()`)
- need reduce full enumeration of finder patterns
- need add finder pattern info to `decode` step
- need to merge the pipeline of the old and new algorithm

[Current results:](https://docs.google.com/spreadsheets/d/1ufKyR-Zs-IGXwvqPgftssmTlceVjiQX364sbrjr2QU8/edit#gid=1192415584)
+20% total detect, +8% total decode in OpenCV [QR benchmark](https://github.com/opencv/opencv_benchmarks/tree/develop/python_benchmarks/qr_codes) 

![res1](https://user-images.githubusercontent.com/22337800/231228556-191d3eae-a318-44e1-af99-e7d420bf6248.png)


78.4% detect, 58.7% decode vs 58.5 detect, 50.5% decode in default

[main.py.txt](https://github.com/opencv/opencv/files/10762369/main.py.txt)

![res2](https://user-images.githubusercontent.com/22337800/231229123-ed7f1eda-159a-444b-a3ff-f107d8eb4a20.png)


add new info to [google docs](https://docs.google.com/spreadsheets/d/1ufKyR-Zs-IGXwvqPgftssmTlceVjiQX364sbrjr2QU8/edit?usp=sharing)


### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

modules/objdetect/include/opencv2/objdetect.hpp

@@ -45,6 +45,7 @@
 #define OPENCV_OBJDETECT_HPP
 
 #include "opencv2/core.hpp"
+#include "opencv2/objdetect/aruco_detector.hpp"
 
 /**
 @defgroup objdetect Object Detection
@@ -763,28 +764,15 @@ class CV_EXPORTS_W QRCodeEncoder {
 
 };
 
-class CV_EXPORTS_W QRCodeDetector
-{
+class CV_EXPORTS_W_SIMPLE QRCodeDetectorBase {
 public:
-    CV_WRAP QRCodeDetector();
-    ~QRCodeDetector();
+    CV_DEPRECATED_EXTERNAL  // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
+    QRCodeDetectorBase();
 
-    /** @brief sets the epsilon used during the horizontal scan of QR code stop marker detection.
-     @param epsX Epsilon neighborhood, which allows you to determine the horizontal pattern
-     of the scheme 1:1:3:1:1 according to QR code standard.
-    */
-    CV_WRAP void setEpsX(double epsX);
-    /** @brief sets the epsilon used during the vertical scan of QR code stop marker detection.
-     @param epsY Epsilon neighborhood, which allows you to determine the vertical pattern
-     of the scheme 1:1:3:1:1 according to QR code standard.
-     */
-    CV_WRAP void setEpsY(double epsY);
-
-    /** @brief use markers to improve the position of the corners of the QR code
-     *
-     * alignmentMarkers using by default
-     */
-    CV_WRAP void setUseAlignmentMarkers(bool useAlignmentMarkers);
+    QRCodeDetectorBase(const QRCodeDetectorBase&) = default;
+    QRCodeDetectorBase(QRCodeDetectorBase&&) = default;
+    QRCodeDetectorBase& operator=(const QRCodeDetectorBase&) = default;
+    QRCodeDetectorBase& operator=(QRCodeDetectorBase&&) = default;
 
     /** @brief Detects QR code in image and returns the quadrangle containing the code.
      @param img grayscale or color (BGR) image containing (or not) QR code.
@@ -799,16 +787,7 @@ class CV_EXPORTS_W QRCodeDetector
      @param points Quadrangle vertices found by detect() method (or some other algorithm).
      @param straight_qrcode The optional output image containing rectified and binarized QR code
      */
-    CV_WRAP std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode = noArray());
-
-    /** @brief Decodes QR code on a curved surface in image once it's found by the detect() method.
-
-     Returns UTF8-encoded output string or empty string if the code cannot be decoded.
-     @param img grayscale or color (BGR) image containing QR code.
-     @param points Quadrangle vertices found by detect() method (or some other algorithm).
-     @param straight_qrcode The optional output image containing rectified and binarized QR code
-     */
-    CV_WRAP cv::String decodeCurved(InputArray img, InputArray points, OutputArray straight_qrcode = noArray());
+    CV_WRAP std::string decode(InputArray img, InputArray points, OutputArray straight_qrcode = noArray()) const;
 
     /** @brief Both detects and decodes QR code
 
@@ -817,16 +796,8 @@ class CV_EXPORTS_W QRCodeDetector
      @param straight_qrcode The optional output image containing rectified and binarized QR code
      */
     CV_WRAP std::string detectAndDecode(InputArray img, OutputArray points=noArray(),
-                                        OutputArray straight_qrcode = noArray());
-
-    /** @brief Both detects and decodes QR code on a curved surface
+                                        OutputArray straight_qrcode = noArray()) const;
 
-     @param img grayscale or color (BGR) image containing QR code.
-     @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found.
-     @param straight_qrcode The optional output image containing rectified and binarized QR code
-     */
-    CV_WRAP std::string detectAndDecodeCurved(InputArray img, OutputArray points=noArray(),
-                                              OutputArray straight_qrcode = noArray());
 
     /** @brief Detects QR codes in image and returns the vector of the quadrangles containing the codes.
      @param img grayscale or color (BGR) image containing (or not) QR codes.
@@ -860,18 +831,109 @@ class CV_EXPORTS_W QRCodeDetector
             OutputArray points = noArray(),
             OutputArrayOfArrays straight_qrcode = noArray()
     ) const;
-
-protected:
     struct Impl;
+protected:
     Ptr<Impl> p;
 };
 
+class CV_EXPORTS_W_SIMPLE QRCodeDetector : public QRCodeDetectorBase
+{
+public:
+    CV_WRAP QRCodeDetector();
+
+    /** @brief sets the epsilon used during the horizontal scan of QR code stop marker detection.
+     @param epsX Epsilon neighborhood, which allows you to determine the horizontal pattern
+     of the scheme 1:1:3:1:1 according to QR code standard.
+    */
+    CV_WRAP QRCodeDetector& setEpsX(double epsX);
+    /** @brief sets the epsilon used during the vertical scan of QR code stop marker detection.
+     @param epsY Epsilon neighborhood, which allows you to determine the vertical pattern
+     of the scheme 1:1:3:1:1 according to QR code standard.
+     */
+    CV_WRAP QRCodeDetector& setEpsY(double epsY);
+
+    /** @brief use markers to improve the position of the corners of the QR code
+     *
+     * alignmentMarkers using by default
+     */
+    CV_WRAP QRCodeDetector& setUseAlignmentMarkers(bool useAlignmentMarkers);
+
+    /** @brief Decodes QR code on a curved surface in image once it's found by the detect() method.
+
+     Returns UTF8-encoded output string or empty string if the code cannot be decoded.
+     @param img grayscale or color (BGR) image containing QR code.
+     @param points Quadrangle vertices found by detect() method (or some other algorithm).
+     @param straight_qrcode The optional output image containing rectified and binarized QR code
+     */
+    CV_WRAP cv::String decodeCurved(InputArray img, InputArray points, OutputArray straight_qrcode = noArray());
+
+    /** @brief Both detects and decodes QR code on a curved surface
+
+     @param img grayscale or color (BGR) image containing QR code.
+     @param points optional output array of vertices of the found QR code quadrangle. Will be empty if not found.
+     @param straight_qrcode The optional output image containing rectified and binarized QR code
+     */
+    CV_WRAP std::string detectAndDecodeCurved(InputArray img, OutputArray points=noArray(),
+                                              OutputArray straight_qrcode = noArray());
+};
+
+class CV_EXPORTS_W_SIMPLE QRCodeDetectorAruco : public QRCodeDetectorBase {
+public:
+    CV_WRAP QRCodeDetectorAruco();
+
+    struct CV_EXPORTS_W_SIMPLE Params {
+        CV_WRAP Params();
+
+        /** @brief The minimum allowed pixel size of a QR module in the smallest image in the image pyramid, default 4.f */
+        CV_PROP_RW float minModuleSizeInPyramid;
+
+        /** @brief The maximum allowed relative rotation for finder patterns in the same QR code, default pi/12 */
+        CV_PROP_RW float maxRotation;
+
+        /** @brief The maximum allowed relative mismatch in module sizes for finder patterns in the same QR code, default 1.75f */
+        CV_PROP_RW float maxModuleSizeMismatch;
+
+        /** @brief The maximum allowed module relative mismatch for timing pattern module, default 2.f
+         *
+         * If relative mismatch of timing pattern module more this value, penalty points will be added.
+         * If a lot of penalty points are added, QR code will be rejected. */
+        CV_PROP_RW float maxTimingPatternMismatch;
+
+        /** @brief The maximum allowed percentage of penalty points out of total pins in timing pattern, default 0.4f */
+        CV_PROP_RW float maxPenalties;
+
+        /** @brief The maximum allowed relative color mismatch in the timing pattern, default 0.2f*/
+        CV_PROP_RW float maxColorsMismatch;
+
+        /** @brief The algorithm find QR codes with almost minimum timing pattern score and minimum size, default 0.9f
+         *
+         * The QR code with the minimum "timing pattern score" and minimum "size" is selected as the best QR code.
+         * If for the current QR code "timing pattern score" * scaleTimingPatternScore < "previous timing pattern score" and "size" < "previous size", then
+         * current QR code set as the best QR code. */
+        CV_PROP_RW float scaleTimingPatternScore;
+    };
+
+    /** @brief QR code detector constructor for Aruco-based algorithm. See cv::QRCodeDetectorAruco::Params */
+    CV_WRAP explicit QRCodeDetectorAruco(const QRCodeDetectorAruco::Params& params);
+
+    /** @brief Detector parameters getter. See cv::QRCodeDetectorAruco::Params */
+    CV_WRAP const QRCodeDetectorAruco::Params& getDetectorParameters() const;
+
+    /** @brief Detector parameters setter. See cv::QRCodeDetectorAruco::Params */
+    CV_WRAP QRCodeDetectorAruco& setDetectorParameters(const QRCodeDetectorAruco::Params& params);
+
+    /** @brief Aruco detector parameters are used to search for the finder patterns. */
+    CV_WRAP aruco::DetectorParameters getArucoParameters();
+
+    /** @brief Aruco detector parameters are used to search for the finder patterns. */
+    CV_WRAP void setArucoParameters(const aruco::DetectorParameters& params);
+};
+
 //! @}
 }
 
 #include "opencv2/objdetect/detection_based_tracker.hpp"
 #include "opencv2/objdetect/face.hpp"
-#include "opencv2/objdetect/aruco_detector.hpp"
 #include "opencv2/objdetect/charuco_detector.hpp"
 
 #endif

modules/objdetect/misc/objc/gen_dict.json

@@ -0,0 +1,7 @@
+{
+    "ManualFuncs" : {
+        "QRCodeDetectorAruco": {
+            "getDetectorParameters": { "declaration" : [""], "implementation" : [""] }
+        }
+    }
+}

modules/objdetect/perf/perf_qrcode_pipeline.cpp

@@ -3,6 +3,7 @@
 // of this distribution and at http://opencv.org/license.html.
 
 #include "perf_precomp.hpp"
+#include "../test/test_qr_utils.hpp"
 
 namespace opencv_test
 {
@@ -23,7 +24,9 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, detect)
     std::vector< Point > corners;
     QRCodeDetector qrcode;
     TEST_CYCLE() ASSERT_TRUE(qrcode.detect(src, corners));
-    SANITY_CHECK(corners);
+    const int pixels_error = 3;
+    check_qr(root, name_current_image, "test_images", corners, {}, pixels_error);
+    SANITY_CHECK_NOTHING();
 }
 
 #ifdef HAVE_QUIRC
@@ -45,75 +48,73 @@ PERF_TEST_P_(Perf_Objdetect_QRCode, decode)
         decoded_info = qrcode.decode(src, corners, straight_barcode);
         ASSERT_FALSE(decoded_info.empty());
     }
-
-    std::vector<uint8_t> decoded_info_uint8_t(decoded_info.begin(), decoded_info.end());
-    SANITY_CHECK(decoded_info_uint8_t);
-    SANITY_CHECK(straight_barcode);
-
+    const int pixels_error = 3;
+    check_qr(root, name_current_image, "test_images", corners, {decoded_info}, pixels_error);
+    SANITY_CHECK_NOTHING();
 }
 #endif
 
-typedef ::perf::TestBaseWithParam< std::string > Perf_Objdetect_QRCode_Multi;
+typedef ::perf::TestBaseWithParam<std::tuple<std::string, std::string>> Perf_Objdetect_QRCode_Multi;
 
-static inline bool compareCorners(const Point2f& corner1, const Point2f& corner2) {
-    return corner1.x == corner2.x ? corner1.y < corner2.y : corner1.x < corner2.x;
-}
+static std::set<std::pair<std::string, std::string>> disabled_samples = {{"5_qrcodes.png", "aruco_based"}};
 
 PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, detectMulti)
 {
-    const std::string name_current_image = GetParam();
+    const std::string name_current_image = get<0>(GetParam());
+    const std::string method = get<1>(GetParam());
     const std::string root = "cv/qrcode/multiple/";
 
     std::string image_path = findDataFile(root + name_current_image);
     Mat src = imread(image_path);
     ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
-    std::vector<Point2f> corners;
-    QRCodeDetector qrcode;
+    std::vector<Point> corners;
+    QRCodeDetectorBase qrcode = QRCodeDetector();
+    if (method == "aruco_based") {
+        qrcode = QRCodeDetectorAruco();
+    }
     TEST_CYCLE() ASSERT_TRUE(qrcode.detectMulti(src, corners));
-    sort(corners.begin(), corners.end(), compareCorners);
-    SANITY_CHECK(corners);
-}
-
-static inline bool compareQR(const pair<string, Mat>& v1, const pair<string, Mat>& v2) {
-    return v1.first < v2.first;
+    const int pixels_error = 7;
+    check_qr(root, name_current_image, "multiple_images", corners, {}, pixels_error, true);
+    SANITY_CHECK_NOTHING();
 }
 
 #ifdef HAVE_QUIRC
 PERF_TEST_P_(Perf_Objdetect_QRCode_Multi, decodeMulti)
 {
-    const std::string name_current_image = GetParam();
+    const std::string name_current_image = get<0>(GetParam());
+    std::string method = get<1>(GetParam());
     const std::string root = "cv/qrcode/multiple/";
     std::string image_path = findDataFile(root + name_current_image);
     Mat src = imread(image_path);
     ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path;
-    QRCodeDetector qrcode;
+    if (disabled_samples.find({name_current_image, method}) != disabled_samples.end()) {
+        throw SkipTestException(name_current_image + " is disabled sample for method " + method);
+    }
+    QRCodeDetectorBase qrcode = QRCodeDetector();
+    if (method == "aruco_based") {
+        qrcode = QRCodeDetectorAruco();
+    }
     std::vector<Point2f> corners;
     ASSERT_TRUE(qrcode.detectMulti(src, corners));
     std::vector<Mat> straight_barcode;
     std::vector< cv::String > decoded_info;
     TEST_CYCLE()
     {
         ASSERT_TRUE(qrcode.decodeMulti(src, corners, decoded_info, straight_barcode));
-        for(size_t i = 0; i < decoded_info.size(); i++)
-        {
-            ASSERT_FALSE(decoded_info[i].empty());
-        }
+    }
+    ASSERT_TRUE(decoded_info.size() > 0ull);
+    for(size_t i = 0; i < decoded_info.size(); i++) {
+        ASSERT_FALSE(decoded_info[i].empty());
     }
     ASSERT_EQ(decoded_info.size(), straight_barcode.size());
-    vector<pair<string, Mat> > result;
-    for (size_t i = 0ull;  i < decoded_info.size(); i++) {
-        result.push_back(make_pair(decoded_info[i], straight_barcode[i]));
+    vector<Point> corners_result(corners.size());
+    for (size_t i = 0ull; i < corners_result.size(); i++) {
+        corners_result[i] = corners[i];
     }
 
-    sort(result.begin(), result.end(), compareQR);
-    vector<vector<uint8_t> > decoded_info_sort;
-    vector<Mat> straight_barcode_sort;
-    for (size_t i = 0ull;  i < result.size(); i++) {
-        vector<uint8_t> tmp(result[i].first.begin(), result[i].first.end());
-        decoded_info_sort.push_back(tmp);
-        straight_barcode_sort.push_back(result[i].second);
-    }
-    SANITY_CHECK(decoded_info_sort);
+    const int pixels_error = 7;
+    check_qr(root, name_current_image, "multiple_images", corners_result, decoded_info, pixels_error, true);
+    SANITY_CHECK_NOTHING();
 }
 #endif
 
@@ -127,11 +128,10 @@ INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode,
 // version_5_right.jpg DISABLED after tile fix, PR #22025
 
 INSTANTIATE_TEST_CASE_P(/*nothing*/, Perf_Objdetect_QRCode_Multi,
-    ::testing::Values(
-      "2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png",
-      "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png"
-    )
-);
+    testing::Combine(testing::Values("2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", "4_qrcodes.png",
+                                     "5_qrcodes.png", "6_qrcodes.png", "7_qrcodes.png", "8_close_qrcodes.png"),
+                     testing::Values("contours_based", "aruco_based")));
+
 
 typedef ::perf::TestBaseWithParam< tuple< std::string, Size > > Perf_Objdetect_Not_QRCode;