Merge pull request #182 from artoolkitx/phil-misc

Phil misc

Examples/2d tracking example/macOS/artoolkitX 2d Tracking Example.xcodeproj/xcshareddata/xcschemes/artoolkitX 2d Tracking Example.xcscheme

@@ -59,6 +59,16 @@
             ReferencedContainer = "container:artoolkitX 2d Tracking Example.xcodeproj">
          </BuildableReference>
       </BuildableProductRunnable>
+      <CommandLineArguments>
+         <CommandLineArgument
+            argument = "--vconf &quot;-source=2&quot;"
+            isEnabled = "NO">
+         </CommandLineArgument>
+         <CommandLineArgument
+            argument = "--vconf &quot;-uid=0x2100000046d09a6 -preset=high&quot;"
+            isEnabled = "NO">
+         </CommandLineArgument>
+      </CommandLineArguments>
    </LaunchAction>
    <ProfileAction
       buildConfiguration = "Release"

Release Notes.md

@@ -1,9 +1,22 @@
 # artoolkitX Release Notes
 --------------------------
 
+
+## Version 1.1.19
+### 2024-05-03
+
+ * 2D tracker updates:
+    - Added new utility check_live_2d_tracking to display 2D tracker features during live tracking session.
+    - Added support for SIFT feature type.
+    - Allow control of minimum required features and RANSAC threshold in feature detector.
+    - Other minor code and documentation improvements.
+    - Fix for case where no 2D tracker supported (thanks to Frederic Alinhac).
+ * Android: Update to artoolkitx-built opencv-4.7.0. Ensure consistency between manual build and gradle-initiated build by specifying ndk version.
+ * Fix: added jepg read buffer overflow protection.
+
 ## Version 1.1.18
 ### 2024-01-17
- 
+
  * mac OS: use ld_classic for backwards compatibility.
  * In the 2D image tracker, incoming images will only be downsampled for the feature tracking stage if they meet a minimum image size. This will help tracking when the source image stream is low resolution.
 

Source/ARX/AR/CMakeLists.txt

@@ -142,8 +142,12 @@ if(HAVE_OPENCV)
     else()
         set (LINK_LIST "")
         foreach(x ${OpenCV_LIBS})
-            string(REPLACE ".lib" "341d.lib" x_debug ${x})
-            list (APPEND LINK_LIST debug ${x_debug} optimized ${x})
+            if(${x} MATCHES ".*ippicvmt.lib")
+                list (APPEND LINK_LIST ${x})
+            else()
+                string(REPLACE ".lib" "d.lib" x_debug ${x})
+                list (APPEND LINK_LIST debug ${x_debug} optimized ${x})
+            endif()
         endforeach()
         target_link_libraries(AR
             ${LINK_LIST}

Source/ARX/AR/matrixCode.c

@@ -115,7 +115,7 @@ int encodeMatrixCode(const AR_MATRIX_CODE_TYPE matrixCodeType, uint64_t in, uint
         int barcode_dimensions = matrixCodeType & AR_MATRIX_CODE_TYPE_SIZE_MASK;
         int length = barcode_dimensions*barcode_dimensions - 3;
         if (length > 63) return 0;
-        uint64_t codeCount = 1u << length;
+        uint64_t codeCount = 1ull << length;
         if (in >= codeCount) return 0;
         *out_bits_p = (uint8_t *)malloc(length);
         return (unpack_number(in, length, *out_bits_p));

Source/ARX/AR2/jpeg.c

@@ -193,6 +193,13 @@ static unsigned char *jpgread (FILE *fp, int *w, int *h, int *nc, float *dpi)
         jpeg_destroy_decompress(&cinfo);
         return NULL;
     }
+    if (cinfo.num_components < 1 || cinfo.num_components > 4 ||
+        cinfo.image_width < 0 || cinfo.image_width > 32767 ||
+        cinfo.image_height < 0 || cinfo.image_height > 32767) {
+        ARLOGe("Error: Improbable values in JPEG header.");
+        jpeg_destroy_decompress(&cinfo);
+        return NULL;
+    }
 
     /* Start decompressor */
     (void) jpeg_start_decompress(&cinfo);

Source/ARX/ARTracker2d.cpp

@@ -428,5 +428,35 @@ PlanarTracker::FeatureDetectorType ARTracker2d::getDetectorType(void)
     return m_2DTracker->GetFeatureDetector();
 }
 
+void ARTracker2d::setMinRequiredDetectedFeatures(int num)
+{
+    m_2DTracker->SetMinRequiredDetectedFeatures(num);
+}
+
+int ARTracker2d::getMinRequiredDetectedFeatures(void)
+{
+    return m_2DTracker->GetMinRequiredDetectedFeatures();
+}
+
+void ARTracker2d::setHomographyEstimationRANSACThreshold(double thresh)
+{
+    m_2DTracker->SetHomographyEstimationRANSACThreshold(thresh);
+}
+
+double ARTracker2d::getHomographyEstimationRANSACThreshold(void)
+{
+    return m_2DTracker->GetHomographyEstimationRANSACThreshold();
+}
+
+void ARTracker2d::setTrackerVisualizationActive(bool active)
+{
+    m_2DTracker->SetTrackerVisualizationActive(active);
+}
+
+void *ARTracker2d::getTrackerVisualization(void)
+{
+    return m_2DTracker->GetTrackerVisualization();
+}
+
 #endif // HAVE_2D
 

Source/ARX/ARUtil/include/ARX/ARUtil/thread_sub.h

@@ -60,57 +60,73 @@ typedef struct _THREAD_HANDLE_T THREAD_HANDLE_T;
 //
 
 // Setup.
-ARUTIL_EXTERN THREAD_HANDLE_T *threadInit( int ID, void *arg, void *(*start_routine)(THREAD_HANDLE_T*) ); // Create a new thread, and launch start_routine() on it. Returns NULL in case of failure.
-ARUTIL_EXTERN int threadFree( THREAD_HANDLE_T **flag ); // Frees structures associated with the thread handle pointed to by the location pointed to by flag. Thread should already have terminated (i.e. threadWaitQuit() has returned). Location pointed to by flag is set to NULL. 
+
+/// \brief Client-side, set up. Create a new thread, and launch start_routine() on it. Returns NULL in case of failure.
+///
+/// Example client structure:
+/// \code
+///    THREAD_HANDLE_T *threadHandle = threadInit(0, NULL, worker);
+///    if (!threadHandle) {
+///        fprintf(stderr, "Error starting thread.\n");
+///        exit(-1);
+///    }
+///
+///    // Ready to do some work:
+///    threadStartSignal(threadHandle);
+///
+///    // Wait for the results.
+///    threadEndWait(threadHandle);
+///
+///    // If all done, quit and cleanup.
+///    threadWaitQuit(threadHandle);
+///    threadFree(&threadHandle);
+/// \endcode
+ARUTIL_EXTERN THREAD_HANDLE_T *threadInit( int ID, void *arg, void *(*start_routine)(THREAD_HANDLE_T*) );
+/// Client-side, set up. Frees structures associated with the thread handle pointed to by the location pointed to by flag. Thread should already have terminated (i.e. threadWaitQuit() has returned). Location pointed to by flag is set to NULL.
+ARUTIL_EXTERN int threadFree( THREAD_HANDLE_T **flag );
 
 // Communication.
-ARUTIL_EXTERN int threadStartSignal( THREAD_HANDLE_T *flag ); // Send the worker thread the "start processing" request.
-ARUTIL_EXTERN int threadGetStatus( THREAD_HANDLE_T *flag );   // Find out (without waiting) whether a worker has ended. 0 = not started or started but not yet ended, 1 = ended.
-ARUTIL_EXTERN int threadGetBusyStatus( THREAD_HANDLE_T *flag ); // Find out if a worker is currently busy. 0 = worker not started or worker ended, 1 = worker started but not yet ended.
-ARUTIL_EXTERN int threadEndWait( THREAD_HANDLE_T *flag );     // Wait for thread to end processing.
-ARUTIL_EXTERN int threadWaitQuit( THREAD_HANDLE_T *flag );    // Tell a thread waiting for the "start processing" request to quit (exit), and wait until this has happened.
 
-// Example client structure:
-//
-//    THREAD_HANDLE_T *threadHandle = threadInit(0, NULL, worker);
-//    if (!threadHandle) {
-//        fprintf(stderr, "Error starting thread.\n");
-//        exit(-1);
-//    }
-//
-//    // Ready to do some work:
-//    threadStartSignal(threadHandle);
-//
-//    // Wait for the results.
-//    threadEndWait(threadHandle);
-//
-//    // If all done, quit and cleanup.
-//    threadWaitQuit(threadHandle);
-//    threadFree(&threadHandle);
+/// Client-side, communication. Send the worker thread the "start processing" request.
+ARUTIL_EXTERN int threadStartSignal( THREAD_HANDLE_T *flag );
+/// Client-side, communication. Find out (without waiting) whether a worker has ended. 0 = not started or started but not yet ended, 1 = ended.
+ARUTIL_EXTERN int threadGetStatus( THREAD_HANDLE_T *flag );
+/// Client-side, communication. Find out if a worker is currently busy. 0 = worker not started or worker ended, 1 = worker started but not yet ended.
+ARUTIL_EXTERN int threadGetBusyStatus( THREAD_HANDLE_T *flag );
+/// Client-side, communication. Wait for thread to end processing.
+ARUTIL_EXTERN int threadEndWait( THREAD_HANDLE_T *flag );
+/// Client-side, communication. Tell a thread waiting for the "start processing" request to quit (exit), and wait until this has happened.
+ARUTIL_EXTERN int threadWaitQuit( THREAD_HANDLE_T *flag );
 
 //
 // Worker-side.
 //
-
-ARUTIL_EXTERN int threadStartWait( THREAD_HANDLE_T *flag );   // Wait for the "start processing" request. Returns 0 if worker should start, or -1 if worker should quit (exit).
-ARUTIL_EXTERN int threadEndSignal( THREAD_HANDLE_T *flag );   // Notify anyone waiting that the worker has ended.
-ARUTIL_EXTERN int threadGetID( THREAD_HANDLE_T *flag );       // Get the 'ID' value passed to the thread's start routine.
-ARUTIL_EXTERN void *threadGetArg( THREAD_HANDLE_T *flag );    // Get the 'arg' value passed to the thread's start routine.
 
-// Example worker structure:
-//
-//    void *worker(THREAD_HANDLE_T *threadHandle)
-//    {
-//        void *arg = threadHandle->arg; // Get data passed to the worker.
-//        while (threadStartWait(threadHandle) == 0) {
-//            // Do work, probably on arg.
-//            threadEndSignal(threadHandle);
-//        }
-//        // Do cleanup.
-//        return (NULL);
-//    }
+/// \brief Worker-side. Wait for the "start processing" request. Returns 0 if worker should start, or -1 if worker should quit (exit).
+///
+/// Example worker structure:
+/// \code
+///    void *worker(THREAD_HANDLE_T *threadHandle)
+///    {
+///        void *arg = threadHandle->arg; // Get data passed to the worker.
+///        while (threadStartWait(threadHandle) == 0) {
+///            // Do work, probably on arg.
+///            threadEndSignal(threadHandle);
+///        }
+///        // Do cleanup.
+///        return (NULL);
+///    }
+/// \endcode
+ARUTIL_EXTERN int threadStartWait( THREAD_HANDLE_T *flag );
+/// Worker-side. Notify anyone waiting that the worker has ended.
+ARUTIL_EXTERN int threadEndSignal( THREAD_HANDLE_T *flag );
+/// Worker-side. Get the 'ID' value passed to the thread's start routine.
+ARUTIL_EXTERN int threadGetID( THREAD_HANDLE_T *flag );
+/// Worker-side. Get the 'arg' value passed to the thread's start routine.
+ARUTIL_EXTERN void *threadGetArg( THREAD_HANDLE_T *flag );
 
-ARUTIL_EXTERN int threadGetCPU(void); // Returns the number of online CPUs in the system.
+/// Returns the number of online CPUs in the system.
+ARUTIL_EXTERN int threadGetCPU(void);
 
 
 #ifdef __cplusplus

Source/ARX/ARVideo/Image/videoImage.c

@@ -129,6 +129,13 @@ static int jpegRead(FILE *fp, unsigned char *buf, int bufWidth, int bufHeight, A
         jpeg_destroy_decompress(&cinfo);
         return (FALSE);
     }
+    if (cinfo.image_width < 0 || cinfo.image_width > 32767 ||
+        cinfo.image_height < 0 || cinfo.image_height > 32767) {
+        ARLOGe("JPEG image size too large.");
+        jpeg_destroy_decompress(&cinfo);
+        return (FALSE);
+    }
+
     if (pixFormat != bufPixFormat) { // Pixel format conversion required?
         if (bufPixFormat == AR_PIXEL_FORMAT_MONO) { // If converting to mono, let libjpeg handle it.
             cinfo.out_color_space = JCS_GRAYSCALE;

Source/ARX/ARVideo/cparamSearch.c

@@ -206,6 +206,11 @@ CPARAM_SEARCH_STATE cparamSearch(const char *device_id, int camera_index, int ca
 {
     struct _CPARAM_SEARCH_DATA *searchData;
     struct _CPARAM_SEARCH_DATA *tail;
+
+    if (!device_id || !device_id[0]) {
+        ARLOGe("cparamSearch called without device_id.\n");
+        return (CPARAM_SEARCH_STATE_FAILED_ERROR);
+    }
 
     // Create a new search request.
     searchData = (struct _CPARAM_SEARCH_DATA *)calloc(1, sizeof(struct _CPARAM_SEARCH_DATA));

Source/ARX/ARX_c.cpp

@@ -439,6 +439,7 @@ void arwSetTrackerOptionInt(int option, int value)
             case 1: type = PlanarTracker::FeatureDetectorType::ORB; break;
             case 2: type = PlanarTracker::FeatureDetectorType::Brisk; break;
             case 3: type = PlanarTracker::FeatureDetectorType::Kaze; break;
+            case 4: type = PlanarTracker::FeatureDetectorType::SIFT; break;
             default: return;
         }
         gARTK->get2dTracker()->setDetectorType(type);
@@ -511,6 +512,7 @@ int arwGetTrackerOptionInt(int option)
             case PlanarTracker::FeatureDetectorType::ORB: return 1;
             case PlanarTracker::FeatureDetectorType::Brisk: return 2;
             case PlanarTracker::FeatureDetectorType::Kaze: return 3;
+            case PlanarTracker::FeatureDetectorType::SIFT: return 4;
             default: return -1;
         }
 #endif

Source/ARX/KPM/kpmMatching.cpp

@@ -104,6 +104,12 @@ static unsigned char *kpmReadJPEGMono(FILE *fp, int *width, int *height)
         jpeg_destroy_decompress(&cinfo);
         return (NULL);
     }
+    if (cinfo.image_width < 0 || cinfo.image_width > 32767 ||
+        cinfo.image_height < 0 || cinfo.image_height > 32767) {
+        ARLOGe("JPEG image size too large.");
+        jpeg_destroy_decompress(&cinfo);
+        return NULL;
+    }
     cinfo.out_color_space = JCS_GRAYSCALE;// Converting to mono, let libjpeg handle it.
 
     // Start decompression. This gives us access to the JPEG size.

Source/ARX/OCVT/CMakeLists.txt

@@ -11,6 +11,7 @@ set(SOURCE
     TrackableInfo.h
     TrackedPoint.h
     TrackingPointSelector.h
+    TrackerVisualization.h
     OCVConfig.cpp
     HarrisDetector.cpp
     OCVFeatureDetector.cpp
@@ -45,7 +46,7 @@ if(NOT ARX_TARGET_PLATFORM_WINDOWS)
 else()
 	set (LINK_LIST "")
 	foreach(x ${OpenCV_LIBS})
-        if(${x} MATCHES "^ippicv")
+        if(${x} MATCHES ".*ippicvmt.lib")
             list (APPEND LINK_LIST ${x})
         else()
 		    string(REPLACE ".lib" "d.lib" x_debug ${x})

Source/ARX/OCVT/HarrisDetector.cpp

@@ -52,7 +52,7 @@ std::vector<cv::Point2f> HarrisDetector::FindCorners(cv::Mat gray)
     maskRoi.setTo(cv::Scalar(255));
 
     std::vector<cv::Point2f> trackablePointsWarped;
-    goodFeaturesToTrack(gray, trackablePointsWarped, featureDetectMaxFeatures, 0.1, 10, mask, 3, false, 0.04);
+    goodFeaturesToTrack(gray, trackablePointsWarped, markerTemplateCountMax, 0.1, 10, mask, 3, false, 0.04);
 
     mask.release();
 

Source/ARX/OCVT/OCVConfig.cpp

@@ -36,18 +36,17 @@
 
 #include "OCVConfig.h"
 
-int minRequiredDetectedFeatures = 50;
-int markerTemplateWidth = 15;
-int maxLevel = 3; ///< Maximum number of levels in optical flow image pyramid.
+int minRequiredDetectedFeatures = 50; ///< Minimum number of detected features required to consider a target matched.
+const int markerTemplateWidth = 15; ///< Width in pixels of image patches used in template matching.
 const cv::Size subPixWinSize(10,10);
 const cv::Size winSize(31,31); ///< Window size to use in optical flow search.
 cv::TermCriteria termcrit(cv::TermCriteria::COUNT|cv::TermCriteria::EPS,20,0.03);
-const int featureDetectMaxFeatures = 300;
-int searchRadius = 15;
-int match_method = cv::TM_SQDIFF_NORMED;
+const int markerTemplateCountMax = 300; ///< Maximum number of Harris corners to use as template locations.  If <= 0, no limit on the maximum is set and all detected corners will be used.
+const int searchRadius = 15;
+const int match_method = cv::TM_SQDIFF_NORMED;
 const cv::Size featureImageMinSize(640, 480); ///< Minimum size when downscaling incoming images used for feature tracking.
 PlanarTracker::FeatureDetectorType defaultDetectorType = PlanarTracker::FeatureDetectorType::Akaze;
 const double nn_match_ratio = 0.8f; ///< Nearest-neighbour matching ratio
-const double ransac_thresh = 2.5f; ///< RANSAC inlier threshold
+double ransac_thresh = 2.5f; ///< RANSAC inlier threshold
 cv::RNG rng( 0xFFFFFFFF );
-int harrisBorder = 10;
+const int harrisBorder = 10; ///< Harris corners within this many pixels of the border of the image will be ignored.

Source/ARX/OCVT/OCVConfig.h

@@ -61,20 +61,24 @@
 #include <opencv2/calib3d.hpp>
 #include <ARX/OCVT/PlanarTracker.h>
 
-OCV_EXTERN extern int minRequiredDetectedFeatures;
-OCV_EXTERN extern int markerTemplateWidth;
-OCV_EXTERN extern int maxLevel;  ///< Maximum number of levels in optical flow image pyramid.
+/** @file */
+
+/// @def k_OCVTOpticalFlowMaxPyrLevel Maximum number of levels in optical flow image pyramid (0 = base level only).
+#define k_OCVTOpticalFlowMaxPyrLevel 3
+
+OCV_EXTERN extern int minRequiredDetectedFeatures; ///< Minimum number of detected features required to consider a target matched.
+OCV_EXTERN extern const int markerTemplateWidth; ///< Width in pixels of image patches used in template matching.
 OCV_EXTERN extern const cv::Size subPixWinSize;
-OCV_EXTERN extern const cv::Size winSize;
+OCV_EXTERN extern const cv::Size winSize; ///< Window size to use in optical flow search.
 OCV_EXTERN extern cv::TermCriteria termcrit;
-OCV_EXTERN extern const int featureDetectMaxFeatures;
-OCV_EXTERN extern int searchRadius;
-OCV_EXTERN extern int match_method;
+OCV_EXTERN extern const int markerTemplateCountMax; ///< Maximum number of Harris corners to use as template locations.  If <= 0, no limit on the maximum is set and all detected corners will be used.
+OCV_EXTERN extern const int searchRadius;
+OCV_EXTERN extern const int match_method;
 OCV_EXTERN extern const cv::Size featureImageMinSize; ///< Minimum size when downscaling incoming images used for feature tracking.
 OCV_EXTERN extern PlanarTracker::FeatureDetectorType defaultDetectorType;
 OCV_EXTERN extern const double nn_match_ratio; ///< Nearest-neighbour matching ratio
-OCV_EXTERN extern const double ransac_thresh; ///< RANSAC inlier threshold
+OCV_EXTERN extern double ransac_thresh; ///< RANSAC inlier threshold
 OCV_EXTERN extern cv::RNG rng;
-OCV_EXTERN extern int harrisBorder; ///< Harris corners within this many pixels of the border of the image will be ignored.
+OCV_EXTERN extern const int harrisBorder; ///< Harris corners within this many pixels of the border of the image will be ignored.
 
 #endif // OCV_CONFIG_H

Source/ARX/OCVT/OCVFeatureDetector.cpp

@@ -57,6 +57,9 @@ void OCVFeatureDetector::SetFeatureDetector(PlanarTracker::FeatureDetectorType d
         case PlanarTracker::FeatureDetectorType::Kaze:
             CreateKazeFeatureDetector();
             break;
+        case PlanarTracker::FeatureDetectorType::SIFT:
+            CreateSIFTFeatureDetector();
+            break;
     }
 }
 
@@ -84,6 +87,17 @@ void OCVFeatureDetector::CreateORBFeatureDetector()
     _matcher = cv::BFMatcher::create();
 }
 
+void OCVFeatureDetector::CreateSIFTFeatureDetector()
+{
+#if CV_VERSION_MAJOR > 4 || (CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR >= 4)
+    _featureDetector = cv::SIFT::create();
+#else
+    // For versions without SIFT, BRISK is best alternative.
+    _featureDetector = cv::BRISK::create();
+#endif
+    _matcher = cv::BFMatcher::create();
+}
+
 bool OCVFeatureDetector::AddDescriptorsToDictionary(int id, cv::Mat descriptors)
 {
     if (_visualDictionary.find(id) == _visualDictionary.end()) {