tracking: add ChArUco boards + cv: update to 3.2.0

.gitignore

@@ -2,3 +2,5 @@
 Build
 Intermediate
 Binaries
+
+*.kdev4

Source/AugmentedUnreality/AURDriver.cpp

@@ -16,6 +16,7 @@ limitations under the License.
 
 #include "AugmentedUnreality.h"
 #include "AURDriver.h"
+#include "tracking/AURFiducialPattern.h"
 
 UAURDriver* UAURDriver::CurrentDriver = nullptr;
 UAURDriver::FAURDriverInstanceChange UAURDriver::OnDriverInstanceChange;
@@ -191,13 +192,13 @@ bool UAURDriver::OpenVideoSourceDefault()
 	return false;
 }
 
-bool UAURDriver::RegisterBoard(AAURMarkerBoardDefinitionBase * board_actor, bool use_as_viewpoint_origin)
+bool UAURDriver::RegisterBoard(AAURFiducialPattern * board_actor, bool use_as_viewpoint_origin)
 {
 	UE_LOG(LogAUR, Error, TEXT("UAURDriver::RegisterBoard: Not implemented"))
 	return false;
 }
 
-void UAURDriver::UnregisterBoard(AAURMarkerBoardDefinitionBase * board_actor)
+void UAURDriver::UnregisterBoard(AAURFiducialPattern * board_actor)
 {
 	UE_LOG(LogAUR, Error, TEXT("UAURDriver::UnregisterBoard: Not implemented"))
 }
@@ -315,7 +316,7 @@ UAURDriver * UAURDriver::GetCurrentDriver()
 	return CurrentDriver;
 }
 
-void UAURDriver::RegisterBoardForTracking(AAURMarkerBoardDefinitionBase * board_actor, bool use_as_viewpoint_origin)
+void UAURDriver::RegisterBoardForTracking(AAURFiducialPattern * board_actor, bool use_as_viewpoint_origin)
 {
 	RegisteredBoards.AddUnique(BoardRegistration(board_actor, use_as_viewpoint_origin));
 
@@ -325,7 +326,7 @@ void UAURDriver::RegisterBoardForTracking(AAURMarkerBoardDefinitionBase * board_
 	}
 }
 
-void UAURDriver::UnregisterBoardForTracking(AAURMarkerBoardDefinitionBase * board_actor)
+void UAURDriver::UnregisterBoardForTracking(AAURFiducialPattern * board_actor)
 {
 	RegisteredBoards.RemoveAll([&](BoardRegistration const & entry) {
 		return entry.Board == board_actor;
@@ -359,9 +360,11 @@ void UAURDriver::UnbindOnDriverInstanceChange(UObject * SlotOwner)
 
 void UAURDriver::RegisterDriver(UAURDriver* driver)
 {
+	UE_LOG(LogAUR, Log, TEXT("UAURDriver::RegisterDriver: Register driver: %s"), *driver->GetFullName())
+
 	if (CurrentDriver)
 	{
-		UE_LOG(LogAUR, Warning, TEXT("UAURDriver::Initialize: CurrentDriver is not null, replacing"))
+		UE_LOG(LogAUR, Warning, TEXT("UAURDriver::RegisterDriver: CurrentDriver is not null, replacing"))
 	}
 
 	CurrentDriver = driver;
@@ -376,6 +379,8 @@ void UAURDriver::RegisterDriver(UAURDriver* driver)
 
 void UAURDriver::UnregisterDriver(UAURDriver* driver)
 {
+	UE_LOG(LogAUR, Log, TEXT("UAURDriver::UnregisterDrive: Remove driver: %s"), *driver->GetFullName());
+
 	if (CurrentDriver == driver)
 	{
 		CurrentDriver = nullptr;

Source/AugmentedUnreality/AURDriver.h

@@ -16,10 +16,11 @@ limitations under the License.
 
 #pragma once
 
-#include "Object.h"
 #include "video_sources/AURVideoSource.h"
 #include "AURDriver.generated.h"
 
+class AAURFiducialPattern;
+
 UENUM(BlueprintType)
 enum class EAURDiagnosticInfoLevel : uint8
 {
@@ -118,7 +119,7 @@ class UAURDriver : public UObject
 	// Switch to the last used video source configuration
 	UFUNCTION(BlueprintCallable, Category = AugmentedReality)
 	bool OpenVideoSourceDefault();
-	
+
 	/** Called when a new viewpoint (camera) position is measured by the tracker */
 	//UPROPERTY(BlueprintAssignable)
 	//FAURDriverViewpointTransformUpdate OnViewpointTransformUpdate;
@@ -204,7 +205,7 @@ class UAURDriver : public UObject
 
 	UFUNCTION(BlueprintCallable, Category = AugmentedReality)
 	virtual FTransform GetCurrentViewportTransform() const;
-	
+
 	/**
 	 * Returns a pointer to FAURVideoFrame containing the current camera frame.
 	 * Do not delete the pointer.
@@ -237,9 +238,9 @@ class UAURDriver : public UObject
 	virtual FString GetDiagnosticText() const;
 
 	// Start tracking a board - called by the static board list mechanism (RegisterBoardForTracking)
-	virtual bool RegisterBoard(AAURMarkerBoardDefinitionBase* board_actor, bool use_as_viewpoint_origin = false);
+	virtual bool RegisterBoard(AAURFiducialPattern* board_actor, bool use_as_viewpoint_origin = false);
 
-	virtual void UnregisterBoard(AAURMarkerBoardDefinitionBase* board_actor);
+	virtual void UnregisterBoard(AAURFiducialPattern* board_actor);
 
 	UFUNCTION(BlueprintCallable, Category = AugmentedReality)
 	static UAURDriver* GetCurrentDriver();
@@ -256,10 +257,10 @@ class UAURDriver : public UObject
 		it will read the list and track the board.
 	**/
 	UFUNCTION(BlueprintCallable, Category = AugmentedReality)
-	static void RegisterBoardForTracking(AAURMarkerBoardDefinitionBase* board_actor, bool use_as_viewpoint_origin = false);
+	static void RegisterBoardForTracking(AAURFiducialPattern* board_actor, bool use_as_viewpoint_origin = false);
 
 	UFUNCTION(BlueprintCallable, Category = AugmentedReality)
-	static void UnregisterBoardForTracking(AAURMarkerBoardDefinitionBase* board_actor);
+	static void UnregisterBoardForTracking(AAURFiducialPattern* board_actor);
 
 	/*
 	 * Add a callback to be notified about a new AURDriver instance being used
@@ -334,10 +335,10 @@ class UAURDriver : public UObject
 	// Global registry of boards to track
 	struct BoardRegistration
 	{
-		AAURMarkerBoardDefinitionBase* Board;
+		AAURFiducialPattern* Board;
 		bool ViewpointOrigin;
 
-		BoardRegistration(AAURMarkerBoardDefinitionBase* board_actor, bool use_as_viewpoint_origin = false)
+		BoardRegistration(AAURFiducialPattern* board_actor, bool use_as_viewpoint_origin = false)
 			: Board(board_actor)
 			, ViewpointOrigin(use_as_viewpoint_origin)
 		{}

Source/AugmentedUnreality/AURDriverOpenCV.cpp

@@ -53,20 +53,20 @@ UAURVideoSource * UAURDriverOpenCV::GetVideoSource()
 void UAURDriverOpenCV::OpenVideoSource(FAURVideoConfiguration const& VideoConfiguration)
 {
 	Super::OpenVideoSource(VideoConfiguration);
-	
+
 	{
 		FScopeLock lock(&VideoSourceLock);
 		NextVideoConfiguration = VideoConfiguration;
 		SwitchToNextVideoSource = true;
 	}
 }
 
-bool UAURDriverOpenCV::RegisterBoard(AAURMarkerBoardDefinitionBase * board_actor, bool use_as_viewpoint_origin)
+bool UAURDriverOpenCV::RegisterBoard(AAURFiducialPattern * board_actor, bool use_as_viewpoint_origin)
 {
 	return Tracker.RegisterBoard(board_actor, use_as_viewpoint_origin);
 }
 
-void UAURDriverOpenCV::UnregisterBoard(AAURMarkerBoardDefinitionBase * board_actor)
+void UAURDriverOpenCV::UnregisterBoard(AAURFiducialPattern * board_actor)
 {
 	Tracker.UnregisterBoard(board_actor);
 }
@@ -226,7 +226,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 	UE_LOG(LogAUR, Log, TEXT("AURDriverOpenCV: Worker thread start"))
 
 	UAURVideoSource* current_video_source = nullptr;
-	
+
 	while (this->bContinue)
 	{
 		// whether we switch to a new vid src in this iteration
@@ -258,7 +258,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 				Driver->SwitchToNextVideoSource = false;
 			}
 		}
-		
+
 		// activate new video source after switch
 		// this is outside the previous block because opening connection can take time
 		// and we don't want to block VideoSourceLock
@@ -284,7 +284,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 
 			// compare the frame size to the size we expect from capture parameters
 			auto frame_size = CapturedFrame.size();
-			
+
 			if (frame_size.width != Driver->FrameResolution.X || frame_size.height != Driver->FrameResolution.Y)
 			{
 				FIntPoint new_camera_res(frame_size.width, frame_size.height);
@@ -295,7 +295,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 				Driver->OnCameraPropertiesChange(new_camera_res);
 			}
 			else
-			{	
+			{
 				if (Driver->IsCalibrationInProgress()) // calibration
 				{
 					if (Driver->WorldReference)
@@ -312,7 +312,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 					{
 						UE_LOG(LogAUR, Error, TEXT("AURDriverOpenCV: WorldReference is null, cannot measure time for calibration"))
 					}
-				} 
+				}
 				else if (this->Driver->bPerformOrientationTracking)
 				{
 					/**
@@ -358,7 +358,7 @@ uint32 UAURDriverOpenCV::FWorkerRunnable::Run()
 		{
 			current_video_source->Disconnect();
 		}
-		
+
 		Driver->VideoSource = nullptr;
 		Driver->NextVideoSource = nullptr;
 		Driver->OnVideoSourceSwitch();

Source/AugmentedUnreality/AURDriverOpenCV.h

@@ -33,7 +33,7 @@ class UAURDriverOpenCV : public UAURDriverThreaded
 	GENERATED_BODY()
 
 public:
-	// ONLY SET THESE PROPERTIES BEFORE CALLING Initialize()	 
+	// ONLY SET THESE PROPERTIES BEFORE CALLING Initialize()
 	UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = AugmentedReality)
 	FArucoTrackerSettings TrackerSettings;
 
@@ -50,12 +50,12 @@ class UAURDriverOpenCV : public UAURDriverThreaded
 	virtual void Tick() override;
 
 	virtual void OpenVideoSource(FAURVideoConfiguration const& VideoConfiguration) override;
-	virtual bool RegisterBoard(AAURMarkerBoardDefinitionBase* board_actor, bool use_as_viewpoint_origin = false) override;
-	virtual void UnregisterBoard(AAURMarkerBoardDefinitionBase* board_actor) override;
+	virtual bool RegisterBoard(AAURFiducialPattern* board_actor, bool use_as_viewpoint_origin = false) override;
+	virtual void UnregisterBoard(AAURFiducialPattern* board_actor) override;
 
 	virtual FVector2D GetFieldOfView() const override;
 	virtual FTransform GetCurrentViewportTransform() const override;
-	
+
 	virtual bool IsConnected() const override;
 	virtual bool IsCalibrated() const override;
 	virtual float GetCalibrationProgress() const override;

Source/AugmentedUnreality/AugmentedUnreality.Build.cs

@@ -62,7 +62,7 @@ protected string BinariesDir
 		"libc++abi.so"
 	};
 
-	protected string OpenCVVersion = "310";
+	protected string OpenCVVersion = "320";
 
 	public AugmentedUnreality(TargetInfo Target)
 	{

Source/AugmentedUnreality/tracking/AURArucoTracker.cpp

@@ -29,7 +29,7 @@ FAURArucoTracker::FAURArucoTracker()
 {
 	ViewpointTransformCamera = CameraAdditionalRotation * ViewpointTransform;
 
-	cv::aur::setLogCallback([](cv::aur::LogLevel level, std::string msg) {
+	cv::aur::setLogCallback([](cv::aur::LogLevel level, std::string const& msg) {
 		switch(level)
 		{
 			case cv::aur::LogLevel::Log:
@@ -45,6 +45,7 @@ FAURArucoTracker::FAURArucoTracker()
 				;
 		}
 	});
+
 }
 
 void FAURArucoTracker::SetSettings(FArucoTrackerSettings const& settings)
@@ -62,7 +63,7 @@ bool FAURArucoTracker::DetectMarkers(cv::Mat_<cv::Vec3b>& image, bool draw_found
 {
 	// this outside of lock so doesn't block
 	TrackerModule.processFrame(image);
-	
+
 	{
 		FScopeLock lock(&PoseLock);
 
@@ -90,17 +91,25 @@ bool FAURArucoTracker::DetectMarkers(cv::Mat_<cv::Vec3b>& image, bool draw_found
 				t_mat.M[3][r] = detected_trans(r);
 			}
 
-			// Smoothly merge measured transform with current transform
-			tbi->CurrentTransform.BlendWith(FTransform(t_mat), (1.0 - Settings.SmoothingStrength));
+			if (!t_mat.ContainsNaN())
+			{
+
+				// Smoothly merge measured transform with current transform
+				tbi->CurrentTransform.BlendWith(FTransform(t_mat), (1.0 - Settings.SmoothingStrength));
+
+				// This board provides the location of the camera
+				if (tbi->UseAsViewpointOrigin)
+				{
+					ViewpointTransform = tbi->CurrentTransform;
+					ViewpointTransformCamera = CameraAdditionalRotation * ViewpointTransform;
+				}
 
-			// This board provides the location of the camera
-			if (tbi->UseAsViewpointOrigin)
+				DetectedBoards.Add(tbi);
+			}
+			else
 			{
-				ViewpointTransform = tbi->CurrentTransform;
-				ViewpointTransformCamera = CameraAdditionalRotation * ViewpointTransform;
+				UE_LOG(LogAUR, Warning, TEXT("Wrong transform matrix %s"), *t_mat.ToString());
 			}
-
-			DetectedBoards.Add(tbi);
 		}
 	}
 
@@ -151,20 +160,19 @@ void FAURArucoTracker::PublishTransformUpdate(TrackedBoardInfo * tracking_info)
 	}
 }
 
-bool FAURArucoTracker::RegisterBoard(AAURMarkerBoardDefinitionBase* board_actor, bool use_as_viewpoint_origin)
+bool FAURArucoTracker::RegisterBoard(AAURFiducialPattern* board_actor, bool use_as_viewpoint_origin)
 {
 	if (!board_actor)
 	{
 		UE_LOG(LogAUR, Error, TEXT("AURArucoTracker::RegisterBoard board_actor is null"));
 		return false;
 	}
 
-	TSharedPtr<FFreeFormBoardData> board_data = board_actor->GetBoardData();
-
 	// Save marker images
 	board_actor->SaveMarkerFiles();
 
-	cv::aur::TrackerAruco::TrackedPose* pose_handle = TrackerModule.registerPoseToTrack(board_actor->GetBoardDef());
+	UE_LOG(LogAUR, Log, TEXT("AURArucoTracker::RegisterBoard %s"), *AActor::GetDebugName(board_actor));
+	cv::aur::TrackedPose* pose_handle = TrackerModule.registerPoseToTrack(board_actor->GetPatternDefinition());
 
 	if(!pose_handle)
 	{
@@ -184,9 +192,9 @@ bool FAURArucoTracker::RegisterBoard(AAURMarkerBoardDefinitionBase* board_actor,
 	return true;
 }
 
-void FAURArucoTracker::UnregisterBoard(AAURMarkerBoardDefinitionBase * board_actor)
+void FAURArucoTracker::UnregisterBoard(AAURFiducialPattern* board_actor)
 {
-	int board_id = board_actor->GetBoardData()->GetMinMarkerId();
+	int board_id = board_actor->GetPatternDefinition()->getMinMarkerId();
 
 	if (!TrackedBoardsById.Contains(board_id))
 	{
@@ -230,12 +238,12 @@ void FAURArucoTracker::SetDiagnosticInfoLevel(EAURDiagnosticInfoLevel NewLevel)
 		2 - show boards and positions of detected markers
 	*/
 	SetBoardVisibility(NewLevel >= EAURDiagnosticInfoLevel::AURD_Basic);
-	
+
 	// Pass to cv::aur module
 	cv::aur::DiagnosticLevel lvl = cv::aur::DiagnosticLevel::Silent;
 	if(NewLevel == EAURDiagnosticInfoLevel::AURD_Basic) lvl = cv::aur::DiagnosticLevel::Basic;
 	else if (NewLevel == EAURDiagnosticInfoLevel::AURD_Advanced) lvl = cv::aur::DiagnosticLevel::Full;
-	
+
 	TrackerModule.setDiagnosticLevel(lvl);
 }