diff --git a/README.md b/README.md index 599d844..ea83b73 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ 3D reconstruction and Structure from Motion are an essential parts of many algorithms of today's research in Visual Odometry, SLAM and localization tasks. In this project I've build the sparse 3D reconstruction from known poses implementing Structure from Motion pipeline with bundle andjustment on C++ and OpenGL visualization. +![3D Reconstruction Fly](./results/3d_reconstruction_fly.gif) + Previously, I've explored the Apolloscape dataset in the [project for localization task](https://github.com/bexcite/apolloscape-loc) via building Pytorch reader and training PoseNet to directly regress the car pose. # Dependencies & Build @@ -20,6 +22,19 @@ cmake .. make ``` +# Quick Run + +Quick test of visualization on stored tiny reconstruction of just 10 images: +``` +./bin/3d_recon --restore=../results/sfm_out_sample.bin +``` + +Controls: +- W,A,S,D - move camera around +- Z,X - camera texture transparency +- J,K,L - change camera forward, backward, lateral +- 0 - return to the last camera (useful when you fly out and want to return) + # Data Folder Download Apolloscape ZPark dataset from its official [page](http://apolloscape.auto/scene.html) and unpack it into a folder. Then symbolically link it to `./data/apolloscape` from the project root directory: diff --git a/include/cv_gl/camera.h b/include/cv_gl/camera.h index 42b6e65..879c19d 100644 --- a/include/cv_gl/camera.h +++ b/include/cv_gl/camera.h @@ -47,7 +47,7 @@ std::ostream& operator<<(std::ostream& os, const CameraIntrinsics& intr); // camera constant const float kYaw = 0.0f; const float kPitch = 0.0f; // -5.0f 5.0f -const float kSpeed = 60.5f; // 2.5f +const float kSpeed = 2.5f; // 2.5f 60.5f const float kSensitivity = 0.1f; const float kZoom = 45.0f; diff --git a/include/cv_gl/sfm.h b/include/cv_gl/sfm.h index 08ddb34..f9f84e2 100644 --- a/include/cv_gl/sfm.h +++ b/include/cv_gl/sfm.h @@ -98,14 +98,18 @@ class SfM3D { int MapSize() const; void RestoreImages(); + void ClearImages(); void SetProcStatus(SfMStatus proc_status); bool IsFinished(); void EmitMapUpdate(); + // void SetResizeScale(const double scale) { resize_scale_ = scale; } + double repr_error_thresh; double max_merge_dist; + double resize_scale = 0.08; // TODO: https://www.patrikhuber.ch/blog/6-serialising-opencv-matrices-using-boost-and-cereal @@ -122,6 +126,10 @@ class SfM3D { archive(image_data_); archive(cameras_); // archive(images_); + archive(resize_scale); + archive(repr_error_thresh); + archive(max_merge_dist); + archive(images_resized_); archive(image_features_); archive(image_pairs_); archive(image_matches_); @@ -173,7 +181,7 @@ class SfM3D { std::unordered_set todo_views_; Map3D map_; - const double resize_scale_ = 0.08; + bool use_cache = true; // Preprocessing storage diff --git a/results/3d_reconstruction_fly.gif b/results/3d_reconstruction_fly.gif new file mode 100644 index 0000000..50322ff Binary files /dev/null and b/results/3d_reconstruction_fly.gif differ diff --git a/results/sfm_out_sample.bin b/results/sfm_out_sample.bin new file mode 100644 index 0000000..6ec2572 Binary files /dev/null and b/results/sfm_out_sample.bin differ diff --git a/src/apps/3d_recon.cpp b/src/apps/3d_recon.cpp index 14c46dd..0694b8b 100644 --- a/src/apps/3d_recon.cpp +++ b/src/apps/3d_recon.cpp @@ -71,25 +71,29 @@ DEFINE_int32(pairs_look_back, 2, "Number of images to look back for" DEFINE_bool(cameraimage, true, "Render camera images"); DEFINE_bool(camera, true, "Render cameras"); DEFINE_bool(viz, true, "Open 3D visualization of the map"); +DEFINE_double(viz_image_scale, 0.15, "Image resize ratio for texture visualization"); DEFINE_bool(matches_cache, true, "Use cached matches and store newly computed" " into cache"); -DEFINE_double(matches_line_dist_thresh, 10.0, "Max distance to the epiline" +DEFINE_double(matches_line_dist_thresh, 40.0, "Max distance to the epiline" " between matched corresponding points"); DEFINE_int32(matches_num_thresh, 7, "Min number of matches betwee image pairs"); DEFINE_bool(features_cache, true, "Use cached features and store new features" " into cache"); -DEFINE_double(sfm_repr_error_thresh, 1.0, "Max reprojection error allowed" +DEFINE_double(sfm_repr_error_thresh, 40.0, "Max reprojection error allowed" " during points triangulation"); -DEFINE_double(sfm_max_merge_dist, 1.0, "Maximum distance between points" +DEFINE_double(sfm_max_merge_dist, 3.0, "Maximum distance between points" " from different views that we merge into one point"); DEFINE_string(restore, "", "--restore=\"\" Saved SfM serialization" " to continue SfM reconstruction pipeline"); DEFINE_string(output, "sfm_out.bin", "--output=\"\" : Destination" " for SfM serialization"); +DEFINE_bool(save_images, false, "Saved resized images to the serialized archive"); + + DEFINE_bool(h, false, "Show help"); DECLARE_bool(help); @@ -168,6 +172,7 @@ int main(int argc, char* argv[]) { SfM3D sfm(camera_intrs); sfm.repr_error_thresh = FLAGS_sfm_repr_error_thresh; sfm.max_merge_dist = FLAGS_sfm_max_merge_dist; + sfm.resize_scale = FLAGS_viz_image_scale; if (FLAGS_restore.empty()) { // Create new run @@ -441,7 +446,7 @@ int main(int argc, char* argv[]) { // std::cout << "AddProcessInput!! == J = " << dt // << ", time = " << gl_window.GetTime() << std::endl; // do { - current_camera_id = (current_camera_id + 1) % cameras_size; + current_camera_id = (current_camera_id + 2) % cameras_size; // } while (!used_views.count(current_camera_id)); change_camera(current_camera_id); @@ -458,11 +463,32 @@ int main(int argc, char* argv[]) { // std::cout << "AddProcessInput!! == K = " << dt // << ", time = " << gl_window.GetTime() << std::endl; // do { - current_camera_id = (current_camera_id + cameras_size - 1) % cameras_size; + current_camera_id = (current_camera_id + cameras_size - 2) % cameras_size; + // } while (!used_views.count(current_camera_id)); + + change_camera(current_camera_id); + }); + gl_window.AddProcessInput(GLFW_KEY_L, [¤t_camera_id, + &last_camera_change, &cameras, &gl_window, // &used_views, + &change_camera] (float dt) { + int cameras_size = cameras->GetChildrenSize(); + const double key_rate = 0.2; + double now = gl_window.GetTime(); + if (now - last_camera_change < key_rate) return; + last_camera_change = now; + std::cout << "AddProcessInput!! == L = " << dt + << ", time = " << gl_window.GetTime() << std::endl; + // do { + if (current_camera_id % 2 == 0) { + current_camera_id = (current_camera_id + 1) % cameras_size; + } else { + current_camera_id = current_camera_id - 1; + } // } while (!used_views.count(current_camera_id)); change_camera(current_camera_id); }); + // <<<<<< Change Camera Events @@ -760,6 +786,10 @@ void StoreSfM(SfM3D& sfm) { output_file = FLAGS_restore; } + if (!FLAGS_save_images) { + sfm.ClearImages(); + } + std::cout << "Serializing SFM!!!! to: " << output_file << std::endl; std::ofstream file(output_file, std::ios::binary); cereal::BinaryOutputArchive archive(file); diff --git a/src/sfm.cpp b/src/sfm.cpp index 0c6cc40..7aca5d7 100644 --- a/src/sfm.cpp +++ b/src/sfm.cpp @@ -161,7 +161,7 @@ void SfM3D::ExtractFeatures() { } ss << std::endl; - cv::resize(img, img, cv::Size(), resize_scale_, resize_scale_); + cv::resize(img, img, cv::Size(), resize_scale, resize_scale); images_resized_[idx] = img; image_features_[idx] = features; @@ -207,7 +207,7 @@ void SfM3D::ExtractFeatures() { } std::cout << std::endl; - cv::resize(img, img, cv::Size(), resize_scale_, resize_scale_); + cv::resize(img, img, cv::Size(), resize_scale, resize_scale); images_resized_.push_back(img); // images_.push_back(); @@ -1170,7 +1170,7 @@ bool SfM3D::GetMapPointsVec(std::vector& glm_points) { ::GetKeyPointColors( images_resized_[img_id], kp, - p3dc, true, kp.angle - orig_angle, resize_scale_); + p3dc, true, kp.angle - orig_angle, resize_scale); // std::cout << " oo: " << glm::to_string(p3dc.color) << std::endl; // std::cout << " tl: " << glm::to_string(p3dc.color_tl) << std::endl; @@ -1247,7 +1247,7 @@ bool SfM3D::GetMapCamerasWithPointsVec(MapCameras& map_cameras) { glm::vec3 v_color = ::GetGlmColorFromImage( images_resized_[img_id], image_features_[img_id].keypoints[view.second], - resize_scale_); + resize_scale); // std::cout << "o: " << glm::to_string(v_color) << std::endl; p3d.color += v_color; } @@ -1355,7 +1355,13 @@ bool SfM3D::IsPairInOrder(const int p1, const int p2) { } void SfM3D::RestoreImages() { - images_resized_.clear(); + + if (!images_resized_.empty()) { + std::cout << "Images exist. Skip RestoreImages.\n"; + return; + } + + // images_resized_.clear(); assert(image_data_.size() == cameras_.size()); @@ -1387,7 +1393,7 @@ void SfM3D::RestoreImages() { cout_mu.unlock(); ImageData& im_data = image_data_[idx]; - cv::Mat img = ::LoadImage(im_data, resize_scale_); + cv::Mat img = ::LoadImage(im_data, resize_scale); images_resized_[idx] = img; } @@ -1398,20 +1404,6 @@ void SfM3D::RestoreImages() { for(int i = 0; i < capacity; ++i) { resize_threads[i].join(); } - - /* - for (size_t i = 0; i < image_data_.size(); ++i) { - std::cout << "Restore image " << i << " out of " << image_data_.size() - <(t1 - t0); @@ -1420,6 +1412,10 @@ void SfM3D::RestoreImages() { }; +void SfM3D::ClearImages() { + images_resized_.clear(); +} + void SfM3D::Print(std::ostream& os) const { os << "SfM3D: intrinsics_.size = " << intrinsics_.size() << ", " << "image_data_.size = " << image_data_.size() << ", "