Added visual gif

README.md

@@ -2,6 +2,8 @@
 
 3D reconstruction and Structure from Motion are an essential parts of many algorithms of today's research in Visual Odometry, SLAM and localization tasks. In this project I've build the sparse 3D reconstruction from known poses implementing Structure from Motion pipeline with bundle andjustment on C++ and OpenGL visualization.
 
+![3D Reconstruction Fly](./results/3d_reconstruction_fly.gif)
+
 Previously, I've explored the Apolloscape dataset in the [project for localization task](https://github.com/bexcite/apolloscape-loc) via building Pytorch reader and training PoseNet to directly regress the car pose.
 
 # Dependencies & Build
@@ -20,6 +22,19 @@ cmake ..
 make
 ```
 
+# Quick Run
+
+Quick test of visualization on stored tiny reconstruction of just 10 images:
+```
+./bin/3d_recon --restore=../results/sfm_out_sample.bin
+```
+
+Controls:
+- W,A,S,D - move camera around
+- Z,X - camera texture transparency
+- J,K,L - change camera forward, backward, lateral
+- 0 - return to the last camera (useful when you fly out and want to return)
+
 # Data Folder
 
 Download Apolloscape ZPark dataset from its official [page](http://apolloscape.auto/scene.html) and unpack it into a folder. Then symbolically link it to `./data/apolloscape` from the project root directory:

include/cv_gl/camera.h

@@ -47,7 +47,7 @@ std::ostream& operator<<(std::ostream& os, const CameraIntrinsics& intr);
 // camera constant
 const float kYaw = 0.0f;
 const float kPitch = 0.0f;  // -5.0f    5.0f
-const float kSpeed = 60.5f; // 2.5f
+const float kSpeed = 2.5f; // 2.5f 60.5f
 const float kSensitivity = 0.1f;
 const float kZoom = 45.0f;
 

include/cv_gl/sfm.h

@@ -98,14 +98,18 @@ class SfM3D {
   int MapSize() const;
 
   void RestoreImages();
+  void ClearImages();
 
   void SetProcStatus(SfMStatus proc_status);
   bool IsFinished();
   void EmitMapUpdate();
 
+  // void SetResizeScale(const double scale) { resize_scale_ = scale; }
+
 
   double repr_error_thresh;
   double max_merge_dist;
+  double resize_scale = 0.08;
 
 
   // TODO: https://www.patrikhuber.ch/blog/6-serialising-opencv-matrices-using-boost-and-cereal
@@ -122,6 +126,10 @@ class SfM3D {
     archive(image_data_);
     archive(cameras_);
     // archive(images_);
+    archive(resize_scale);
+    archive(repr_error_thresh);
+    archive(max_merge_dist);
+    archive(images_resized_);
     archive(image_features_);
     archive(image_pairs_);
     archive(image_matches_);
@@ -173,7 +181,7 @@ class SfM3D {
   std::unordered_set<int> todo_views_;
   Map3D map_;
 
-  const double resize_scale_ = 0.08;
+
   bool use_cache = true;
 
   // Preprocessing storage

src/apps/3d_recon.cpp

@@ -71,25 +71,29 @@ DEFINE_int32(pairs_look_back, 2, "Number of images to look back for"
 DEFINE_bool(cameraimage, true, "Render camera images");
 DEFINE_bool(camera, true, "Render cameras");
 DEFINE_bool(viz, true, "Open 3D visualization of the map");
+DEFINE_double(viz_image_scale, 0.15, "Image resize ratio for texture visualization");
 
 
 DEFINE_bool(matches_cache, true, "Use cached matches and store newly computed"
     " into cache");
-DEFINE_double(matches_line_dist_thresh, 10.0, "Max distance to the epiline"
+DEFINE_double(matches_line_dist_thresh, 40.0, "Max distance to the epiline"
     " between matched corresponding points");
 DEFINE_int32(matches_num_thresh, 7, "Min number of matches betwee image pairs");
 DEFINE_bool(features_cache, true, "Use cached features and store new features"
     " into cache");
 
-DEFINE_double(sfm_repr_error_thresh, 1.0, "Max reprojection error allowed"
+DEFINE_double(sfm_repr_error_thresh, 40.0, "Max reprojection error allowed"
     " during points triangulation");
-DEFINE_double(sfm_max_merge_dist, 1.0, "Maximum distance between points"
+DEFINE_double(sfm_max_merge_dist, 3.0, "Maximum distance between points"
     " from different views that we merge into one point");
 
 DEFINE_string(restore, "", "--restore=\"<filename>\" Saved SfM serialization"
                            " to continue SfM reconstruction pipeline");
 DEFINE_string(output, "sfm_out.bin", "--output=\"<filename>\" : Destination"
                       " for SfM serialization");
+DEFINE_bool(save_images, false, "Saved resized images to the serialized archive");
+
+
 DEFINE_bool(h, false, "Show help");
 
 DECLARE_bool(help);
@@ -168,6 +172,7 @@ int main(int argc, char* argv[]) {
   SfM3D sfm(camera_intrs);
   sfm.repr_error_thresh = FLAGS_sfm_repr_error_thresh;
   sfm.max_merge_dist = FLAGS_sfm_max_merge_dist;
+  sfm.resize_scale = FLAGS_viz_image_scale;
 
   if (FLAGS_restore.empty()) {
     // Create new run
@@ -441,7 +446,7 @@ int main(int argc, char* argv[]) {
     // std::cout << "AddProcessInput!! == J = " << dt 
     //           << ", time = " << gl_window.GetTime() << std::endl;
     // do {
-      current_camera_id = (current_camera_id + 1) % cameras_size;
+      current_camera_id = (current_camera_id + 2) % cameras_size;
     // } while (!used_views.count(current_camera_id));
 
     change_camera(current_camera_id);
@@ -458,11 +463,32 @@ int main(int argc, char* argv[]) {
     // std::cout << "AddProcessInput!! == K = " << dt 
     //           << ", time = " << gl_window.GetTime() << std::endl;
     // do {
-      current_camera_id = (current_camera_id + cameras_size - 1) % cameras_size;
+      current_camera_id = (current_camera_id + cameras_size - 2) % cameras_size;
+    // } while (!used_views.count(current_camera_id));
+
+    change_camera(current_camera_id);
+  });
+  gl_window.AddProcessInput(GLFW_KEY_L, [&current_camera_id,
+      &last_camera_change, &cameras, &gl_window, // &used_views,
+      &change_camera] (float dt) {
+    int cameras_size = cameras->GetChildrenSize();
+    const double key_rate = 0.2;
+    double now = gl_window.GetTime();
+    if (now - last_camera_change < key_rate) return;
+    last_camera_change = now;
+    std::cout << "AddProcessInput!! == L = " << dt 
+              << ", time = " << gl_window.GetTime() << std::endl;
+    // do {
+    if (current_camera_id % 2 == 0) {
+      current_camera_id = (current_camera_id + 1) % cameras_size;
+    } else {
+      current_camera_id = current_camera_id - 1;
+    }
     // } while (!used_views.count(current_camera_id));
 
     change_camera(current_camera_id);
   });
+
   // <<<<<< Change Camera Events
 
 
@@ -760,6 +786,10 @@ void StoreSfM(SfM3D& sfm) {
     output_file = FLAGS_restore;
   }
 
+  if (!FLAGS_save_images) {
+    sfm.ClearImages();
+  }
+
   std::cout << "Serializing SFM!!!! to: " << output_file << std::endl;
   std::ofstream file(output_file, std::ios::binary);
   cereal::BinaryOutputArchive archive(file);

src/sfm.cpp

@@ -161,7 +161,7 @@ void SfM3D::ExtractFeatures() {
         }
         ss << std::endl;
 
-        cv::resize(img, img, cv::Size(), resize_scale_, resize_scale_);
+        cv::resize(img, img, cv::Size(), resize_scale, resize_scale);
         images_resized_[idx] = img;
         image_features_[idx] = features;
 
@@ -207,7 +207,7 @@ void SfM3D::ExtractFeatures() {
     }
     std::cout << std::endl;
 
-    cv::resize(img, img, cv::Size(), resize_scale_, resize_scale_);
+    cv::resize(img, img, cv::Size(), resize_scale, resize_scale);
     images_resized_.push_back(img);
 
     // images_.push_back();
@@ -1170,7 +1170,7 @@ bool SfM3D::GetMapPointsVec(std::vector<Point3DColor>& glm_points) {
       ::GetKeyPointColors(
         images_resized_[img_id],
         kp,
-        p3dc, true, kp.angle - orig_angle, resize_scale_);
+        p3dc, true, kp.angle - orig_angle, resize_scale);
 
       // std::cout << " oo: " << glm::to_string(p3dc.color) << std::endl;
       // std::cout << " tl: " << glm::to_string(p3dc.color_tl) << std::endl;
@@ -1247,7 +1247,7 @@ bool SfM3D::GetMapCamerasWithPointsVec(MapCameras& map_cameras) {
       glm::vec3 v_color = ::GetGlmColorFromImage(
         images_resized_[img_id],
         image_features_[img_id].keypoints[view.second],
-        resize_scale_);
+        resize_scale);
       // std::cout << "o: " << glm::to_string(v_color) << std::endl;
       p3d.color += v_color;
     }
@@ -1355,7 +1355,13 @@ bool SfM3D::IsPairInOrder(const int p1, const int p2) {
 }
 
 void SfM3D::RestoreImages() {
-  images_resized_.clear();
+
+  if (!images_resized_.empty()) {
+    std::cout << "Images exist. Skip RestoreImages.\n";
+    return;
+  }
+
+  // images_resized_.clear();
 
   assert(image_data_.size() == cameras_.size());
 
@@ -1387,7 +1393,7 @@ void SfM3D::RestoreImages() {
         cout_mu.unlock();
 
         ImageData& im_data = image_data_[idx];
-        cv::Mat img = ::LoadImage(im_data, resize_scale_);
+        cv::Mat img = ::LoadImage(im_data, resize_scale);
         images_resized_[idx] = img;
 
       }
@@ -1398,20 +1404,6 @@ void SfM3D::RestoreImages() {
   for(int i = 0; i < capacity; ++i) {
     resize_threads[i].join();
   }
-
-  /*
-  for (size_t i = 0; i < image_data_.size(); ++i) {
-    std::cout << "Restore image " << i << " out of " << image_data_.size()
-              <<std::endl;
-    ImageData& im_data = image_data_[i];
-    // boost::filesystem::path full_image_path = boost::filesystem::path(im_data.image_dir)
-    //     / boost::filesystem::path(im_data.filename);
-    // cv::Mat img = cv::imread(full_image_path.string().c_str());
-    cv::Mat img = ::LoadImage(im_data, resize_scale_);
-    images_resized_.push_back(img);
-  }
-  */
-
 
   auto t1 = high_resolution_clock::now();
   auto dur = duration_cast<microseconds>(t1 - t0);
@@ -1420,6 +1412,10 @@ void SfM3D::RestoreImages() {
 
 };
 
+void SfM3D::ClearImages() {
+  images_resized_.clear();
+}
+
 void SfM3D::Print(std::ostream& os) const {
   os << "SfM3D: intrinsics_.size = " << intrinsics_.size() << ", "
      << "image_data_.size = " << image_data_.size() << ", "