diff --git a/environment_v2_linux_cpuonly.yml b/environment_v2_linux_cpuonly.yml new file mode 100644 index 000000000..2e1e18020 --- /dev/null +++ b/environment_v2_linux_cpuonly.yml @@ -0,0 +1,56 @@ +name: gtsfm-v2 +channels: + # for priority order, we prefer pytorch as the highest priority as it supplies + # latest stable packages for numerous deep learning based methods. conda-forge + # supplies higher versions of packages like opencv compared to the defaults + # channel. + - pytorch + - conda-forge +dependencies: + # python essentials + - python + - pip + # formatting and dev environment + - black + - coverage + - mypy + - pylint + - pytest + - flake8 + - isort + # dask and related + - dask # same as dask[complete] pip distribution + - asyncssh + - python-graphviz + # core functionality and APIs + - matplotlib + - networkx + - numpy + - nodejs + - pandas + - pillow + - scikit-learn + - seaborn + - scipy + - hydra-core + - gtsam + # 3rd party algorithms for different modules + - cpuonly # replacement of cudatoolkit for cpu only machines + - pytorch + - torchvision + - kornia + - pycolmap + - opencv + # io + - h5py + - plotly + - tabulate + - simplejson + - open3d + - colour + - pydot + - trimesh + # testing + - parameterized + # - pip: + # - pydegensac diff --git a/gtsfm/averaging/rotation/rotation_averaging_base.py b/gtsfm/averaging/rotation/rotation_averaging_base.py index 28d4c38ac..51d982a2b 100644 --- a/gtsfm/averaging/rotation/rotation_averaging_base.py +++ b/gtsfm/averaging/rotation/rotation_averaging_base.py @@ -26,12 +26,13 @@ class RotationAveragingBase(GTSFMProcess): rotations. """ + @staticmethod def get_ui_metadata() -> UiMetadata: """Returns data needed to display node and edge info for this process in the process graph.""" return UiMetadata( display_name="Rotation Averaging", - input_products=("View-Graph Relative Rotations", "Relative Pose Priors"), + input_products=("View-Graph Relative Rotations", "Relative Pose Priors", "Verified Correspondences"), output_products=("Global Rotations",), parent_plate="Sparse Reconstruction", ) @@ -64,8 +65,8 @@ def _run_rotation_averaging_base( num_images: int, i2Ri1_dict: Dict[Tuple[int, int], Optional[Rot3]], i1Ti2_priors: Dict[Tuple[int, int], PosePrior], - wTi_gt: List[Optional[Pose3]], v_corr_idxs: Dict[Tuple[int, int], np.ndarray], + wTi_gt: List[Optional[Pose3]], ) -> Tuple[List[Optional[Rot3]], GtsfmMetricsGroup]: """Runs rotation averaging and computes metrics. @@ -73,8 +74,8 @@ def _run_rotation_averaging_base( num_images: Number of poses. i2Ri1_dict: Relative rotations as dictionary (i1, i2): i2Ri1. i1Ti2_priors: Priors on relative poses as dictionary(i1, i2): PosePrior on i1Ti2. - wTi_gt: Ground truth global rotations to compare against. v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. + wTi_gt: Ground truth global rotations to compare against. Returns: Global rotations for each camera pose, i.e. wRi, as a list. The number of entries in the list is @@ -121,8 +122,8 @@ def create_computation_graph( num_images: int, i2Ri1_graph: Delayed, i1Ti2_priors: Dict[Tuple[int, int], PosePrior], - gt_wTi_list: List[Optional[Pose3]], v_corr_idxs: Dict[Tuple[int, int], np.ndarray], + gt_wTi_list: List[Optional[Pose3]], ) -> Tuple[Delayed, Delayed]: """Create the computation graph for performing rotation averaging. @@ -130,8 +131,8 @@ def create_computation_graph( num_images: Number of poses. i2Ri1_graph: Dictionary of relative rotations as a delayed task. i1Ti2_priors: Priors on relative poses as (i1, i2): PosePrior on i1Ti2. - gt_wTi_list: Ground truth poses, to be used for evaluation. v_corr_idxs: Dict mapping image pair indices (i1, i2) to indices of verified correspondences. + gt_wTi_list: Ground truth poses, to be used for evaluation. Returns: Global rotations wrapped using dask.delayed. @@ -141,8 +142,8 @@ def create_computation_graph( num_images, i2Ri1_dict=i2Ri1_graph, i1Ti2_priors=i1Ti2_priors, - wTi_gt=gt_wTi_list, v_corr_idxs=v_corr_idxs, + wTi_gt=gt_wTi_list, ) return wRis, metrics diff --git a/gtsfm/averaging/rotation/shonan.py b/gtsfm/averaging/rotation/shonan.py index 0917f8c6b..cdccabe2f 100644 --- a/gtsfm/averaging/rotation/shonan.py +++ b/gtsfm/averaging/rotation/shonan.py @@ -16,10 +16,7 @@ import gtsam import numpy as np from gtsam import ( - BetweenFactorPose3, - BetweenFactorPose3s, LevenbergMarquardtParams, - Pose3, Rot3, ShonanAveraging3, ShonanAveragingParameters3, @@ -31,6 +28,7 @@ from gtsfm.averaging.rotation.rotation_averaging_base import RotationAveragingBase from gtsfm.common.pose_prior import PosePrior +ROT3_DOF = 3 POSE3_DOF = 6 logger = logger_utils.get_logger() @@ -42,7 +40,10 @@ class ShonanRotationAveraging(RotationAveragingBase): """Performs Shonan rotation averaging.""" def __init__( - self, two_view_rotation_sigma: float = _DEFAULT_TWO_VIEW_ROTATION_SIGMA, use_mst_init: bool = True + self, + two_view_rotation_sigma: float = _DEFAULT_TWO_VIEW_ROTATION_SIGMA, + weight_by_inliers: bool = True, + use_mst_init: bool = True, ) -> None: """Initializes module. @@ -50,12 +51,15 @@ def __init__( Args: two_view_rotation_sigma: Covariance to use (lower values -> more strictly adhere to input measurements). + weight_by_inliers: Whether to weight pairwise costs according to an uncertainty equal to the inverse number + of inlier correspondences per edge. """ super().__init__() + self._p_min = 3 + self._p_max = 64 self._two_view_rotation_sigma = two_view_rotation_sigma + self._weight_by_inliers = weight_by_inliers self._use_mst_init = use_mst_init - self._p_min = 5 - self._p_max = 64 def __get_shonan_params(self) -> ShonanAveragingParameters3: lm_params = LevenbergMarquardtParams.CeresDefaults() @@ -64,30 +68,34 @@ def __get_shonan_params(self) -> ShonanAveragingParameters3: shonan_params.setCertifyOptimality(True) return shonan_params - def __between_factors_from_2view_relative_rotations( - self, i2Ri1_dict: Dict[Tuple[int, int], Rot3], old_to_new_idxs: Dict[int, int] - ) -> BetweenFactorPose3s: + def __measurements_from_2view_relative_rotations( + self, + i2Ri1_dict: Dict[Tuple[int, int], Rot3], + num_correspondences_dict: Dict[Tuple[int, int], int], + ) -> gtsam.BinaryMeasurementsRot3: """Create between factors from relative rotations computed by the 2-view estimator.""" # TODO: how to weight the noise model on relative rotations compared to priors? - noise_model = gtsam.noiseModel.Isotropic.Sigma(POSE3_DOF, self._two_view_rotation_sigma) - between_factors = BetweenFactorPose3s() + # Default noise model if `self._weight_by_inliers` is False, or zero correspondences on edge. + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, self._two_view_rotation_sigma) + measurements = gtsam.BinaryMeasurementsRot3() for (i1, i2), i2Ri1 in i2Ri1_dict.items(): - if i2Ri1 is not None: + if i2Ri1 is None: + continue + if self._weight_by_inliers and num_correspondences_dict[(i1, i2)] > 0: # ignore translation during rotation averaging - i2Ti1 = Pose3(i2Ri1, np.zeros(3)) - i2_ = old_to_new_idxs[i2] - i1_ = old_to_new_idxs[i1] - between_factors.append(BetweenFactorPose3(i2_, i1_, i2Ti1, noise_model)) + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, 1 / num_correspondences_dict[(i1, i2)]) - return between_factors + measurements.append(gtsam.BinaryMeasurementRot3(i2, i1, i2Ri1, noise_model)) - def _between_factors_from_pose_priors( + return measurements + + def _measurements_from_pose_priors( self, i1Ti2_priors: Dict[Tuple[int, int], PosePrior], old_to_new_idxs: Dict[int, int] - ) -> BetweenFactorPose3s: + ) -> gtsam.BinaryMeasurementsRot3: """Create between factors from the priors on relative poses.""" - between_factors = BetweenFactorPose3s() + measurements = gtsam.BinaryMeasurementsRot3() def get_isotropic_noise_model_sigma(covariance: np.ndarray) -> float: """Get the sigma to be used for the isotropic noise model. @@ -100,13 +108,13 @@ def get_isotropic_noise_model_sigma(covariance: np.ndarray) -> float: i1_ = old_to_new_idxs[i1] i2_ = old_to_new_idxs[i2] noise_model_sigma = get_isotropic_noise_model_sigma(i1Ti2_prior.covariance) - noise_model = gtsam.noiseModel.Isotropic.Sigma(POSE3_DOF, noise_model_sigma) - between_factors.append(BetweenFactorPose3(i1_, i2_, i1Ti2_prior.value, noise_model)) + noise_model = gtsam.noiseModel.Isotropic.Sigma(ROT3_DOF, noise_model_sigma) + measurements.append(gtsam.BinaryMeasurementRot3(i1_, i2_, i1Ti2_prior.value.rotation(), noise_model)) - return between_factors + return measurements def _run_with_consecutive_ordering( - self, num_connected_nodes: int, between_factors: BetweenFactorPose3s, initial: Optional[Values] + self, num_connected_nodes: int, measurements: gtsam.BinaryMeasurementsRot3, initial: Optional[Values] ) -> List[Optional[Rot3]]: """Run the rotation averaging on a connected graph w/ N keys ordered consecutively [0,...,N-1]. @@ -117,7 +125,7 @@ def _run_with_consecutive_ordering( Args: num_connected_nodes: Number of unique connected nodes (i.e. images) in the graph (<= the number of images in the dataset) - between_factors: BetweenFactorPose3s created from relative rotations from 2-view estimator and the priors. + measurements: BinaryMeasurementsRot3 created from relative rotations from 2-view estimator and the priors. Returns: Global rotations for each **CONNECTED** camera pose, i.e. wRi, as a list. The number of entries in @@ -127,10 +135,10 @@ def _run_with_consecutive_ordering( logger.info( "Running Shonan with %d constraints on %d nodes", - len(between_factors), + len(measurements), num_connected_nodes, ) - shonan = ShonanAveraging3(between_factors, self.__get_shonan_params()) + shonan = ShonanAveraging3(measurements, self.__get_shonan_params()) if initial is None: logger.info("Using random initialization for Shonan") @@ -192,36 +200,51 @@ def run_rotation_averaging( return wRi_list nodes_with_edges = sorted(list(self._nodes_with_edges(i2Ri1_dict, i1Ti2_priors))) - old_to_new_idxes = {old_idx: i for i, old_idx in enumerate(nodes_with_edges)} + old_to_new_idxs = {old_idx: i for i, old_idx in enumerate(nodes_with_edges)} - i2Ri1_dict_ = {(old_to_new_idxes[i1], old_to_new_idxes[i2]): i2Ri1 for (i1, i2), i2Ri1 in i2Ri1_dict.items()} + i2Ri1_dict_remapped = { + (old_to_new_idxs[i1], old_to_new_idxs[i2]): i2Ri1 for (i1, i2), i2Ri1 in i2Ri1_dict.items() + } num_correspondences_dict: Dict[Tuple[int, int], int] = { - (old_to_new_idxes[i1], old_to_new_idxes[i2]): len(v_corr_idxs[(i1, i2)]) + (old_to_new_idxs[i1], old_to_new_idxs[i2]): len(v_corr_idxs[(i1, i2)]) for (i1, i2) in v_corr_idxs.keys() if (i1, i2) in i2Ri1_dict } + # Use negative of the number of correspondences as the edge weight. initial_values: Optional[Values] = None if self._use_mst_init: logger.info("Using MST initialization for Shonan") wRi_initial_ = rotation_util.initialize_global_rotations_using_mst( len(nodes_with_edges), - i2Ri1_dict_, - edge_weights={(i1, i2): -num_correspondences_dict.get((i1, i2), 0) for i1, i2 in i2Ri1_dict_.keys()}, + i2Ri1_dict_remapped, + edge_weights={ + (i1, i2): -num_correspondences_dict.get((i1, i2), 0) for i1, i2 in i2Ri1_dict_remapped.keys() + }, ) initial_values = Values() for i, wRi in enumerate(wRi_initial_): initial_values.insert(i, wRi) - between_factors: BetweenFactorPose3s = self.__between_factors_from_2view_relative_rotations( - i2Ri1_dict, old_to_new_idxes - ) - between_factors.extend(self._between_factors_from_pose_priors(i1Ti2_priors, old_to_new_idxes)) - - wRi_list_subset = self._run_with_consecutive_ordering( - num_connected_nodes=len(nodes_with_edges), between_factors=between_factors, initial=initial_values - ) - + def _create_factors_and_run() -> List[Rot3]: + measurements: gtsam.BinaryMeasurementsRot3 = self.__measurements_from_2view_relative_rotations( + i2Ri1_dict=i2Ri1_dict_remapped, num_correspondences_dict=num_correspondences_dict + ) + measurements.extend(self._measurements_from_pose_priors(i1Ti2_priors, old_to_new_idxs)) + wRi_list_subset = self._run_with_consecutive_ordering( + num_connected_nodes=len(nodes_with_edges), measurements=measurements, initial=initial_values + ) + return wRi_list_subset + + try: + wRi_list_subset = _create_factors_and_run() + except RuntimeError: + logger.exception("Shonan failed") + if self._weight_by_inliers is True: + logger.info("Reattempting Shonan without inlier-weighted costs...") + # At times, Shonan's `SparseMinimumEigenValue` fails to compute minimum eigenvalue. + self._weight_by_inliers = False + wRi_list_subset = _create_factors_and_run() wRi_list = [None] * num_images for remapped_i, original_i in enumerate(nodes_with_edges): wRi_list[original_i] = wRi_list_subset[remapped_i] diff --git a/gtsfm/configs/unified.yaml b/gtsfm/configs/unified.yaml index 8fef52586..493af70cd 100644 --- a/gtsfm/configs/unified.yaml +++ b/gtsfm/configs/unified.yaml @@ -65,15 +65,17 @@ SceneOptimizer: # comment out to not run view_graph_estimator: _target_: gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator.CycleConsistentRotationViewGraphEstimator - edge_error_aggregation_criterion: MEDIAN_EDGE_ERROR + edge_error_aggregation_criterion: MIN_EDGE_ERROR rot_avg_module: _target_: gtsfm.averaging.rotation.shonan.ShonanRotationAveraging + weight_by_inliers: True trans_avg_module: _target_: gtsfm.averaging.translation.averaging_1dsfm.TranslationAveraging1DSFM robust_measurement_noise: True projection_sampling_method: SAMPLE_INPUT_MEASUREMENTS + reject_outliers: True data_association_module: _target_: gtsfm.data_association.data_assoc.DataAssociation diff --git a/gtsfm/frontend/verifier/verifier_base.py b/gtsfm/frontend/verifier/verifier_base.py index ef670398c..af1a45d2b 100644 --- a/gtsfm/frontend/verifier/verifier_base.py +++ b/gtsfm/frontend/verifier/verifier_base.py @@ -33,6 +33,12 @@ def get_ui_metadata() -> UiMetadata: parent_plate="Two-View Estimator", ) + def __repr__(self) -> str: + return ( + f"{type(self).__name__}" + + f"__use_intrinsics{self._use_intrinsics_in_verification}_{self._estimation_threshold_px}px" + ) + def __init__( self, use_intrinsics_in_verification: bool, diff --git a/gtsfm/multi_view_optimizer.py b/gtsfm/multi_view_optimizer.py index c55e0633b..c5d23e6d2 100644 --- a/gtsfm/multi_view_optimizer.py +++ b/gtsfm/multi_view_optimizer.py @@ -21,11 +21,15 @@ from gtsfm.common.keypoints import Keypoints from gtsfm.common.pose_prior import PosePrior from gtsfm.common.sfm_track import SfmTrack2d +from gtsfm.common.two_view_estimation_report import TwoViewEstimationReport +from gtsfm.data_association.cpp_dsf_tracks_estimator import CppDsfTracksEstimator from gtsfm.data_association.data_assoc import DataAssociation from gtsfm.evaluation.metrics import GtsfmMetricsGroup +from gtsfm.view_graph_estimator.cycle_consistent_rotation_estimator import ( + CycleConsistentRotationViewGraphEstimator, + EdgeErrorAggregationCriterion, +) from gtsfm.view_graph_estimator.view_graph_estimator_base import ViewGraphEstimatorBase -from gtsfm.data_association.cpp_dsf_tracks_estimator import CppDsfTracksEstimator -from gtsfm.common.two_view_estimation_report import TwoViewEstimationReport class MultiViewOptimizer: @@ -44,6 +48,10 @@ def __init__( self.ba_optimizer = bundle_adjustment_module self._run_view_graph_estimator: bool = self.view_graph_estimator is not None + self.view_graph_estimator_v2 = CycleConsistentRotationViewGraphEstimator( + edge_error_aggregation_criterion=EdgeErrorAggregationCriterion.MEDIAN_EDGE_ERROR + ) + def __repr__(self) -> str: return f""" MultiviewOptimizer: @@ -114,6 +122,21 @@ def create_computation_graph( two_view_reports_dict, debug_output_dir, ) + ( + viewgraph_i2Ri1_graph, + viewgraph_i2Ui1_graph, + viewgraph_v_corr_idxs_graph, + viewgraph_two_view_reports_graph, + viewgraph_estimation_metrics, + ) = self.view_graph_estimator_v2.create_computation_graph( + viewgraph_i2Ri1_graph, + viewgraph_i2Ui1_graph, + all_intrinsics, + viewgraph_v_corr_idxs_graph, + keypoints_list, + viewgraph_two_view_reports_graph, + debug_output_dir / "2", + ) else: viewgraph_i2Ri1_graph = dask.delayed(i2Ri1_dict) viewgraph_i2Ui1_graph = dask.delayed(i2Ui1_dict) diff --git a/gtsfm/two_view_estimator_cacher.py b/gtsfm/two_view_estimator_cacher.py index 3557631bd..a245948eb 100644 --- a/gtsfm/two_view_estimator_cacher.py +++ b/gtsfm/two_view_estimator_cacher.py @@ -38,6 +38,7 @@ class TwoViewEstimatorCacher(TwoViewEstimator): def __init__(self, two_view_estimator_obj: TwoViewEstimator) -> None: self._two_view_estimator = two_view_estimator_obj + self._verifier_key = self._two_view_estimator._verifier.__repr__() def __repr__(self) -> str: return self._two_view_estimator.__repr__() @@ -62,7 +63,8 @@ def __generate_cache_key( numpy_arrays_to_hash.append(keypoints_i2.coordinates[sampled_idxs[:, 1]].flatten()) # Hash the concatenation of all the numpy arrays. - return cache_utils.generate_hash_for_numpy_array(np.concatenate(numpy_arrays_to_hash)) + input_key = cache_utils.generate_hash_for_numpy_array(np.concatenate(numpy_arrays_to_hash)) + return f"{self._verifier_key}_{input_key}" def __load_result_from_cache( self, keypoints_i1: Keypoints, keypoints_i2: Keypoints, putative_corr_idxs: np.ndarray diff --git a/gtsfm/utils/geometry_comparisons.py b/gtsfm/utils/geometry_comparisons.py index 1dddccb03..87289ed0c 100644 --- a/gtsfm/utils/geometry_comparisons.py +++ b/gtsfm/utils/geometry_comparisons.py @@ -31,7 +31,7 @@ def compare_rotations( aTi_list: 1st list of rotations. bTi_list: 2nd list of rotations. angular_error_threshold_degrees: Threshold for angular error between two rotations. - + Returns: Result of the comparison. """ @@ -56,6 +56,7 @@ def compare_rotations( relative_rotations_angles = np.array( [compute_relative_rotation_angle(aRi, aRi_) for (aRi, aRi_) in zip(aRi_list, aRi_list_)], dtype=np.float32 ) + print(relative_rotations_angles) return np.all(relative_rotations_angles < angular_error_threshold_degrees) diff --git a/gtsfm/utils/rotation.py b/gtsfm/utils/rotation.py index b3ba2a529..cde9b51b9 100644 --- a/gtsfm/utils/rotation.py +++ b/gtsfm/utils/rotation.py @@ -10,14 +10,16 @@ from gtsam import Rot3 -def random_rotation() -> Rot3: +def random_rotation(angle_scale_factor: float = 0.1) -> Rot3: """Sample a random rotation by generating a sample from the 4d unit sphere.""" - q = np.random.randn(4) + q = np.random.rand(4) # make unit-length quaternion q /= np.linalg.norm(q) qw, qx, qy, qz = q R = Rot3(qw, qx, qy, qz) - return R + axis, angle = R.axisAngle() + angle = angle * angle_scale_factor + return Rot3.AxisAngle(axis.point3(), angle) def initialize_global_rotations_using_mst( diff --git a/rtf_vis_tool/package-lock.json b/rtf_vis_tool/package-lock.json index c6183bf76..26ec825ec 100644 --- a/rtf_vis_tool/package-lock.json +++ b/rtf_vis_tool/package-lock.json @@ -5808,20 +5808,20 @@ "dev": true }, "node_modules/body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", - "raw-body": "2.5.1", + "qs": "6.11.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -6637,9 +6637,9 @@ } }, "node_modules/content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "engines": { "node": ">= 0.6" } @@ -6658,9 +6658,9 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -8782,16 +8782,16 @@ } }, "node_modules/express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -8807,7 +8807,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -9096,9 +9096,9 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -16257,9 +16257,9 @@ } }, "node_modules/qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "dependencies": { "side-channel": "^1.0.4" }, @@ -16395,9 +16395,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -19496,9 +19496,9 @@ } }, "node_modules/webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "dependencies": { "colorette": "^2.0.10", "memfs": "^3.4.3", @@ -24377,20 +24377,20 @@ "dev": true }, "body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", - "raw-body": "2.5.1", + "qs": "6.11.0", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -25037,9 +25037,9 @@ } }, "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "convert-source-map": { "version": "1.8.0", @@ -25057,9 +25057,9 @@ } }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -26635,16 +26635,16 @@ } }, "express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -26660,7 +26660,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -26898,9 +26898,9 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" }, "follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "for-each": { "version": "0.3.3", @@ -32072,9 +32072,9 @@ "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" }, "qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "requires": { "side-channel": "^1.0.4" } @@ -32168,9 +32168,9 @@ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "requires": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -34547,9 +34547,9 @@ } }, "webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "requires": { "colorette": "^2.0.10", "memfs": "^3.4.3", diff --git a/scripts/benchmark_wildcat.sh b/scripts/benchmark_wildcat.sh index 12bd5f2ea..0da043052 100755 --- a/scripts/benchmark_wildcat.sh +++ b/scripts/benchmark_wildcat.sh @@ -7,218 +7,242 @@ now=$(date +"%Y%m%d_%H%M%S") datasets=( - # Tanks and Temples Dataset. - barn-tanks-and-temples-410 - # Olsson Datasets. - # See https://www.maths.lth.se/matematiklth/personal/calle/dataset/dataset.html - ecole-superieure-de-guerre-35 - fort-channing-gate-singapore-27 - skansen-kronan-gothenburg-131 - nijo-castle-gate-19 - kings-college-cambridge-328 - spilled-blood-cathedral-st-petersburg-781 - palace-fine-arts-281 - # 1dsfm Datasets - gendarmenmarkt-1463 - # Other. - skydio-crane-mast-501 - # Astrovision Datasets. - 2011205_rc3 - # Colmap Datasets. - south-building-128 - gerrard-hall-100 - ) + # Tanks and Temples Dataset. + barn-tanks-and-temples-410 + # Olsson Datasets. + # See https://www.maths.lth.se/matematiklth/personal/calle/dataset/dataset.html + ecole-superieure-de-guerre-35 + #fort-channing-gate-singapore-27 + skansen-kronan-gothenburg-131 + #nijo-castle-gate-19 + kings-college-cambridge-328 + spilled-blood-cathedral-st-petersburg-781 + palace-fine-arts-281 + # 1dsfm Datasets + # gendarmenmarkt-1463 + # Other. + skydio-crane-mast-501 + # Astrovision Datasets. + 2011205_rc3 + # Colmap Datasets. + south-building-128 + gerrard-hall-100 +) max_frame_lookahead_sizes=( - 0 - 5 - 10 - 15 - ) + # 0 + # 5 + 10 + #15 +) num_matched_sizes=( - 0 - 5 - 10 - 15 - 20 - 25 - ) + # 0 + 5 + # 10 + # 15 + # 20 + # 25 +) correspondence_generator_config_names=( - sift - lightglue - superglue - loftr - ) + sift + lightglue + superglue + loftr + disk +) if [[ $CLUSTER_CONFIG ]] then - CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" + CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" else - CLUSTER_ARGS="" + CLUSTER_ARGS="" fi for num_matched in ${num_matched_sizes[@]}; do - for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do - for dataset in ${datasets[@]}; do - if [[ $dataset == *"gendarmenmarkt-1463"* && $max_frame_lookahead != 0 ]] - then - # Gendarmenmarkt images have no natural order. - continue - fi - - if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] - then - # Matches must come from at least some retriever. - continue - fi - - for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do - - if [[ $correspondence_generator_config_name == *"sift"* ]] - then - num_workers=10 - elif [[ $correspondence_generator_config_name == *"lightglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"superglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"loftr"* ]] - then - num_workers=1 - fi - - echo "Dataset: ${dataset}" - echo "Num matched: ${num_matched}" - echo "Max frame lookahead: ${max_frame_lookahead}" - echo "Correspondence Generator: ${correspondence_generator_config_name}" - echo "Num workers: ${num_workers}" - - if [[ $dataset == *"barn-tanks-and-temples-410"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/Barn - colmap_files_dirpath=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/colmap_gt_2023_11_15_2250_highquality - elif [[ $dataset == *"palace-fine-arts-281"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/palace-fine-arts-281 - - elif [[ $dataset == *"ecole-superieure-de-guerre-35"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35 - - elif [[ $dataset == *"fort-channing-gate-singapore-27"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/fort-channing-gate-singapore-27 - - elif [[ $dataset == *"skansen-kronan-gothenburg-131"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131 - - elif [[ $dataset == *"nijo-castle-gate-19"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/nijo-castle-gate-19 - - elif [[ $dataset == *"kings-college-cambridge-328"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/kings-college-cambridge-328 - - elif [[ $dataset == *"spilled-blood-cathedral-st-petersburg-781"* ]] - then - loader=olsson - dataset_root=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781 - - elif [[ $dataset == *"skydio-crane-mast-501"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-crane-mast-501-images - colmap_files_dirpath=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-501-colmap-pseudo-gt - elif [[ $dataset == *"2011205_rc3"* ]] - then - loader=astrovision - data_dir=/usr/local/gtsfm-data/2011205_rc3 - elif [[ $dataset == *"south-building-128"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/south-building-128/images - colmap_files_dirpath=/usr/local/gtsfm-data/south-building-128/colmap-2023-07-28-txt - elif [[ $dataset == *"gerrard-hall-100"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/gerrard-hall-100/images - colmap_files_dirpath=/usr/local/gtsfm-data/gerrard-hall-100/colmap-3.7-sparse-txt-2023-07-27 - elif [[ $dataset == *"gendarmenmarkt-1463"* ]] - then - loader=colmap - images_dir=/usr/local/gtsfm-data/Gendarmenmarkt/images - colmap_files_dirpath=/usr/local/gtsfm-data/Gendarmenmarkt/gendarmenmark/im_size_full/0 - fi - - OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} - mkdir -p $OUTPUT_ROOT - - if [[ $loader == *"olsson"* ]] - then - python gtsfm/runner/run_scene_optimizer_olssonloader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --dataset_root $dataset_root \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - elif [[ $loader == *"colmap"* ]] - then - python gtsfm/runner/run_scene_optimizer_colmaploader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --images_dir $images_dir \ - --colmap_files_dirpath $colmap_files_dirpath \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - elif [[ $loader == *"astrovision"* ]] - then - python gtsfm/runner/run_scene_optimizer_astrovision.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --data_dir $data_dir \ - --num_workers $num_workers \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - fi - done - done - done + for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do + for dataset in ${datasets[@]}; do + if [[ $dataset == *"gendarmenmarkt-1463"* && $max_frame_lookahead != 0 ]] + then + # Gendarmenmarkt images have no natural order. + continue + fi + + if [[ $dataset == *"gendarmenmarkt-1463"* ]] + then + INTRINSICS_ARGS="" + else + INTRINSICS_ARGS="--share_intrinsics" + fi + + if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] + then + # Matches must come from at least some retriever. + continue + fi + + for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do + + if [[ $correspondence_generator_config_name == *"sift"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"lightglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"superglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"disk"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"loftr"* ]] + then + num_workers=1 + fi + + echo "Dataset: ${dataset}" + echo "Num matched: ${num_matched}" + echo "Max frame lookahead: ${max_frame_lookahead}" + echo "Correspondence Generator: ${correspondence_generator_config_name}" + echo "Num workers: ${num_workers}" + echo "Intrinsics: ${INTRINSICS_ARGS}" + + if [[ $dataset == *"barn-tanks-and-temples-410"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/Barn + colmap_files_dirpath=/usr/local/gtsfm-data/Tanks_and_Temples_Barn_410/colmap_gt_2023_11_15_2250_highquality + + elif [[ $dataset == *"palace-fine-arts-281"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/palace-fine-arts-281 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/palace_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/palace-fine-arts-281/images + + elif [[ $dataset == *"ecole-superieure-de-guerre-35"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/ecole_superieure_colmap_gt_2023_11_22/txt_gt + images_dir=/usr/local/gtsfm-data/ecole-superieure-de-guerre-35/images + + elif [[ $dataset == *"fort-channing-gate-singapore-27"* ]] + then + loader=olsson + dataset_root=/usr/local/gtsfm-data/fort-channing-gate-singapore-27 + + elif [[ $dataset == *"skansen-kronan-gothenburg-131"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/skansen_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/skansen-kronan-gothenburg-131/images + + elif [[ $dataset == *"nijo-castle-gate-19"* ]] + then + loader=olsson + dataset_root=/usr/local/gtsfm-data/nijo-castle-gate-19 + + elif [[ $dataset == *"kings-college-cambridge-328"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/kings-college-cambridge-328 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/kings_college_colmap_gt_2023_11_22 + images_dir=/usr/local/gtsfm-data/kings-college-cambridge-328/images + + elif [[ $dataset == *"spilled-blood-cathedral-st-petersburg-781"* ]] + then + # loader=olsson + # dataset_root=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781 + loader=colmap + colmap_files_dirpath=/usr/local/gtsfm-data/colmap-spilled-blood-gt + images_dir=/usr/local/gtsfm-data/spilled-blood-cathedral-st-petersburg-781/images + + elif [[ $dataset == *"skydio-crane-mast-501"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-crane-mast-501-images + colmap_files_dirpath=/usr/local/gtsfm-data/skydio-crane-mast-501/skydio-501-colmap-pseudo-gt + elif [[ $dataset == *"2011205_rc3"* ]] + then + loader=astrovision + data_dir=/usr/local/gtsfm-data/2011205_rc3 + elif [[ $dataset == *"south-building-128"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/south-building-128/images + colmap_files_dirpath=/usr/local/gtsfm-data/south-building-128/colmap-2023-07-28-txt + elif [[ $dataset == *"gerrard-hall-100"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/gerrard-hall-100/images + colmap_files_dirpath=/usr/local/gtsfm-data/gerrard-hall-100/colmap-3.7-sparse-txt-2023-07-27 + elif [[ $dataset == *"gendarmenmarkt-1463"* ]] + then + loader=colmap + images_dir=/usr/local/gtsfm-data/Gendarmenmarkt/images + colmap_files_dirpath=/usr/local/gtsfm-data/Gendarmenmarkt/gendarmenmark/im_size_full/0 + fi + + OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} + mkdir -p $OUTPUT_ROOT + + if [[ $loader == *"olsson"* ]] + then + python gtsfm/runner/run_scene_optimizer_olssonloader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --dataset_root $dataset_root \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + elif [[ $loader == *"colmap"* ]] + then + python gtsfm/runner/run_scene_optimizer_colmaploader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --images_dir $images_dir \ + --colmap_files_dirpath $colmap_files_dirpath \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + elif [[ $loader == *"astrovision"* ]] + then + python gtsfm/runner/run_scene_optimizer_astrovision.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --data_dir $data_dir \ + --num_workers $num_workers \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $INTRINSICS_ARGS $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + fi + done + done + done done - diff --git a/scripts/eth3d_benchmark.sh b/scripts/eth3d_benchmark.sh index e6c6f9afe..32ba6543c 100755 --- a/scripts/eth3d_benchmark.sh +++ b/scripts/eth3d_benchmark.sh @@ -6,112 +6,118 @@ CLUSTER_CONFIG=$2 now=$(date +"%Y%m%d_%H%M%S") -ETH3D_ROOT=/home/tdriver6/Downloads/eth3d_datasets +ETH3D_ROOT=/usr/local/gtsfm-data/eth3d_datasets/multi_view_training_dslr_undistorted +#ETH3D_ROOT=/home/tdriver6/Downloads/eth3d_datasets # Includes all "high-resolution multi-view" datasets from 'training' split (i.e. w/ public GT data) # See https://www.eth3d.net/datasets for more information. datasets=( - courtyard - delivery_area - electro - facade - kicker - meadow - office - pipes - playground - relief_2 - relief - terrace - terrains - ) + courtyard + delivery_area + electro + facade + kicker + meadow + office + pipes + playground + relief_2 + relief + terrace + terrains +) max_frame_lookahead_sizes=( - #0 - #5 - 10 - #15 - ) + #0 + #5 + 10 + #15 +) num_matched_sizes=( - 5 - #10 - #15 - #20 - #25 - ) + 5 + #10 + #15 + #20 + #25 +) correspondence_generator_config_names=( - sift - #disk - #lightglue - #superglue - #loftr - ) + sift + disk + lightglue + superglue + loftr +) if [[ $CLUSTER_CONFIG ]] then - CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" + CLUSTER_ARGS="--cluster_config $CLUSTER_CONFIG" else - CLUSTER_ARGS="" + CLUSTER_ARGS="" fi for num_matched in ${num_matched_sizes[@]}; do - for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do - for dataset in ${datasets[@]}; do - if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] - then - # Matches must come from at least some retriever. - continue - fi - - for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do - - if [[ $correspondence_generator_config_name == *"sift"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"lightglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"superglue"* ]] - then - num_workers=1 - elif [[ $correspondence_generator_config_name == *"loftr"* ]] - then - num_workers=1 - fi - - echo "Dataset: ${dataset}" - echo "Num matched: ${num_matched}" - echo "Max frame lookahead: ${max_frame_lookahead}" - echo "Correspondence Generator: ${correspondence_generator_config_name}" - echo "Num workers: ${num_workers}" - - images_dir="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/images" - colmap_files_dirpath="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/dslr_calibration_undistorted" - - OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} - mkdir -p $OUTPUT_ROOT - - python gtsfm/runner/run_scene_optimizer_colmaploader.py \ - --mvs_off \ - --config unified \ - --correspondence_generator_config_name $correspondence_generator_config_name \ - --share_intrinsics \ - --images_dir $images_dir \ - --colmap_files_dirpath $colmap_files_dirpath \ - --num_workers 1 \ - --num_matched $num_matched \ - --max_frame_lookahead $max_frame_lookahead \ - --worker_memory_limit "32GB" \ - --output_root $OUTPUT_ROOT \ - --max_resolution 760 \ - $CLUSTER_ARGS \ - 2>&1 | tee $OUTPUT_ROOT/out.log - done - done - done + for max_frame_lookahead in ${max_frame_lookahead_sizes[@]}; do + for dataset in ${datasets[@]}; do + if [[ $num_matched == 0 && $max_frame_lookahead == 0 ]] + then + # Matches must come from at least some retriever. + continue + fi + + for correspondence_generator_config_name in ${correspondence_generator_config_names[@]}; do + + if [[ $correspondence_generator_config_name == *"sift"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"lightglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"superglue"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"loftr"* ]] + then + num_workers=1 + elif [[ $correspondence_generator_config_name == *"disk"* ]] + then + num_workers=1 + fi + + echo "Dataset: ${dataset}" + echo "Num matched: ${num_matched}" + echo "Max frame lookahead: ${max_frame_lookahead}" + echo "Correspondence Generator: ${correspondence_generator_config_name}" + echo "Num workers: ${num_workers}" + + images_dir="${ETH3D_ROOT}/${dataset}/images" + colmap_files_dirpath="${ETH3D_ROOT}/${dataset}/dslr_calibration_undistorted" + # images_dir="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/images" + # colmap_files_dirpath="${ETH3D_ROOT}/${dataset}_dslr_undistorted/${dataset}/dslr_calibration_undistorted" + + OUTPUT_ROOT=${USER_ROOT}/${now}/${now}__${dataset}__results__num_matched${num_matched}__maxframelookahead${max_frame_lookahead}__760p__unified_${correspondence_generator_config_name} + mkdir -p $OUTPUT_ROOT + + python gtsfm/runner/run_scene_optimizer_colmaploader.py \ + --mvs_off \ + --config unified \ + --correspondence_generator_config_name $correspondence_generator_config_name \ + --share_intrinsics \ + --images_dir $images_dir \ + --colmap_files_dirpath $colmap_files_dirpath \ + --num_workers 1 \ + --num_matched $num_matched \ + --max_frame_lookahead $max_frame_lookahead \ + --worker_memory_limit "32GB" \ + --output_root $OUTPUT_ROOT \ + --max_resolution 760 \ + $CLUSTER_ARGS \ + 2>&1 | tee $OUTPUT_ROOT/out.log + done + done + done done diff --git a/tests/averaging/rotation/test_shonan.py b/tests/averaging/rotation/test_shonan.py index 77bc0cde3..0556b0ede 100644 --- a/tests/averaging/rotation/test_shonan.py +++ b/tests/averaging/rotation/test_shonan.py @@ -7,18 +7,22 @@ import random import unittest from typing import Dict, List, Tuple +from pathlib import Path import dask import numpy as np from gtsam import Pose3, Rot3 import gtsfm.utils.geometry_comparisons as geometry_comparisons +import gtsfm.utils.io as io_utils import gtsfm.utils.rotation as rotation_util import tests.data.sample_poses as sample_poses from gtsfm.averaging.rotation.shonan import ShonanRotationAveraging from gtsfm.common.pose_prior import PosePrior, PosePriorType ROTATION_ANGLE_ERROR_THRESHOLD_DEG = 2 +TEST_DATA_ROOT = Path(__file__).resolve().parent.parent.parent / "data" +LARGE_PROBLEM_BAL_FILE = TEST_DATA_ROOT / "problem-394-100368-pre.txt" class TestShonanRotationAveraging(unittest.TestCase): @@ -40,7 +44,7 @@ def __execute_test(self, i2Ri1_input: Dict[Tuple[int, int], Rot3], wRi_expected: wRi_expected: Expected global rotations. """ i1Ti2_priors: Dict[Tuple[int, int], PosePrior] = {} - v_corr_idxs = {(i1, i2): _generate_corr_idxs(random.randint(0, 100)) for i1, i2 in i2Ri1_input.keys()} + v_corr_idxs = _create_dummy_correspondences(i2Ri1_input) wRi_computed = self.obj.run_rotation_averaging(len(wRi_expected), i2Ri1_input, i1Ti2_priors, v_corr_idxs) self.assertTrue( geometry_comparisons.compare_rotations(wRi_computed, wRi_expected, ROTATION_ANGLE_ERROR_THRESHOLD_DEG) @@ -101,12 +105,7 @@ def test_simple_with_prior(self): type=PosePriorType.SOFT_CONSTRAINT, ) } - - v_corr_idxs = { - (0, 1): _generate_corr_idxs(1), - (0, 2): _generate_corr_idxs(1), - } - + v_corr_idxs = _create_dummy_correspondences(i2Ri1_dict) wRi_computed = self.obj.run_rotation_averaging(len(expected_wRi_list), i2Ri1_dict, i1Ti2_priors, v_corr_idxs) self.assertTrue( geometry_comparisons.compare_rotations(wRi_computed, expected_wRi_list, ROTATION_ANGLE_ERROR_THRESHOLD_DEG) @@ -121,21 +120,18 @@ def test_computation_graph(self): (0, 1): Rot3.RzRyRx(0, np.deg2rad(30), 0), (1, 2): Rot3.RzRyRx(0, 0, np.deg2rad(20)), } - v_corr_idxs = { - (0, 1): _generate_corr_idxs(200), - (1, 2): _generate_corr_idxs(500), - } i2Ri1_graph = dask.delayed(i2Ri1_dict) - # use the GTSAM API directly (without dask) for rotation averaging + # Use the GTSAM API directly (without dask) for rotation averaging i1Ti2_priors: Dict[Tuple[int, int], PosePrior] = {} + v_corr_idxs = _create_dummy_correspondences(i2Ri1_dict) expected_wRi_list = self.obj.run_rotation_averaging(num_poses, i2Ri1_dict, i1Ti2_priors, v_corr_idxs) - # use dask's computation graph + # Use dask's computation graph gt_wTi_list = [None] * len(expected_wRi_list) rotations_graph, _ = self.obj.create_computation_graph( - num_poses, i2Ri1_graph, i1Ti2_priors, gt_wTi_list, v_corr_idxs + num_poses, i2Ri1_graph, i1Ti2_priors, v_corr_idxs=v_corr_idxs, gt_wTi_list=gt_wTi_list ) with dask.config.set(scheduler="single-threaded"): @@ -184,6 +180,7 @@ def test_nonconsecutive_indices(self): } relative_pose_priors: Dict[Tuple[int, int], PosePrior] = {} + v_corr_idxs = _create_dummy_correspondences(i2Ri1_input) wRi_computed = self.obj.run_rotation_averaging(num_images, i2Ri1_input, relative_pose_priors, v_corr_idxs) wRi_expected = [None, wTi1.rotation(), wTi2.rotation(), wTi3.rotation()] self.assertTrue( @@ -223,10 +220,61 @@ def test_initialization(self) -> None: ) ) + def test_initialization_big(self): + """Test that the result of Shonan is not dependent on the initialization on a bigger dataset.""" + gt_data = io_utils.read_bal(str(LARGE_PROBLEM_BAL_FILE)) + poses = gt_data.get_camera_poses()[:15] + pairs: List[Tuple[int, int]] = [] + for i in range(len(poses)): + for j in range(i + 1, min(i + 5, len(poses))): + pairs.append((i, j)) + + i2Ri1_dict_noisefree, _ = sample_poses.convert_data_for_rotation_averaging( + poses, sample_poses.generate_relative_from_global(poses, pairs) + ) + v_corr_idxs = {pair: _generate_corr_idxs(random.randint(1, 10)) for pair in i2Ri1_dict_noisefree.keys()} + + # Add noise to the relative rotations + i2Ri1_dict_noisy = { + pair: i2Ri1 * rotation_util.random_rotation(angle_scale_factor=0.5) + for pair, i2Ri1 in i2Ri1_dict_noisefree.items() + } + + wRi_computed_with_random_init = self.obj.run_rotation_averaging( + num_images=len(poses), + i2Ri1_dict=i2Ri1_dict_noisy, + i1Ti2_priors={}, + v_corr_idxs=v_corr_idxs, + ) + + shonan_mst_init = ShonanRotationAveraging(use_mst_init=True) + wRi_computed_with_mst_init = shonan_mst_init.run_rotation_averaging( + num_images=len(poses), + i2Ri1_dict=i2Ri1_dict_noisy, + i1Ti2_priors={}, + v_corr_idxs=v_corr_idxs, + ) + + self.assertTrue( + geometry_comparisons.compare_rotations( + wRi_computed_with_random_init, wRi_computed_with_mst_init, angular_error_threshold_degrees=0.1 + ) + ) + def _generate_corr_idxs(num_corrs: int) -> np.ndarray: return np.random.randint(low=0, high=10000, size=(num_corrs, 2)) +def _create_dummy_correspondences(i2Ri1_dict: Dict[Tuple[int, int], Rot3]) -> Dict[Tuple[int, int], np.ndarray]: + """Create dummy verified correspondences for each edge in view graph.""" + # Assume image has shape (img_h, img_w) = (1000,1000) + img_h = 1000 + v_corr_idxs_dict = { + (i1, i2): np.random.randint(low=0, high=img_h, size=(i1 + i2, 2)) for i1, i2 in i2Ri1_dict.keys() + } + return v_corr_idxs_dict + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_two_view_estimator_cacher.py b/tests/test_two_view_estimator_cacher.py index c95be44c0..386b0310f 100644 --- a/tests/test_two_view_estimator_cacher.py +++ b/tests/test_two_view_estimator_cacher.py @@ -45,6 +45,12 @@ def test_cache_miss( # Mock the underlying two-view estimator which is used on cache miss. underlying_estimator_mock = MagicMock() underlying_estimator_mock.run_2view.return_value = self.dummy_output + verifier_key = "Ransac__use_intrinsicsTrue_4px" + + def new_repr(self) -> str: + return verifier_key + + underlying_estimator_mock._verifier.__repr__ = new_repr cacher = TwoViewEstimatorCacher(two_view_estimator_obj=underlying_estimator_mock) @@ -69,11 +75,10 @@ def test_cache_miss( generate_hash_for_numpy_array_mock.assert_called() # Assert that read function was called once and write function was called once. - cache_path = ROOT_PATH / "cache" / "two_view_estimator" / "numpy_key.pbz2" + cache_path = ROOT_PATH / "cache" / "two_view_estimator" / f"{verifier_key}_numpy_key.pbz2" read_mock.assert_called_once_with(cache_path) write_mock.assert_called_once() - @patch("gtsfm.utils.cache.generate_hash_for_numpy_array", return_value="numpy_key") @patch("gtsfm.utils.io.read_from_bz2_file", return_value=_DUMMY_OUTPUT) @patch("gtsfm.utils.io.write_to_bz2_file") @@ -89,7 +94,12 @@ def test_cache_hit( # Mock the underlying two-view estimator which is used on cache miss. underlying_estimator_mock = MagicMock() underlying_estimator_mock.run_2view.return_value = self.dummy_output + verifier_key = "Ransac__use_intrinsicsTrue_4px" + + def new_repr(self) -> str: + return verifier_key + underlying_estimator_mock._verifier.__repr__ = new_repr cacher = TwoViewEstimatorCacher(two_view_estimator_obj=underlying_estimator_mock) result = cacher.run_2view( @@ -114,7 +124,7 @@ def test_cache_hit( generate_hash_for_numpy_array_mock.assert_called() # Assert that the read function was called once. - cache_path = ROOT_PATH / "cache" / "two_view_estimator" / "numpy_key.pbz2" + cache_path = ROOT_PATH / "cache" / "two_view_estimator" / f"{verifier_key}_numpy_key.pbz2" read_mock.assert_called_once_with(cache_path) # Assert that the write function was not called (as cache is mocked to already exist).