Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow cleanup of SQLite tables using new database_cleaner command #1136

Merged
merged 3 commits into from Mar 3, 2021
Merged

Allow cleanup of SQLite tables using new database_cleaner command #1136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b094d6e
Allow cleanup of SQLite tables using new database_cleaner command
Feb 23, 2021
701e281
Change type option to string
Feb 24, 2021
83884ef
Add Database unit tests for clearing tables
Mar 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
53 changes: 53 additions & 0 deletions src/base/database.cc
View file
Open in desktop
Expand Up @@ -275,6 +275,9 @@ void Database::Open(const std::string& path) {
// Disabled by default
SQLITE3_EXEC(database_, "PRAGMA foreign_keys=ON", nullptr);

// Enable auto vacuum to reduce DB file size
SQLITE3_EXEC(database_, "PRAGMA auto_vacuum=1", nullptr);

CreateTables();
UpdateSchema();
PrepareSQLStatements();
Expand All @@ -283,6 +286,7 @@ void Database::Open(const std::string& path) {
void Database::Close() {
if (database_ != nullptr) {
FinalizeSQLStatements();
SQLITE3_EXEC(database_, "VACUUM", nullptr);
sqlite3_close_v2(database_);
database_ = nullptr;
}
Expand Down Expand Up @@ -820,6 +824,35 @@ void Database::DeleteInlierMatches(const image_t image_id1,
SQLITE3_CALL(sqlite3_reset(sql_stmt_delete_two_view_geometry_));
}

void Database::ClearAllTables() const {
ClearMatches();
ClearTwoViewGeometries();
ClearDescriptors();
ClearKeypoints();
ClearImages();
ClearCameras();
}

void Database::ClearCameras() const {
SQLITE3_CALL(sqlite3_step(sql_stmt_clear_cameras_));
SQLITE3_CALL(sqlite3_reset(sql_stmt_clear_cameras_));
}

void Database::ClearImages() const {
SQLITE3_CALL(sqlite3_step(sql_stmt_clear_images_));
SQLITE3_CALL(sqlite3_reset(sql_stmt_clear_images_));
}

void Database::ClearDescriptors() const {
SQLITE3_CALL(sqlite3_step(sql_stmt_clear_descriptors_));
SQLITE3_CALL(sqlite3_reset(sql_stmt_clear_descriptors_));
}

void Database::ClearKeypoints() const {
SQLITE3_CALL(sqlite3_step(sql_stmt_clear_keypoints_));
SQLITE3_CALL(sqlite3_reset(sql_stmt_clear_keypoints_));
}

void Database::ClearMatches() const {
SQLITE3_CALL(sqlite3_step(sql_stmt_clear_matches_));
SQLITE3_CALL(sqlite3_reset(sql_stmt_clear_matches_));
Expand Down Expand Up @@ -1145,6 +1178,26 @@ void Database::PrepareSQLStatements() {
//////////////////////////////////////////////////////////////////////////////
// clear_*
//////////////////////////////////////////////////////////////////////////////
sql = "DELETE FROM cameras;";
SQLITE3_CALL(sqlite3_prepare_v2(database_, sql.c_str(), -1,
&sql_stmt_clear_cameras_, 0));
sql_stmts_.push_back(sql_stmt_clear_cameras_);

sql = "DELETE FROM images;";
SQLITE3_CALL(sqlite3_prepare_v2(database_, sql.c_str(), -1,
&sql_stmt_clear_images_, 0));
sql_stmts_.push_back(sql_stmt_clear_images_);

sql = "DELETE FROM descriptors;";
SQLITE3_CALL(sqlite3_prepare_v2(database_, sql.c_str(), -1,
&sql_stmt_clear_descriptors_, 0));
sql_stmts_.push_back(sql_stmt_clear_descriptors_);

sql = "DELETE FROM keypoints;";
SQLITE3_CALL(sqlite3_prepare_v2(database_, sql.c_str(), -1,
&sql_stmt_clear_keypoints_, 0));
sql_stmts_.push_back(sql_stmt_clear_keypoints_);

sql = "DELETE FROM matches;";
SQLITE3_CALL(sqlite3_prepare_v2(database_, sql.c_str(), -1,
&sql_stmt_clear_matches_, 0));
Expand Down
19 changes: 19 additions & 0 deletions src/base/database.h
View file
Open in desktop
Expand Up @@ -201,6 +201,21 @@ class Database {
void DeleteInlierMatches(const image_t image_id1,
const image_t image_id2) const;

// Clear all database tables
void ClearAllTables() const;

// Clear the entire cameras table
void ClearCameras() const;

// Clear the entire images, keypoints, and descriptors tables
void ClearImages() const;

// Clear the entire descriptors table
void ClearDescriptors() const;

// Clear the entire keypoints table
void ClearKeypoints() const;
ahojnnes marked this conversation as resolved.
Show resolved Hide resolved

// Clear the entire matches table.
void ClearMatches() const;

Expand Down Expand Up @@ -313,6 +328,10 @@ class Database {
sqlite3_stmt* sql_stmt_delete_two_view_geometry_ = nullptr;

// clear_*
sqlite3_stmt* sql_stmt_clear_cameras_ = nullptr;
sqlite3_stmt* sql_stmt_clear_images_ = nullptr;
sqlite3_stmt* sql_stmt_clear_descriptors_ = nullptr;
sqlite3_stmt* sql_stmt_clear_keypoints_ = nullptr;
sqlite3_stmt* sql_stmt_clear_matches_ = nullptr;
sqlite3_stmt* sql_stmt_clear_two_view_geometries_ = nullptr;
};
Expand Down
18 changes: 18 additions & 0 deletions src/base/database_test.cc
View file
Open in desktop
Expand Up @@ -148,6 +148,8 @@ BOOST_AUTO_TEST_CASE(TestCamera) {
BOOST_CHECK_EQUAL(database.ReadAllCameras()[0].CameraId(), camera.CameraId());
BOOST_CHECK_EQUAL(database.ReadAllCameras()[1].CameraId(),
camera2.CameraId());
database.ClearCameras();
BOOST_CHECK_EQUAL(database.NumCameras(), 0);
}

BOOST_AUTO_TEST_CASE(TestImage) {
Expand Down Expand Up @@ -210,6 +212,8 @@ BOOST_AUTO_TEST_CASE(TestImage) {
BOOST_CHECK_EQUAL(database.ExistsImage(image.ImageId()), true);
BOOST_CHECK_EQUAL(database.ExistsImage(image2.ImageId()), true);
BOOST_CHECK_EQUAL(database.ReadAllImages().size(), 2);
database.ClearImages();
BOOST_CHECK_EQUAL(database.NumImages(), 0);
}

BOOST_AUTO_TEST_CASE(TestKeypoints) {
Expand Down Expand Up @@ -245,6 +249,10 @@ BOOST_AUTO_TEST_CASE(TestKeypoints) {
BOOST_CHECK_EQUAL(database.NumKeypoints(), 30);
BOOST_CHECK_EQUAL(database.MaxNumKeypoints(), 20);
BOOST_CHECK_EQUAL(database.NumKeypointsForImage(image.ImageId()), 20);
database.ClearKeypoints();
BOOST_CHECK_EQUAL(database.NumKeypoints(), 0);
BOOST_CHECK_EQUAL(database.MaxNumKeypoints(), 0);
BOOST_CHECK_EQUAL(database.NumKeypointsForImage(image.ImageId()), 0);
}

BOOST_AUTO_TEST_CASE(TestDescriptors) {
Expand Down Expand Up @@ -278,6 +286,10 @@ BOOST_AUTO_TEST_CASE(TestDescriptors) {
BOOST_CHECK_EQUAL(database.NumDescriptors(), 30);
BOOST_CHECK_EQUAL(database.MaxNumDescriptors(), 20);
BOOST_CHECK_EQUAL(database.NumDescriptorsForImage(image.ImageId()), 20);
database.ClearDescriptors();
BOOST_CHECK_EQUAL(database.NumDescriptors(), 0);
BOOST_CHECK_EQUAL(database.MaxNumDescriptors(), 0);
BOOST_CHECK_EQUAL(database.NumDescriptorsForImage(image.ImageId()), 0);
}

BOOST_AUTO_TEST_CASE(TestMatches) {
Expand Down Expand Up @@ -478,4 +490,10 @@ BOOST_AUTO_TEST_CASE(TestMerge) {
BOOST_CHECK(!merged_database.ExistsMatches(2, 3));
BOOST_CHECK(!merged_database.ExistsMatches(2, 4));
BOOST_CHECK(merged_database.ExistsMatches(3, 4));
merged_database.ClearAllTables();
BOOST_CHECK_EQUAL(merged_database.NumCameras(), 0);
BOOST_CHECK_EQUAL(merged_database.NumImages(), 0);
BOOST_CHECK_EQUAL(merged_database.NumKeypoints(), 0);
BOOST_CHECK_EQUAL(merged_database.NumDescriptors(), 0);
BOOST_CHECK_EQUAL(merged_database.NumMatches(), 0);
}
42 changes: 42 additions & 0 deletions src/exe/colmap.cc
View file
Open in desktop
Expand Up @@ -226,6 +226,47 @@ int RunColorExtractor(int argc, char** argv) {
return EXIT_SUCCESS;
}

int RunDatabaseCleaner(int argc, char** argv) {
std::string type;

OptionManager options;
options.AddRequiredOption("type", &type, "{all, images, features, matches}");
options.AddDatabaseOptions();
options.Parse(argc, argv);

StringToLower(&type);
Database database(*options.database_path);
PrintHeading1("Clearing database");
{
DatabaseTransaction transaction(&database);
if (type == "all") {
PrintHeading2("Clearing all tables");
database.ClearAllTables();
} else if (type == "images") {
PrintHeading2("Clearing Images and all dependent tables");
database.ClearImages();
database.ClearTwoViewGeometries();
database.ClearMatches();
} else if (type == "features") {
PrintHeading2("Clearing image features and matches");
database.ClearDescriptors();
database.ClearKeypoints();
database.ClearTwoViewGeometries();
database.ClearMatches();
} else if (type == "matches") {
PrintHeading2("Clearing image matches");
database.ClearTwoViewGeometries();
database.ClearMatches();
} else {
std::cout << "ERROR: Invalid cleanup type; no changes in database"
<< std::endl;
return EXIT_FAILURE;
}
}

return EXIT_SUCCESS;
}

int RunDatabaseCreator(int argc, char** argv) {
OptionManager options;
options.AddDatabaseOptions();
Expand Down Expand Up @@ -2140,6 +2181,7 @@ int main(int argc, char** argv) {
commands.emplace_back("automatic_reconstructor", &RunAutomaticReconstructor);
commands.emplace_back("bundle_adjuster", &RunBundleAdjuster);
commands.emplace_back("color_extractor", &RunColorExtractor);
commands.emplace_back("database_cleaner", &RunDatabaseCleaner);
commands.emplace_back("database_creator", &RunDatabaseCreator);
commands.emplace_back("database_merger", &RunDatabaseMerger);
commands.emplace_back("delaunay_mesher", &RunDelaunayMesher);
Expand Down