Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for excluding tables #182

Merged
merged 10 commits into from
Jul 13, 2022
Merged

Support for excluding tables #182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
2b8d50d
add new command-line option for skipping tables to the relevant commands
alexbruy Jul 5, 2022
a0243b1
add support for skipping tables
alexbruy Jul 5, 2022
579685c
test for schema() call with --skip-tables
alexbruy Jul 5, 2022
1fbfdf3
tests for geodiff calls with skipped tables set
alexbruy Jul 5, 2022
b00d9dd
add new API call to C and Python wrappers
alexbruy Jul 5, 2022
cb15e06
tests for Python API call to set ignored tables
alexbruy Jul 6, 2022
b93ddd0
fix cppcheck issues
alexbruy Jul 7, 2022
a025317
fix changeset creation with skipped tables
alexbruy Jul 8, 2022
4770ffd
address review
alexbruy Jul 12, 2022
71ab657
more review
alexbruy Jul 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions geodiff/src/drivers/postgresdriver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,9 @@ std::vector<std::string> PostgresDriver::listTables( bool useModified )
if ( startsWith( res.value( i, 0 ), "gpkg_" ) )
continue;

if ( context()->isTableSkipped( res.value( i, 0 ) ) )
continue;

tables.push_back( res.value( i, 0 ) );
}

Expand Down Expand Up @@ -820,6 +823,12 @@ void PostgresDriver::applyChangeset( ChangesetReader &reader )
if ( startsWith( tableName, "gpkg_" ) )
continue; // skip any changes to GPKG meta tables

// skip table if necessary
if ( context()->isTableSkipped( tableName ) )
{
continue;
}

if ( tableName != lastTableName )
{
lastTableName = tableName;
Expand Down
9 changes: 9 additions & 0 deletions geodiff/src/drivers/sqlitedriver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,9 @@ std::vector<std::string> SqliteDriver::listTables( bool useModified )
if ( tableName == "sqlite_sequence" )
continue;

if ( context()->isTableSkipped( tableName ) )
continue;

tableNames.push_back( tableName );
}
if ( rc != SQLITE_DONE )
Expand Down Expand Up @@ -778,6 +781,12 @@ void SqliteDriver::applyChangeset( ChangesetReader &reader )
if ( startsWith( tableName, "gpkg_" ) )
continue; // skip any changes to GPKG meta tables

// skip table if necessary
if ( context()->isTableSkipped( tableName ) )
{
continue;
}

if ( tableName != lastTableName )
{
lastTableName = tableName;
Expand Down
123 changes: 117 additions & 6 deletions geodiff/src/geodiff-cli.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include <fstream>
#include <vector>
#include <sstream>
#include "geodiffutils.hpp"
#include "geodiffcontext.hpp"
#include "driver.h"
Expand Down Expand Up @@ -66,7 +67,7 @@ static bool isOption( const std::string &str )
return str.size() > 0 && str[0] == '-';
}

static bool parseDriverOption( const std::vector<std::string> &args, size_t &i, const std::string &cmdName, std::string &driverName, std::string &driverOptions )
static bool parseDriverOption( const std::vector<std::string> &args, size_t &i, const std::string &cmdName, std::string &driverName, std::string &driverOptions, std::string &tablesToSkip )
{
for ( ; i < args.size(); ++i )
{
Expand All @@ -85,6 +86,17 @@ static bool parseDriverOption( const std::vector<std::string> &args, size_t &i,
i += 2;
continue;
}
else if ( args[i] == "--skip-tables" )
{
if ( i + 1 >= args.size() )
{
std::cout << "Error: missing arguments for skip-tables option" << std::endl;
return 1;
}
tablesToSkip = args[i + 1];
i += 1;
continue;
}
else
{
std::cout << "Error: unknown option '" << args[i] << "' for '" << cmdName << "' command." << std::endl;
Expand All @@ -94,6 +106,20 @@ static bool parseDriverOption( const std::vector<std::string> &args, size_t &i,
return true;
}

static std::vector<std::string> parseIgnoredTables( std::string &tablesToSkip )
{
std::vector<std::string> tables;

std::istringstream strm( tablesToSkip );
std::string s;
while ( getline( strm, s, ';' ) )
{
tables.push_back( s );
}

return tables;
}


static int handleCmdDiff( GEODIFF_ContextH context, const std::vector<std::string> &args )
{
Expand All @@ -104,6 +130,7 @@ static int handleCmdDiff( GEODIFF_ContextH context, const std::vector<std::strin
bool printOutput = true;
std::string db1, db2, chOutput;
std::string driver1Name = "sqlite", driver2Name = "sqlite", driver1Options, driver2Options;
std::string tablesToSkip;
size_t i = 1;

// parse options
Expand Down Expand Up @@ -142,6 +169,17 @@ static int handleCmdDiff( GEODIFF_ContextH context, const std::vector<std::strin
i += 2;
continue;
}
else if ( args[i] == "--skip-tables" )
{
if ( i + 1 >= args.size() )
{
std::cout << "Error: missing arguments for skip-tables option" << std::endl;
return 1;
}
tablesToSkip = args[i + 1];
i += 1;
continue;
}
else
{
std::cout << "Error: unknown option '" << args[i] << "' for 'diff' command." << std::endl;
Expand All @@ -156,6 +194,11 @@ static int handleCmdDiff( GEODIFF_ContextH context, const std::vector<std::strin
return 1;
}

// set tables to skip
Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

// parse required arguments
if ( !parseRequiredArgument( db1, args, i, "DB_1", "diff" ) )
return 1;
Expand Down Expand Up @@ -264,9 +307,10 @@ static int handleCmdApply( GEODIFF_ContextH context, const std::vector<std::stri
size_t i = 1;
std::string db, changeset;
std::string driverName = "sqlite", driverOptions;
std::string tablesToSkip;

// parse options
if ( !parseDriverOption( args, i, "apply", driverName, driverOptions ) )
if ( !parseDriverOption( args, i, "apply", driverName, driverOptions, tablesToSkip ) )
return 1;

// parse required arguments
Expand All @@ -277,6 +321,10 @@ static int handleCmdApply( GEODIFF_ContextH context, const std::vector<std::stri
if ( !checkNoExtraArguments( args, i, "apply" ) )
return 1;

Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

int ret = GEODIFF_applyChangesetEx( context, driverName.data(), driverOptions.data(), db.data(), changeset.data() );
if ( ret != GEODIFF_SUCCESS )
{
Expand All @@ -294,9 +342,10 @@ static int handleCmdRebaseDiff( GEODIFF_ContextH context, const std::vector<std
size_t i = 1;
std::string dbBase, chBaseOur, chBaseTheir, chRebased, conflict;
std::string driverName = "sqlite", driverOptions;
std::string tablesToSkip;

// parse options
if ( !parseDriverOption( args, i, "rebase-diff", driverName, driverOptions ) )
if ( !parseDriverOption( args, i, "rebase-diff", driverName, driverOptions, tablesToSkip ) )
return 1;

if ( !parseRequiredArgument( dbBase, args, i, "DB_BASE", "rebase-diff" ) )
Expand All @@ -312,6 +361,10 @@ static int handleCmdRebaseDiff( GEODIFF_ContextH context, const std::vector<std
if ( !checkNoExtraArguments( args, i, "rebase-diff" ) )
return 1;

Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

int ret = GEODIFF_createRebasedChangesetEx(
context,
driverName.data(), driverOptions.data(),
Expand All @@ -333,9 +386,10 @@ static int handleCmdRebaseDb( GEODIFF_ContextH context, const std::vector<std::s
size_t i = 1;
std::string dbBase, dbOur, chBaseTheir, conflict;
std::string driverName = "sqlite", driverOptions;
std::string tablesToSkip;

// parse options
if ( !parseDriverOption( args, i, "rebase-db", driverName, driverOptions ) )
if ( !parseDriverOption( args, i, "rebase-db", driverName, driverOptions, tablesToSkip ) )
return 1;

if ( !parseRequiredArgument( dbBase, args, i, "DB_BASE", "rebase-db" ) )
Expand All @@ -349,6 +403,11 @@ static int handleCmdRebaseDb( GEODIFF_ContextH context, const std::vector<std::s
if ( !checkNoExtraArguments( args, i, "rebase-db" ) )
return 1;

// set tables to skip
Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

int ret = GEODIFF_rebaseEx( context,
driverName.data(), driverOptions.data(), dbBase.data(), dbOur.data(),
chBaseTheir.data(), conflict.data() );
Expand Down Expand Up @@ -522,6 +581,7 @@ static int handleCmdCopy( GEODIFF_ContextH context, const std::vector<std::strin
size_t i = 1;
std::string chInput, chOutput;
std::string driver1Name = "sqlite", driver1Options, driver2Name = "sqlite", driver2Options;
std::string tablesToSkip;

// parse options
for ( ; i < args.size(); ++i )
Expand Down Expand Up @@ -549,6 +609,16 @@ static int handleCmdCopy( GEODIFF_ContextH context, const std::vector<std::strin
i += 2;
continue;
}
else if ( args[i] == "--skip-tables" )
{
if ( i + 1 >= args.size() )
{
std::cout << "Error: missing arguments for skip-tables option" << std::endl;
return 1;
}
tablesToSkip = args[i + 1];
i += 1;
}
else
{
std::cout << "Error: unknown option '" << args[i] << "' for 'copy' command." << std::endl;
Expand All @@ -563,8 +633,18 @@ static int handleCmdCopy( GEODIFF_ContextH context, const std::vector<std::strin
if ( !checkNoExtraArguments( args, i, "copy" ) )
return 1;

Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

if ( driver1Name == "sqlite" && driver2Name == "sqlite" )
{
if ( !tablesToSkip.empty() )
{
std::cout << "Source and destination drivers are \"sqlite\". Option \"--skip-tables\" will be ignored." << std::endl;
alexbruy marked this conversation as resolved.
Show resolved Hide resolved
return 1;
}

int ret = GEODIFF_makeCopySqlite( context, chInput.data(), chOutput.data() );
if ( ret != GEODIFF_SUCCESS )
{
Expand Down Expand Up @@ -595,9 +675,10 @@ static int handleCmdSchema( GEODIFF_ContextH context, const std::vector<std::str
bool printOutput = true;
std::string db, schemaJson;
std::string driverName = "sqlite", driverOptions;
std::string tablesToSkip;

// parse options
if ( !parseDriverOption( args, i, "schema", driverName, driverOptions ) )
if ( !parseDriverOption( args, i, "schema", driverName, driverOptions, tablesToSkip ) )
return 1;

// parse required arguments
Expand All @@ -614,6 +695,10 @@ static int handleCmdSchema( GEODIFF_ContextH context, const std::vector<std::str
return 1;
}

Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

std::string json;
TmpFile tmpJson;
if ( printOutput )
Expand Down Expand Up @@ -650,9 +735,10 @@ static int handleCmdDump( GEODIFF_ContextH context, const std::vector<std::strin
size_t i = 1;
std::string db, chOutput;
std::string driverName = "sqlite", driverOptions;
std::string tablesToSkip;

// parse options
if ( !parseDriverOption( args, i, "dump", driverName, driverOptions ) )
if ( !parseDriverOption( args, i, "dump", driverName, driverOptions, tablesToSkip ) )
return 1;

if ( !parseRequiredArgument( db, args, i, "DB", "dump" ) )
Expand All @@ -662,6 +748,10 @@ static int handleCmdDump( GEODIFF_ContextH context, const std::vector<std::strin
if ( !checkNoExtraArguments( args, i, "dump" ) )
return 1;

Context *ctx = static_cast<Context *>( context );
std::vector<std::string> tables = parseIgnoredTables( tablesToSkip );
ctx->setTablesToSkip( tables );

int ret = GEODIFF_dumpData( context, driverName.data(), driverOptions.data(), db.data(), chOutput.data() );
if ( ret != GEODIFF_SUCCESS )
{
Expand Down Expand Up @@ -744,6 +834,9 @@ Create and apply changesets (diffs):\n\
--driver-2 NAME DRIVER_OPTIONS\n\
Use driver NAME just for the second database. This allows\n\
creation of changesets across datasets in two different drivers.\n\
--skip-tables TABLES\n\
Ignore specified tables when creating a changeset. Tables are defined as\n\
a semicolon separated list of names.\n\
\n\
geodiff apply [OPTIONS...] DB CH_INPUT\n\
\n\
Expand All @@ -754,6 +847,9 @@ Create and apply changesets (diffs):\n\
--driver NAME DRIVER_OPTIONS\n\
Use driver NAME instead of the default 'sqlite' for the\n\
database. Driver-specific options are provided in CONN_OPTIONS.\n\
--skip-tables TABLES\n\
Ignore specified tables when applying a changeset. Tables are defined as\n\
a semicolon separated list of names.\n\
\n\
Rebasing:\n\
\n\
Expand All @@ -772,6 +868,9 @@ Rebasing:\n\
--driver NAME DRIVER_OPTIONS\n\
Use driver NAME instead of the default 'sqlite' for both\n\
databases. Driver-specific options are provided in CONN_OPTIONS.\n\
--skip-tables TABLES\n\
Ignore specified tables when creating a rebased changeset. Tables are\n\
defined as a semicolon separated list of names.\n\
\n\
geodiff rebase-db [OPTIONS...] DB_BASE DB_OUR CH_BASE_THEIR CONFLICT\n\
\n\
Expand All @@ -786,6 +885,9 @@ Rebasing:\n\
--driver NAME DRIVER_OPTIONS\n\
Use driver NAME instead of the default 'sqlite' for all three\n\
databases. Driver-specific options are provided in CONN_OPTIONS.\n\
--skip-tables TABLES\n\
Ignore specified tables when rebasing. Tables are defined as\n\
a semicolon separated list of names.\n\
\n\
Utilities:\n\
\n\
Expand Down Expand Up @@ -825,6 +927,9 @@ Utilities:\n\
--driver-2 NAME DRIVER_OPTIONS\n\
Use driver NAME just for the second database. This allows\n\
creation of changesets across datasets in two different drivers.\n\
--skip-tables TABLES\n\
Ignore specified tables when copying the database. Tables are defined\n\
as a semicolon separated list of names.\n\
\n\
geodiff schema [OPTIONS...] DB [SCHEMA_JSON]\n\
\n\
Expand All @@ -836,6 +941,9 @@ Utilities:\n\
--driver NAME DRIVER_OPTIONS\n\
Use driver NAME instead of the default 'sqlite' for the\n\
database. Driver-specific options are provided in CONN_OPTIONS.\n\
--skip-tables TABLES\n\
Ignore specified tables when writing a schema. Tables are defined\n\
as a semicolon separated list of names.\n\
\n\
geodiff dump [OPTIONS...] DB CH_OUTPUT\n\
\n\
Expand All @@ -845,6 +953,9 @@ Utilities:\n\
--driver NAME DRIVER_OPTIONS\n\
Use driver NAME instead of the default 'sqlite' for the\n\
database. Driver-specific options are provided in CONN_OPTIONS.\n\
--skip-tables TABLES\n\
Ignore specified tables when dumping the database content. Tables\n\
are defined a semicolon separated list of names.\n\
\n\
geodiff drivers\n\
\n\
Expand Down
Loading