Skip to content

Commit

Permalink
0002822: DBCompare should support where clauses
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmichalek
mmichalek committed Nov 15, 2016
1 parent 8c96083 commit 941e90b
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 85 deletions.
97 changes: 74 additions & 23 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java
View file
Expand Up @@ -21,18 +21,25 @@
package org.jumpmind.symmetric;

import java.io.File;
import java.io.FileInputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.ListIterator;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.jumpmind.symmetric.io.DbCompare;
import org.jumpmind.symmetric.io.DbCompareConfig;
import org.jumpmind.symmetric.io.DbCompareReport;

public class DbCompareCommand extends AbstractCommandLauncher {

public DbCompareCommand() {
super("dbcompare", "[tablename...]", "DbCompare.Option.");
}
Expand All @@ -49,72 +56,76 @@ protected boolean requiresPropertiesFile(CommandLine line) {

@Override
protected boolean executeWithOptions(CommandLine line) throws Exception {

String source = line.getOptionValue('s');
if (source == null) {
source = line.getOptionValue(OPTION_SOURCE);
}
if (StringUtils.isEmpty(source)) {
throw new ParseException(String.format("-source properties file is required."));
}

File sourceProperies = new File(source);
if (!sourceProperies.exists()) {
throw new SymmetricException("Source properties file '" + sourceProperies + "' does not exist.");
}

String target = line.getOptionValue('t');
if (target == null) {
target = line.getOptionValue(OPTION_TARGET);
}
if (StringUtils.isEmpty(target)) {
throw new ParseException(String.format("-target properties file is required."));
}

File targetProperties = new File(target);
if (!targetProperties.exists()) {
throw new SymmetricException("Target properties file '" + targetProperties + "' does not exist.");
}

ISymmetricEngine sourceEngine = new ClientSymmetricEngine(sourceProperies);
ISymmetricEngine targetEngine = new ClientSymmetricEngine(targetProperties);

DbCompare dbCompare = new DbCompare(sourceEngine, targetEngine);


DbCompareConfig config = new DbCompareConfig();

if (line.hasOption(OPTION_OUTPUT_SQL)) {
dbCompare.setSqlDiffFileName(line.getOptionValue(OPTION_OUTPUT_SQL));
config.setSqlDiffFileName(line.getOptionValue(OPTION_OUTPUT_SQL));
}
if (line.hasOption(OPTION_USE_SYM_CONFIG)) {
dbCompare.setUseSymmetricConfig(Boolean.valueOf(line.getOptionValue(OPTION_USE_SYM_CONFIG)));
config.setUseSymmetricConfig(Boolean.valueOf(line.getOptionValue(OPTION_USE_SYM_CONFIG)));
}
if (line.hasOption(OPTION_EXCLUDE)) {
dbCompare.setExcludedTableNames(Arrays.asList(line.getOptionValue(OPTION_EXCLUDE).split(",")));
config.setExcludedTableNames(Arrays.asList(line.getOptionValue(OPTION_EXCLUDE).split(",")));
}

config.setWhereClauses(parseWhereClauses(line));

if (!CollectionUtils.isEmpty(line.getArgList())) {
dbCompare.setIncludedTableNames(Arrays.asList(line.getArgList().get(0).toString().split(",")));
config.setIncludedTableNames(Arrays.asList(line.getArgList().get(0).toString().split(",")));
}

String numericScaleArg = line.getOptionValue(OPTION_NUMERIC_SCALE);
if (!StringUtils.isEmpty(numericScaleArg)) {
try {
dbCompare.setNumericScale(Integer.parseInt(numericScaleArg.trim()));
config.setNumericScale(Integer.parseInt(numericScaleArg.trim()));
} catch (Exception ex) {
throw new ParseException("Failed to parse arg [" + numericScaleArg + "] " + ex);
}
}


ISymmetricEngine sourceEngine = new ClientSymmetricEngine(sourceProperies);
ISymmetricEngine targetEngine = new ClientSymmetricEngine(targetProperties);

DbCompare dbCompare = new DbCompare(sourceEngine, targetEngine, config);
DbCompareReport report = dbCompare.compare();

return false;
}

public static void main(String[] args) throws Exception {
new DbCompareCommand().execute(args);
}

protected static void initFromServerProperties() {
}

private static final String OPTION_SOURCE = "source";

private static final String OPTION_TARGET = "target";
Expand All @@ -124,8 +135,10 @@ protected static void initFromServerProperties() {
private static final String OPTION_USE_SYM_CONFIG = "use-sym-config";

private static final String OPTION_OUTPUT_SQL = "output-sql";

private static final String OPTION_NUMERIC_SCALE = "numeric-scale";

private static final String OPTION_CONFIG_PROPERTIES = "config";

@Override
protected void printHelp(CommandLine cmd, Options options) {
Expand All @@ -143,6 +156,44 @@ protected void buildOptions(Options options) {
addOption(options, null, OPTION_USE_SYM_CONFIG, true);
addOption(options, null, OPTION_OUTPUT_SQL, true);
addOption(options, null, OPTION_NUMERIC_SCALE, true);
addOption(options, null, OPTION_CONFIG_PROPERTIES, true);
}

protected Map<String, String> parseWhereClauses(CommandLine line) {
String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES);
Map<String, String> whereClauses = new HashMap<String, String>();
if (!StringUtils.isEmpty(configPropertiesFile)) {
Properties props = new Properties();
try {
props.load(new FileInputStream(configPropertiesFile));
} catch (Exception ex) {
String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath();
throw new SymmetricException("Could not load config properties file '" + configPropertiesFile +
"' at '" + qualifiedFileName + "' ", ex);
}
for (Object key : props.keySet()) {
String arg = key.toString();
if (arg.endsWith(DbCompareConfig.WHERE_CLAUSE)) {
whereClauses.put(arg, props.getProperty(arg));
}
}
}

return whereClauses;
}

static String stripLeadingHyphens(String str) {
if (str == null) {
return null;
}
if (str.startsWith("--")) {
return str.substring(2, str.length());
}
else if (str.startsWith("-")) {
return str.substring(1, str.length());
}

return str;
}

}
2 changes: 1 addition & 1 deletion symmetric-client/src/main/resources/symmetric-messages.properties
View file
Expand Up @@ -177,7 +177,7 @@ DbCompare.Option.target=The target database engine properties file for compariso
DbCompare.Option.use-sym-config=true|false. If true, sym_trigger, sym_transform, etc. will be consulted to build up the datamodel to compare. Default is true.
DbCompare.Option.numeric-scale=When comparing decimals, how many decimal places to consider while doing the comparison. Remaining digits will be rounded. Default is 3.
DbCompare.Option.output-sql=An output file for SQL statements that if executed on the target, should bring it into sync with the source.

DbCompare.Option.config=A reference to a properties file path containing additional configuration for dbcompare. This properties file supports table where clauses in the format [table].[source|target].where_clause. A property just called "where_clause" will specify a where clause used on all tables. Or for a table called item, use item.where_clause=field > 5000 or for a target table item, you can item.target.where_clause=field > now()-2, for example.

DbSql.Option.sql=Run this sql statement in the shell

Expand Down
84 changes: 23 additions & 61 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java
View file
Expand Up @@ -64,30 +64,18 @@ public Row mapRow(Row row) {

private ISymmetricEngine sourceEngine;
private ISymmetricEngine targetEngine;

private String sqlDiffFileName;
private List<String> includedTableNames;
private List<String> excludedTableNames;
private boolean useSymmetricConfig = true;
private DbCompareConfig config;
private DbValueComparator dbValueComparator;
private int numericScale = 3;

public int getNumericScale() {
return numericScale;
}

public void setNumericScale(int numericScale) {
this.numericScale = numericScale;
}

public DbCompare(ISymmetricEngine sourceEngine, ISymmetricEngine targetEngine) {
public DbCompare(ISymmetricEngine sourceEngine, ISymmetricEngine targetEngine, DbCompareConfig config) {
this.config = config;
this.sourceEngine = sourceEngine;
this.targetEngine = targetEngine;
dbValueComparator = new DbValueComparator(sourceEngine, targetEngine);
}

public DbCompareReport compare() {
dbValueComparator.setNumericScale(numericScale);
dbValueComparator.setNumericScale(config.getNumericScale());

OutputStream sqlDiffOutput = getSqlDiffOutputStream();

Expand Down Expand Up @@ -119,6 +107,7 @@ public DbCompareReport compare() {
}

protected OutputStream getSqlDiffOutputStream() {
String sqlDiffFileName = config.getSqlDiffFileName();
if (!StringUtils.isEmpty(sqlDiffFileName) && !sqlDiffFileName.contains("%t")) {
try {
return new FirstUseFileOutputStream(sqlDiffFileName);
Expand All @@ -131,6 +120,7 @@ protected OutputStream getSqlDiffOutputStream() {
}

protected OutputStream getSqlDiffOutputStream(DbCompareTables tables) {
String sqlDiffFileName = config.getSqlDiffFileName();
if (!StringUtils.isEmpty(sqlDiffFileName)) {
// allow file per table.
String fileNameFormatted = sqlDiffFileName.replace("%t", "%s");
Expand Down Expand Up @@ -166,8 +156,6 @@ protected TableReport compareTables(DbCompareTables tables, OutputStream sqlDiff
int counter = 0;
long startTime = System.currentTimeMillis();

boolean localStreamCreated = false;

DbCompareDiffWriter diffWriter = null;
OutputStream stream = null;
if (sqlDiffOutput != null) {
Expand Down Expand Up @@ -244,8 +232,9 @@ protected int comparePk(DbCompareTables tables, DbCompareRow sourceCompareRow, D
}

protected String getSourceComparisonSQL(DbCompareTables tables, IDatabasePlatform platform) {
String whereClause = config.getSourceWhereClause(tables.getSourceTable().getName());
return getComparisonSQL(tables.getSourceTable(),
tables.getSourceTable().getPrimaryKeyColumns(), platform);
tables.getSourceTable().getPrimaryKeyColumns(), platform, whereClause);
}

protected String getTargetComparisonSQL(DbCompareTables tables, IDatabasePlatform platform) {
Expand All @@ -259,18 +248,19 @@ protected String getTargetComparisonSQL(DbCompareTables tables, IDatabasePlatfor
mappedPkColumns.add(targetColumn);
}
}

return getComparisonSQL(tables.getTargetTable(), tables.getTargetTable().getPrimaryKeyColumns(), platform);

String whereClause = config.getTargetWhereClause(tables.getTargetTable().getName());
return getComparisonSQL(tables.getTargetTable(), tables.getTargetTable().getPrimaryKeyColumns(), platform, whereClause);
}

protected String getComparisonSQL(Table table, Column[] sortByColumns, IDatabasePlatform platform) {
protected String getComparisonSQL(Table table, Column[] sortByColumns, IDatabasePlatform platform, String whereClause) {
DmlStatement statement = platform.createDmlStatement(DmlType.SELECT,
table.getCatalog(), table.getSchema(), table.getName(),
null, table.getColumns(),
null, null);

StringBuilder sql = new StringBuilder(statement.getSql());
sql.append("1=1 ");
sql.append(whereClause).append(" ");

sql.append(buildOrderBy(table, sortByColumns, platform));
log.info("Comparison SQL: {}", sql);
Expand All @@ -292,7 +282,7 @@ protected String buildOrderBy(Table table, Column[] sortByColumns, IDatabasePlat

protected List<DbCompareTables> getTablesToCompare() {
List<DbCompareTables> tablesToCompare;
if (useSymmetricConfig) {
if (config.isUseSymmetricConfig()) {
tablesToCompare = loadTablesFromConfig();
} else {
tablesToCompare = loadTablesFromArguments();
Expand Down Expand Up @@ -403,7 +393,7 @@ protected boolean mapPrimaryKey(DbCompareTables tables) {

protected Table loadTargetTable(DbCompareTables tables) {
Table targetTable = null;
if (useSymmetricConfig) {
if (config.isUseSymmetricConfig()) {
TransformTableNodeGroupLink transform = getTransformFor(tables.getSourceTable());
if (transform != null) {
targetTable = loadTargetTableUsingTransform(transform);
Expand Down Expand Up @@ -451,19 +441,19 @@ protected Table cloneTable(Table table) {
}

protected List<DbCompareTables> loadTablesFromArguments() {
if (CollectionUtils.isEmpty(includedTableNames)) {
if (CollectionUtils.isEmpty(config.getIncludedTableNames())) {
throw new RuntimeException("includedTableNames not provided, includedTableNames must be provided "
+ "when not comparing using SymmetricDS config.");
}

return loadTables(includedTableNames);
return loadTables(config.getIncludedTableNames());
}

protected List<String> filterTables(List<String> tables) {
List<String> filteredTables = new ArrayList<String>(tables.size());

if (!CollectionUtils.isEmpty(includedTableNames)) {
for (String includedTableName : includedTableNames) {
if (!CollectionUtils.isEmpty(config.getIncludedTableNames())) {
for (String includedTableName : config.getIncludedTableNames()) {
for (String tableName : tables) {
if (compareTableNames(tableName, includedTableName)) {
filteredTables.add(tableName);
Expand All @@ -474,10 +464,10 @@ protected List<String> filterTables(List<String> tables) {
filteredTables.addAll(tables);
}

if (!CollectionUtils.isEmpty(excludedTableNames)) {
if (!CollectionUtils.isEmpty(config.getExcludedTableNames())) {
List<String> excludedTables = new ArrayList<String>(filteredTables);

for (String excludedTableName : excludedTableNames) {
for (String excludedTableName : config.getExcludedTableNames()) {
for (String tableName : filteredTables) {
if (compareTableNames(tableName, excludedTableName)) {
excludedTables.remove(tableName);
Expand Down Expand Up @@ -505,30 +495,6 @@ protected boolean compareTableNames(String sourceTableName, String targetTableNa
}
}

public List<String> getIncludedTableNames() {
return includedTableNames;
}

public void setIncludedTableNames(List<String> includedTableNames) {
this.includedTableNames = includedTableNames;
}

public List<String> getExcludedTableNames() {
return excludedTableNames;
}

public void setExcludedTableNames(List<String> excludedTableNames) {
this.excludedTableNames = excludedTableNames;
}

public boolean isUseSymmetricConfig() {
return useSymmetricConfig;
}

public void setUseSymmetricConfig(boolean useSymmetricConfig) {
this.useSymmetricConfig = useSymmetricConfig;
}

class CountingSqlReadCursor implements ISqlReadCursor<Row>, Closeable {

ISqlReadCursor<Row> wrapped;
Expand All @@ -553,11 +519,7 @@ public void close() {
}
}

public String getSqlDiffFileName() {
return sqlDiffFileName;
}

public void setSqlDiffFileName(String sqlDiffFileName) {
this.sqlDiffFileName = sqlDiffFileName;
public DbCompareConfig getConfig() {
return config;
}
}

0 comments on commit 941e90b

Please sign in to comment.