diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java index f76a1d71fe..12eae408c0 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java @@ -22,9 +22,10 @@ import java.io.File; import java.io.FileInputStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.ListIterator; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -40,6 +41,8 @@ public class DbCompareCommand extends AbstractCommandLauncher { + private Properties configProperties; + public DbCompareCommand() { super("dbcompare", "[tablename...]", "DbCompare.Option."); } @@ -94,8 +97,12 @@ protected boolean executeWithOptions(CommandLine line) throws Exception { if (line.hasOption(OPTION_EXCLUDE)) { config.setExcludedTableNames(Arrays.asList(line.getOptionValue(OPTION_EXCLUDE).split(","))); } + if (line.hasOption(OPTION_TARGET_TABLES)) { + config.setTargetTableNames(Arrays.asList(line.getOptionValue(OPTION_TARGET_TABLES).split(","))); + } config.setWhereClauses(parseWhereClauses(line)); + config.setTablesToExcludedColumns(parseExcludedColumns(line)); if (!CollectionUtils.isEmpty(line.getArgList())) { config.setIncludedTableNames(Arrays.asList(line.getArgList().get(0).toString().split(","))); @@ -132,12 +139,14 @@ protected static void initFromServerProperties() { private static final String OPTION_EXCLUDE = "exclude"; + private static final String OPTION_TARGET_TABLES = "target-tables"; + private static final String OPTION_USE_SYM_CONFIG = "use-sym-config"; private static final String OPTION_OUTPUT_SQL = "output-sql"; private static final String OPTION_NUMERIC_SCALE = "numeric-scale"; - + private static final String OPTION_CONFIG_PROPERTIES = "config"; @Override @@ -153,6 +162,7 @@ protected void buildOptions(Options options) { addOption(options, "s", OPTION_SOURCE, true); addOption(options, "t", OPTION_TARGET, true); addOption(options, null, OPTION_EXCLUDE, true); + addOption(options, null, OPTION_TARGET_TABLES, true); addOption(options, null, OPTION_USE_SYM_CONFIG, true); addOption(options, null, OPTION_OUTPUT_SQL, true); addOption(options, null, OPTION_NUMERIC_SCALE, true); @@ -160,17 +170,9 @@ protected void buildOptions(Options options) { } protected Map parseWhereClauses(CommandLine line) { - String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES); + Properties props = getConfigProperties(line); Map whereClauses = new HashMap(); - if (!StringUtils.isEmpty(configPropertiesFile)) { - Properties props = new Properties(); - try { - props.load(new FileInputStream(configPropertiesFile)); - } catch (Exception ex) { - String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath(); - throw new SymmetricException("Could not load config properties file '" + configPropertiesFile + - "' at '" + qualifiedFileName + "' ", ex); - } + if (props != null) { for (Object key : props.keySet()) { String arg = key.toString(); if (arg.endsWith(DbCompareConfig.WHERE_CLAUSE)) { @@ -178,10 +180,53 @@ protected Map parseWhereClauses(CommandLine line) { } } } - + return whereClauses; } + protected Map> parseExcludedColumns(CommandLine line) { + Properties props = getConfigProperties(line); + Map> tablesToExcludedColumns = new HashMap>(); + if (props != null) { + for (Object key : props.keySet()) { + String arg = key.toString(); + if (arg.endsWith(DbCompareConfig.EXCLUDED_COLUMN)) { + List excludedColumns = tablesToExcludedColumns.get(key); + if (excludedColumns == null) { + excludedColumns = new ArrayList(); + tablesToExcludedColumns.put(key.toString(), excludedColumns); + } + excludedColumns.addAll(Arrays.asList(props.getProperty(arg).split(","))); + } + } + } + + return tablesToExcludedColumns; + + } + + protected Properties getConfigProperties(CommandLine line) { + if (configProperties != null) { + return configProperties; + } else { + String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES); + if (!StringUtils.isEmpty(configPropertiesFile)) { + Properties props = new Properties(); + try { + props.load(new FileInputStream(configPropertiesFile)); + configProperties = props; + return configProperties; + } catch (Exception ex) { + String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath(); + throw new SymmetricException("Could not load config properties file '" + configPropertiesFile + + "' at '" + qualifiedFileName + "' ", ex); + } + } + } + + return null; + } + static String stripLeadingHyphens(String str) { if (str == null) { return null; diff --git a/symmetric-client/src/main/resources/symmetric-messages.properties b/symmetric-client/src/main/resources/symmetric-messages.properties index 3a10786632..44635d60a6 100644 --- a/symmetric-client/src/main/resources/symmetric-messages.properties +++ b/symmetric-client/src/main/resources/symmetric-messages.properties @@ -174,6 +174,7 @@ DbCompare.Option.exclude=A comma-separated list of table names to exclude from c DbCompare.Option.output=A file name to output delta SQL (insert/update/delete statements) that would bring the target into sync with the source. You can use the %t pattern to use the table name as part of the file and generate a file per table. (E.g. /output/%t.diff.sql) DbCompare.Option.source=The source database engine properties file for comparison. DbCompare.Option.target=The target database engine properties file for comparison. +DbCompare.Option.target-tables=A comma-seperated list of table names to use for comparison on the target side. Use with use-sym-config=false. DbCompare.Option.use-sym-config=true|false. If true, sym_trigger, sym_transform, etc. will be consulted to build up the datamodel to compare. Default is true. DbCompare.Option.numeric-scale=When comparing decimals, how many decimal places to consider while doing the comparison. Remaining digits will be rounded. Default is 3. DbCompare.Option.output-sql=An output file for SQL statements that if executed on the target, should bring it into sync with the source. diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java index 831e292607..20045367ae 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java @@ -41,6 +41,7 @@ import org.jumpmind.db.sql.ISqlRowMapper; import org.jumpmind.db.sql.Row; import org.jumpmind.symmetric.ISymmetricEngine; +import org.jumpmind.symmetric.SymmetricException; import org.jumpmind.symmetric.common.TableConstants; import org.jumpmind.symmetric.io.DbCompareReport.TableReport; import org.jumpmind.symmetric.model.Trigger; @@ -306,16 +307,24 @@ protected List loadTablesFromConfig() { } } - return loadTables(tableNames); + return loadTables(tableNames, config.getTargetTableNames()); } - protected List loadTables(List tableNames) { + protected List loadTables(List tableNames, List targetTableNames) { List compareTables = new ArrayList(1); List filteredTablesNames = filterTables(tableNames); + + if (!CollectionUtils.isEmpty(targetTableNames) && filteredTablesNames.size() != targetTableNames.size()) { + throw new SymmetricException("Table names must be the same length as the list of target " + + "table names. Check your arguments. table names = " + + filteredTablesNames + " target table names = " + targetTableNames); + } - for (String tableName : filteredTablesNames) { + for (int i = 0; i < filteredTablesNames.size(); i++) { + + String tableName = filteredTablesNames.get(i); Table sourceTable = null; Map tableNameParts = sourceEngine.getDatabasePlatform().parseQualifiedTableName(tableName); if (tableNameParts.size() == 1) { @@ -326,19 +335,25 @@ protected List loadTables(List tableNames) { } if (sourceTable == null) { - log.warn("No source table found for table name {}", tableName); + log.warn("No source table found for name {}", tableName); continue; } DbCompareTables tables = new DbCompareTables(sourceTable, null); - Table targetTable = loadTargetTable(tables); + String targetTableName = null; + if (!CollectionUtils.isEmpty(targetTableNames)) { + targetTableName = targetTableNames.get(i); + } + + Table targetTable = loadTargetTable(tables, targetTableName); if (targetTable == null) { - log.warn("No target table found for table {}", tableName); + log.warn("No target table found for name {}", tableName); continue; } tables.applyColumnMappings(); + tables.filterExcludedColumns(config); if (tables.getSourceTable().getPrimaryKeyColumnCount() == 0) { log.warn("Source table {} doesn't have any primary key columns and will not be considered in the comparison.", sourceTable); @@ -393,7 +408,7 @@ protected boolean mapPrimaryKey(DbCompareTables tables) { return true; } - protected Table loadTargetTable(DbCompareTables tables) { + protected Table loadTargetTable(DbCompareTables tables, String targetTableNameOverride) { Table targetTable = null; String catalog = targetEngine.getDatabasePlatform().getDefaultCatalog(); @@ -414,9 +429,20 @@ protected Table loadTargetTable(DbCompareTables tables) { schema = triggerRouter.getTargetSchema(schema); } } - - targetTable = targetEngine.getDatabasePlatform(). - getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true); + + if (StringUtils.isEmpty(targetTableNameOverride)) { + targetTable = targetEngine.getDatabasePlatform(). + getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true); + } else { + try { + targetTable = (Table) tables.getSourceTable().clone(); + } catch (CloneNotSupportedException ex) { + throw new SymmetricException("Exception while cloning " + tables.getSourceTable()); + } + targetTable.setCatalog(""); + targetTable.setSchema(""); + targetTable.setName(targetTableNameOverride); + } tables.setTargetTable(targetTable); return targetTable; @@ -473,7 +499,7 @@ protected List loadTablesFromArguments() { + "when not comparing using SymmetricDS config."); } - return loadTables(config.getIncludedTableNames()); + return loadTables(config.getIncludedTableNames(), config.getTargetTableNames()); } protected List filterTables(List tables) { diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java index 7787cc5ae1..c1b9a2232a 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java @@ -20,7 +20,7 @@ */ package org.jumpmind.symmetric.io; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -29,13 +29,16 @@ public class DbCompareConfig { public final static String WHERE_CLAUSE = "where_clause"; + public final static String EXCLUDED_COLUMN = "exclude_columns"; private String sqlDiffFileName; private List includedTableNames; + private List targetTableNames; private List excludedTableNames; private boolean useSymmetricConfig = true; private int numericScale = 3; - private Map whereClauses = new HashMap(); + private Map whereClauses = new LinkedHashMap(); + private Map> tablesToExcludedColumns = new LinkedHashMap>(); public String getSourceWhereClause(String tableName) { return getWhereClause(tableName, "source"); @@ -62,6 +65,24 @@ protected String getWhereClause(String tableName, String sourceOrTarget) { return "1=1"; } + protected boolean shouldIncludeColumn(String tableName, String columnName) { + String tableNameLower = tableName.toLowerCase(); + String columnNameLower = columnName.toLowerCase(); + String[] keys = { + tableNameLower + "." + EXCLUDED_COLUMN, + EXCLUDED_COLUMN + }; + + for (String key : keys) { + if (tablesToExcludedColumns.containsKey(key)) { + List exludedColumnNames = tablesToExcludedColumns.get(key); + return !exludedColumnNames.contains(columnNameLower); + } + } + + return true; + } + public String getSqlDiffFileName() { return sqlDiffFileName; } @@ -99,4 +120,20 @@ public Map getWhereClauses() { public void setWhereClauses(Map whereClauses) { this.whereClauses = new CaseInsensitiveMap(whereClauses); } + + public List getTargetTableNames() { + return targetTableNames; + } + + public void setTargetTableNames(List targetTableNames) { + this.targetTableNames = targetTableNames; + } + + public Map> getTablesToExcludedColumns() { + return tablesToExcludedColumns; + } + + public void setTablesToExcludedColumns(Map> tablesToExcludedColumns) { + this.tablesToExcludedColumns = tablesToExcludedColumns; + } } diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java index 4326a0c8a6..03ea119696 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java @@ -28,6 +28,7 @@ import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Column; import org.jumpmind.db.model.Table; +import org.jumpmind.symmetric.SymmetricException; import org.jumpmind.symmetric.io.data.transform.ColumnPolicy; import org.jumpmind.symmetric.io.data.transform.TransformColumn; import org.jumpmind.symmetric.service.impl.TransformService.TransformTableNodeGroupLink; @@ -123,4 +124,20 @@ public Table getTargetTable() { public void setTargetTable(Table targetTable) { this.targetTable = targetTable; } + + public void filterExcludedColumns(DbCompareConfig config) { + filterExcludedColumns(config, sourceTable); + filterExcludedColumns(config, targetTable); + } + + private void filterExcludedColumns(DbCompareConfig config, Table table) { + for (Column column : table.getColumnsAsList()) { + if (! config.shouldIncludeColumn(table.getName(), column.getName())) { + if (table.getPrimaryKeyColumnsAsList().contains(column)) { + throw new SymmetricException("Invalid config - cannot exclude a primary key column: " + column + " (Table " + table + ")"); + } + table.removeColumn(column); + } + } + } }