Skip to content

Commit

Permalink
Merge branch '3.8' of https://github.com/JumpMind/symmetric-ds into 3.8
Browse files Browse the repository at this point in the history
  • Loading branch information
klementinastojanovska committed Jul 31, 2017
2 parents 06cd222 + e6c67fc commit f1ec560
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 26 deletions.
71 changes: 58 additions & 13 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java
View file
Expand Up @@ -22,9 +22,10 @@

import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.ListIterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;

Expand All @@ -40,6 +41,8 @@

public class DbCompareCommand extends AbstractCommandLauncher {

private Properties configProperties;

public DbCompareCommand() {
super("dbcompare", "[tablename...]", "DbCompare.Option.");
}
Expand Down Expand Up @@ -94,8 +97,12 @@ protected boolean executeWithOptions(CommandLine line) throws Exception {
if (line.hasOption(OPTION_EXCLUDE)) {
config.setExcludedTableNames(Arrays.asList(line.getOptionValue(OPTION_EXCLUDE).split(",")));
}
if (line.hasOption(OPTION_TARGET_TABLES)) {
config.setTargetTableNames(Arrays.asList(line.getOptionValue(OPTION_TARGET_TABLES).split(",")));
}

config.setWhereClauses(parseWhereClauses(line));
config.setTablesToExcludedColumns(parseExcludedColumns(line));

if (!CollectionUtils.isEmpty(line.getArgList())) {
config.setIncludedTableNames(Arrays.asList(line.getArgList().get(0).toString().split(",")));
Expand Down Expand Up @@ -132,12 +139,14 @@ protected static void initFromServerProperties() {

private static final String OPTION_EXCLUDE = "exclude";

private static final String OPTION_TARGET_TABLES = "target-tables";

private static final String OPTION_USE_SYM_CONFIG = "use-sym-config";

private static final String OPTION_OUTPUT_SQL = "output-sql";

private static final String OPTION_NUMERIC_SCALE = "numeric-scale";

private static final String OPTION_CONFIG_PROPERTIES = "config";

@Override
Expand All @@ -153,35 +162,71 @@ protected void buildOptions(Options options) {
addOption(options, "s", OPTION_SOURCE, true);
addOption(options, "t", OPTION_TARGET, true);
addOption(options, null, OPTION_EXCLUDE, true);
addOption(options, null, OPTION_TARGET_TABLES, true);
addOption(options, null, OPTION_USE_SYM_CONFIG, true);
addOption(options, null, OPTION_OUTPUT_SQL, true);
addOption(options, null, OPTION_NUMERIC_SCALE, true);
addOption(options, null, OPTION_CONFIG_PROPERTIES, true);
}

protected Map<String, String> parseWhereClauses(CommandLine line) {
String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES);
Properties props = getConfigProperties(line);
Map<String, String> whereClauses = new HashMap<String, String>();
if (!StringUtils.isEmpty(configPropertiesFile)) {
Properties props = new Properties();
try {
props.load(new FileInputStream(configPropertiesFile));
} catch (Exception ex) {
String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath();
throw new SymmetricException("Could not load config properties file '" + configPropertiesFile +
"' at '" + qualifiedFileName + "' ", ex);
}
if (props != null) {
for (Object key : props.keySet()) {
String arg = key.toString();
if (arg.endsWith(DbCompareConfig.WHERE_CLAUSE)) {
whereClauses.put(arg, props.getProperty(arg));
}
}
}

return whereClauses;
}

protected Map<String, List<String>> parseExcludedColumns(CommandLine line) {
Properties props = getConfigProperties(line);
Map<String, List<String>> tablesToExcludedColumns = new HashMap<String, List<String>>();
if (props != null) {
for (Object key : props.keySet()) {
String arg = key.toString();
if (arg.endsWith(DbCompareConfig.EXCLUDED_COLUMN)) {
List<String> excludedColumns = tablesToExcludedColumns.get(key);
if (excludedColumns == null) {
excludedColumns = new ArrayList<String>();
tablesToExcludedColumns.put(key.toString(), excludedColumns);
}
excludedColumns.addAll(Arrays.asList(props.getProperty(arg).split(",")));
}
}
}

return tablesToExcludedColumns;

}

protected Properties getConfigProperties(CommandLine line) {
if (configProperties != null) {
return configProperties;
} else {
String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES);
if (!StringUtils.isEmpty(configPropertiesFile)) {
Properties props = new Properties();
try {
props.load(new FileInputStream(configPropertiesFile));
configProperties = props;
return configProperties;
} catch (Exception ex) {
String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath();
throw new SymmetricException("Could not load config properties file '" + configPropertiesFile +
"' at '" + qualifiedFileName + "' ", ex);
}
}
}

return null;
}

static String stripLeadingHyphens(String str) {
if (str == null) {
return null;
Expand Down
1 change: 1 addition & 0 deletions symmetric-client/src/main/resources/symmetric-messages.properties
View file
Expand Up @@ -174,6 +174,7 @@ DbCompare.Option.exclude=A comma-separated list of table names to exclude from c
DbCompare.Option.output=A file name to output delta SQL (insert/update/delete statements) that would bring the target into sync with the source. You can use the %t pattern to use the table name as part of the file and generate a file per table. (E.g. /output/%t.diff.sql)
DbCompare.Option.source=The source database engine properties file for comparison.
DbCompare.Option.target=The target database engine properties file for comparison.
DbCompare.Option.target-tables=A comma-seperated list of table names to use for comparison on the target side. Use with use-sym-config=false.
DbCompare.Option.use-sym-config=true|false. If true, sym_trigger, sym_transform, etc. will be consulted to build up the datamodel to compare. Default is true.
DbCompare.Option.numeric-scale=When comparing decimals, how many decimal places to consider while doing the comparison. Remaining digits will be rounded. Default is 3.
DbCompare.Option.output-sql=An output file for SQL statements that if executed on the target, should bring it into sync with the source.
Expand Down
48 changes: 37 additions & 11 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java
View file
Expand Up @@ -41,6 +41,7 @@
import org.jumpmind.db.sql.ISqlRowMapper;
import org.jumpmind.db.sql.Row;
import org.jumpmind.symmetric.ISymmetricEngine;
import org.jumpmind.symmetric.SymmetricException;
import org.jumpmind.symmetric.common.TableConstants;
import org.jumpmind.symmetric.io.DbCompareReport.TableReport;
import org.jumpmind.symmetric.model.Trigger;
Expand Down Expand Up @@ -306,16 +307,24 @@ protected List<DbCompareTables> loadTablesFromConfig() {
}
}

return loadTables(tableNames);
return loadTables(tableNames, config.getTargetTableNames());
}

protected List<DbCompareTables> loadTables(List<String> tableNames) {
protected List<DbCompareTables> loadTables(List<String> tableNames, List <String> targetTableNames) {

List<DbCompareTables> compareTables = new ArrayList<DbCompareTables>(1);

List<String> filteredTablesNames = filterTables(tableNames);

if (!CollectionUtils.isEmpty(targetTableNames) && filteredTablesNames.size() != targetTableNames.size()) {
throw new SymmetricException("Table names must be the same length as the list of target "
+ "table names. Check your arguments. table names = "
+ filteredTablesNames + " target table names = " + targetTableNames);
}

for (String tableName : filteredTablesNames) {
for (int i = 0; i < filteredTablesNames.size(); i++) {

String tableName = filteredTablesNames.get(i);
Table sourceTable = null;
Map<String, String> tableNameParts = sourceEngine.getDatabasePlatform().parseQualifiedTableName(tableName);
if (tableNameParts.size() == 1) {
Expand All @@ -326,19 +335,25 @@ protected List<DbCompareTables> loadTables(List<String> tableNames) {
}

if (sourceTable == null) {
log.warn("No source table found for table name {}", tableName);
log.warn("No source table found for name {}", tableName);
continue;
}

DbCompareTables tables = new DbCompareTables(sourceTable, null);

Table targetTable = loadTargetTable(tables);
String targetTableName = null;
if (!CollectionUtils.isEmpty(targetTableNames)) {
targetTableName = targetTableNames.get(i);
}

Table targetTable = loadTargetTable(tables, targetTableName);
if (targetTable == null) {
log.warn("No target table found for table {}", tableName);
log.warn("No target table found for name {}", tableName);
continue;
}

tables.applyColumnMappings();
tables.filterExcludedColumns(config);

if (tables.getSourceTable().getPrimaryKeyColumnCount() == 0) {
log.warn("Source table {} doesn't have any primary key columns and will not be considered in the comparison.", sourceTable);
Expand Down Expand Up @@ -393,7 +408,7 @@ protected boolean mapPrimaryKey(DbCompareTables tables) {
return true;
}

protected Table loadTargetTable(DbCompareTables tables) {
protected Table loadTargetTable(DbCompareTables tables, String targetTableNameOverride) {
Table targetTable = null;

String catalog = targetEngine.getDatabasePlatform().getDefaultCatalog();
Expand All @@ -414,9 +429,20 @@ protected Table loadTargetTable(DbCompareTables tables) {
schema = triggerRouter.getTargetSchema(schema);
}
}

targetTable = targetEngine.getDatabasePlatform().
getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true);

if (StringUtils.isEmpty(targetTableNameOverride)) {
targetTable = targetEngine.getDatabasePlatform().
getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true);
} else {
try {
targetTable = (Table) tables.getSourceTable().clone();
} catch (CloneNotSupportedException ex) {
throw new SymmetricException("Exception while cloning " + tables.getSourceTable());
}
targetTable.setCatalog("");
targetTable.setSchema("");
targetTable.setName(targetTableNameOverride);
}
tables.setTargetTable(targetTable);

return targetTable;
Expand Down Expand Up @@ -473,7 +499,7 @@ protected List<DbCompareTables> loadTablesFromArguments() {
+ "when not comparing using SymmetricDS config.");
}

return loadTables(config.getIncludedTableNames());
return loadTables(config.getIncludedTableNames(), config.getTargetTableNames());
}

protected List<String> filterTables(List<String> tables) {
Expand Down
41 changes: 39 additions & 2 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java
View file
Expand Up @@ -20,7 +20,7 @@
*/
package org.jumpmind.symmetric.io;

import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

Expand All @@ -29,13 +29,16 @@
public class DbCompareConfig {

public final static String WHERE_CLAUSE = "where_clause";
public final static String EXCLUDED_COLUMN = "exclude_columns";

private String sqlDiffFileName;
private List<String> includedTableNames;
private List<String> targetTableNames;
private List<String> excludedTableNames;
private boolean useSymmetricConfig = true;
private int numericScale = 3;
private Map<String, String> whereClauses = new HashMap<String, String>();
private Map<String, String> whereClauses = new LinkedHashMap<String, String>();
private Map<String, List<String>> tablesToExcludedColumns = new LinkedHashMap<String, List<String>>();

public String getSourceWhereClause(String tableName) {
return getWhereClause(tableName, "source");
Expand All @@ -62,6 +65,24 @@ protected String getWhereClause(String tableName, String sourceOrTarget) {
return "1=1";
}

protected boolean shouldIncludeColumn(String tableName, String columnName) {
String tableNameLower = tableName.toLowerCase();
String columnNameLower = columnName.toLowerCase();
String[] keys = {
tableNameLower + "." + EXCLUDED_COLUMN,
EXCLUDED_COLUMN
};

for (String key : keys) {
if (tablesToExcludedColumns.containsKey(key)) {
List<String> exludedColumnNames = tablesToExcludedColumns.get(key);
return !exludedColumnNames.contains(columnNameLower);
}
}

return true;
}

public String getSqlDiffFileName() {
return sqlDiffFileName;
}
Expand Down Expand Up @@ -99,4 +120,20 @@ public Map<String, String> getWhereClauses() {
public void setWhereClauses(Map<String, String> whereClauses) {
this.whereClauses = new CaseInsensitiveMap(whereClauses);
}

public List<String> getTargetTableNames() {
return targetTableNames;
}

public void setTargetTableNames(List<String> targetTableNames) {
this.targetTableNames = targetTableNames;
}

public Map<String, List<String>> getTablesToExcludedColumns() {
return tablesToExcludedColumns;
}

public void setTablesToExcludedColumns(Map<String, List<String>> tablesToExcludedColumns) {
this.tablesToExcludedColumns = tablesToExcludedColumns;
}
}
17 changes: 17 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java
View file
Expand Up @@ -28,6 +28,7 @@
import org.apache.commons.lang.StringUtils;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.symmetric.SymmetricException;
import org.jumpmind.symmetric.io.data.transform.ColumnPolicy;
import org.jumpmind.symmetric.io.data.transform.TransformColumn;
import org.jumpmind.symmetric.service.impl.TransformService.TransformTableNodeGroupLink;
Expand Down Expand Up @@ -123,4 +124,20 @@ public Table getTargetTable() {
public void setTargetTable(Table targetTable) {
this.targetTable = targetTable;
}

public void filterExcludedColumns(DbCompareConfig config) {
filterExcludedColumns(config, sourceTable);
filterExcludedColumns(config, targetTable);
}

private void filterExcludedColumns(DbCompareConfig config, Table table) {
for (Column column : table.getColumnsAsList()) {
if (! config.shouldIncludeColumn(table.getName(), column.getName())) {
if (table.getPrimaryKeyColumnsAsList().contains(column)) {
throw new SymmetricException("Invalid config - cannot exclude a primary key column: " + column + " (Table " + table + ")");
}
table.removeColumn(column);
}
}
}
}

0 comments on commit f1ec560

Please sign in to comment.