Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/3.8' into 3.9
Browse files Browse the repository at this point in the history 
Conflicts:
	symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/NodeService.java
  • Loading branch information
@chenson42
chenson42 committed Aug 1, 2017
2 parents f844c8e + 0679d7e commit 740bba8
Show file tree
Hide file tree
Showing 14 changed files with 230 additions and 138 deletions.
71 changes: 58 additions & 13 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbCompareCommand.java
View file
Expand Up @@ -22,9 +22,10 @@

import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.ListIterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;

Expand All @@ -40,6 +41,8 @@

public class DbCompareCommand extends AbstractCommandLauncher {

private Properties configProperties;

public DbCompareCommand() {
super("dbcompare", "[tablename...]", "DbCompare.Option.");
}
Expand Down Expand Up @@ -94,8 +97,12 @@ protected boolean executeWithOptions(CommandLine line) throws Exception {
if (line.hasOption(OPTION_EXCLUDE)) {
config.setExcludedTableNames(Arrays.asList(line.getOptionValue(OPTION_EXCLUDE).split(",")));
}
if (line.hasOption(OPTION_TARGET_TABLES)) {
config.setTargetTableNames(Arrays.asList(line.getOptionValue(OPTION_TARGET_TABLES).split(",")));
}

config.setWhereClauses(parseWhereClauses(line));
config.setTablesToExcludedColumns(parseExcludedColumns(line));

if (!CollectionUtils.isEmpty(line.getArgList())) {
config.setIncludedTableNames(Arrays.asList(line.getArgList().get(0).toString().split(",")));
Expand Down Expand Up @@ -132,12 +139,14 @@ protected static void initFromServerProperties() {

private static final String OPTION_EXCLUDE = "exclude";

private static final String OPTION_TARGET_TABLES = "target-tables";

private static final String OPTION_USE_SYM_CONFIG = "use-sym-config";

private static final String OPTION_OUTPUT_SQL = "output-sql";

private static final String OPTION_NUMERIC_SCALE = "numeric-scale";

private static final String OPTION_CONFIG_PROPERTIES = "config";

@Override
Expand All @@ -153,35 +162,71 @@ protected void buildOptions(Options options) {
addOption(options, "s", OPTION_SOURCE, true);
addOption(options, "t", OPTION_TARGET, true);
addOption(options, null, OPTION_EXCLUDE, true);
addOption(options, null, OPTION_TARGET_TABLES, true);
addOption(options, null, OPTION_USE_SYM_CONFIG, true);
addOption(options, null, OPTION_OUTPUT_SQL, true);
addOption(options, null, OPTION_NUMERIC_SCALE, true);
addOption(options, null, OPTION_CONFIG_PROPERTIES, true);
}

protected Map<String, String> parseWhereClauses(CommandLine line) {
String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES);
Properties props = getConfigProperties(line);
Map<String, String> whereClauses = new HashMap<String, String>();
if (!StringUtils.isEmpty(configPropertiesFile)) {
Properties props = new Properties();
try {
props.load(new FileInputStream(configPropertiesFile));
} catch (Exception ex) {
String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath();
throw new SymmetricException("Could not load config properties file '" + configPropertiesFile +
"' at '" + qualifiedFileName + "' ", ex);
}
if (props != null) {
for (Object key : props.keySet()) {
String arg = key.toString();
if (arg.endsWith(DbCompareConfig.WHERE_CLAUSE)) {
whereClauses.put(arg, props.getProperty(arg));
}
}
}

return whereClauses;
}

protected Map<String, List<String>> parseExcludedColumns(CommandLine line) {
Properties props = getConfigProperties(line);
Map<String, List<String>> tablesToExcludedColumns = new HashMap<String, List<String>>();
if (props != null) {
for (Object key : props.keySet()) {
String arg = key.toString();
if (arg.endsWith(DbCompareConfig.EXCLUDED_COLUMN)) {
List<String> excludedColumns = tablesToExcludedColumns.get(key);
if (excludedColumns == null) {
excludedColumns = new ArrayList<String>();
tablesToExcludedColumns.put(key.toString(), excludedColumns);
}
excludedColumns.addAll(Arrays.asList(props.getProperty(arg).split(",")));
}
}
}

return tablesToExcludedColumns;

}

protected Properties getConfigProperties(CommandLine line) {
if (configProperties != null) {
return configProperties;
} else {
String configPropertiesFile = line.getOptionValue(OPTION_CONFIG_PROPERTIES);
if (!StringUtils.isEmpty(configPropertiesFile)) {
Properties props = new Properties();
try {
props.load(new FileInputStream(configPropertiesFile));
configProperties = props;
return configProperties;
} catch (Exception ex) {
String qualifiedFileName = new File(configPropertiesFile).getAbsolutePath();
throw new SymmetricException("Could not load config properties file '" + configPropertiesFile +
"' at '" + qualifiedFileName + "' ", ex);
}
}
}

return null;
}

static String stripLeadingHyphens(String str) {
if (str == null) {
return null;
Expand Down
1 change: 1 addition & 0 deletions symmetric-client/src/main/resources/symmetric-messages.properties
View file
Expand Up @@ -174,6 +174,7 @@ DbCompare.Option.exclude=A comma-separated list of table names to exclude from c
DbCompare.Option.output=A file name to output delta SQL (insert/update/delete statements) that would bring the target into sync with the source. You can use the %t pattern to use the table name as part of the file and generate a file per table. (E.g. /output/%t.diff.sql)
DbCompare.Option.source=The source database engine properties file for comparison.
DbCompare.Option.target=The target database engine properties file for comparison.
DbCompare.Option.target-tables=A comma-seperated list of table names to use for comparison on the target side. Use with use-sym-config=false.
DbCompare.Option.use-sym-config=true|false. If true, sym_trigger, sym_transform, etc. will be consulted to build up the datamodel to compare. Default is true.
DbCompare.Option.numeric-scale=When comparing decimals, how many decimal places to consider while doing the comparison. Remaining digits will be rounded. Default is 3.
DbCompare.Option.output-sql=An output file for SQL statements that if executed on the target, should bring it into sync with the source.
Expand Down
48 changes: 37 additions & 11 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompare.java
View file
Expand Up @@ -41,6 +41,7 @@
import org.jumpmind.db.sql.ISqlRowMapper;
import org.jumpmind.db.sql.Row;
import org.jumpmind.symmetric.ISymmetricEngine;
import org.jumpmind.symmetric.SymmetricException;
import org.jumpmind.symmetric.common.TableConstants;
import org.jumpmind.symmetric.io.DbCompareReport.TableReport;
import org.jumpmind.symmetric.model.Trigger;
Expand Down Expand Up @@ -309,16 +310,24 @@ protected List<DbCompareTables> loadTablesFromConfig() {
}
}

return loadTables(tableNames);
return loadTables(tableNames, config.getTargetTableNames());
}

protected List<DbCompareTables> loadTables(List<String> tableNames) {
protected List<DbCompareTables> loadTables(List<String> tableNames, List <String> targetTableNames) {

List<DbCompareTables> compareTables = new ArrayList<DbCompareTables>(1);

List<String> filteredTablesNames = filterTables(tableNames);

if (!CollectionUtils.isEmpty(targetTableNames) && filteredTablesNames.size() != targetTableNames.size()) {
throw new SymmetricException("Table names must be the same length as the list of target "
+ "table names. Check your arguments. table names = "
+ filteredTablesNames + " target table names = " + targetTableNames);
}

for (String tableName : filteredTablesNames) {
for (int i = 0; i < filteredTablesNames.size(); i++) {

String tableName = filteredTablesNames.get(i);
Table sourceTable = null;
Map<String, String> tableNameParts = sourceEngine.getDatabasePlatform().parseQualifiedTableName(tableName);
if (tableNameParts.size() == 1) {
Expand All @@ -329,19 +338,25 @@ protected List<DbCompareTables> loadTables(List<String> tableNames) {
}

if (sourceTable == null) {
log.warn("No source table found for table name {}", tableName);
log.warn("No source table found for name {}", tableName);
continue;
}

DbCompareTables tables = new DbCompareTables(sourceTable, null);

Table targetTable = loadTargetTable(tables);
String targetTableName = null;
if (!CollectionUtils.isEmpty(targetTableNames)) {
targetTableName = targetTableNames.get(i);
}

Table targetTable = loadTargetTable(tables, targetTableName);
if (targetTable == null) {
log.warn("No target table found for table {}", tableName);
log.warn("No target table found for name {}", tableName);
continue;
}

tables.applyColumnMappings();
tables.filterExcludedColumns(config);

if (tables.getSourceTable().getPrimaryKeyColumnCount() == 0) {
log.warn("Source table {} doesn't have any primary key columns and will not be considered in the comparison.", sourceTable);
Expand Down Expand Up @@ -396,7 +411,7 @@ protected boolean mapPrimaryKey(DbCompareTables tables) {
return true;
}

protected Table loadTargetTable(DbCompareTables tables) {
protected Table loadTargetTable(DbCompareTables tables, String targetTableNameOverride) {
Table targetTable = null;

String catalog = targetEngine.getDatabasePlatform().getDefaultCatalog();
Expand All @@ -417,9 +432,20 @@ protected Table loadTargetTable(DbCompareTables tables) {
schema = triggerRouter.getTargetSchema(schema);
}
}

targetTable = targetEngine.getDatabasePlatform().
getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true);

if (StringUtils.isEmpty(targetTableNameOverride)) {
targetTable = targetEngine.getDatabasePlatform().
getTableFromCache(catalog, schema, tables.getSourceTable().getName(), true);
} else {
try {
targetTable = (Table) tables.getSourceTable().clone();
} catch (CloneNotSupportedException ex) {
throw new SymmetricException("Exception while cloning " + tables.getSourceTable());
}
targetTable.setCatalog("");
targetTable.setSchema("");
targetTable.setName(targetTableNameOverride);
}
tables.setTargetTable(targetTable);

return targetTable;
Expand Down Expand Up @@ -476,7 +502,7 @@ protected List<DbCompareTables> loadTablesFromArguments() {
+ "when not comparing using SymmetricDS config.");
}

return loadTables(config.getIncludedTableNames());
return loadTables(config.getIncludedTableNames(), config.getTargetTableNames());
}

protected List<String> filterTables(List<String> tables) {
Expand Down
41 changes: 39 additions & 2 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareConfig.java
View file
Expand Up @@ -20,7 +20,7 @@
*/
package org.jumpmind.symmetric.io;

import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

Expand All @@ -29,13 +29,16 @@
public class DbCompareConfig {

public final static String WHERE_CLAUSE = "where_clause";
public final static String EXCLUDED_COLUMN = "exclude_columns";

private String sqlDiffFileName;
private List<String> includedTableNames;
private List<String> targetTableNames;
private List<String> excludedTableNames;
private boolean useSymmetricConfig = true;
private int numericScale = 3;
private Map<String, String> whereClauses = new HashMap<String, String>();
private Map<String, String> whereClauses = new LinkedHashMap<String, String>();
private Map<String, List<String>> tablesToExcludedColumns = new LinkedHashMap<String, List<String>>();

public String getSourceWhereClause(String tableName) {
return getWhereClause(tableName, "source");
Expand All @@ -62,6 +65,24 @@ protected String getWhereClause(String tableName, String sourceOrTarget) {
return "1=1";
}

protected boolean shouldIncludeColumn(String tableName, String columnName) {
String tableNameLower = tableName.toLowerCase();
String columnNameLower = columnName.toLowerCase();
String[] keys = {
tableNameLower + "." + EXCLUDED_COLUMN,
EXCLUDED_COLUMN
};

for (String key : keys) {
if (tablesToExcludedColumns.containsKey(key)) {
List<String> exludedColumnNames = tablesToExcludedColumns.get(key);
return !exludedColumnNames.contains(columnNameLower);
}
}

return true;
}

public String getSqlDiffFileName() {
return sqlDiffFileName;
}
Expand Down Expand Up @@ -99,4 +120,20 @@ public Map<String, String> getWhereClauses() {
public void setWhereClauses(Map<String, String> whereClauses) {
this.whereClauses = new CaseInsensitiveMap(whereClauses);
}

public List<String> getTargetTableNames() {
return targetTableNames;
}

public void setTargetTableNames(List<String> targetTableNames) {
this.targetTableNames = targetTableNames;
}

public Map<String, List<String>> getTablesToExcludedColumns() {
return tablesToExcludedColumns;
}

public void setTablesToExcludedColumns(Map<String, List<String>> tablesToExcludedColumns) {
this.tablesToExcludedColumns = tablesToExcludedColumns;
}
}
17 changes: 17 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/io/DbCompareTables.java
View file
Expand Up @@ -28,6 +28,7 @@
import org.apache.commons.lang.StringUtils;
import org.jumpmind.db.model.Column;
import org.jumpmind.db.model.Table;
import org.jumpmind.symmetric.SymmetricException;
import org.jumpmind.symmetric.io.data.transform.ColumnPolicy;
import org.jumpmind.symmetric.io.data.transform.TransformColumn;
import org.jumpmind.symmetric.service.impl.TransformService.TransformTableNodeGroupLink;
Expand Down Expand Up @@ -123,4 +124,20 @@ public Table getTargetTable() {
public void setTargetTable(Table targetTable) {
this.targetTable = targetTable;
}

public void filterExcludedColumns(DbCompareConfig config) {
filterExcludedColumns(config, sourceTable);
filterExcludedColumns(config, targetTable);
}

private void filterExcludedColumns(DbCompareConfig config, Table table) {
for (Column column : table.getColumnsAsList()) {
if (! config.shouldIncludeColumn(table.getName(), column.getName())) {
if (table.getPrimaryKeyColumnsAsList().contains(column)) {
throw new SymmetricException("Invalid config - cannot exclude a primary key column: " + column + " (Table " + table + ")");
}
table.removeColumn(column);
}
}
}
}
2 changes: 1 addition & 1 deletion symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/ContextService.java
View file
Expand Up @@ -92,7 +92,7 @@ public void save(ISqlTransaction transaction, String name, String value) {
}
} else {
int count = sqlTemplate.update(getSql("updateSql"), value, name);
if (count == 0) {
if (count <= 0) {
sqlTemplate.update(getSql("insertSql"), name, value);
}
}
Expand Down
6 changes: 3 additions & 3 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/GroupletService.java
View file
Expand Up @@ -267,7 +267,7 @@ public void saveGrouplet(Grouplet grouplet) {
grouplet.getCreateTime(), grouplet.getLastUpdateBy(),
grouplet.getLastUpdateTime(), grouplet.getGroupletId() }, new int[] {
Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR,
Types.TIMESTAMP, Types.VARCHAR }) == 0) {
Types.TIMESTAMP, Types.VARCHAR }) <= 0) {
grouplet.setCreateTime(new Date());
sqlTemplate.update(
getSql("insertGroupletSql"),
Expand Down Expand Up @@ -311,7 +311,7 @@ public void saveGroupletLink(Grouplet grouplet, GroupletLink link) {
new Object[] { link.getCreateTime(), link.getLastUpdateBy(),
link.getLastUpdateTime(), grouplet.getGroupletId(), link.getExternalId() },
new int[] { Types.TIMESTAMP, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR,
Types.VARCHAR }) == 0) {
Types.VARCHAR }) <= 0) {
link.setCreateTime(new Date());
sqlTemplate.update(getSql("insertGroupletLinkSql"), new Object[] {
link.getCreateTime(), link.getLastUpdateBy(), link.getLastUpdateTime(),
Expand All @@ -338,7 +338,7 @@ public void saveTriggerRouterGrouplet(Grouplet grouplet,
triggerRouterGrouplet.getAppliesWhen().name(),
triggerRouterGrouplet.getTriggerId(), triggerRouterGrouplet.getRouterId() },
new int[] { Types.TIMESTAMP, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR,
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR }) == 0) {
Types.VARCHAR, Types.VARCHAR, Types.VARCHAR }) <= 0) {
triggerRouterGrouplet.setCreateTime(new Date());
sqlTemplate.update(
getSql("insertTriggerRouterGroupletSql"),
Expand Down

0 comments on commit 740bba8

Please sign in to comment.