Permalink
Browse files

Merge branch '3.10' of https://github.com/JumpMind/symmetric-ds.git i…

…nto 3.10
  • Loading branch information...
jumpmind-josh committed Jan 9, 2019
2 parents 0c2ea02 + 45ef15b commit 61bbc6eff9dc7cf0dc693f37733486cdbb5ed960
@@ -152,7 +152,7 @@ public void exportTestDatabaseSQL() throws Exception {
return;
}

final int EXPECTED_VARCHAR_MAX_COUNT = engine.getDatabasePlatform().getName().equals(DatabaseNamesConstants.SQLITE) ? 314 : 59;
final int EXPECTED_VARCHAR_MAX_COUNT = engine.getDatabasePlatform().getName().equals(DatabaseNamesConstants.SQLITE) ? 315 : 60;
final String EXPECTED_VARCHAR_MAX_STRING = "varchar(" + Integer.MAX_VALUE + ")";
final int actualVarcharMaxCount = StringUtils.countMatches(output, EXPECTED_VARCHAR_MAX_STRING);
String msg = String.format("Expected %s, but got %s in the following output %s",
@@ -167,6 +167,7 @@ private ParameterConstants() {

public final static String CREATE_TABLE_WITHOUT_DEFAULTS = "create.table.without.defaults";
public final static String CREATE_TABLE_WITHOUT_FOREIGN_KEYS = "create.table.without.foreign.keys";
public final static String CREATE_TABLE_WITHOUT_INDEXES = "create.table.without.indexes";
public final static String CREATE_TABLE_WITHOUT_PK_IF_SOURCE_WITHOUT_PK = "create.table.without.pk.if.source.without.pk";

public final static String CREATE_TABLE_NOT_NULL_COLUMNS = "create.table.not.null.columns.supported";
@@ -2441,6 +2441,7 @@ public CsvData next() {

boolean excludeDefaults = parameterService.is(ParameterConstants.CREATE_TABLE_WITHOUT_DEFAULTS, false);
boolean excludeForeignKeys = parameterService.is(ParameterConstants.CREATE_TABLE_WITHOUT_FOREIGN_KEYS, false);
boolean excludeIndexes = parameterService.is(ParameterConstants.CREATE_TABLE_WITHOUT_INDEXES, false);

/*
* Force a reread of table so new columns are picked up. A create
@@ -2475,6 +2476,9 @@ public CsvData next() {
if (excludeForeignKeys) {
copyTargetTable.removeAllForeignKeys();
}
if (excludeIndexes) {
copyTargetTable.removeAllIndexes();
}

if (parameterService.is(ParameterConstants.CREATE_TABLE_WITHOUT_PK_IF_SOURCE_WITHOUT_PK, false)
&& sourceTable.getPrimaryKeyColumnCount() == 0
@@ -119,7 +119,7 @@ public FileSyncService(ISymmetricEngine engine) {
}

public boolean refreshFromDatabase() {
Date date1 = sqlTemplate.queryForObject(getSql("selectMaxTriggerLastUpdateTime"), Date.class);
Date date1 = sqlTemplate.queryForObject(getSql("selectMaxFileTriggerLastUpdateTime"), Date.class);
Date date2 = sqlTemplate.queryForObject(getSql("selectMaxRouterLastUpdateTime"), Date.class);
Date date3 = sqlTemplate.queryForObject(getSql("selectMaxFileTriggerRouterLastUpdateTime"), Date.class);
Date date = maxDate(date1, date2, date3);
@@ -1383,6 +1383,13 @@ create.table.without.defaults=false
# Type: boolean
create.table.without.foreign.keys=false

# If set to true, when a table's schema is sent to the target database, indexes will not
# be included.
#
# DatabaseOverridable: true
# Type: boolean
create.table.without.indexes=false

# If set to true, when a table's schema is sent to the target database it will not have all
# columns set as the primary key if the source does not have any primary keys.
#
@@ -1828,7 +1835,7 @@ job.file.sync.pull.period.time.ms=60000
# DatabaseOverridable: true
# Tags: filesync
# Type: boolean
file.sync.fast.scan=false
file.sync.fast.scan=true

# Calculate a checksum for each file (using CRC32), which is used to detect a file collision
# if the target file has a different checksum. If you don't need to detect conflicts,
@@ -478,23 +478,23 @@ public Object getParsedDefaultValue() {
switch (mappedTypeCode) {
case Types.TINYINT:
case Types.SMALLINT:
return new Short(defaultValue);
return new Short(getCleanDefaultValue());
case Types.INTEGER:
try {
return new Integer(defaultValue);
return new Integer(getCleanDefaultValue());
} catch (NumberFormatException e) {
return new Long(defaultValue);
return new Long(getCleanDefaultValue());
}
case Types.BIGINT:
return new Long(defaultValue);
return new Long(getCleanDefaultValue());
case Types.DECIMAL:
case Types.NUMERIC:
return new BigDecimal(defaultValue);
return new BigDecimal(getCleanDefaultValue());
case Types.REAL:
return new Float(defaultValue);
return new Float(getCleanDefaultValue());
case Types.DOUBLE:
case Types.FLOAT:
return new Double(defaultValue);
return new Double(getCleanDefaultValue());
case Types.DATE:
return Date.valueOf(defaultValue);
case Types.TIME:
@@ -518,6 +518,10 @@ public Object getParsedDefaultValue() {
}
return defaultValue;
}

private String getCleanDefaultValue() {
return defaultValue.replace("'", "");
}

public void removePlatformColumn(String databaseName) {
if (platformColumns != null) {
@@ -138,6 +138,10 @@ public void removeAllForeignKeys() {
foreignKeys.clear();
}

public void removeAllIndexes() {
indices.clear();
}

/**
* Returns the catalog of this table as read from the database.
*
@@ -117,7 +117,8 @@
public static final String NCLOB = "NCLOB";
public static final String IMAGE = "IMAGE";
public static final String DATETIME2 = "DATETIME2";

public static final String TSVECTOR = "TSVECTOR";

/** Maps type names to the corresponding {@link java.sql.Types} constants. */
private static HashMap<String, Integer> _typeNameToTypeCode = new HashMap<String, Integer>();

@@ -125,6 +125,8 @@ protected void appendColumnParameter(StringBuilder sql, Column column) {
column.getJdbcTypeName().toUpperCase().contains(TypeMap.GEOMETRY) ||
column.getJdbcTypeName().toUpperCase().contains(TypeMap.GEOGRAPHY))) {
sql.append("ST_GEOMFROMTEXT(?)").append(",");
} else if (column.getJdbcTypeName().toUpperCase().contains(TypeMap.TSVECTOR)) {
sql.append("cast(? as tsvector)").append(",");
} else {
super.appendColumnParameter(sql, column);
}
@@ -148,6 +150,8 @@ protected void appendColumnEquals(StringBuilder sql, Column column) {
column.getJdbcTypeName().toUpperCase().contains(TypeMap.GEOGRAPHY)) {
sql.append(quote).append(column.getName()).append(quote)
.append(" = ST_GEOMFROMTEXT(?)");
} else if (column.getJdbcTypeName().toUpperCase().contains(TypeMap.TSVECTOR)) {
sql.append(quote).append(column.getName()).append(quote).append(" = cast(? as tsvector)");
} else {
super.appendColumnEquals(sql, column);
}
@@ -24,9 +24,12 @@

import org.jumpmind.db.DbTestUtils;
import org.jumpmind.db.platform.ase.AseDatabasePlatform;
import org.jumpmind.db.platform.greenplum.GreenplumPlatform;
import org.jumpmind.db.platform.informix.InformixDatabasePlatform;
import org.jumpmind.db.platform.redshift.RedshiftDatabasePlatform;
import org.jumpmind.db.platform.sqlanywhere.SqlAnywhereDatabasePlatform;
import org.jumpmind.db.platform.sqlite.SqliteDatabasePlatform;
import org.jumpmind.db.platform.voltdb.VoltDbDatabasePlatform;
import org.jumpmind.symmetric.io.AbstractWriterTest;
import org.jumpmind.symmetric.io.data.CsvData;
import org.jumpmind.symmetric.io.data.DataEventType;
@@ -67,8 +70,11 @@ public static void setup() throws Exception {
// Ignore SQLite because it doesn't enforce FKs, and its unique constraint error message doesn't tell you the index name
// TODO: Sybase ASE has metadata problem in the DDL reader for exported keys
// TODO: Informix has metadata problem in the DDL reader for exported keys
// TODO: Untested on Volt, Greenplum, Redshift
shouldTest = !(platform instanceof SqliteDatabasePlatform || platform instanceof SqlAnywhereDatabasePlatform ||
platform instanceof AseDatabasePlatform || platform instanceof InformixDatabasePlatform);
platform instanceof AseDatabasePlatform || platform instanceof InformixDatabasePlatform ||
platform instanceof VoltDbDatabasePlatform || platform instanceof GreenplumPlatform ||
platform instanceof RedshiftDatabasePlatform);
}

@Test
@@ -475,7 +475,7 @@ protected String getResultSetCatalogName() {
* @return The database model
*/
public Database readTables(final String catalog, final String schema, final String[] tableTypes) {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return postprocessModelFromDatabase(sqlTemplate
.execute(new IConnectionCallback<Database>() {
public Database execute(Connection connection) throws SQLException {
@@ -565,7 +565,7 @@ public int compare(Table obj1, Table obj2) {
public Table readTable(final String catalog, final String schema, final String table) {
try {
log.debug("reading table: " + table);
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return postprocessTableFromDatabase(sqlTemplate.execute(new IConnectionCallback<Table>() {
public Table execute(Connection connection) throws SQLException {
DatabaseMetaDataWrapper metaData = new DatabaseMetaDataWrapper();
@@ -1438,7 +1438,7 @@ protected String unescape(String text, String unescaped, String escaped) {
}

public List<String> getTableTypes() {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return sqlTemplate.execute(new IConnectionCallback<List<String>>() {
public List<String> execute(Connection connection) throws SQLException {
ArrayList<String> types = new ArrayList<String>();
@@ -1458,7 +1458,7 @@ protected String unescape(String text, String unescaped, String escaped) {
}

public List<String> getCatalogNames() {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return sqlTemplate.execute(new IConnectionCallback<List<String>>() {
public List<String> execute(Connection connection) throws SQLException {
ArrayList<String> catalogs = new ArrayList<String>();
@@ -1481,7 +1481,7 @@ protected String unescape(String text, String unescaped, String escaped) {
}

public List<String> getSchemaNames(final String catalog) {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return sqlTemplate.execute(new IConnectionCallback<List<String>>() {
public List<String> execute(Connection connection) throws SQLException {
ArrayList<String> schemas = new ArrayList<String>();
@@ -1525,7 +1525,7 @@ protected IConnectionHandler getConnectionHandler(String catalog) {

public List<String> getTableNames(final String catalog, final String schema,
final String[] tableTypes) {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
List<String> list = sqlTemplate.execute(new IConnectionCallback<List<String>>() {
public List<String> execute(Connection connection) throws SQLException {
ArrayList<String> list = new ArrayList<String>();
@@ -1550,7 +1550,7 @@ protected IConnectionHandler getConnectionHandler(String catalog) {
}

public List<String> getColumnNames(final String catalog, final String schema, final String tableName) {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return sqlTemplate.execute(new IConnectionCallback<List<String>>() {
public List<String> execute(Connection connection) throws SQLException {
ArrayList<String> list = new ArrayList<String>();
@@ -1589,7 +1589,7 @@ public Trigger getTriggerFor(Table table, String triggerName) {
@Override
public Collection<ForeignKey> getExportedKeys(Table table) {
try {
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();
return sqlTemplate.execute(new IConnectionCallback<Collection<ForeignKey>>() {
public Collection<ForeignKey> execute(Connection connection) throws SQLException {
connection.getMetaData().getExportedKeys(table.getCatalog(), table.getSchema(), table.getName());
@@ -1715,22 +1715,23 @@ public Trigger getTriggerFor(Table table, String triggerName) {
whereSql = whereSql.substring(0, whereSql.length() - delimiter.length());
}

Row foreignRow = new Row(foreignTable.getColumnCount());
if (foreignTable.getForeignKeyCount() > 0) {
DmlStatement selectSt = platform.createDmlStatement(DmlType.SELECT, foreignTable, null);
Object[] keys = whereRow.toArray(foreignTable.getPrimaryKeyColumnNames());
Map<String, Object> values = platform.getSqlTemplate().queryForMap(selectSt.getSql(), keys);
Map<String, Object> values = platform.getSqlTemplateDirty().queryForMap(selectSt.getSql(), keys);
if (values == null) {
log.warn(
"Unable to reload rows for missing foreign key data for table '{}', parent data not found. Using sql='{}' with keys '{}'",
foreignTable.getName(), selectSt.getSql(), keys);
} else {
Row foreignRow = new Row(foreignTable.getColumnCount());
foreignRow.putAll(values);
TableRow foreignTableRow = new TableRow(foreignTable, foreignRow, whereSql, referenceColumnName, fk.getName());
fkDepList.add(foreignTableRow);
log.debug("Add foreign table reference '{}' whereSql='{}'", foreignTable.getName(), whereSql);
}
}

TableRow foreignTableRow = new TableRow(foreignTable, foreignRow, whereSql, referenceColumnName, fk.getName());
fkDepList.add(foreignTableRow);
log.debug("Add foreign table reference '{}' whereSql='{}'", foreignTable.getName(), whereSql);
} else {
log.debug("The foreign table reference was null for {}", foreignTable.getName());
}
@@ -300,21 +300,15 @@ public static synchronized IDatabasePlatform createNewPlatformInstance(DataSourc
private static boolean isGreenplumDatabase(Connection connection) {
Statement stmt = null;
ResultSet rs = null;
String productName = null;
boolean isGreenplum = false;
int greenplumCount = 0;
try {
stmt = connection.createStatement();
rs = stmt.executeQuery(GreenplumPlatform.SQL_GET_GREENPLUM_NAME);
while (rs.next()) {
productName = rs.getString(1);
}
if (productName != null && productName.equalsIgnoreCase("Greenplum")) {
isGreenplum = true;
rs = stmt.executeQuery(GreenplumPlatform.SQL_GET_GREENPLUM_COUNT);
if (rs.next()) {
greenplumCount = rs.getInt(1);
}
} catch (SQLException ex) {
// ignore the exception, if it is caught, then this is most likely
// not
// a greenplum database
throw new RuntimeException(ex);
} finally {
try {
if (rs != null) {
@@ -326,7 +320,7 @@ private static boolean isGreenplumDatabase(Connection connection) {
} catch (SQLException ex) {
}
}
return isGreenplum;
return greenplumCount > 0;
}

private static boolean isRedshiftDatabase(Connection connection) {
@@ -31,7 +31,7 @@
public class GreenplumPlatform extends PostgreSqlDatabasePlatform {

/* PostgreSql can be either PostgreSql or Greenplum. Metadata queries to determine which one */
public static final String SQL_GET_GREENPLUM_NAME = "select gpname from gp_id";
public static final String SQL_GET_GREENPLUM_COUNT = "select count(*) from information_schema.tables where table_name = 'gp_id'";
public static final String SQL_GET_GREENPLUM_VERSION = "select productversion from gp_version_at_initdb";

public GreenplumPlatform(DataSource dataSource, SqlTemplateSettings settings) {
@@ -281,16 +281,15 @@ protected void removePlatformSizeAndDecimal(Column column) {
args.add(schema);
}

return platform.getSqlTemplate().queryWithHandler(sql.toString(), new StringMapper(),
return platform.getSqlTemplateDirty().queryWithHandler(sql.toString(), new StringMapper(),
new ChangeCatalogConnectionHandler(catalog) ,args.toArray(new Object[args.size()]));
}

@Override
public List<Trigger> getTriggers(final String catalog, final String schema,
final String tableName) throws SqlException {
log.debug("Reading triggers for: " + tableName);
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform
.getSqlTemplate();
JdbcSqlTemplate sqlTemplate = (JdbcSqlTemplate) platform.getSqlTemplateDirty();

String sql = "select "
+ "TRIG.name, "

0 comments on commit 61bbc6e

Please sign in to comment.