Skip to content

Commit

Permalink
0000904: DbExport does not respect the compatible setting for data
Browse files Browse the repository at this point in the history
  • Loading branch information
@chenson42
chenson42 committed Nov 21, 2012
1 parent b82325f commit 03f3bd4
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 75 deletions.
11 changes: 7 additions & 4 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java
View file
Expand Up @@ -212,8 +212,7 @@ protected void writeTable(final WriterWrapper writerWrapper, Table table, String

if (!noData) {
if (sql == null) {
sql = DmlStatementFactory.createDmlStatement(compatible
.toString().toLowerCase(), DmlType.SELECT_ALL, table).getSql();
sql = platform.createDmlStatement(DmlType.SELECT_ALL, table).getSql();
}

platform.getSqlTemplate().query(sql, new ISqlRowMapper<Object>() {
Expand Down Expand Up @@ -411,7 +410,11 @@ protected void startTable(Table table) {
csvWriter = new CsvWriter(writer, ',');
csvWriter.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH);
} else if (format == Format.SQL) {
insertSql = platform.createDmlStatement(DmlType.INSERT, table).getSql();
Table targetTable = table.copy();
targetTable.setSchema(schema);
targetTable.setCatalog(catalog);
insertSql = DmlStatementFactory.createDmlStatement(compatible
.toString().toLowerCase(), DmlType.INSERT, targetTable).getSql();
}

if (!noCreateInfo) {
Expand Down Expand Up @@ -469,7 +472,7 @@ protected void writeRow(Row row) {
if (format == Format.CSV) {
csvWriter.writeRecord(values, true);
} else if (format == Format.SQL) {
write(platform.replaceSql(insertSql, BinaryEncoding.HEX, columns, row,
write(platform.replaceSql(insertSql, BinaryEncoding.HEX, table, row,
useVariableDates), "\n");

} else if (format == Format.XML) {
Expand Down
117 changes: 59 additions & 58 deletions symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java
View file
Expand Up @@ -30,6 +30,8 @@ public class DbExportImportTest extends AbstractServiceTest {

private static final String SELECT_FROM_TEST_DB_IMPORT_1_ORDER_BY_ID = "select * from test_db_import_1 order by id";

private static final String TEST_TS_W_TZ = "test_ts_w_tz";

@Test
public void exportTableInAnotherSchemaOnH2() throws Exception {
if (getPlatform().getName().equals(DatabaseNamesConstants.H2)) {
Expand Down Expand Up @@ -106,7 +108,50 @@ public void exportThenImportXml() throws Exception {

}

protected void recreateImportTable() throws Exception {
@Test
public void testExportTimestampWithTimeZone() throws Exception {
if (createAndFillTimestampWithTimeZoneTable()) {
ISymmetricEngine engine = getSymmetricEngine();
DataSource ds = engine.getDataSource();

DbExport export = new DbExport(ds);
export.setCompatible(Compatible.POSTGRES);
export.setFormat(Format.SQL);
String sql = export.exportTables(new String[] { TEST_TS_W_TZ });
final String EXPECTED_POSTGRES = "insert into \"test_ts_w_tz\"(\"id\", \"tz\") (select 1,cast('1973-06-08 07:00:00.000000 -04:00' as timestamp with time zone) where (select 1 from \"test_ts_w_tz\" where \"id\" = 1) is null);";
Assert.assertTrue("Expected the following sql:\n" +sql + "\n\n to contain:\n" +EXPECTED_POSTGRES, sql.contains(EXPECTED_POSTGRES));

export.setCompatible(Compatible.ORACLE);
sql = export.exportTables(new String[] { TEST_TS_W_TZ });
final String EXPECTED_ORACLE = "insert into \"test_ts_w_tz\" (\"id\", \"tz\") values (1,TO_TIMESTAMP_TZ('1973-06-08 07:00:00.000000 -04:00', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'));";
Assert.assertTrue("Expected the following sql:\n" +sql + "\n\n to contain:\n" +EXPECTED_ORACLE, sql.contains(EXPECTED_ORACLE));

}
}

protected boolean createAndFillTimestampWithTimeZoneTable() {
ISymmetricEngine engine = getSymmetricEngine();
String dbName = engine.getDatabasePlatform().getName();
if (dbName.equals(DatabaseNamesConstants.ORACLE)
|| dbName.equals(DatabaseNamesConstants.POSTGRESQL)) {
ISqlTemplate template = engine.getSqlTemplate();
try {
template.update(String.format("drop table \"%s\"", TEST_TS_W_TZ));
} catch (Exception ex) {
}
String createSql = String.format(
"create table \"%s\" (\"id\" integer, \"tz\" timestamp with time zone, primary key (\"id\"))",
TEST_TS_W_TZ);
template.update(createSql);
template.update(String.format("insert into \"%s\" values(1, {ts '1973-06-08 07:00:00.000'})", TEST_TS_W_TZ));
return true;
} else {
return false;
}

}

protected void recreateImportTable() {
ISymmetricEngine engine = getSymmetricEngine();
DataSource ds = engine.getDataSource();
DbImport reCreateTablesImport = new DbImport(ds);
Expand Down Expand Up @@ -172,7 +217,7 @@ public void importSymXmlData() throws Exception {
assertCountDbImportTableRecords(0);

DbImport importCsv = new DbImport(ds);
importCsv.setFormat(DbImport.Format.SYM_XML);
importCsv.setFormat(DbImport.Format.SYM_XML);
importCsv.importTables(getClass().getResourceAsStream(FILE));

assertCountDbImportTableRecords(2);
Expand All @@ -193,7 +238,7 @@ public void importSymXmlData() throws Exception {
assertCountDbImportTableRecords(2);

}

@Test
public void importXmlData() throws Exception {
final String FILE = "/test-dbimport-1-xml-1.xml";
Expand All @@ -203,7 +248,7 @@ public void importXmlData() throws Exception {
DbImport importer = new DbImport(ds);
importer.setFormat(DbImport.Format.XML);
importer.setDropIfExists(true);
importer.setAlterCaseToMatchDatabaseDefaultCase(true);
importer.setAlterCaseToMatchDatabaseDefaultCase(true);
importer.importTables(getClass().getResourceAsStream(FILE));

assertCountDbImportTableRecords(3);
Expand All @@ -213,7 +258,7 @@ public void importXmlData() throws Exception {

assertCountDbImportTableRecords(3);

}
}

@Test
public void exportThenImportCsv() throws Exception {
Expand Down Expand Up @@ -262,73 +307,29 @@ public void exportThenImportCsv() throws Exception {
// TODO test force
}

@Test
public void testExportTimestampWithTimezone() throws Exception {
ISymmetricEngine engine = getSymmetricEngine();
IDatabasePlatform platform = engine.getSymmetricDialect().getPlatform();
DataSource ds = engine.getDataSource();
ISqlTemplate template = platform.getSqlTemplate();
if (platform.getName() == DatabaseNamesConstants.ORACLE
|| platform.getName() == DatabaseNamesConstants.POSTGRESQL) {
try {
template.update("drop table test");
} catch (Exception ex) {
}

template.update("create table TEST (ID integer, LAST_UPDATE_TIME timestamp with time zone, primary key (ID))");
Object[] args = { 1, "2012-10-18 13:14:11.111000 -04:00" };

if (platform.getName() == DatabaseNamesConstants.ORACLE) {
template.update(
"insert into TEST (ID, LAST_UPDATE_TIME) values (?, TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'))",
args);
} else if (platform.getName() == DatabaseNamesConstants.POSTGRESQL) {
template.update(
"insert into TEST (ID, LAST_UPDATE_TIME) values (?, cast(? as timestamp with time zone))",
args);
}
Table table = platform.getTableFromCache("TEST", true);

DbExport export = new DbExport(ds);
export.setFormat(Format.CSV);
export.setNoCreateInfo(true);
export.setNoData(false);
String csvOutput = export.exportTables(new String[] { table.getName() });
if (platform.getName() == DatabaseNamesConstants.ORACLE) {
Assert.assertEquals(
"ID,LAST_UPDATE_TIME\n" + "1,2012-10-18 13:14:11.111000 -04:00",
csvOutput.trim());
} else if (platform.getName() == DatabaseNamesConstants.POSTGRESQL) {
Assert.assertEquals("id,last_update_time\n" + "1,2012-10-18 13:14:11.111000 -4:00",
csvOutput.trim());

}
}
}

@Test
public void testExportCsvToDirectory() throws Exception {
ISymmetricEngine engine = getSymmetricEngine();
IDatabasePlatform platform = engine.getSymmetricDialect().getPlatform();
DataSource ds = engine.getDataSource();

DbImport importXml = new DbImport(ds);
importXml.setFormat(DbImport.Format.XML);
importXml.importTables(getClass().getResourceAsStream("/test-dbexportimport-3-tables.xml"));

File dir = new File("target/test");
FileUtils.deleteDirectory(dir);
Assert.assertFalse(dir.exists());

DbExport exportCsv = new DbExport(ds);
exportCsv.setComments(true);
exportCsv.setFormat(Format.CSV);
exportCsv.setDir(dir.getAbsolutePath());
exportCsv.exportTables(new String[] {"a", "b", "c"});
exportCsv.exportTables(new String[] { "a", "b", "c" });

Assert.assertTrue(dir.exists());
Assert.assertTrue(dir.isDirectory());
Assert.assertTrue(dir.isDirectory());

File a = new File(dir, platform.getTableFromCache("a", false).getName() + ".csv");
Assert.assertTrue(a.exists());
Assert.assertTrue(a.isFile());
Expand All @@ -337,7 +338,7 @@ public void testExportCsvToDirectory() throws Exception {
Assert.assertEquals("id,string_value", lines.get(5));
Assert.assertEquals("1,This is a test of a", lines.get(6));
Assert.assertEquals("2,This is a test of a", lines.get(7));

File b = new File(dir, platform.getTableFromCache("b", false).getName() + ".csv");
Assert.assertTrue(b.exists());
Assert.assertTrue(b.isFile());
Expand All @@ -347,7 +348,7 @@ public void testExportCsvToDirectory() throws Exception {
Assert.assertEquals("1,This is a test of b", lines.get(6));
Assert.assertEquals("2,This is a test of b", lines.get(7));
Assert.assertEquals("3,This is line 3 of b", lines.get(8));

File c = new File(dir, platform.getTableFromCache("c", false).getName() + ".csv");
Assert.assertTrue(c.exists());
Assert.assertTrue(c.isFile());
Expand Down
4 changes: 4 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java
View file
Expand Up @@ -931,5 +931,9 @@ public Logger getLog() {
public <T> T getDataSource() {
return (T) getSymmetricDialect().getPlatform().getDataSource();
}

public IDatabasePlatform getDatabasePlatform() {
return getSymmetricDialect().getPlatform();
}

}
3 changes: 3 additions & 0 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/ISymmetricEngine.java
View file
Expand Up @@ -22,6 +22,7 @@
import java.util.Date;
import java.util.Properties;

import org.jumpmind.db.platform.IDatabasePlatform;
import org.jumpmind.db.sql.ISqlTemplate;
import org.jumpmind.symmetric.db.ISymmetricDialect;
import org.jumpmind.symmetric.ext.IExtensionPointManager;
Expand Down Expand Up @@ -277,4 +278,6 @@ public interface ISymmetricEngine {

public <T> T getDataSource();

public IDatabasePlatform getDatabasePlatform();

}
Expand Down
17 changes: 13 additions & 4 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java
View file
Expand Up @@ -430,13 +430,19 @@ public String[] getStringValues(BinaryEncoding encoding, Column[] metaData, Row
return values;
}

public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, Row row,
public String replaceSql(String sql, BinaryEncoding encoding, Table table, Row row,
boolean useVariableDates) {
final String QUESTION_MARK = "<!QUESTION_MARK!>";
String newSql = sql;
String quote = getDatabaseInfo().getValueQuoteToken();
String regex = "\\?";
for (int i = 0; i < metaData.length; i++) {
Column column = metaData[i];

List<Column> columnsToProcess = new ArrayList<Column>();
columnsToProcess.addAll(table.getColumnsAsList());
columnsToProcess.addAll(table.getPrimaryKeyColumnsAsList());

for (int i = 0; i < columnsToProcess.size(); i++) {
Column column = columnsToProcess.get(i);
String name = column.getName();
int type = column.getJdbcTypeCode();

Expand All @@ -447,13 +453,14 @@ public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData,
value = value.replace("\\", "\\\\");
value = value.replace("$", "\\$");
value = value.replace("'", "''");
value = value.replace("?", QUESTION_MARK);
newSql = newSql.replaceFirst(regex, quote + value + quote);
} catch (RuntimeException ex) {
log.error("Failed to replace ? in {" + sql + "} with " + name + "="
+ row.getString(name));
throw ex;
}
} else if (column.getMappedTypeCode() == -101) {
} else if (column.isTimestampWithTimezone()) {
newSql = newSql.replaceFirst(regex, quote + row.getString(name) + quote);
} else if (type == Types.DATE || type == Types.TIMESTAMP || type == Types.TIME) {
Date date = row.getDateTime(name);
Expand Down Expand Up @@ -485,6 +492,8 @@ public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData,
newSql = newSql.replaceFirst(regex, "null");
}
}

newSql = newSql.replace(QUESTION_MARK, "?");
return newSql + getDatabaseInfo().getSqlCommandDelimiter();
}

Expand Down
2 changes: 1 addition & 1 deletion symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java
View file
Expand Up @@ -122,7 +122,7 @@ public Object[] getObjectValues(BinaryEncoding encoding, String[] values,

public String[] getStringValues(BinaryEncoding encoding, Column[] metaData, Row row, boolean useVariableDates);

public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, Row row, boolean useVariableDates);
public String replaceSql(String sql, BinaryEncoding encoding, Table table, Row row, boolean useVariableDates);

public Database readDatabaseFromXml(String filePath, boolean alterCaseToMatchDatabaseDefaultCase);

Expand Down
12 changes: 4 additions & 8 deletions symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java
View file
Expand Up @@ -147,16 +147,12 @@ protected void appendColumnNameForSql(StringBuilder sql, Column column, boolean
if (select && column.isTimestampWithTimezone()) {
sql.append(
" case " +
" when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") < -9 then " +
" to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')|| " +
" lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),3,'0')||':'|| " +
" lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " +
" when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") >= 0 then " +
" to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'+'|| " +
" lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),2,'0')||':'|| " +
" when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") < 0 then " +
" to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'-'|| " +
" lpad(cast(abs(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(")) as varchar),2,'0')||':'|| " +
" lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " +
" else " +
" to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')|| " +
" to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'+'|| " +
" lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),2,'0')||':'|| " +
" lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " +
" end as ").append(columnName);
Expand Down

0 comments on commit 03f3bd4

Please sign in to comment.