From 03f3bd445aad2fc5d32391aca7643e439aebadf8 Mon Sep 17 00:00:00 2001 From: chenson42 Date: Wed, 21 Nov 2012 00:35:45 +0000 Subject: [PATCH] 0000904: DbExport does not respect the compatible setting for data --- .../java/org/jumpmind/symmetric/DbExport.java | 11 +- .../symmetric/DbExportImportTest.java | 117 +++++++++--------- .../symmetric/AbstractSymmetricEngine.java | 4 + .../jumpmind/symmetric/ISymmetricEngine.java | 3 + .../db/platform/AbstractDatabasePlatform.java | 17 ++- .../db/platform/IDatabasePlatform.java | 2 +- .../postgresql/PostgreSqlDmlStatement.java | 12 +- 7 files changed, 91 insertions(+), 75 deletions(-) diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java index 9eb9a96ad0..59b516b30e 100644 --- a/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java +++ b/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java @@ -212,8 +212,7 @@ protected void writeTable(final WriterWrapper writerWrapper, Table table, String if (!noData) { if (sql == null) { - sql = DmlStatementFactory.createDmlStatement(compatible - .toString().toLowerCase(), DmlType.SELECT_ALL, table).getSql(); + sql = platform.createDmlStatement(DmlType.SELECT_ALL, table).getSql(); } platform.getSqlTemplate().query(sql, new ISqlRowMapper() { @@ -411,7 +410,11 @@ protected void startTable(Table table) { csvWriter = new CsvWriter(writer, ','); csvWriter.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH); } else if (format == Format.SQL) { - insertSql = platform.createDmlStatement(DmlType.INSERT, table).getSql(); + Table targetTable = table.copy(); + targetTable.setSchema(schema); + targetTable.setCatalog(catalog); + insertSql = DmlStatementFactory.createDmlStatement(compatible + .toString().toLowerCase(), DmlType.INSERT, targetTable).getSql(); } if (!noCreateInfo) { @@ -469,7 +472,7 @@ protected void writeRow(Row row) { if (format == Format.CSV) { csvWriter.writeRecord(values, true); } else if (format == Format.SQL) { - write(platform.replaceSql(insertSql, BinaryEncoding.HEX, columns, row, + write(platform.replaceSql(insertSql, BinaryEncoding.HEX, table, row, useVariableDates), "\n"); } else if (format == Format.XML) { diff --git a/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java b/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java index 447dca5b41..f598434066 100644 --- a/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java +++ b/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java @@ -30,6 +30,8 @@ public class DbExportImportTest extends AbstractServiceTest { private static final String SELECT_FROM_TEST_DB_IMPORT_1_ORDER_BY_ID = "select * from test_db_import_1 order by id"; + private static final String TEST_TS_W_TZ = "test_ts_w_tz"; + @Test public void exportTableInAnotherSchemaOnH2() throws Exception { if (getPlatform().getName().equals(DatabaseNamesConstants.H2)) { @@ -106,7 +108,50 @@ public void exportThenImportXml() throws Exception { } - protected void recreateImportTable() throws Exception { + @Test + public void testExportTimestampWithTimeZone() throws Exception { + if (createAndFillTimestampWithTimeZoneTable()) { + ISymmetricEngine engine = getSymmetricEngine(); + DataSource ds = engine.getDataSource(); + + DbExport export = new DbExport(ds); + export.setCompatible(Compatible.POSTGRES); + export.setFormat(Format.SQL); + String sql = export.exportTables(new String[] { TEST_TS_W_TZ }); + final String EXPECTED_POSTGRES = "insert into \"test_ts_w_tz\"(\"id\", \"tz\") (select 1,cast('1973-06-08 07:00:00.000000 -04:00' as timestamp with time zone) where (select 1 from \"test_ts_w_tz\" where \"id\" = 1) is null);"; + Assert.assertTrue("Expected the following sql:\n" +sql + "\n\n to contain:\n" +EXPECTED_POSTGRES, sql.contains(EXPECTED_POSTGRES)); + + export.setCompatible(Compatible.ORACLE); + sql = export.exportTables(new String[] { TEST_TS_W_TZ }); + final String EXPECTED_ORACLE = "insert into \"test_ts_w_tz\" (\"id\", \"tz\") values (1,TO_TIMESTAMP_TZ('1973-06-08 07:00:00.000000 -04:00', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'));"; + Assert.assertTrue("Expected the following sql:\n" +sql + "\n\n to contain:\n" +EXPECTED_ORACLE, sql.contains(EXPECTED_ORACLE)); + + } + } + + protected boolean createAndFillTimestampWithTimeZoneTable() { + ISymmetricEngine engine = getSymmetricEngine(); + String dbName = engine.getDatabasePlatform().getName(); + if (dbName.equals(DatabaseNamesConstants.ORACLE) + || dbName.equals(DatabaseNamesConstants.POSTGRESQL)) { + ISqlTemplate template = engine.getSqlTemplate(); + try { + template.update(String.format("drop table \"%s\"", TEST_TS_W_TZ)); + } catch (Exception ex) { + } + String createSql = String.format( + "create table \"%s\" (\"id\" integer, \"tz\" timestamp with time zone, primary key (\"id\"))", + TEST_TS_W_TZ); + template.update(createSql); + template.update(String.format("insert into \"%s\" values(1, {ts '1973-06-08 07:00:00.000'})", TEST_TS_W_TZ)); + return true; + } else { + return false; + } + + } + + protected void recreateImportTable() { ISymmetricEngine engine = getSymmetricEngine(); DataSource ds = engine.getDataSource(); DbImport reCreateTablesImport = new DbImport(ds); @@ -172,7 +217,7 @@ public void importSymXmlData() throws Exception { assertCountDbImportTableRecords(0); DbImport importCsv = new DbImport(ds); - importCsv.setFormat(DbImport.Format.SYM_XML); + importCsv.setFormat(DbImport.Format.SYM_XML); importCsv.importTables(getClass().getResourceAsStream(FILE)); assertCountDbImportTableRecords(2); @@ -193,7 +238,7 @@ public void importSymXmlData() throws Exception { assertCountDbImportTableRecords(2); } - + @Test public void importXmlData() throws Exception { final String FILE = "/test-dbimport-1-xml-1.xml"; @@ -203,7 +248,7 @@ public void importXmlData() throws Exception { DbImport importer = new DbImport(ds); importer.setFormat(DbImport.Format.XML); importer.setDropIfExists(true); - importer.setAlterCaseToMatchDatabaseDefaultCase(true); + importer.setAlterCaseToMatchDatabaseDefaultCase(true); importer.importTables(getClass().getResourceAsStream(FILE)); assertCountDbImportTableRecords(3); @@ -213,7 +258,7 @@ public void importXmlData() throws Exception { assertCountDbImportTableRecords(3); - } + } @Test public void exportThenImportCsv() throws Exception { @@ -262,73 +307,29 @@ public void exportThenImportCsv() throws Exception { // TODO test force } - @Test - public void testExportTimestampWithTimezone() throws Exception { - ISymmetricEngine engine = getSymmetricEngine(); - IDatabasePlatform platform = engine.getSymmetricDialect().getPlatform(); - DataSource ds = engine.getDataSource(); - ISqlTemplate template = platform.getSqlTemplate(); - if (platform.getName() == DatabaseNamesConstants.ORACLE - || platform.getName() == DatabaseNamesConstants.POSTGRESQL) { - try { - template.update("drop table test"); - } catch (Exception ex) { - } - - template.update("create table TEST (ID integer, LAST_UPDATE_TIME timestamp with time zone, primary key (ID))"); - Object[] args = { 1, "2012-10-18 13:14:11.111000 -04:00" }; - - if (platform.getName() == DatabaseNamesConstants.ORACLE) { - template.update( - "insert into TEST (ID, LAST_UPDATE_TIME) values (?, TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM'))", - args); - } else if (platform.getName() == DatabaseNamesConstants.POSTGRESQL) { - template.update( - "insert into TEST (ID, LAST_UPDATE_TIME) values (?, cast(? as timestamp with time zone))", - args); - } - Table table = platform.getTableFromCache("TEST", true); - - DbExport export = new DbExport(ds); - export.setFormat(Format.CSV); - export.setNoCreateInfo(true); - export.setNoData(false); - String csvOutput = export.exportTables(new String[] { table.getName() }); - if (platform.getName() == DatabaseNamesConstants.ORACLE) { - Assert.assertEquals( - "ID,LAST_UPDATE_TIME\n" + "1,2012-10-18 13:14:11.111000 -04:00", - csvOutput.trim()); - } else if (platform.getName() == DatabaseNamesConstants.POSTGRESQL) { - Assert.assertEquals("id,last_update_time\n" + "1,2012-10-18 13:14:11.111000 -4:00", - csvOutput.trim()); - - } - } - } - @Test public void testExportCsvToDirectory() throws Exception { ISymmetricEngine engine = getSymmetricEngine(); IDatabasePlatform platform = engine.getSymmetricDialect().getPlatform(); DataSource ds = engine.getDataSource(); - + DbImport importXml = new DbImport(ds); importXml.setFormat(DbImport.Format.XML); importXml.importTables(getClass().getResourceAsStream("/test-dbexportimport-3-tables.xml")); - + File dir = new File("target/test"); FileUtils.deleteDirectory(dir); Assert.assertFalse(dir.exists()); - + DbExport exportCsv = new DbExport(ds); exportCsv.setComments(true); exportCsv.setFormat(Format.CSV); exportCsv.setDir(dir.getAbsolutePath()); - exportCsv.exportTables(new String[] {"a", "b", "c"}); - + exportCsv.exportTables(new String[] { "a", "b", "c" }); + Assert.assertTrue(dir.exists()); - Assert.assertTrue(dir.isDirectory()); - + Assert.assertTrue(dir.isDirectory()); + File a = new File(dir, platform.getTableFromCache("a", false).getName() + ".csv"); Assert.assertTrue(a.exists()); Assert.assertTrue(a.isFile()); @@ -337,7 +338,7 @@ public void testExportCsvToDirectory() throws Exception { Assert.assertEquals("id,string_value", lines.get(5)); Assert.assertEquals("1,This is a test of a", lines.get(6)); Assert.assertEquals("2,This is a test of a", lines.get(7)); - + File b = new File(dir, platform.getTableFromCache("b", false).getName() + ".csv"); Assert.assertTrue(b.exists()); Assert.assertTrue(b.isFile()); @@ -347,7 +348,7 @@ public void testExportCsvToDirectory() throws Exception { Assert.assertEquals("1,This is a test of b", lines.get(6)); Assert.assertEquals("2,This is a test of b", lines.get(7)); Assert.assertEquals("3,This is line 3 of b", lines.get(8)); - + File c = new File(dir, platform.getTableFromCache("c", false).getName() + ".csv"); Assert.assertTrue(c.exists()); Assert.assertTrue(c.isFile()); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java index 9963c95ef6..7b5759c3e1 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/AbstractSymmetricEngine.java @@ -931,5 +931,9 @@ public Logger getLog() { public T getDataSource() { return (T) getSymmetricDialect().getPlatform().getDataSource(); } + + public IDatabasePlatform getDatabasePlatform() { + return getSymmetricDialect().getPlatform(); + } } diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/ISymmetricEngine.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/ISymmetricEngine.java index a8f30086a6..cfc9da23df 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/ISymmetricEngine.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/ISymmetricEngine.java @@ -22,6 +22,7 @@ import java.util.Date; import java.util.Properties; +import org.jumpmind.db.platform.IDatabasePlatform; import org.jumpmind.db.sql.ISqlTemplate; import org.jumpmind.symmetric.db.ISymmetricDialect; import org.jumpmind.symmetric.ext.IExtensionPointManager; @@ -277,4 +278,6 @@ public interface ISymmetricEngine { public T getDataSource(); + public IDatabasePlatform getDatabasePlatform(); + } \ No newline at end of file diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java index d411586330..c545f1b583 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/AbstractDatabasePlatform.java @@ -430,13 +430,19 @@ public String[] getStringValues(BinaryEncoding encoding, Column[] metaData, Row return values; } - public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, Row row, + public String replaceSql(String sql, BinaryEncoding encoding, Table table, Row row, boolean useVariableDates) { + final String QUESTION_MARK = ""; String newSql = sql; String quote = getDatabaseInfo().getValueQuoteToken(); String regex = "\\?"; - for (int i = 0; i < metaData.length; i++) { - Column column = metaData[i]; + + List columnsToProcess = new ArrayList(); + columnsToProcess.addAll(table.getColumnsAsList()); + columnsToProcess.addAll(table.getPrimaryKeyColumnsAsList()); + + for (int i = 0; i < columnsToProcess.size(); i++) { + Column column = columnsToProcess.get(i); String name = column.getName(); int type = column.getJdbcTypeCode(); @@ -447,13 +453,14 @@ public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, value = value.replace("\\", "\\\\"); value = value.replace("$", "\\$"); value = value.replace("'", "''"); + value = value.replace("?", QUESTION_MARK); newSql = newSql.replaceFirst(regex, quote + value + quote); } catch (RuntimeException ex) { log.error("Failed to replace ? in {" + sql + "} with " + name + "=" + row.getString(name)); throw ex; } - } else if (column.getMappedTypeCode() == -101) { + } else if (column.isTimestampWithTimezone()) { newSql = newSql.replaceFirst(regex, quote + row.getString(name) + quote); } else if (type == Types.DATE || type == Types.TIMESTAMP || type == Types.TIME) { Date date = row.getDateTime(name); @@ -485,6 +492,8 @@ public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, newSql = newSql.replaceFirst(regex, "null"); } } + + newSql = newSql.replace(QUESTION_MARK, "?"); return newSql + getDatabaseInfo().getSqlCommandDelimiter(); } diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java index 0d3c654e16..1a6b2c225c 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/IDatabasePlatform.java @@ -122,7 +122,7 @@ public Object[] getObjectValues(BinaryEncoding encoding, String[] values, public String[] getStringValues(BinaryEncoding encoding, Column[] metaData, Row row, boolean useVariableDates); - public String replaceSql(String sql, BinaryEncoding encoding, Column[] metaData, Row row, boolean useVariableDates); + public String replaceSql(String sql, BinaryEncoding encoding, Table table, Row row, boolean useVariableDates); public Database readDatabaseFromXml(String filePath, boolean alterCaseToMatchDatabaseDefaultCase); diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java index b9f7e5b3a8..c6d32f5198 100644 --- a/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java +++ b/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java @@ -147,16 +147,12 @@ protected void appendColumnNameForSql(StringBuilder sql, Column column, boolean if (select && column.isTimestampWithTimezone()) { sql.append( " case " + - " when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") < -9 then " + - " to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')|| " + - " lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),3,'0')||':'|| " + - " lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " + - " when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") >= 0 then " + - " to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'+'|| " + - " lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),2,'0')||':'|| " + + " when extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") < 0 then " + + " to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'-'|| " + + " lpad(cast(abs(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(")) as varchar),2,'0')||':'|| " + " lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " + " else " + - " to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')|| " + + " to_char(").append(quote).append(columnName).append(quote).append(", 'YYYY-MM-DD HH24:MI:SS.US ')||'+'|| " + " lpad(cast(extract(timezone_hour from ").append(quote).append(columnName).append(quote).append(") as varchar),2,'0')||':'|| " + " lpad(cast(extract(timezone_minute from ").append(quote).append(columnName).append(quote).append(") as varchar), 2, '0') " + " end as ").append(columnName);