0000915: backslash (\) is exported as doubule backslash (\\) in CSV f…

…ormat
JumpMind · Nov 21, 2012 · 184c8ea · 184c8ea
1 parent 03f3bd4
commit 184c8ea
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 36 deletions.
diff --git a/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java b/symmetric-client/src/main/java/org/jumpmind/symmetric/DbExport.java
@@ -409,6 +409,9 @@ protected void startTable(Table table) {
                 if (format == Format.CSV && csvWriter == null) {
                     csvWriter = new CsvWriter(writer, ',');
                     csvWriter.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH);
+                    csvWriter.setTextQualifier('\"');
+                    csvWriter.setUseTextQualifier(true);
+                    csvWriter.setForceQualifier(true);                    
                 } else if (format == Format.SQL) {
                     Table targetTable = table.copy();
                     targetTable.setSchema(schema);

diff --git a/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java b/symmetric-client/src/test/java/org/jumpmind/symmetric/DbExportImportTest.java
@@ -131,7 +131,8 @@ public void testExportTimestampWithTimeZone() throws Exception {
 
     protected boolean createAndFillTimestampWithTimeZoneTable() {
         ISymmetricEngine engine = getSymmetricEngine();
-        String dbName = engine.getDatabasePlatform().getName();
+        IDatabasePlatform platform = engine.getDatabasePlatform();
+        String dbName = platform.getName();
         if (dbName.equals(DatabaseNamesConstants.ORACLE)
                 || dbName.equals(DatabaseNamesConstants.POSTGRESQL)) {
             ISqlTemplate template = engine.getSqlTemplate();
@@ -142,8 +143,9 @@ protected boolean createAndFillTimestampWithTimeZoneTable() {
             String createSql = String.format(
                     "create table \"%s\" (\"id\" integer, \"tz\" timestamp with time zone, primary key (\"id\"))",
                     TEST_TS_W_TZ);
-            template.update(createSql);            
-            template.update(String.format("insert into \"%s\" values(1, {ts '1973-06-08 07:00:00.000'})", TEST_TS_W_TZ));
+            template.update(createSql);          
+            DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, platform.getTableFromCache(TEST_TS_W_TZ, true));            
+            template.update(statement.getSql(), statement.getValueArray(new Object[] {1, "1973-06-08 07:00:00.000 -04:00"}, new Object[] {1}));
             return true;
         } else {
             return false;
@@ -306,6 +308,41 @@ public void exportThenImportCsv() throws Exception {
 
         // TODO test force
     }
+
+    @Test
+    public void exportThenImportCsvWithBackslashes() throws Exception {
+        ISymmetricEngine engine = getSymmetricEngine();
+        DataSource ds = engine.getDataSource();
+        IDatabasePlatform platform = engine.getSymmetricDialect().getPlatform();
+        Database testTables = platform.readDatabaseFromXml("/test-dbimport.xml", true);
+        Table table = testTables.findTable("test_db_import_1", false);
+
+        recreateImportTable();
+
+        DbImport importCsv = new DbImport(ds);
+        importCsv.setFormat(DbImport.Format.SQL);
+        importCsv.importTables(getClass().getResourceAsStream("/test-dbimport-1-backslashes.sql"));
+
+        assertCountDbImportTableRecords(1);
+
+        DbExport export = new DbExport(ds);
+        export.setFormat(Format.CSV);
+        export.setNoCreateInfo(true);
+        export.setNoData(false);
+        String csvOutput = export.exportTables(new String[] { table.getName() });
+
+        ISqlTemplate sqlTemplate = platform.getSqlTemplate();
+
+        List<Row> rowsBeforeImport = sqlTemplate.query(SELECT_FROM_TEST_DB_IMPORT_1_ORDER_BY_ID);
+
+        recreateImportTable();
+
+        importCsv.setFormat(DbImport.Format.CSV);
+        importCsv.importTables(csvOutput, table.getName());
+
+        compareRows(rowsBeforeImport, sqlTemplate.query(SELECT_FROM_TEST_DB_IMPORT_1_ORDER_BY_ID));
+
+    }    
 
     @Test
     public void testExportCsvToDirectory() throws Exception {
@@ -335,28 +372,28 @@ public void testExportCsvToDirectory() throws Exception {
         Assert.assertTrue(a.isFile());
         List<String> lines = FileUtils.readLines(a);
         Assert.assertEquals(9, lines.size());
-        Assert.assertEquals("id,string_value", lines.get(5));
-        Assert.assertEquals("1,This is a test of a", lines.get(6));
-        Assert.assertEquals("2,This is a test of a", lines.get(7));
+        Assert.assertEquals("\"id\",\"string_value\"", lines.get(5));
+        Assert.assertEquals("\"1\",\"This is a test of a\"", lines.get(6));
+        Assert.assertEquals("\"2\",\"This is a test of a\"", lines.get(7));
 
         File b = new File(dir, platform.getTableFromCache("b", false).getName() + ".csv");
         Assert.assertTrue(b.exists());
         Assert.assertTrue(b.isFile());
         lines = FileUtils.readLines(b);
         Assert.assertEquals(10, lines.size());
-        Assert.assertEquals("id,string_value", lines.get(5));
-        Assert.assertEquals("1,This is a test of b", lines.get(6));
-        Assert.assertEquals("2,This is a test of b", lines.get(7));
-        Assert.assertEquals("3,This is line 3 of b", lines.get(8));
+        Assert.assertEquals("\"id\",\"string_value\"", lines.get(5));
+        Assert.assertEquals("\"1\",\"This is a test of b\"", lines.get(6));
+        Assert.assertEquals("\"2\",\"This is a test of b\"", lines.get(7));
+        Assert.assertEquals("\"3\",\"This is line 3 of b\"", lines.get(8));
 
         File c = new File(dir, platform.getTableFromCache("c", false).getName() + ".csv");
         Assert.assertTrue(c.exists());
         Assert.assertTrue(c.isFile());
         lines = FileUtils.readLines(c);
         Assert.assertEquals(9, lines.size());
-        Assert.assertEquals("id,string_value", lines.get(5));
-        Assert.assertEquals("1,This is a test of c", lines.get(6));
-        Assert.assertEquals("2,This is a test of c", lines.get(7));
+        Assert.assertEquals("\"id\",\"string_value\"", lines.get(5));
+        Assert.assertEquals("\"1\",\"This is a test of c\"", lines.get(6));
+        Assert.assertEquals("\"2\",\"This is a test of c\"", lines.get(7));
 
     }
 

diff --git a/symmetric-client/src/test/resources/test-dbimport-1-backslashes.sql b/symmetric-client/src/test/resources/test-dbimport-1-backslashes.sql
@@ -0,0 +1 @@
+insert into test_db_import_1 (id,string_value,string_required_value,char_value,char_required_value,date_value,time_value,boolean_value,integer_value,decimal_value,double_value) values (5,'a\b\\c\\\d','junk','j','j',{d '1989-09-21'},{ts '1997-11-06 08:02:33.324'},0,1,1,1);
diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java
@@ -120,8 +120,6 @@ public void insertSqlEvent(Node targetNode, TriggerHistory triggerHistory, Strin
 
     public Map<String, String> getRowDataAsMap(Data data);
 
-    public void setRowDataFromMap(Data data, Map<String, String> map);
-
     public void addReloadListener(IReloadListener listener);
 
     public void addHeartbeatListener(IHeartbeatListener listener);

diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java
@@ -920,21 +920,6 @@ public Map<String, String> getRowDataAsMap(Data data) {
         return map;
     }
 
-    public void setRowDataFromMap(Data data, Map<String, String> map) {
-        String[] columnNames = CsvUtils.tokenizeCsvData(data.getTriggerHistory().getColumnNames());
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        CsvWriter writer = new CsvWriter(new OutputStreamWriter(out), ',');
-        writer.setEscapeMode(CsvWriter.ESCAPE_MODE_BACKSLASH);
-        for (String columnName : columnNames) {
-            try {
-                writer.write(map.get(columnName.toLowerCase()), true);
-            } catch (IOException e) {
-            }
-        }
-        writer.close();
-        data.setRowData(out.toString());
-    }
-
     /**
      * Get a list of {@link IHeartbeatListener}s that are ready for a heartbeat
      * according to

diff --git a/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/platform/postgresql/PostgreSqlDmlStatement.java
@@ -67,10 +67,11 @@ public Column[] getMetaData() {
         }
     }
 
+    @SuppressWarnings("unchecked")
     @Override
-    public String[] getValueArray(String[] columnValues, String[] keyValues) {
+    public <T> T[] getValueArray(T[] columnValues, T[] keyValues) {
         if (dmlType == DmlType.INSERT) {
-            return (String[]) ArrayUtils.addAll(columnValues, keyValues);
+            return (T[]) ArrayUtils.addAll(columnValues, keyValues);
         } else {
             return super.getValueArray(columnValues, keyValues);
         }

diff --git a/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java b/symmetric-db/src/main/java/org/jumpmind/db/sql/DmlStatement.java
@@ -310,11 +310,12 @@ public Column[] getMetaData() {
     public Column[] getKeys() {
         return keys;
     }
-
-    public String[] getValueArray(String[] columnValues, String[] keyValues) {
+
+    @SuppressWarnings("unchecked")
+    public <T> T[] getValueArray(T[] columnValues, T[] keyValues) {
         switch (dmlType) {
             case UPDATE:
-                return (String[]) ArrayUtils.addAll(columnValues, keyValues);
+                return (T[]) ArrayUtils.addAll(columnValues, keyValues);
             case INSERT:
                 return columnValues;
             case DELETE:
@@ -323,7 +324,7 @@ public String[] getValueArray(String[] columnValues, String[] keyValues) {
                 break;
         }
         return null;
-    }
+    }    
 
     public Object[] buildArgsFrom(Map<String, Object> params) {
         Object[] args = null;

diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvUtils.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/CsvUtils.java
@@ -63,6 +63,9 @@ public static String[] tokenizeCsvData(String csvData) {
         return tokens;
     }
 
+    /**
+     * This escapes backslashes but doesn't wrap the data in a text qualifier.
+     */
     public static String escapeCsvData(String data) {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         CsvWriter writer = new CsvWriter(new OutputStreamWriter(out), ',');