Skip to content
Permalink
Browse files

0003994: DbExport CSV file should escape quote characters with double

quotes
  • Loading branch information...
jaredfrees committed Jun 11, 2019
1 parent 1a11fba commit 9e85e2d3e9bc94bf7ebe4930916a2f8c81ccf45f
@@ -131,7 +131,7 @@ protected boolean executeWithOptions(CommandLine line) throws Exception {

if (line.hasOption(OPTION_FORMAT)) {
dbExport.setFormat(Format.valueOf(line.getOptionValue(OPTION_FORMAT).toUpperCase()));
if (dbExport.getFormat() == Format.CSV && line.getArgs().length > 1
if ((dbExport.getFormat() == Format.CSV || dbExport.getFormat() == Format.CSV_DQUOTE) && line.getArgs().length > 1
&& StringUtils.isBlank(dbExport.getDir())) {
throw new ParseException(
"When exporting multiple tables to CSV format you must designate a directory where the files will be written");
@@ -48,6 +48,14 @@ public static CsvReader getCsvReader(Reader reader) {
csvReader.setCaptureRawRecord(false);
return csvReader;
}

public static CsvReader getCsvReaderDquote(Reader reader) {
CsvReader csvReader = new CsvReader(reader);
csvReader.setEscapeMode(CsvWriter.ESCAPE_MODE_DOUBLED);
csvReader.setSafetySwitch(false);
csvReader.setCaptureRawRecord(false);
return csvReader;
}

public static String[] tokenizeCsvData(String csvData) {
String[] tokens = null;
@@ -60,7 +60,7 @@
public class DbExport {

public enum Format {
SQL, CSV, XML, SYM_XML
SQL, CSV, XML, SYM_XML, CSV_DQUOTE
};

public enum Compatible {
@@ -528,6 +528,12 @@ protected void startTable(Table table) {
csvWriter.setTextQualifier('\"');
csvWriter.setUseTextQualifier(true);
csvWriter.setForceQualifier(true);
} else if (format == Format.CSV_DQUOTE && csvWriter == null) {
csvWriter = new CsvWriter(writer, ',');
csvWriter.setEscapeMode(CsvWriter.ESCAPE_MODE_DOUBLED);
csvWriter.setTextQualifier('\"');
csvWriter.setUseTextQualifier(true);
csvWriter.setForceQualifier(true);
} else if (format == Format.SQL) {
if (table.getCatalog() != null
&& table.getCatalog().equals(platform.getDefaultCatalog())) {
@@ -579,7 +585,7 @@ else if (addDropTable) {
writeComment("Table Columns: " + java.util.Arrays.deepToString(table.getColumnNames()));
writeComment("Started on " + df.format(new Date()));

if (format == Format.CSV) {
if (format == Format.CSV || format == Format.CSV_DQUOTE) {
csvWriter.writeRecord(table.getColumnNames());
} else if (!noData && format == Format.XML) {
write("<table_data name=\"", table.getName(), "\">\n");
@@ -604,7 +610,7 @@ protected void writeComment(String commentStr) {
if (writer != null) {
try {
if (comments) {
if (format == Format.CSV) {
if (format == Format.CSV || format == Format.CSV_DQUOTE) {
write("# ", commentStr, "\n");
} else if (format == Format.XML) {
write("<!-- ", commentStr, " -->\n");
@@ -624,7 +630,7 @@ protected void writeRow(Row row) {
String[] values = platform.getStringValues(BinaryEncoding.HEX, columns, row,
useVariableDates, false);
try {
if (format == Format.CSV) {
if (format == Format.CSV || format == Format.CSV_DQUOTE) {
csvWriter.writeRecord(values, true);
} else if (format == Format.SQL) {
write(insertSql.buildDynamicSql(BinaryEncoding.HEX, row, useVariableDates,
@@ -52,6 +52,7 @@
import org.jumpmind.db.platform.IDatabasePlatform;
import org.jumpmind.db.util.BinaryEncoding;
import org.jumpmind.exception.IoException;
import org.jumpmind.symmetric.io.data.reader.CsvDquoteDataReader;
import org.jumpmind.symmetric.io.data.reader.CsvTableDataReader;
import org.jumpmind.symmetric.io.data.reader.SqlDataReader;
import org.jumpmind.symmetric.io.data.reader.SymXmlDataReader;
@@ -70,7 +71,7 @@
public class DbImport {

public enum Format {
SQL, CSV, XML, SYM_XML
SQL, CSV, XML, SYM_XML, CSV_DQUOTE
};

private Format format = Format.SQL;
@@ -156,6 +157,12 @@ public void importTables(InputStream in, String tableName) {
} else {
throw new RuntimeException("Table name argument is required when importing CSV.");
}
} else if (format == Format.CSV_DQUOTE) {
if (StringUtils.isNotBlank(tableName)) {
importTablesFromCsvDquote(in, tableName);
} else {
throw new RuntimeException("Table name argument is required when importing CSV.");
}
} else if (format == Format.XML) {
importTablesFromXml(in);
} else if (format == Format.SYM_XML) {
@@ -207,6 +214,19 @@ protected void importTablesFromCsv(InputStream in, String tableName) {
dataProcessor.process();
}

protected void importTablesFromCsvDquote(InputStream in, String tableName) {
DefaultDatabaseWriter writer = new DefaultDatabaseWriter(symmetricPlatform, buildDatabaseWriterSettings());
Table table = writer.getPlatform(tableName).readTableFromDatabase(catalog, schema, tableName);
if (table == null) {
throw new RuntimeException("Unable to find table '" + tableName + "' in the database.");
}

CsvDquoteDataReader reader = new CsvDquoteDataReader(BinaryEncoding.HEX, table.getCatalog(),
table.getSchema(), table.getName(), in);
DataProcessor dataProcessor = new DataProcessor(reader, writer, "import");
dataProcessor.process();
}

protected void importTablesFromXml(InputStream in) {
XmlDataReader reader = new XmlDataReader(in);
DefaultDatabaseWriter writer = new DefaultDatabaseWriter(symmetricPlatform, buildDatabaseWriterSettings());
@@ -0,0 +1,63 @@
/**
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU General Public License, version 3.0 (GPLv3)
* (the "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU General Public License,
* version 3.0 (GPLv3) along with this library; if not, see
* <http://www.gnu.org/licenses/>.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jumpmind.symmetric.io.data.reader;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;

import org.jumpmind.db.model.Column;
import org.jumpmind.db.util.BinaryEncoding;
import org.jumpmind.exception.IoException;
import org.jumpmind.symmetric.io.data.CsvUtils;

/**
* Read CSV formatted data for a single table. Requires that the column names be
* the header of the CSV.
*/
public class CsvDquoteDataReader extends CsvTableDataReader {

public CsvDquoteDataReader(BinaryEncoding binaryEncoding, String catalogName, String schemaName,
String tableName, InputStream is) {
super(binaryEncoding, catalogName, schemaName, tableName, is);
}

public CsvDquoteDataReader(BinaryEncoding binaryEncoding, String catalogName, String schemaName,
String tableName, Reader reader) {
super(binaryEncoding, catalogName, schemaName, tableName, reader);
}

@Override
protected void init() {
try {
this.csvReader = CsvUtils.getCsvReaderDquote(reader);
this.csvReader.setUseComments(true);
this.csvReader.readHeaders();
String[] columnNames = this.csvReader.getHeaders();
for (String columnName : columnNames) {
table.addColumn(new Column(columnName));
}
} catch (IOException e) {
throw new IoException(e);
}
}

}
@@ -58,6 +58,7 @@
import com.vaadin.ui.VerticalLayout;
import com.vaadin.ui.themes.ValoTheme;

@SuppressWarnings("deprecation")
public class DbExportDialog extends ResizableWindow {

private static final String EXPORT_TO_THE_SQL_EDITOR = "Export to the SQL Editor";
@@ -68,6 +69,10 @@

final Logger log = LoggerFactory.getLogger(getClass());

enum DbExportFormat {
SQL, XML, CSV, SYM_XML, CSV_DQUOTE;
}

private AbstractSelect formatSelect;

private AbstractSelect compatibilitySelect;
@@ -280,6 +285,7 @@ public void valueChange(ValueChangeEvent event) {
quotedIdentifiers.setEnabled(true);
break;
case CSV:
case CSV_DQUOTE:
compatibilitySelect.setEnabled(false);
compatibilitySelect.setNullSelectionAllowed(true);
compatibilitySelect.setValue(null);
@@ -472,7 +478,10 @@ public Resource getFileDownloadResource() {

private StreamResource createResource() {

final String format = (String) formatSelect.getValue().toString();
String format = (String) formatSelect.getValue().toString();
if (format.equals("CSV_DQUOTE")) {
format = "CSV";
}
StreamSource ss = new StreamSource() {
private static final long serialVersionUID = 1L;

@@ -501,11 +510,4 @@ public InputStream getStream() {
"table-export-%s." + format.toLowerCase(), datetime));
return sr;
}

enum DbExportFormat {

SQL, XML, CSV, SYM_XML;

}

}
@@ -68,7 +68,7 @@
final Logger log = LoggerFactory.getLogger(getClass());

enum DbImportFormat {
SQL, XML, CSV, SYM_XML;
SQL, XML, CSV, SYM_XML, CSV_DQUOTE;
}

private Set<Table> selectedTablesSet;
@@ -157,6 +157,7 @@ protected void createImportLayout() {
alterCase.setEnabled(true);
break;
case CSV:
case CSV_DQUOTE:
listOfTablesSelect.setEnabled(true);
alter.setEnabled(false);
alterCase.setEnabled(false);
@@ -296,7 +297,7 @@ protected void deleteFileAndResource() {
}

protected void doDbImport() throws FileNotFoundException {
if (format.toString().equals("CSV")) {
if (format.toString().equals("CSV") || format.toString().equals("CSV_DQUOTE")) {
dbImport.importTables(new BufferedInputStream(new FileInputStream(file)),
listOfTablesSelect.getValue().toString());
} else {
@@ -323,7 +324,7 @@ protected void createDbImport() {
protected boolean importButtonEnable() {
if (formatSelect.getValue() != null) {
if (!commitField.getValue().equals("")) {
if (formatSelect.getValue().toString().equals("CSV")) {
if (formatSelect.getValue().toString().equals("CSV") || formatSelect.getValue().toString().equals("CSV_DQUOTE")) {
if (listOfTablesSelect.getValue() != null) {
return true;
}

0 comments on commit 9e85e2d

Please sign in to comment.
You can’t perform that action at this time.