From bb960e587d9d57fdbba3c96b5c1a467166f95016 Mon Sep 17 00:00:00 2001 From: Benjamin BONNET Date: Wed, 17 Jan 2018 10:10:30 +0100 Subject: [PATCH 1/2] SQOOP-3278:use writer instead of stream and set encoding --- .../netezza/NetezzaExternalTableExportMapper.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java index aa058d14e..625e9275e 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java @@ -18,13 +18,13 @@ package org.apache.sqoop.mapreduce.db.netezza; -import java.io.BufferedOutputStream; +import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.sql.Connection; import java.sql.SQLException; @@ -62,7 +62,7 @@ public abstract class NetezzaExternalTableExportMapper extends private DBConfiguration dbc; private File fifoFile; private Connection con; - private OutputStream recordWriter; + private BufferedWriter recordWriter; public static final Log LOG = LogFactory .getLog(NetezzaExternalTableImportMapper.class.getName()); private NetezzaJDBCStatementRunner extTableThread; @@ -218,7 +218,9 @@ private void initNetezzaExternalTableExport(Context context) extTableThread.start(); // We start the JDBC thread first in this case as we want the FIFO reader to // be running. - recordWriter = new BufferedOutputStream(new FileOutputStream(nf.getFile())); + final String encoding = conf + .get(DirectNetezzaManager.NETEZZA_TABLE_ENCODING_OPT); + recordWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(nf.getFile()),encoding)); counter.startClock(); } @@ -264,7 +266,7 @@ protected void writeTextRecord(Text record) throws IOException, String outputStr = record.toString() + "\n"; byte[] outputBytes = outputStr.getBytes("UTF-8"); counter.addBytes(outputBytes.length); - recordWriter.write(outputBytes, 0, outputBytes.length); + recordWriter.write(outputStr); } protected void writeSqoopRecord(SqoopRecord sqr) throws IOException, @@ -272,7 +274,7 @@ protected void writeSqoopRecord(SqoopRecord sqr) throws IOException, String outputStr = sqr.toString(this.outputDelimiters); byte[] outputBytes = outputStr.getBytes("UTF-8"); counter.addBytes(outputBytes.length); - recordWriter.write(outputBytes, 0, outputBytes.length); + recordWriter.write(outputStr); } } From e65d324d001d538c2f9bdf946aa7865e879f4222 Mon Sep 17 00:00:00 2001 From: Benjamin BONNET Date: Wed, 17 Jan 2018 10:11:19 +0100 Subject: [PATCH 2/2] remove useless imports --- .../db/netezza/NetezzaExternalTableExportMapper.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java index 625e9275e..41a2ecdda 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableExportMapper.java @@ -20,21 +20,15 @@ import java.io.BufferedWriter; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.sql.Connection; import java.sql.SQLException; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.sqoop.io.NamedFifo;