TEIIDDES-1540 Handle Non-UTF-8 Multi-byte characters imported from cs…

…v file
Teiid-Designer · Feb 5, 2013 · 117dee7 · 117dee7
1 parent 24bc298
commit 117dee7
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 13 deletions.
diff --git a/...ation.ui/src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataFileInfo.java b/...ation.ui/src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataFileInfo.java
@@ -9,11 +9,11 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.StringTokenizer;
-
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
 import org.teiid.core.designer.util.CoreArgCheck;
@@ -159,7 +159,9 @@ private static String getString( final String id ) {
 	private Collection<TeiidColumnInfo> columnInfoList;
 
 	private boolean ignoreReload = false;
-
+
+	private String charset = "UTF-8";  //$NON-NLS-1$
+
 	/**
 	 * 
 	 * @param dataFile the Teiid-formatted data file
@@ -171,6 +173,22 @@ public TeiidMetadataFileInfo(File dataFile) {
 		initialize();
 	}
 
+	/**
+	 * 
+	 * @param dataFile the Teiid-formatted data file
+	 * @param charset charset
+	 */
+	public TeiidMetadataFileInfo(File dataFile, String charset) {
+	    super(dataFile, true);
+
+	    CoreArgCheck.isNotNull(dataFile, "dataFile is null"); //$NON-NLS-1$
+	    if(charset != null && charset.length() > 0) {
+	        this.charset = charset;
+	    }
+
+	    initialize();
+	}
+
 	/**
 	 * 
 	 * @param info the data file info object
@@ -363,13 +381,15 @@ private void loadHeader() {
 		Collection<String> lines = new ArrayList<String>(7);
 
         if(getDataFile() != null && getDataFile().exists()){
-            FileReader fr=null;
+            FileInputStream stream = null;
+            InputStreamReader reader = null;
             BufferedReader in=null;
 
             try{
             	int iLines = 0;
-                fr=new FileReader(getDataFile());
-                in = new BufferedReader(fr);
+                stream = new FileInputStream(getDataFile());
+                reader = new InputStreamReader(stream, this.charset);
+                in = new BufferedReader(reader);                
                 String str;
                 while ((str = in.readLine()) != null) {
                 	Collection<String> strings = getRowsFromLine(str);
@@ -387,12 +407,14 @@ private void loadHeader() {
             }
             finally{
                 try{
-                    fr.close();
+                    stream.close();
+                }catch(java.io.IOException e){}
+                try{
+                    reader.close();
                 }catch(java.io.IOException e){}
                 try{
                     in.close();
                 }catch(java.io.IOException e){}
-
             }
         }
 
@@ -978,4 +1000,9 @@ public String getSqlString(String relationalModelName) {
     	return finalSQLString;
 
     }
+
+    public String getCharset() {
+        return this.charset;
+    }
+
 }
diff --git a/.../src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataImportFormatPage.java b/.../src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataImportFormatPage.java
@@ -177,6 +177,12 @@ private void setThisPageComplete(String message, int severity) {
 	private void synchronizeUI() {
 		synchronizing = true;
 
+		String charset = this.info.getFileInfo(this.dataFileInfo.getDataFile()).getCharset();
+		if (!charset.equals(this.dataFileInfo.getCharset())) {
+		    this.dataFileInfo = this.info.getFileInfo(this.dataFileInfo.getDataFile());
+		    loadFileContentsViewers();
+		}
+
 		selectedFileText.setText(dataFileInfo.getDataFile().getName());
 
 		boolean isDelimitedOption = this.dataFileInfo.doUseDelimitedColumns();

diff --git a/.../src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataImportSourcePage.java b/.../src/org/teiid/designer/transformation/ui/wizards/file/TeiidMetadataImportSourcePage.java
@@ -132,14 +132,14 @@ public class TeiidMetadataImportSourcePage extends AbstractWizardPage implements
 	private static final String INCLCOLUMNNAME = "INCLCOLUMNNAME"; //$NON-NLS-1$
 	private static final String VALUE_YES = "YES"; //$NON-NLS-1$
 	private static final String VALUE_COMMA = "COMMA"; //$NON-NLS-1$
+	private static final String CHARSET = "CHARSET"; //$NON-NLS-1$
 
 	private static final String EMPTY_STRING = ""; //$NON-NLS-1$
 	private static final String DOT_XML = ".XML"; //$NON-NLS-1$
 	private static final String DOT_TXT = ".TXT"; //$NON-NLS-1$
 	private static final String DOT_CSV = ".CSV"; //$NON-NLS-1$
 	private static final String DOT_XML_LOWER = ".xml"; //$NON-NLS-1$
 
-	private static final String DEFAULT_EXTENSION = ".xmi"; //$NON-NLS-1$
 	private static final String GET_TEXT_FILES = "getTextFiles"; //$NON-NLS-1$
 
 	private static final String ODA_FLAT_FILE_ID = "org.eclipse.datatools.connectivity.oda.flatfile"; //$NON-NLS-1$
@@ -185,13 +185,16 @@ private static String getString(final String id) {
     IConnectionInfoHelper connectionInfoHelper;
 
 	/**
+	 * @param info the TeiidMetadataImportInfo
 	 * @since 4.0
 	 */
 	public TeiidMetadataImportSourcePage(TeiidMetadataImportInfo info) {
 		this(null, info);
 	}
 
 	/**
+	 * @param selection the selection
+	 * @param info the TeiidMetadataImportInfo
 	 * @since 4.0
 	 */
 	public TeiidMetadataImportSourcePage(Object selection,	TeiidMetadataImportInfo info) {
@@ -507,6 +510,10 @@ void profileModified() {
 						this.dataFileFolderText.setText(location);
 						this.dataFileFolderText.setToolTipText(home);
 					}
+					String charset = (String) props.get(CHARSET);
+					if (charset != null) {
+					    this.profileInfo.charset = charset;
+					}					
 					clearFileListViewer();
 					loadFileListViewer();
 
@@ -567,9 +574,9 @@ private void loadFileListViewer() {
 						if (data != null && data instanceof File) {
 							File theFile = (File) data;
 							if (!theFile.isDirectory()) {
-								if (this.info.getFileInfo(theFile) == null) {
-									TeiidMetadataFileInfo fileInfo = new TeiidMetadataFileInfo(theFile);
-									this.info.addFileInfo(fileInfo);
+								TeiidMetadataFileInfo fileInfo = this.info.getFileInfo(theFile);
+								if (fileInfo == null || !fileInfo.getCharset().equals(this.profileInfo.charset)) {
+								    this.info.addFileInfo(new TeiidMetadataFileInfo(theFile, this.profileInfo.charset));
 								}
 								this.info.validate();
 							}
@@ -663,7 +670,7 @@ private void loadFileListViewer() {
      * If the path begins with a "/", we need to strip off since this will be changed to an underscore and create an invalid model
      * name. Also, we need to remove any periods.
      * 
-     * @param newUrl
+     * @param newUrl the file url
      * @return filePath - reformatted string used for generating the new file name
      */
     public static String formatPath( URL newUrl ) {
@@ -1189,6 +1196,7 @@ class ConnectionProfileInfo  {
     	public boolean columnsInFirstLine = false;
     	public String home;
     	public String delimiterType = VALUE_COMMA;
+    	public String charset="UTF-8";  //$NON-NLS-1$
     }
 
 /**