Bug 582462 Character encodings for output and input files

Make some more encodings explicit. Task-Url: https://bugs.eclipse.org/bugs/show_bug.cgi?id=582260 Change-Id: I71b7def6a6a4d17cb2f83289ade5be3f6af51e91
eclipse-mat · Nov 1, 2023 · e222bdd · e222bdd
1 parent 2da241d
commit e222bdd
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 32 deletions.
diff --git a/plugins/org.eclipse.mat.report/src/org/eclipse/mat/util/ConsoleProgressListener.java b/plugins/org.eclipse.mat.report/src/org/eclipse/mat/util/ConsoleProgressListener.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2008, 2010 SAP AG.
+ * Copyright (c) 2008, 2023 SAP AG.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -13,7 +13,9 @@
 package org.eclipse.mat.util;
 
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.Charset;
 
 import org.eclipse.mat.report.internal.Messages;
 
@@ -32,7 +34,7 @@ public class ConsoleProgressListener implements IProgressListener
 
     public ConsoleProgressListener(OutputStream out)
     {
-        this(new PrintWriter(out));
+        this(new PrintWriter(new OutputStreamWriter(out, Charset.defaultCharset())));
     }
 
     public ConsoleProgressListener(PrintWriter out)

diff --git a/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/JUnit4TestRunner.java b/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/JUnit4TestRunner.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2008,2020 SAP AG and IBM Corporation.
+ * Copyright (c) 2008,2023 SAP AG and IBM Corporation.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -18,16 +18,11 @@
 import java.io.FileOutputStream;
 import java.io.PrintStream;
 import java.lang.reflect.InvocationTargetException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 
-import junit.framework.AssertionFailedError;
-import junit.framework.JUnit4TestAdapter;
-import junit.framework.Test;
-import junit.framework.TestListener;
-import junit.framework.TestResult;
-
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter;
 import org.apache.tools.ant.taskdefs.optional.junit.JUnitTest;
@@ -39,6 +34,12 @@
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleException;
 
+import junit.framework.AssertionFailedError;
+import junit.framework.JUnit4TestAdapter;
+import junit.framework.Test;
+import junit.framework.TestListener;
+import junit.framework.TestResult;
+
 public class JUnit4TestRunner implements IApplication
 {
     private String bundleName;
@@ -61,8 +62,8 @@ public Object start(IApplicationContext context) throws Exception
 
             ByteArrayOutputStream errStrm = new ByteArrayOutputStream();
             ByteArrayOutputStream outStrm = new ByteArrayOutputStream();
-            System.setOut(new PrintStream(outStrm));
-            System.setErr(new PrintStream(errStrm));
+            System.setOut(new PrintStream(outStrm, false, StandardCharsets.UTF_8.name()));
+            System.setErr(new PrintStream(errStrm, false, StandardCharsets.UTF_8.name()));
 
             long start = System.currentTimeMillis();
 
@@ -84,7 +85,7 @@ public Object start(IApplicationContext context) throws Exception
             antTest.setCounts(junitTestResult.runCount(), junitTestResult.failureCount(), junitTestResult.errorCount());
             antTest.setRunTime(System.currentTimeMillis() - start);
 
-            sendOutAndErr(new String(outStrm.toByteArray()), new String(errStrm.toByteArray()));
+            sendOutAndErr(new String(outStrm.toByteArray(), StandardCharsets.UTF_8.name()), new String(errStrm.toByteArray(), StandardCharsets.UTF_8.name()));
 
             errStrm.close();
             outStrm.close();

diff --git a/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/regression/TestApplication.java b/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/regression/TestApplication.java
@@ -26,7 +26,8 @@
 import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.io.PrintWriter;
-import com.ibm.icu.text.SimpleDateFormat;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -55,6 +56,8 @@
 import org.osgi.framework.Bundle;
 import org.xml.sax.helpers.AttributesImpl;
 
+import com.ibm.icu.text.SimpleDateFormat;
+
 public class TestApplication
 {
     private File dumpDir;
@@ -88,7 +91,7 @@ public void run()
         {
             try
             {
-                InputStreamReader isr = new InputStreamReader(is);
+                InputStreamReader isr = new InputStreamReader(is, Charset.defaultCharset());
                 BufferedReader br = new BufferedReader(isr);
                 String line = null;
                 while ((line = br.readLine()) != null)
@@ -395,7 +398,7 @@ private void generateXMLReport(List<TestSuiteResult> testResults)
         try
         {
             File resultFile = new File(dumpDir, RegTestUtils.RESULT_FILENAME);
-            PrintWriter out = new PrintWriter(resultFile);
+            PrintWriter out = new PrintWriter(resultFile, StandardCharsets.UTF_8.name());
 
             StreamResult streamResult = new StreamResult(out);
             SAXTransformerFactory tf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
@@ -537,7 +540,7 @@ private void generatePerformanceReport(List<TestSuiteResult> results) throws IOE
 
         try
         {
-            out = new PrintStream(new FileOutputStream(report));
+            out = new PrintStream(report, StandardCharsets.UTF_8.name());
 
             // add heading
             out.append("Heap Dump").append(RegTestUtils.SEPARATOR) //

diff --git a/.../org.eclipse.mat.tests/src/org/eclipse/mat/tests/regression/comparator/CSVComparator.java b/.../org.eclipse.mat.tests/src/org/eclipse/mat/tests/regression/comparator/CSVComparator.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2008,2022 SAP AG and IBM Corporation.
+ * Copyright (c) 2008,2023 SAP AG and IBM Corporation.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -15,11 +15,16 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.mat.tests.regression.Difference;
 import org.eclipse.mat.util.MessageUtil;
 
@@ -39,9 +44,26 @@ public List<Difference> compare(File baseline, File testFile) throws Exception
                             + testFile.length()));
             System.err.println(MessageUtil.format("ERROR: ({0}) Files have different lengths", testName));
         }
+        /*
+         * Currently Report CSV will have been written in the workspace encoding.
+         * Should the CSV have always been written in UTF-8 ?
+         */
+        Charset cs;
+        try
+        {
+            String encoding = ResourcesPlugin.getEncoding();
+            if (encoding != null)
+                cs = Charset.forName(encoding);
+            else
+                cs = StandardCharsets.UTF_8;
+        }
+        catch (UnsupportedCharsetException e)
+        {
+            cs = StandardCharsets.UTF_8;
+        }
         try (
-            BufferedReader baselineReader = new BufferedReader(new FileReader(baseline.getAbsolutePath()), 1024);
-            BufferedReader testFileReader = new BufferedReader(new FileReader(testFile.getAbsolutePath()), 1024);)
+            BufferedReader baselineReader = new BufferedReader(new InputStreamReader(new FileInputStream(baseline.getAbsolutePath()), cs), 1024);
+            BufferedReader testFileReader = new BufferedReader(new InputStreamReader(new FileInputStream(testFile.getAbsolutePath()), cs), 1024);)
         {
             String baseLine;
             String testLine;

diff --git a/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/snapshot/TestInstanceSizes.java b/plugins/org.eclipse.mat.tests/src/org/eclipse/mat/tests/snapshot/TestInstanceSizes.java
@@ -15,8 +15,10 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
@@ -25,13 +27,12 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.junit.Assert;
-
 import org.eclipse.mat.snapshot.ISnapshot;
 import org.eclipse.mat.snapshot.model.IClass;
 import org.eclipse.mat.snapshot.model.IPrimitiveArray;
 import org.eclipse.mat.tests.TestSnapshots;
 import org.eclipse.mat.util.MessageUtil;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -81,7 +82,7 @@ public void testHistogramSizes() throws Exception
     /**
      * Convert jmap histogram class name to MAT name See
      * {@link org.eclipse.mat.hprof.Pass1Parser}.
-     * 
+     *
      * @param className
      * @return
      */
@@ -131,12 +132,12 @@ private void doTest(File histogramFile, ISnapshot snapshot) throws Exception
 		BufferedReader in = null;
 		try
 		{
-			in = new BufferedReader(new FileReader(histogramFile));
-			
+			in = new BufferedReader(new InputStreamReader(new FileInputStream(histogramFile), StandardCharsets.UTF_8));
+
 			String line;
 			int errorCount = 0;
 			StringBuilder errorMessage = new StringBuilder();
-			
+
 			while ((line = in.readLine()) != null)
 			{
 				StringTokenizer tokenizer = new StringTokenizer(line);
@@ -149,10 +150,10 @@ private void doTest(File histogramFile, ISnapshot snapshot) throws Exception
 				String className = tokenizer.nextToken();
 //				System.out.println(className + " " + instanceSize);
 				className = fixArrayName(className);
-				
+
 				if (className.startsWith("<") || className.equals("java.lang.Class"))
 					continue;
-				
+
 				Collection<IClass> classes = snapshot.getClassesByName(className, false);
 				if (classes == null || classes.size() == 0)
 				{
@@ -182,7 +183,7 @@ private void doTest(File histogramFile, ISnapshot snapshot) throws Exception
 				    }
 				}
 			}
-			
+
 			Assert.assertEquals("Unexpected number of failures: for the following classes the instance size isn't correct\r\n" + errorMessage.toString(), expectedErrors, errorCount);
 		}
 		finally

diff --git a/plugins/org.eclipse.mat.ui/src/org/eclipse/mat/ui/internal/views/SnapshotHistoryView.java b/plugins/org.eclipse.mat.ui/src/org/eclipse/mat/ui/internal/views/SnapshotHistoryView.java
@@ -23,6 +23,7 @@
 import java.io.Reader;
 import java.io.Serializable;
 import java.nio.CharBuffer;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Pattern;
@@ -510,12 +511,14 @@ protected IStatus run(IProgressMonitor monitor)
                                         cb.append(launchCmd);
                                         int pos = cb.position();
                                         cb.append('\n');
-                                        try (Reader rd = new InputStreamReader(p.getInputStream()))
+                                        // OS encoding
+                                        try (Reader rd = new InputStreamReader(p.getInputStream(), Charset.defaultCharset()))
                                         {
                                             rd.read(cb);
                                         }
                                         cb.append('\n');
-                                        try (Reader rd = new InputStreamReader(p.getErrorStream()))
+                                        // OS encoding
+                                        try (Reader rd = new InputStreamReader(p.getErrorStream(), Charset.defaultCharset()))
                                         {
                                             rd.read(cb);
                                         }