From e97000151bfd781f9e63554fd989e86656790023 Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Wed, 26 Dec 2018 18:30:24 +0300 Subject: [PATCH 1/5] IGNITE-10732 Force -Dfile.encoding=UTF-8 in linux/windows batch runners, C# and C++. --- bin/include/parseargs.sh | 10 ++++++++++ .../startup/cmdline/CommandLineTransformer.java | 10 +++++++++- .../GridCommandLineTransformerSelfTest.java | 6 +++--- modules/platforms/cpp/core/src/ignition.cpp | 17 ++++++++++++++++- .../Apache.Ignite.Core/Impl/IgniteManager.cs | 12 +++++++++--- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/bin/include/parseargs.sh b/bin/include/parseargs.sh index 3ab255eae1659..5c9d826bcf003 100755 --- a/bin/include/parseargs.sh +++ b/bin/include/parseargs.sh @@ -51,3 +51,13 @@ do esac shift done + +# +# Set 'file.encoding' to UTF-8 default if not specified otherwise +# +case "${JVM_OPTS}" in + *-Dfile.encoding=*) + ;; + *) + JVM_OPTS="${JVM_OPTS} -Dfile.encoding=UTF-8";; +esac diff --git a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java index f4aad172b05bf..e449bfa3a6075 100644 --- a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java +++ b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java @@ -204,10 +204,15 @@ private void addArgWithValue(StringBuilder sb, String arg, Object val) { * @param args Collection of unknown (from JCommander point of view) arguments. */ private void parseJvmOptionsAndSpringConfig(Iterable args) { + boolean hadFileEncoding = false; + for (String arg : args) { if (arg.startsWith(JVM_OPTION_PREFIX)) { String jvmOpt = arg.substring(JVM_OPTION_PREFIX.length()); + if (jvmOpt.startsWith("-Dfile.encoding=")) + hadFileEncoding = true; + if (!checkJVMOptionIsSupported(jvmOpt)) throw new RuntimeException(JVM_OPTION_PREFIX + " JVM parameters for Ignite batch scripts " + "with double quotes are not supported. " + @@ -222,6 +227,9 @@ private void parseJvmOptionsAndSpringConfig(Iterable args) { throw new RuntimeException("Unrecognised parameter has been found: " + arg); } } + + if (!hadFileEncoding) + jvmOptions = (jvmOptions.isEmpty() ? "" : (jvmOptions + " ")) + "-Dfile.encoding=UTF-8"; } /** @@ -234,4 +242,4 @@ private void parseJvmOptionsAndSpringConfig(Iterable args) { private boolean checkJVMOptionIsSupported(String jvmOpt) { return !(jvmOpt.contains("-XX:OnError") || jvmOpt.contains("-XX:OnOutOfMemoryError")); } -} \ No newline at end of file +} diff --git a/modules/core/src/test/java/org/apache/ignite/startup/cmdline/GridCommandLineTransformerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/startup/cmdline/GridCommandLineTransformerSelfTest.java index d08ed6bdb236d..fbcb547d96fb5 100644 --- a/modules/core/src/test/java/org/apache/ignite/startup/cmdline/GridCommandLineTransformerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/startup/cmdline/GridCommandLineTransformerSelfTest.java @@ -36,7 +36,7 @@ public class GridCommandLineTransformerSelfTest extends GridCommonAbstractTest { public void testTransformIfNoArguments() throws Exception { assertEquals( "\"INTERACTIVE=0\" \"QUIET=-DIGNITE_QUIET=true\" \"NO_PAUSE=0\" " + - "\"NO_JMX=0\" \"JVM_XOPTS=\" \"CONFIG=\"", + "\"NO_JMX=0\" \"JVM_XOPTS=-Dfile.encoding=UTF-8\" \"CONFIG=\"", CommandLineTransformer.transform()); } @@ -111,7 +111,7 @@ public void testTransformIfSeveralArgumentsWithoutDashPrefix() throws Exception public void testTransformIfOnlyPathToConfigSpecified() throws Exception { assertEquals( "\"INTERACTIVE=0\" \"QUIET=-DIGNITE_QUIET=true\" \"NO_PAUSE=0\" \"NO_JMX=0\" " + - "\"JVM_XOPTS=\" \"CONFIG=c:\\qw.xml\"", + "\"JVM_XOPTS=-Dfile.encoding=UTF-8\" \"CONFIG=c:\\qw.xml\"", CommandLineTransformer.transform("c:\\qw.xml")); } @@ -122,7 +122,7 @@ public void testTransformIfOnlyPathToConfigSpecified() throws Exception { public void testTransformIfAllSupportedArguments() throws Exception { assertEquals( "\"INTERACTIVE=1\" \"QUIET=-DIGNITE_QUIET=false\" \"NO_PAUSE=1\" \"NO_JMX=1\" " + - "\"JVM_XOPTS=-Xmx1g -Xms1m\" " + + "\"JVM_XOPTS=-Xmx1g -Xms1m -Dfile.encoding=UTF-8\" " + "\"CONFIG=\"c:\\path to\\русский каталог\"\"", CommandLineTransformer.transform("-i", "-np", "-v", "-J-Xmx1g", "-J-Xms1m", "-nojmx", "\"c:\\path to\\русский каталог\"")); diff --git a/modules/platforms/cpp/core/src/ignition.cpp b/modules/platforms/cpp/core/src/ignition.cpp index 7d90a52c77682..6a43ab11a360f 100644 --- a/modules/platforms/cpp/core/src/ignition.cpp +++ b/modules/platforms/cpp/core/src/ignition.cpp @@ -111,6 +111,10 @@ namespace ignite int idx = 0; + std::string fileEncParam = "-Dfile.encoding="; + + bool hadFileEnc = false; + // 1. Set classpath. std::string cpFull = "-Djava.class.path=" + cp; @@ -131,8 +135,19 @@ namespace ignite opts[idx++] = CopyChars(xmxStr.c_str()); // 4. Set the rest options. - for (std::list::const_iterator i = cfg.jvmOpts.begin(); i != cfg.jvmOpts.end(); ++i) + for (std::list::const_iterator i = cfg.jvmOpts.begin(); i != cfg.jvmOpts.end(); ++i) { + if (i->find(fileEncParam) == 0) + hadFileEnc = true; + opts[idx++] = CopyChars(i->c_str()); + } + + // 5. Set file.encoding. + if (!hadFileEnc) { + std::string fileEncFull = fileEncParam + "UTF-8"; + + opts[idx++] = CopyChars(fileEncFull.c_str()); + } } /** diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs index 47260a75b146c..00f48d7965b48 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs @@ -40,6 +40,9 @@ internal static class IgniteManager /** Java Command line argument: Xmx. Case sensitive. */ private const string CmdJvmMaxMemJava = "-Xmx"; + /** Java Command line argument: file.encoding. Case sensitive. */ + private const string CmdJvmFileEncoding = "-Dfile.encoding="; + /** Monitor for DLL load synchronization. */ private static readonly object SyncRoot = new object(); @@ -93,7 +96,7 @@ internal static UnmanagedCallbacks CreateJvmContext(IgniteConfiguration cfg, ILo return cbs; } } - + /// /// Memory manager attached to currently running JVM. /// @@ -133,14 +136,17 @@ private static IList GetMergedJvmOptions(IgniteConfiguration cfg) var jvmOpts = cfg.JvmOptions == null ? new List() : cfg.JvmOptions.ToList(); // JvmInitialMemoryMB / JvmMaxMemoryMB have lower priority than CMD_JVM_OPT - if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMinMemJava, StringComparison.OrdinalIgnoreCase)) && + if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMinMemJava, StringComparison.Ordinal)) && cfg.JvmInitialMemoryMb != IgniteConfiguration.DefaultJvmInitMem) jvmOpts.Add(string.Format(CultureInfo.InvariantCulture, "{0}{1}m", CmdJvmMinMemJava, cfg.JvmInitialMemoryMb)); - if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMaxMemJava, StringComparison.OrdinalIgnoreCase)) && + if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMaxMemJava, StringComparison.Ordinal)) && cfg.JvmMaxMemoryMb != IgniteConfiguration.DefaultJvmMaxMem) jvmOpts.Add(string.Format(CultureInfo.InvariantCulture, "{0}{1}m", CmdJvmMaxMemJava, cfg.JvmMaxMemoryMb)); + if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmFileEncoding, StringComparison.Ordinal))) + jvmOpts.Add(string.Format(CultureInfo.InvariantCulture, "{0}UTF-8", CmdJvmFileEncoding)); + return jvmOpts; } From bf9d038ba7e6b84ac5a6d9ffd575e9e1c70887a6 Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Sat, 29 Dec 2018 15:42:03 +0300 Subject: [PATCH 2/5] Add warning when default encoding is not UTF-8. --- .../apache/ignite/internal/IgniteKernal.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 3a3af8e72e157..9633f896741a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -29,6 +29,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.lang.reflect.Constructor; +import java.nio.charset.Charset; import java.text.DateFormat; import java.text.DecimalFormat; import java.util.ArrayList; @@ -40,6 +41,7 @@ import java.util.HashSet; import java.util.List; import java.util.ListIterator; +import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -1077,6 +1079,9 @@ public void start( gw.writeUnlock(); } + // Check whether UTF-8 is the default character encoding. + checkFileEncoding(); + // Check whether physical RAM is not exceeded. checkPhysicalRam(); @@ -1405,6 +1410,20 @@ private void validateCommon(IgniteConfiguration cfg) { A.ensure(cfg.getNetworkSendRetryCount() > 0, "cfg.getNetworkSendRetryCount() > 0"); } + /** + * Check whether UTF-8 is the default character encoding. + * Differing character encodings across cluster may lead to erratic behavior. + */ + private void checkFileEncoding() { + String encodingDisplayName = Charset.defaultCharset().displayName(Locale.ENGLISH); + + if (!"UTF-8".equals(encodingDisplayName)) { + U.quietAndWarn(log, "Default character encoding is " + encodingDisplayName + + ". Specify UTF-8 character encoding by setting -Dfile.encoding=UTF-8 JVM parameter. " + + "Differing character encodings across cluster may lead to erratic behavior."); + } + } + /** * Checks whether physical RAM is not exceeded. */ From b427a4488fc5294f389abc0ccd53d3ba1b3c1f29 Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Wed, 9 Jan 2019 13:25:26 +0300 Subject: [PATCH 3/5] Fix shell indentation --- bin/include/parseargs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/include/parseargs.sh b/bin/include/parseargs.sh index 5c9d826bcf003..1efb48edd0793 100755 --- a/bin/include/parseargs.sh +++ b/bin/include/parseargs.sh @@ -56,8 +56,8 @@ done # Set 'file.encoding' to UTF-8 default if not specified otherwise # case "${JVM_OPTS}" in - *-Dfile.encoding=*) - ;; - *) - JVM_OPTS="${JVM_OPTS} -Dfile.encoding=UTF-8";; + *-Dfile.encoding=*) + ;; + *) + JVM_OPTS="${JVM_OPTS} -Dfile.encoding=UTF-8";; esac From e2a5c1cc5cbd6128f3b46ab817980b20a0f6483b Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Wed, 9 Jan 2019 17:57:00 +0300 Subject: [PATCH 4/5] C++ fix as per @isapego --- modules/platforms/cpp/core/src/ignition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/platforms/cpp/core/src/ignition.cpp b/modules/platforms/cpp/core/src/ignition.cpp index 6a43ab11a360f..3891be1e586f4 100644 --- a/modules/platforms/cpp/core/src/ignition.cpp +++ b/modules/platforms/cpp/core/src/ignition.cpp @@ -136,7 +136,7 @@ namespace ignite // 4. Set the rest options. for (std::list::const_iterator i = cfg.jvmOpts.begin(); i != cfg.jvmOpts.end(); ++i) { - if (i->find(fileEncParam) == 0) + if (i->find(fileEncParam) != std::string::npos) hadFileEnc = true; opts[idx++] = CopyChars(i->c_str()); From 10ec21516325361f46036f52f21b70ae364405eb Mon Sep 17 00:00:00 2001 From: Ilya Kasnacheev Date: Thu, 10 Jan 2019 13:27:12 +0300 Subject: [PATCH 5/5] Revert C# to legacy behavior for compatibility reasons. --- .../platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs index 00f48d7965b48..e2a4b9f99bc87 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/IgniteManager.cs @@ -136,11 +136,11 @@ private static IList GetMergedJvmOptions(IgniteConfiguration cfg) var jvmOpts = cfg.JvmOptions == null ? new List() : cfg.JvmOptions.ToList(); // JvmInitialMemoryMB / JvmMaxMemoryMB have lower priority than CMD_JVM_OPT - if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMinMemJava, StringComparison.Ordinal)) && + if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMinMemJava, StringComparison.OrdinalIgnoreCase)) && cfg.JvmInitialMemoryMb != IgniteConfiguration.DefaultJvmInitMem) jvmOpts.Add(string.Format(CultureInfo.InvariantCulture, "{0}{1}m", CmdJvmMinMemJava, cfg.JvmInitialMemoryMb)); - if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMaxMemJava, StringComparison.Ordinal)) && + if (!jvmOpts.Any(opt => opt.StartsWith(CmdJvmMaxMemJava, StringComparison.OrdinalIgnoreCase)) && cfg.JvmMaxMemoryMb != IgniteConfiguration.DefaultJvmMaxMem) jvmOpts.Add(string.Format(CultureInfo.InvariantCulture, "{0}{1}m", CmdJvmMaxMemJava, cfg.JvmMaxMemoryMb));