From b240d16372f3d32ced5c0c0451daeb4751fc9cf3 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Fri, 16 Jun 2023 10:23:48 +0200 Subject: [PATCH 1/5] [hotfix] Fix license indentation --- flink-dist/src/main/resources/flink-conf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-dist/src/main/resources/flink-conf.yaml b/flink-dist/src/main/resources/flink-conf.yaml index 5d4407d5f7869..87dae7fe49920 100644 --- a/flink-dist/src/main/resources/flink-conf.yaml +++ b/flink-dist/src/main/resources/flink-conf.yaml @@ -13,7 +13,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License. ################################################################################ From a15e013d972174f5cfe3c110170feaccc388e70f Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Fri, 16 Jun 2023 16:21:25 +0200 Subject: [PATCH 2/5] [hotfix][yarn][tests] Simplify string construction --- .../clusterframework/BootstrapToolsTest.java | 84 +++++++------------ 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java index 1ad0a1f1c0906..158d97089f342 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java @@ -179,21 +179,7 @@ public void testGetTaskManagerShellCommand() { final String redirects = "1> ./logs/taskmanager.out 2> ./logs/taskmanager.err"; assertEquals( - java - + " " - + jvmmem - + "" - + // jvmOpts - "" - + // logging - " " - + mainClass - + " " - + dynamicConfigs - + " " - + basicArgs - + " " - + redirects, + String.join(" ", java, jvmmem, mainClass, dynamicConfigs, basicArgs, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -206,9 +192,7 @@ public void testGetTaskManagerShellCommand() { "")); assertEquals( - java + " " + jvmmem + "" + // jvmOpts - "" + // logging - " " + mainClass + " " + args + " " + redirects, + String.join(" ", java, jvmmem, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -222,9 +206,7 @@ public void testGetTaskManagerShellCommand() { final String krb5 = "-Djava.security.krb5.conf=krb5.conf"; assertEquals( - java + " " + jvmmem + " " + krb5 + // jvmOpts - "" + // logging - " " + mainClass + " " + args + " " + redirects, + String.join(" ", java, jvmmem, krb5, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -238,9 +220,7 @@ public void testGetTaskManagerShellCommand() { // logback only, with/out krb5 assertEquals( - java + " " + jvmmem + "" + // jvmOpts - " " + logfile + " " + logback + " " + mainClass + " " + args + " " - + redirects, + String.join(" ", java, jvmmem, logfile, logback, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -253,9 +233,7 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - java + " " + jvmmem + " " + krb5 + // jvmOpts - " " + logfile + " " + logback + " " + mainClass + " " + args + " " - + redirects, + String.join(" ", java, jvmmem, krb5, logfile, logback, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -269,9 +247,7 @@ public void testGetTaskManagerShellCommand() { // log4j, with/out krb5 assertEquals( - java + " " + jvmmem + "" + // jvmOpts - " " + logfile + " " + log4j + " " + mainClass + " " + args + " " - + redirects, + String.join(" ", java, jvmmem, logfile, log4j, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -284,9 +260,7 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - java + " " + jvmmem + " " + krb5 + // jvmOpts - " " + logfile + " " + log4j + " " + mainClass + " " + args + " " - + redirects, + String.join(" ", java, jvmmem, krb5, logfile, log4j, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -300,9 +274,7 @@ public void testGetTaskManagerShellCommand() { // logback + log4j, with/out krb5 assertEquals( - java + " " + jvmmem + "" + // jvmOpts - " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args - + " " + redirects, + String.join(" ", java, jvmmem, logfile, logback, log4j, mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -315,9 +287,9 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - java + " " + jvmmem + " " + krb5 + // jvmOpts - " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args - + " " + redirects, + String.join( + " ", java, jvmmem, krb5, logfile, logback, log4j, mainClass, args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -332,8 +304,9 @@ public void testGetTaskManagerShellCommand() { // logback + log4j, with/out krb5, different JVM opts cfg.setString(CoreOptions.FLINK_JVM_OPTIONS, jvmOpts); assertEquals( - java + " " + jvmmem + " " + jvmOpts + " " + logfile + " " + logback + " " + log4j - + " " + mainClass + " " + args + " " + redirects, + String.join( + " ", java, jvmmem, jvmOpts, logfile, logback, log4j, mainClass, args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -346,9 +319,9 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - java + " " + jvmmem + " " + jvmOpts + " " + krb5 + // jvmOpts - " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args - + " " + redirects, + String.join( + " ", java, jvmmem, jvmOpts, krb5, logfile, logback, log4j, mainClass, args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -363,8 +336,9 @@ public void testGetTaskManagerShellCommand() { // logback + log4j, with/out krb5, different JVM opts cfg.setString(CoreOptions.FLINK_TM_JVM_OPTIONS, tmJvmOpts); assertEquals( - java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + logfile + " " - + logback + " " + log4j + " " + mainClass + " " + args + " " + redirects, + String.join( + " ", java, jvmmem, jvmOpts, tmJvmOpts, logfile, logback, log4j, mainClass, + args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -377,9 +351,9 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - java + " " + jvmmem + " " + jvmOpts + " " + tmJvmOpts + " " + krb5 + // jvmOpts - " " + logfile + " " + logback + " " + log4j + " " + mainClass + " " + args - + " " + redirects, + String.join( + " ", java, jvmmem, jvmOpts, tmJvmOpts, krb5, logfile, logback, log4j, + mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -397,9 +371,9 @@ public void testGetTaskManagerShellCommand() { ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% 1 %jvmmem% 2 %jvmopts% 3 %logging% 4 %class% 5 %args% 6 %redirects%"); assertEquals( - java + " 1 " + jvmmem + " 2 " + jvmOpts + " " + tmJvmOpts + " " + krb5 + // jvmOpts - " 3 " + logfile + " " + logback + " " + log4j + " 4 " + mainClass + " 5 " - + args + " 6 " + redirects, + String.join( + " ", java, "1", jvmmem, "2", jvmOpts, tmJvmOpts, krb5, "3", logfile, + logback, log4j, "4", mainClass, "5", args, "6", redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -415,9 +389,9 @@ public void testGetTaskManagerShellCommand() { ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, "%java% %logging% %jvmopts% %jvmmem% %class% %args% %redirects%"); assertEquals( - java + " " + logfile + " " + logback + " " + log4j + " " + jvmOpts + " " + tmJvmOpts - + " " + krb5 + // jvmOpts - " " + jvmmem + " " + mainClass + " " + args + " " + redirects, + String.join( + " ", java, logfile, logback, log4j, jvmOpts, tmJvmOpts, krb5, jvmmem, + mainClass, args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, From eebfe4928b529cc84726f83337ca327c6cb8f590 Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Fri, 16 Jun 2023 14:46:30 +0200 Subject: [PATCH 3/5] [hotfix][yarn] Set -XX:+IgnoreUnrecognizedVMOptions for task managers --- .../clusterframework/BootstrapTools.java | 5 + .../clusterframework/BootstrapToolsTest.java | 186 ++++++++++++++++-- 2 files changed, 172 insertions(+), 19 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java index 6f85dde369b1a..f53b19f654f90 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.clusterframework; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; @@ -61,6 +62,9 @@ public class BootstrapTools { private static final Escaper WINDOWS_DOUBLE_QUOTE_ESCAPER = Escapers.builder().addEscape('"', "\\\"").addEscape('^', "\"^^\"").build(); + @VisibleForTesting + static final String IGNORE_UNRECOGNIZED_VM_OPTIONS = "-XX:+IgnoreUnrecognizedVMOptions"; + /** * Writes a Flink YAML config file from a Flink Configuration object. * @@ -191,6 +195,7 @@ public static String getTaskManagerShellCommand( if (flinkConfig.getString(CoreOptions.FLINK_TM_JVM_OPTIONS).length() > 0) { javaOpts += " " + flinkConfig.getString(CoreOptions.FLINK_TM_JVM_OPTIONS); } + javaOpts += " " + IGNORE_UNRECOGNIZED_VM_OPTIONS; // krb5.conf file will be available as local resource in JM/TM container if (hasKrb5) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java index 158d97089f342..2012494d5bbcd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/BootstrapToolsTest.java @@ -179,7 +179,15 @@ public void testGetTaskManagerShellCommand() { final String redirects = "1> ./logs/taskmanager.out 2> ./logs/taskmanager.err"; assertEquals( - String.join(" ", java, jvmmem, mainClass, dynamicConfigs, basicArgs, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + mainClass, + dynamicConfigs, + basicArgs, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -192,7 +200,14 @@ public void testGetTaskManagerShellCommand() { "")); assertEquals( - String.join(" ", java, jvmmem, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -206,7 +221,15 @@ public void testGetTaskManagerShellCommand() { final String krb5 = "-Djava.security.krb5.conf=krb5.conf"; assertEquals( - String.join(" ", java, jvmmem, krb5, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -220,7 +243,16 @@ public void testGetTaskManagerShellCommand() { // logback only, with/out krb5 assertEquals( - String.join(" ", java, jvmmem, logfile, logback, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + logfile, + logback, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -233,7 +265,17 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - String.join(" ", java, jvmmem, krb5, logfile, logback, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + logfile, + logback, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -247,7 +289,16 @@ public void testGetTaskManagerShellCommand() { // log4j, with/out krb5 assertEquals( - String.join(" ", java, jvmmem, logfile, log4j, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + logfile, + log4j, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -260,7 +311,17 @@ public void testGetTaskManagerShellCommand() { mainArgs)); assertEquals( - String.join(" ", java, jvmmem, krb5, logfile, log4j, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + logfile, + log4j, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -274,7 +335,17 @@ public void testGetTaskManagerShellCommand() { // logback + log4j, with/out krb5 assertEquals( - String.join(" ", java, jvmmem, logfile, logback, log4j, mainClass, args, redirects), + String.join( + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + logfile, + logback, + log4j, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -288,7 +359,16 @@ public void testGetTaskManagerShellCommand() { assertEquals( String.join( - " ", java, jvmmem, krb5, logfile, logback, log4j, mainClass, args, + " ", + java, + jvmmem, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + logfile, + logback, + log4j, + mainClass, + args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, @@ -305,7 +385,16 @@ public void testGetTaskManagerShellCommand() { cfg.setString(CoreOptions.FLINK_JVM_OPTIONS, jvmOpts); assertEquals( String.join( - " ", java, jvmmem, jvmOpts, logfile, logback, log4j, mainClass, args, + " ", + java, + jvmmem, + jvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + logfile, + logback, + log4j, + mainClass, + args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, @@ -320,7 +409,17 @@ public void testGetTaskManagerShellCommand() { assertEquals( String.join( - " ", java, jvmmem, jvmOpts, krb5, logfile, logback, log4j, mainClass, args, + " ", + java, + jvmmem, + jvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + logfile, + logback, + log4j, + mainClass, + args, redirects), BootstrapTools.getTaskManagerShellCommand( cfg, @@ -337,8 +436,18 @@ public void testGetTaskManagerShellCommand() { cfg.setString(CoreOptions.FLINK_TM_JVM_OPTIONS, tmJvmOpts); assertEquals( String.join( - " ", java, jvmmem, jvmOpts, tmJvmOpts, logfile, logback, log4j, mainClass, - args, redirects), + " ", + java, + jvmmem, + jvmOpts, + tmJvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + logfile, + logback, + log4j, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -352,8 +461,19 @@ public void testGetTaskManagerShellCommand() { assertEquals( String.join( - " ", java, jvmmem, jvmOpts, tmJvmOpts, krb5, logfile, logback, log4j, - mainClass, args, redirects), + " ", + java, + jvmmem, + jvmOpts, + tmJvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + logfile, + logback, + log4j, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -372,8 +492,25 @@ public void testGetTaskManagerShellCommand() { "%java% 1 %jvmmem% 2 %jvmopts% 3 %logging% 4 %class% 5 %args% 6 %redirects%"); assertEquals( String.join( - " ", java, "1", jvmmem, "2", jvmOpts, tmJvmOpts, krb5, "3", logfile, - logback, log4j, "4", mainClass, "5", args, "6", redirects), + " ", + java, + "1", + jvmmem, + "2", + jvmOpts, + tmJvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + "3", + logfile, + logback, + log4j, + "4", + mainClass, + "5", + args, + "6", + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, @@ -390,8 +527,19 @@ public void testGetTaskManagerShellCommand() { "%java% %logging% %jvmopts% %jvmmem% %class% %args% %redirects%"); assertEquals( String.join( - " ", java, logfile, logback, log4j, jvmOpts, tmJvmOpts, krb5, jvmmem, - mainClass, args, redirects), + " ", + java, + logfile, + logback, + log4j, + jvmOpts, + tmJvmOpts, + BootstrapTools.IGNORE_UNRECOGNIZED_VM_OPTIONS, + krb5, + jvmmem, + mainClass, + args, + redirects), BootstrapTools.getTaskManagerShellCommand( cfg, containeredParams, From fb24fb7ff1b24c5f608d320eb2dbfeec4c033cca Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Fri, 16 Jun 2023 16:31:24 +0200 Subject: [PATCH 4/5] [hotfix][build] Set -XX:+IgnoreUnrecognizedVMOptions in surefire VMs --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f818c5a02d802..b38414684ddfc 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ under the License. 2 4 true - -XX:+UseG1GC -Xms256m + -XX:+UseG1GC -Xms256m -XX:+IgnoreUnrecognizedVMOptions 16.1 2.13.4 true From 3ab33d1d1e4171c2e36f16c46f95d6c4d29da49c Mon Sep 17 00:00:00 2001 From: Chesnay Schepler Date: Wed, 17 Nov 2021 12:42:58 +0100 Subject: [PATCH 5/5] [FLINK-25002][build] Add java 17 add-opens/add-exports JVM arguments --- .mvn/jvm.config | 6 ++++++ flink-clients/pom.xml | 7 +++++++ flink-connectors/flink-connector-base/pom.xml | 7 +++++++ flink-connectors/flink-connector-files/pom.xml | 7 +++++++ flink-connectors/flink-connector-kafka/pom.xml | 6 ++++++ flink-core/pom.xml | 18 ++++++++++++++++++ flink-dist/src/main/resources/flink-conf.yaml | 3 +++ .../flink-queryable-state-test/pom.xml | 3 +++ .../test_local_recovery_and_scheduling.sh | 2 ++ flink-examples/flink-examples-table/pom.xml | 7 +++++++ flink-filesystems/flink-hadoop-fs/pom.xml | 11 +++++++++++ flink-formats/flink-avro/pom.xml | 13 +++++++++++++ flink-formats/flink-csv/pom.xml | 7 +++++++ flink-formats/flink-json/pom.xml | 7 +++++++ flink-formats/flink-orc/pom.xml | 9 +++++++++ flink-formats/flink-parquet/pom.xml | 7 +++++++ flink-fs-tests/pom.xml | 7 +++++++ flink-java/pom.xml | 9 +++++++++ flink-kubernetes/pom.xml | 4 ++++ flink-libraries/flink-cep/pom.xml | 7 +++++++ .../flink-state-processing-api/pom.xml | 9 +++++++++ flink-metrics/flink-metrics-jmx/pom.xml | 7 +++++++ flink-python/pom.xml | 9 +++++++++ flink-python/pyflink/pyflink_gateway_server.py | 6 ++++-- .../flink-queryable-state-runtime/pom.xml | 7 +++++++ flink-runtime/pom.xml | 14 ++++++++++++++ .../runtime/testutils/TestJvmProcess.java | 5 +++++ flink-scala/pom.xml | 7 +++++++ .../flink-statebackend-changelog/pom.xml | 7 +++++++ .../flink-statebackend-rocksdb/pom.xml | 7 +++++++ flink-streaming-java/pom.xml | 9 +++++++++ flink-streaming-scala/pom.xml | 7 +++++++ flink-table/flink-sql-client/pom.xml | 9 +++++++++ flink-table/flink-sql-gateway/pom.xml | 9 +++++++++ flink-table/flink-sql-jdbc-driver/pom.xml | 7 +++++++ flink-table/flink-table-planner/pom.xml | 17 +++++++++++++++++ flink-table/flink-table-runtime/pom.xml | 9 +++++++++ .../flink/test/util/TestProcessBuilder.java | 7 +++++++ flink-tests/pom.xml | 9 +++++++++ flink-yarn-tests/pom.xml | 8 ++++++++ flink-yarn/pom.xml | 4 ++++ pom.xml | 10 +++++++++- 42 files changed, 327 insertions(+), 3 deletions(-) diff --git a/.mvn/jvm.config b/.mvn/jvm.config index ac69200fd2b3b..7a099aa85a455 100644 --- a/.mvn/jvm.config +++ b/.mvn/jvm.config @@ -1 +1,7 @@ -XX:+IgnoreUnrecognizedVMOptions +--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED +--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED +--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED +--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml index 05253fcb43725..2926300301237 100644 --- a/flink-clients/pom.xml +++ b/flink-clients/pom.xml @@ -34,6 +34,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-connectors/flink-connector-base/pom.xml b/flink-connectors/flink-connector-base/pom.xml index cb73db400870e..78a1d650276b5 100644 --- a/flink-connectors/flink-connector-base/pom.xml +++ b/flink-connectors/flink-connector-base/pom.xml @@ -33,6 +33,13 @@ jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml index 22931f2094567..ba4fdffb1a4c2 100644 --- a/flink-connectors/flink-connector-files/pom.xml +++ b/flink-connectors/flink-connector-files/pom.xml @@ -34,6 +34,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-connectors/flink-connector-kafka/pom.xml b/flink-connectors/flink-connector-kafka/pom.xml index 607af881a5104..4ddb73007b60d 100644 --- a/flink-connectors/flink-connector-kafka/pom.xml +++ b/flink-connectors/flink-connector-kafka/pom.xml @@ -36,6 +36,12 @@ under the License. 3.2.3 + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED + diff --git a/flink-core/pom.xml b/flink-core/pom.xml index 5ba7b3496395c..b84e666bb3650 100644 --- a/flink-core/pom.xml +++ b/flink-core/pom.xml @@ -33,6 +33,24 @@ under the License. jar + + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-dist/src/main/resources/flink-conf.yaml b/flink-dist/src/main/resources/flink-conf.yaml index 87dae7fe49920..0134a8645e698 100644 --- a/flink-dist/src/main/resources/flink-conf.yaml +++ b/flink-dist/src/main/resources/flink-conf.yaml @@ -16,6 +16,9 @@ # limitations under the License. ################################################################################ +# These parameters are required for Java 17 support. +# They can be safely removed when using Java 8/11. +env.java.opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED #============================================================================== # Common diff --git a/flink-end-to-end-tests/flink-queryable-state-test/pom.xml b/flink-end-to-end-tests/flink-queryable-state-test/pom.xml index ebe055351e271..38cf9a628fa99 100644 --- a/flink-end-to-end-tests/flink-queryable-state-test/pom.xml +++ b/flink-end-to-end-tests/flink-queryable-state-test/pom.xml @@ -107,6 +107,9 @@ org.apache.flink.streaming.tests.queryablestate.QsStateClient + + java.base/java.util + diff --git a/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh b/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh index 0d38beb30e12c..dc6ac2e7426f0 100755 --- a/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh +++ b/flink-end-to-end-tests/test-scripts/test_local_recovery_and_scheduling.sh @@ -73,6 +73,8 @@ function run_local_recovery_test { TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-local-recovery-and-allocation-test/target/StickyAllocationAndLocalRecoveryTestJob.jar # configure for HA create_ha_config + # required for PID business in StickyAllocationAndLocalRecoveryTestJob + set_config_key env.java.opts.taskmanager "--add-opens=java.management/sun.management=ALL-UNNAMED" # Enable debug logging sed -i -e 's/rootLogger.level = .*/rootLogger.level = DEBUG/' "$FLINK_DIR/conf/log4j.properties" diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml index 106a8729d123f..0d7503bd19c5d 100644 --- a/flink-examples/flink-examples-table/pom.xml +++ b/flink-examples/flink-examples-table/pom.xml @@ -32,6 +32,13 @@ under the License. flink-examples-table_${scala.binary.version} jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml index a304bc835a69c..441daf4f0b4e2 100644 --- a/flink-filesystems/flink-hadoop-fs/pom.xml +++ b/flink-filesystems/flink-hadoop-fs/pom.xml @@ -31,6 +31,17 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-formats/flink-avro/pom.xml b/flink-formats/flink-avro/pom.xml index 0c8f0f26b5534..a6186b7f60da0 100644 --- a/flink-formats/flink-avro/pom.xml +++ b/flink-formats/flink-avro/pom.xml @@ -33,6 +33,19 @@ under the License. jar + + --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-formats/flink-csv/pom.xml b/flink-formats/flink-csv/pom.xml index 6fbc6fc146981..63299d7dd3e4b 100644 --- a/flink-formats/flink-csv/pom.xml +++ b/flink-formats/flink-csv/pom.xml @@ -33,6 +33,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml index 35cc4149883b7..3d6ebec629923 100644 --- a/flink-formats/flink-json/pom.xml +++ b/flink-formats/flink-json/pom.xml @@ -33,6 +33,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml index b022194648333..c46b9b205ddae 100644 --- a/flink-formats/flink-orc/pom.xml +++ b/flink-formats/flink-orc/pom.xml @@ -33,6 +33,15 @@ under the License. jar + + --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml index 3a0475cbdfdff..7533b17a55b3b 100644 --- a/flink-formats/flink-parquet/pom.xml +++ b/flink-formats/flink-parquet/pom.xml @@ -33,6 +33,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml index 5cc60a47231bd..84378ac39f53a 100644 --- a/flink-fs-tests/pom.xml +++ b/flink-fs-tests/pom.xml @@ -31,6 +31,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-java/pom.xml b/flink-java/pom.xml index caf89a97bab85..8763de59e1f4b 100644 --- a/flink-java/pom.xml +++ b/flink-java/pom.xml @@ -33,6 +33,15 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-kubernetes/pom.xml b/flink-kubernetes/pom.xml index 92307a5edead3..7faf4a71f8b49 100644 --- a/flink-kubernetes/pom.xml +++ b/flink-kubernetes/pom.xml @@ -32,6 +32,10 @@ under the License. 6.6.2 + --add-opens=java.base/java.util=ALL-UNNAMED + diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml index 4f8c4858414f7..46b8f034a8244 100644 --- a/flink-libraries/flink-cep/pom.xml +++ b/flink-libraries/flink-cep/pom.xml @@ -33,6 +33,13 @@ under the License. Flink : Libraries : CEP jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-libraries/flink-state-processing-api/pom.xml b/flink-libraries/flink-state-processing-api/pom.xml index 86f00f9d879e6..030ce3fd685e9 100644 --- a/flink-libraries/flink-state-processing-api/pom.xml +++ b/flink-libraries/flink-state-processing-api/pom.xml @@ -34,6 +34,15 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED + + + diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml index e4afd54c0676f..0da8376e283ef 100644 --- a/flink-metrics/flink-metrics-jmx/pom.xml +++ b/flink-metrics/flink-metrics-jmx/pom.xml @@ -31,6 +31,13 @@ under the License. flink-metrics-jmx Flink : Metrics : JMX + + --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-python/pom.xml b/flink-python/pom.xml index a027e698877b1..f9fc4ba728c6d 100644 --- a/flink-python/pom.xml +++ b/flink-python/pom.xml @@ -35,6 +35,15 @@ under the License. 5.0.0 + 3.2.3 + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED + diff --git a/flink-python/pyflink/pyflink_gateway_server.py b/flink-python/pyflink/pyflink_gateway_server.py index 63cb142e193cf..07bba321bde0b 100644 --- a/flink-python/pyflink/pyflink_gateway_server.py +++ b/flink-python/pyflink/pyflink_gateway_server.py @@ -254,8 +254,10 @@ def launch_gateway_server_process(env, args): [construct_flink_classpath(env), construct_hadoop_classpath(env)]) if "FLINK_TESTING" in env: classpath = os.pathsep.join([classpath, construct_test_classpath()]) - command = [java_executable, jvm_args, "-XX:+IgnoreUnrecognizedVMOptions"] + jvm_opts \ - + log_settings + ["-cp", classpath, program_args.main_class] + program_args.other_args + command = [java_executable, jvm_args, "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=jdk.proxy2/jdk.proxy2=ALL-UNNAMED"] \ + + jvm_opts + log_settings \ + + ["-cp", classpath, program_args.main_class] + program_args.other_args else: command = [os.path.join(env["FLINK_BIN_DIR"], "flink"), "run"] + program_args.other_args \ + ["-c", program_args.main_class] diff --git a/flink-queryable-state/flink-queryable-state-runtime/pom.xml b/flink-queryable-state/flink-queryable-state-runtime/pom.xml index 9775c67485a20..2b9483e96b4e5 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/pom.xml +++ b/flink-queryable-state/flink-queryable-state-runtime/pom.xml @@ -33,6 +33,13 @@ under the License. Flink : Queryable state : Runtime jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index fb8bb57621a56..cfef09466f271 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -33,6 +33,20 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + + + diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java index 3315cc3577d97..713603987cc50 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestJvmProcess.java @@ -150,6 +150,11 @@ public void startProcess() throws IOException { cmd = ArrayUtils.addAll(cmd, jvmArgs); } + final String moduleConfig = System.getProperty("surefire.module.config"); + if (moduleConfig != null) { + cmd = ArrayUtils.addAll(cmd, moduleConfig.split(" ")); + } + synchronized (createDestroyLock) { checkState(process == null, "process already started"); diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml index 2ae22c36b8307..105e2e7029ad9 100644 --- a/flink-scala/pom.xml +++ b/flink-scala/pom.xml @@ -31,6 +31,13 @@ under the License. Flink : Scala jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-state-backends/flink-statebackend-changelog/pom.xml b/flink-state-backends/flink-statebackend-changelog/pom.xml index 4ac5074befcc7..a969fbb041402 100644 --- a/flink-state-backends/flink-statebackend-changelog/pom.xml +++ b/flink-state-backends/flink-statebackend-changelog/pom.xml @@ -35,6 +35,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-state-backends/flink-statebackend-rocksdb/pom.xml b/flink-state-backends/flink-statebackend-rocksdb/pom.xml index 4f1bd7fba3267..d50bfcd29d8b7 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/pom.xml +++ b/flink-state-backends/flink-statebackend-rocksdb/pom.xml @@ -35,6 +35,13 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml index b0c215c2e3b47..926f4dcaa3a03 100644 --- a/flink-streaming-java/pom.xml +++ b/flink-streaming-java/pom.xml @@ -33,6 +33,15 @@ under the License. jar + + --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml index 58b08d1b47320..26d89ea65f6ad 100644 --- a/flink-streaming-scala/pom.xml +++ b/flink-streaming-scala/pom.xml @@ -31,6 +31,13 @@ under the License. Flink : Streaming Scala jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-table/flink-sql-client/pom.xml b/flink-table/flink-sql-client/pom.xml index 3422a2cf9ba94..f4d8093c1a16d 100644 --- a/flink-table/flink-sql-client/pom.xml +++ b/flink-table/flink-sql-client/pom.xml @@ -35,6 +35,15 @@ under the License. jar + + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED + + + diff --git a/flink-table/flink-sql-gateway/pom.xml b/flink-table/flink-sql-gateway/pom.xml index f61b4005e4bd3..35efa59962374 100644 --- a/flink-table/flink-sql-gateway/pom.xml +++ b/flink-table/flink-sql-gateway/pom.xml @@ -34,6 +34,15 @@ jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + + + diff --git a/flink-table/flink-sql-jdbc-driver/pom.xml b/flink-table/flink-sql-jdbc-driver/pom.xml index fc6b7b285c899..32ed400537fc7 100644 --- a/flink-table/flink-sql-jdbc-driver/pom.xml +++ b/flink-table/flink-sql-jdbc-driver/pom.xml @@ -34,6 +34,13 @@ jar + + --add-opens=java.base/java.util=ALL-UNNAMED + + + org.apache.flink diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index 51255452db53b..70d8157a7e523 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -37,6 +37,23 @@ under the License. jar + + --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED + + + diff --git a/flink-table/flink-table-runtime/pom.xml b/flink-table/flink-table-runtime/pom.xml index a50823b22c913..a043e6518b95d 100644 --- a/flink-table/flink-table-runtime/pom.xml +++ b/flink-table/flink-table-runtime/pom.xml @@ -35,6 +35,15 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + + + diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestProcessBuilder.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestProcessBuilder.java index f015aa88d6323..2fa4b8a8b39ce 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestProcessBuilder.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestProcessBuilder.java @@ -63,6 +63,13 @@ public TestProcessBuilder(String mainClass) throws IOException { jvmArgs.add(getCurrentClasspath()); jvmArgs.add("-XX:+IgnoreUnrecognizedVMOptions"); + final String moduleConfig = System.getProperty("surefire.module.config"); + if (moduleConfig != null) { + for (String moduleArg : moduleConfig.split(" ")) { + addJvmArg(moduleArg); + } + } + this.mainClass = mainClass; } diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 9b9d1ca791a7e..60353f88b0179 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -34,6 +34,15 @@ under the License. jar + + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED + + + diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index da70ef74b3550..2b5ed9c1103e7 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -38,6 +38,14 @@ under the License. true + --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED + diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml index 3268b7acd5078..248fe5bd8e1bb 100644 --- a/flink-yarn/pom.xml +++ b/flink-yarn/pom.xml @@ -34,6 +34,10 @@ under the License. 1.12.319 + --add-opens=java.base/java.util=ALL-UNNAMED + diff --git a/pom.xml b/pom.xml index b38414684ddfc..4688d5f48f47b 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ under the License. 2 4 true - -XX:+UseG1GC -Xms256m -XX:+IgnoreUnrecognizedVMOptions + -XX:+UseG1GC -Xms256m -XX:+IgnoreUnrecognizedVMOptions ${surefire.module.config} 16.1 2.13.4 true @@ -196,6 +196,12 @@ under the License. */ + + + **/*Test.* @@ -1647,6 +1653,8 @@ under the License. unset: don't alter the configuration --> random + + ${surefire.module.config} ${project.basedir} ${test.randomization.seed}