tuner to update jvm version specific options (#1060)

Co-authored-by: ayushis <ayushis@netflix.com>
Netflix · Jun 26, 2023 · 01bf39d · 01bf39d
1 parent e4a6a13
commit 01bf39d
Show file tree

Hide file tree

Showing 7 changed files with 177 additions and 83 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java
@@ -49,6 +49,13 @@ default String getJVMOptionsFileLocation() {
         return getCassHome() + "/conf/jvm-server.options";
     }
 
+    /**
+     * @return Path to jvm8-server.options file. This is used to pass JVM GC options to Cassandra.
+     */
+    default String getJVMVersionOptionsFileLocation() {
+        return getCassHome() + "/conf/jvm8-server.options";
+    }
+
     /**
      * @return Type of garbage collection mechanism to use for Cassandra. Supported values are
      *     CMS,G1GC

diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java
@@ -52,13 +52,16 @@ public JVMOptionsTuner(IConfiguration config) {
      * @throws Exception when encountered with invalid configured GC type. {@link
      *     IConfiguration#getGCType()}
      */
-    public void updateAndSaveJVMOptions(final String outputFile) throws Exception {
-        List<String> configuredJVMOptions = updateJVMOptions();
+    public void updateAndSaveJVMOptions(final String outputFile, final String versionOutputFile)
+            throws Exception {
+        Map<String, List<String>> Options = updateJVMOptions();
 
         if (logger.isInfoEnabled()) {
             StringBuffer buffer = new StringBuffer("\n");
-            configuredJVMOptions.stream().forEach(line -> buffer.append(line).append("\n"));
-            logger.info("Updating jvm.options with following values: " + buffer.toString());
+            Options.get("configuredJVMOptions")
+                    .stream()
+                    .forEach(line -> buffer.append(line).append("\n"));
+            logger.info("Updating jvm-server.options with following values: " + buffer.toString());
         }
 
         // Verify we can write to output file and it is not directory.
@@ -67,8 +70,27 @@ public void updateAndSaveJVMOptions(final String outputFile) throws Exception {
             throw new Exception("Not enough permissions to write to file: " + outputFile);
         }
 
-        // Write jvm.options back to override defaults.
-        Files.write(new File(outputFile).toPath(), configuredJVMOptions);
+        // Write jvm-server.options back to override defaults.
+        Files.write(new File(outputFile).toPath(), Options.get("configuredJVMOptions"));
+
+        // Update jdk version specific options
+        if (logger.isInfoEnabled()) {
+            StringBuffer buffer = new StringBuffer("\n");
+            Options.get("configuredJVMVersionOptions")
+                    .stream()
+                    .forEach(line -> buffer.append(line).append("\n"));
+            logger.info("Updating jvm8-server.options with following values: " + buffer.toString());
+        }
+
+        // Verify we can write to version specific output file and it is not directory.
+        File versionFile = new File(versionOutputFile);
+        if (versionFile.exists() && !versionFile.canWrite()) {
+            throw new Exception("Not enough permissions to write to file: " + versionOutputFile);
+        }
+
+        // Write jvm8-server.options back to override version specific defaults.
+        Files.write(
+                new File(versionOutputFile).toPath(), Options.get("configuredJVMVersionOptions"));
     }
 
     /**
@@ -80,9 +102,13 @@ public void updateAndSaveJVMOptions(final String outputFile) throws Exception {
      * @throws Exception when encountered with invalid configured GC type. {@link
      *     IConfiguration#getGCType()}
      */
-    protected List<String> updateJVMOptions() throws Exception {
+    protected Map<String, List<String>> updateJVMOptions() throws Exception {
         File jvmOptionsFile = new File(config.getJVMOptionsFileLocation());
         validate(jvmOptionsFile);
+
+        File jvmVersionOptionsFile = new File(config.getJVMVersionOptionsFileLocation());
+        validate(jvmVersionOptionsFile);
+
         final GCType configuredGC = config.getGCType();
 
         final Map<String, JVMOption> excludeSet =
@@ -94,7 +120,7 @@ protected List<String> updateJVMOptions() throws Exception {
 
         // Don't use streams for processing as upsertSet jvm options needs to be removed if we find
         // them
-        // already in jvm.options file.
+        // already in jvm-server.options file.
         List<String> optionsFromFile =
                 Files.lines(jvmOptionsFile.toPath()).collect(Collectors.toList());
         List<String> configuredOptions = new LinkedList<>();
@@ -103,7 +129,16 @@ protected List<String> updateJVMOptions() throws Exception {
                     updateConfigurationValue(line, configuredGC, upsertSet, excludeSet));
         }
 
-        // Add all the upserts(inserts only left) from config.
+        // Process the list for version specific options file.
+        List<String> optionsFromVersionFile =
+                Files.lines(jvmVersionOptionsFile.toPath()).collect(Collectors.toList());
+        List<String> configuredVersionOptions = new LinkedList<>();
+        for (String line : optionsFromVersionFile) {
+            configuredVersionOptions.add(
+                    updateConfigurationValue(line, configuredGC, upsertSet, excludeSet));
+        }
+
+        // Add all the upserts(inserts only left) from config to generic options file.
         if (upsertSet != null && !upsertSet.isEmpty()) {
 
             configuredOptions.add("#################");
@@ -118,7 +153,10 @@ protected List<String> updateJVMOptions() throws Exception {
                             .collect(Collectors.toList()));
         }
 
-        return configuredOptions;
+        HashMap<String, List<String>> options = new HashMap<String, List<String>>() {};
+        options.put("configuredJVMOptions", configuredOptions);
+        options.put("configuredJVMVersionOptions", configuredVersionOptions);
+        return options;
     }
 
     private void setHeapSetting(String configuredValue, JVMOption option) {

diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
@@ -251,7 +251,8 @@ public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws I
     public final void updateJVMOptions() throws Exception {
         JVMOptionsTuner tuner = new JVMOptionsTuner(config);
         // Overwrite default jvm.options file.
-        tuner.updateAndSaveJVMOptions(config.getJVMOptionsFileLocation());
+        tuner.updateAndSaveJVMOptions(
+                config.getJVMOptionsFileLocation(), config.getJVMVersionOptionsFileLocation());
     }
 
     public void addExtraCassParams(Map map) {

diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java
@@ -155,6 +155,11 @@ public String getJVMOptionsFileLocation() {
         return "src/test/resources/conf/jvm-server.options";
     }
 
+    @Override
+    public String getJVMVersionOptionsFileLocation() {
+        return "src/test/resources/conf/jvm8-server.options";
+    }
+
     public String getCommitLogBackupPropsFile() {
         return getCassHome() + "/conf/commitlog_archiving.properties";
     }

diff --git a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java
@@ -35,7 +35,7 @@ public class JVMOptionTunerTest {
     @Test
     public void testCMS() throws Exception {
         config = new GCConfiguration(GCType.CMS, null, null, null, null);
-        List<JVMOption> jvmOptionMap = getConfiguredJVMOptions(config);
+        List<JVMOption> jvmOptionMap = getConfiguredJVMVersionOptions(config);
         // Validate that all CMS options should be uncommented.
         long failedVerification =
                 jvmOptionMap
@@ -57,7 +57,7 @@ public void testCMS() throws Exception {
     @Test
     public void testG1GC() throws Exception {
         config = new GCConfiguration(GCType.G1GC, null, null, null, null);
-        List<JVMOption> jvmOptionMap = getConfiguredJVMOptions(config);
+        List<JVMOption> jvmOptionMap = getConfiguredJVMVersionOptions(config);
         // Validate that all G1GC options should be uncommented.
         long failedVerification =
                 jvmOptionMap
@@ -101,9 +101,10 @@ public void testCMSUpsert() throws Exception {
                         xmnOption.getValue(),
                         xmxOption.getValue());
         List<JVMOption> jvmOptions = getConfiguredJVMOptions(config);
+        List<JVMOption> jvmVersionOptions = getConfiguredJVMVersionOptions(config);
 
         // Verify all the options do exist.
-        assertTrue(jvmOptions.contains(option3));
+        assertTrue(jvmVersionOptions.contains(option3));
         assertTrue(jvmOptions.contains(option2));
         assertTrue(jvmOptions.contains(option1));
 
@@ -131,11 +132,12 @@ public void testCMSExclude() throws Exception {
                                 + option3.toJVMOptionString());
         config = new GCConfiguration(GCType.CMS, buffer.toString(), null, "3G", "12G");
         List<JVMOption> jvmOptions = getConfiguredJVMOptions(config);
+        List<JVMOption> jvmVersionOptions = getConfiguredJVMVersionOptions(config);
 
         // Verify all the options do not exist.
-        assertFalse(jvmOptions.contains(option3));
-        assertFalse(jvmOptions.contains(option2));
-        assertFalse(jvmOptions.contains(option1));
+        assertFalse(jvmVersionOptions.contains(option3));
+        assertFalse(jvmVersionOptions.contains(option2));
+        assertFalse(jvmVersionOptions.contains(option1));
 
         // Verify that Xmn is present since CMS needs tuning of young gen heap
         assertTrue(jvmOptions.contains(maxHeap));
@@ -167,16 +169,17 @@ public void testG1GCUpsertExclude() throws Exception {
                 new GCConfiguration(
                         GCType.G1GC, exclude.toString(), upsert.toString(), "3G", "12G");
         List<JVMOption> jvmOptions = getConfiguredJVMOptions(config);
+        List<JVMOption> jvmVersionOptions = getConfiguredJVMVersionOptions(config);
 
         // Verify upserts exist
         assertTrue(jvmOptions.contains(option1));
         assertTrue(jvmOptions.contains(option2));
 
         // Verify exclude exist. This is to prove that if an element is in EXCLUDE, it will always
         // be excluded.
-        assertFalse(jvmOptions.contains(option3));
-        assertFalse(jvmOptions.contains(option4));
-        assertFalse(jvmOptions.contains(option5));
+        assertFalse(jvmVersionOptions.contains(option3));
+        assertFalse(jvmVersionOptions.contains(option4));
+        assertFalse(jvmVersionOptions.contains(option5));
 
         // Verify that Xmn is not present since G1GC autotunes the young gen heap
         assertTrue(jvmOptions.contains(maxHeap));
@@ -193,7 +196,8 @@ private List<JVMOption> getConfiguredJVMOptions(IConfiguration config) throws Ex
     private List<JVMOption> getConfiguredJVMOptions(IConfiguration config, boolean filter)
             throws Exception {
         tuner = new JVMOptionsTuner(config);
-        List<String> configuredJVMOptions = tuner.updateJVMOptions();
+        Map<String, List<String>> options = tuner.updateJVMOptions();
+        List<String> configuredJVMOptions = options.get("configuredJVMOptions");
         if (filter) {
             return configuredJVMOptions
                     .stream()
@@ -206,6 +210,30 @@ private List<JVMOption> getConfiguredJVMOptions(IConfiguration config, boolean f
         }
     }
 
+    private List<JVMOption> getConfiguredJVMVersionOptions(IConfiguration config) throws Exception {
+        return getConfiguredJVMVersionOptions(config, true);
+    }
+
+    private List<JVMOption> getConfiguredJVMVersionOptions(IConfiguration config, boolean filter)
+            throws Exception {
+        tuner = new JVMOptionsTuner(config);
+        Map<String, List<String>> options = tuner.updateJVMOptions();
+        List<String> configuredJVMVersionOptions = options.get("configuredJVMVersionOptions");
+        if (filter) {
+            return configuredJVMVersionOptions
+                    .stream()
+                    .map(JVMOption::parse)
+                    .filter(jvmOption -> (jvmOption != null))
+                    .filter(jvmOption -> !jvmOption.isCommented())
+                    .collect(Collectors.toList());
+        } else {
+            return configuredJVMVersionOptions
+                    .stream()
+                    .map(JVMOption::parse)
+                    .collect(Collectors.toList());
+        }
+    }
+
     private class GCConfiguration extends FakeConfiguration {
         private GCType gcType;
         private String configuredJVMExclude;

diff --git a/priam/src/test/resources/conf/jvm-server.options b/priam/src/test/resources/conf/jvm-server.options
@@ -175,65 +175,4 @@
 # The example below assumes a modern 8-core+ machine for decent
 # times. If in doubt, and if you do not particularly want to tweak, go
 # 100 MB per physical CPU core.
-#-Xmn800M
-
-#################
-#  GC SETTINGS  #
-#################
-
-### CMS Settings
-
--XX:+UseParNewGC
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:SurvivorRatio=8
--XX:MaxTenuringThreshold=1
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
--XX:CMSWaitDuration=10000
--XX:+CMSParallelInitialMarkEnabled
--XX:+CMSEdenChunksRecordAlways
-# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
--XX:+CMSClassUnloadingEnabled
-
-### G1 Settings (experimental, comment previous section and uncomment section below to enable)
-
-## Use the Hotspot garbage-first collector.
-#-XX:+UseG1GC
-#
-## Have the JVM do less remembered set work during STW, instead
-## preferring concurrent GC. Reduces p99.9 latency.
-#-XX:G1RSetUpdatingPauseTimePercent=5
-#
-## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
-## 200ms is the JVM default and lowest viable setting
-## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
-#-XX:MaxGCPauseMillis=500
-
-## Optional G1 Settings
-
-# Save CPU time on large (>= 16GB) heaps by delaying region scanning
-# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
-#-XX:InitiatingHeapOccupancyPercent=70
-
-# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
-# Otherwise equal to the number of cores when 8 or less.
-# Machines with > 10 cores should try setting these to <= full cores.
-#-XX:ParallelGCThreads=16
-# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
-# Setting both to the same value can reduce STW durations.
-#-XX:ConcGCThreads=16
-
-### GC logging options -- uncomment to enable
-
--XX:+PrintGCDetails
--XX:+PrintGCDateStamps
--XX:+PrintHeapAtGC
--XX:+PrintTenuringDistribution
--XX:+PrintGCApplicationStoppedTime
--XX:+PrintPromotionFailure
-#-XX:PrintFLSStatistics=1
-#-Xloggc:/var/log/cassandra/gc.log
--XX:+UseGCLogFileRotation
--XX:NumberOfGCLogFiles=10
--XX:GCLogFileSize=10M
+#-Xmn800M
diff --git a/priam/src/test/resources/conf/jvm8-server.options b/priam/src/test/resources/conf/jvm8-server.options
@@ -0,0 +1,76 @@
+###########################################################################
+#                          jvm8-server.options                            #
+#                                                                         #
+# See jvm-server.options. This file is specific for Java 8 and newer.     #
+###########################################################################
+
+########################
+# GENERAL JVM SETTINGS #
+########################
+
+# allows lowering thread priority without being root on linux - probably
+# not necessary on Windows but doesn't harm anything.
+# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workaround.html
+-XX:ThreadPriorityPolicy=42
+
+#################
+#  GC SETTINGS  #
+#################
+
+### CMS Settings
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:SurvivorRatio=8
+-XX:MaxTenuringThreshold=1
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+-XX:CMSWaitDuration=10000
+-XX:+CMSParallelInitialMarkEnabled
+-XX:+CMSEdenChunksRecordAlways
+## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
+-XX:+CMSClassUnloadingEnabled
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+#-XX:+UseG1GC
+#-XX:+ParallelRefProcEnabled
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+#-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+#-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+### GC logging options -- uncomment to enable
+
+-XX:+PrintGCDetails
+-XX:+PrintGCDateStamps
+-XX:+PrintHeapAtGC
+-XX:+PrintTenuringDistribution
+-XX:+PrintGCApplicationStoppedTime
+-XX:+PrintPromotionFailure
+#-XX:PrintFLSStatistics=1
+#-Xloggc:/var/log/cassandra/gc.log
+-XX:+UseGCLogFileRotation
+-XX:NumberOfGCLogFiles=10
+-XX:GCLogFileSize=10M
+
+# The newline in the end of file is intentional