From 01bf39df2b7758b4ab070ef035667eb3a3f2528a Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Mon, 26 Jun 2023 15:42:38 -0700 Subject: [PATCH] tuner to update jvm version specific options (#1060) Co-authored-by: ayushis --- .../netflix/priam/config/IConfiguration.java | 7 ++ .../netflix/priam/tuner/JVMOptionsTuner.java | 58 +++++++++++--- .../netflix/priam/tuner/StandardTuner.java | 3 +- .../priam/config/FakeConfiguration.java | 5 ++ .../priam/tuner/JVMOptionTunerTest.java | 48 +++++++++--- .../test/resources/conf/jvm-server.options | 63 +-------------- .../test/resources/conf/jvm8-server.options | 76 +++++++++++++++++++ 7 files changed, 177 insertions(+), 83 deletions(-) create mode 100644 priam/src/test/resources/conf/jvm8-server.options diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index db46690e2..cfbbb8659 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -49,6 +49,13 @@ default String getJVMOptionsFileLocation() { return getCassHome() + "/conf/jvm-server.options"; } + /** + * @return Path to jvm8-server.options file. This is used to pass JVM GC options to Cassandra. + */ + default String getJVMVersionOptionsFileLocation() { + return getCassHome() + "/conf/jvm8-server.options"; + } + /** * @return Type of garbage collection mechanism to use for Cassandra. Supported values are * CMS,G1GC diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java index 9f771ad57..de680c8b2 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java @@ -52,13 +52,16 @@ public JVMOptionsTuner(IConfiguration config) { * @throws Exception when encountered with invalid configured GC type. {@link * IConfiguration#getGCType()} */ - public void updateAndSaveJVMOptions(final String outputFile) throws Exception { - List configuredJVMOptions = updateJVMOptions(); + public void updateAndSaveJVMOptions(final String outputFile, final String versionOutputFile) + throws Exception { + Map> Options = updateJVMOptions(); if (logger.isInfoEnabled()) { StringBuffer buffer = new StringBuffer("\n"); - configuredJVMOptions.stream().forEach(line -> buffer.append(line).append("\n")); - logger.info("Updating jvm.options with following values: " + buffer.toString()); + Options.get("configuredJVMOptions") + .stream() + .forEach(line -> buffer.append(line).append("\n")); + logger.info("Updating jvm-server.options with following values: " + buffer.toString()); } // Verify we can write to output file and it is not directory. @@ -67,8 +70,27 @@ public void updateAndSaveJVMOptions(final String outputFile) throws Exception { throw new Exception("Not enough permissions to write to file: " + outputFile); } - // Write jvm.options back to override defaults. - Files.write(new File(outputFile).toPath(), configuredJVMOptions); + // Write jvm-server.options back to override defaults. + Files.write(new File(outputFile).toPath(), Options.get("configuredJVMOptions")); + + // Update jdk version specific options + if (logger.isInfoEnabled()) { + StringBuffer buffer = new StringBuffer("\n"); + Options.get("configuredJVMVersionOptions") + .stream() + .forEach(line -> buffer.append(line).append("\n")); + logger.info("Updating jvm8-server.options with following values: " + buffer.toString()); + } + + // Verify we can write to version specific output file and it is not directory. + File versionFile = new File(versionOutputFile); + if (versionFile.exists() && !versionFile.canWrite()) { + throw new Exception("Not enough permissions to write to file: " + versionOutputFile); + } + + // Write jvm8-server.options back to override version specific defaults. + Files.write( + new File(versionOutputFile).toPath(), Options.get("configuredJVMVersionOptions")); } /** @@ -80,9 +102,13 @@ public void updateAndSaveJVMOptions(final String outputFile) throws Exception { * @throws Exception when encountered with invalid configured GC type. {@link * IConfiguration#getGCType()} */ - protected List updateJVMOptions() throws Exception { + protected Map> updateJVMOptions() throws Exception { File jvmOptionsFile = new File(config.getJVMOptionsFileLocation()); validate(jvmOptionsFile); + + File jvmVersionOptionsFile = new File(config.getJVMVersionOptionsFileLocation()); + validate(jvmVersionOptionsFile); + final GCType configuredGC = config.getGCType(); final Map excludeSet = @@ -94,7 +120,7 @@ protected List updateJVMOptions() throws Exception { // Don't use streams for processing as upsertSet jvm options needs to be removed if we find // them - // already in jvm.options file. + // already in jvm-server.options file. List optionsFromFile = Files.lines(jvmOptionsFile.toPath()).collect(Collectors.toList()); List configuredOptions = new LinkedList<>(); @@ -103,7 +129,16 @@ protected List updateJVMOptions() throws Exception { updateConfigurationValue(line, configuredGC, upsertSet, excludeSet)); } - // Add all the upserts(inserts only left) from config. + // Process the list for version specific options file. + List optionsFromVersionFile = + Files.lines(jvmVersionOptionsFile.toPath()).collect(Collectors.toList()); + List configuredVersionOptions = new LinkedList<>(); + for (String line : optionsFromVersionFile) { + configuredVersionOptions.add( + updateConfigurationValue(line, configuredGC, upsertSet, excludeSet)); + } + + // Add all the upserts(inserts only left) from config to generic options file. if (upsertSet != null && !upsertSet.isEmpty()) { configuredOptions.add("#################"); @@ -118,7 +153,10 @@ protected List updateJVMOptions() throws Exception { .collect(Collectors.toList())); } - return configuredOptions; + HashMap> options = new HashMap>() {}; + options.put("configuredJVMOptions", configuredOptions); + options.put("configuredJVMVersionOptions", configuredVersionOptions); + return options; } private void setHeapSetting(String configuredValue, JVMOption option) { diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index ffdcd0b99..7ad932231 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -251,7 +251,8 @@ public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws I public final void updateJVMOptions() throws Exception { JVMOptionsTuner tuner = new JVMOptionsTuner(config); // Overwrite default jvm.options file. - tuner.updateAndSaveJVMOptions(config.getJVMOptionsFileLocation()); + tuner.updateAndSaveJVMOptions( + config.getJVMOptionsFileLocation(), config.getJVMVersionOptionsFileLocation()); } public void addExtraCassParams(Map map) { diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 6f855b086..81501201d 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -155,6 +155,11 @@ public String getJVMOptionsFileLocation() { return "src/test/resources/conf/jvm-server.options"; } + @Override + public String getJVMVersionOptionsFileLocation() { + return "src/test/resources/conf/jvm8-server.options"; + } + public String getCommitLogBackupPropsFile() { return getCassHome() + "/conf/commitlog_archiving.properties"; } diff --git a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java index 1942704f5..4a53bea1a 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java @@ -35,7 +35,7 @@ public class JVMOptionTunerTest { @Test public void testCMS() throws Exception { config = new GCConfiguration(GCType.CMS, null, null, null, null); - List jvmOptionMap = getConfiguredJVMOptions(config); + List jvmOptionMap = getConfiguredJVMVersionOptions(config); // Validate that all CMS options should be uncommented. long failedVerification = jvmOptionMap @@ -57,7 +57,7 @@ public void testCMS() throws Exception { @Test public void testG1GC() throws Exception { config = new GCConfiguration(GCType.G1GC, null, null, null, null); - List jvmOptionMap = getConfiguredJVMOptions(config); + List jvmOptionMap = getConfiguredJVMVersionOptions(config); // Validate that all G1GC options should be uncommented. long failedVerification = jvmOptionMap @@ -101,9 +101,10 @@ public void testCMSUpsert() throws Exception { xmnOption.getValue(), xmxOption.getValue()); List jvmOptions = getConfiguredJVMOptions(config); + List jvmVersionOptions = getConfiguredJVMVersionOptions(config); // Verify all the options do exist. - assertTrue(jvmOptions.contains(option3)); + assertTrue(jvmVersionOptions.contains(option3)); assertTrue(jvmOptions.contains(option2)); assertTrue(jvmOptions.contains(option1)); @@ -131,11 +132,12 @@ public void testCMSExclude() throws Exception { + option3.toJVMOptionString()); config = new GCConfiguration(GCType.CMS, buffer.toString(), null, "3G", "12G"); List jvmOptions = getConfiguredJVMOptions(config); + List jvmVersionOptions = getConfiguredJVMVersionOptions(config); // Verify all the options do not exist. - assertFalse(jvmOptions.contains(option3)); - assertFalse(jvmOptions.contains(option2)); - assertFalse(jvmOptions.contains(option1)); + assertFalse(jvmVersionOptions.contains(option3)); + assertFalse(jvmVersionOptions.contains(option2)); + assertFalse(jvmVersionOptions.contains(option1)); // Verify that Xmn is present since CMS needs tuning of young gen heap assertTrue(jvmOptions.contains(maxHeap)); @@ -167,6 +169,7 @@ public void testG1GCUpsertExclude() throws Exception { new GCConfiguration( GCType.G1GC, exclude.toString(), upsert.toString(), "3G", "12G"); List jvmOptions = getConfiguredJVMOptions(config); + List jvmVersionOptions = getConfiguredJVMVersionOptions(config); // Verify upserts exist assertTrue(jvmOptions.contains(option1)); @@ -174,9 +177,9 @@ public void testG1GCUpsertExclude() throws Exception { // Verify exclude exist. This is to prove that if an element is in EXCLUDE, it will always // be excluded. - assertFalse(jvmOptions.contains(option3)); - assertFalse(jvmOptions.contains(option4)); - assertFalse(jvmOptions.contains(option5)); + assertFalse(jvmVersionOptions.contains(option3)); + assertFalse(jvmVersionOptions.contains(option4)); + assertFalse(jvmVersionOptions.contains(option5)); // Verify that Xmn is not present since G1GC autotunes the young gen heap assertTrue(jvmOptions.contains(maxHeap)); @@ -193,7 +196,8 @@ private List getConfiguredJVMOptions(IConfiguration config) throws Ex private List getConfiguredJVMOptions(IConfiguration config, boolean filter) throws Exception { tuner = new JVMOptionsTuner(config); - List configuredJVMOptions = tuner.updateJVMOptions(); + Map> options = tuner.updateJVMOptions(); + List configuredJVMOptions = options.get("configuredJVMOptions"); if (filter) { return configuredJVMOptions .stream() @@ -206,6 +210,30 @@ private List getConfiguredJVMOptions(IConfiguration config, boolean f } } + private List getConfiguredJVMVersionOptions(IConfiguration config) throws Exception { + return getConfiguredJVMVersionOptions(config, true); + } + + private List getConfiguredJVMVersionOptions(IConfiguration config, boolean filter) + throws Exception { + tuner = new JVMOptionsTuner(config); + Map> options = tuner.updateJVMOptions(); + List configuredJVMVersionOptions = options.get("configuredJVMVersionOptions"); + if (filter) { + return configuredJVMVersionOptions + .stream() + .map(JVMOption::parse) + .filter(jvmOption -> (jvmOption != null)) + .filter(jvmOption -> !jvmOption.isCommented()) + .collect(Collectors.toList()); + } else { + return configuredJVMVersionOptions + .stream() + .map(JVMOption::parse) + .collect(Collectors.toList()); + } + } + private class GCConfiguration extends FakeConfiguration { private GCType gcType; private String configuredJVMExclude; diff --git a/priam/src/test/resources/conf/jvm-server.options b/priam/src/test/resources/conf/jvm-server.options index 03378b1d3..1e3626994 100644 --- a/priam/src/test/resources/conf/jvm-server.options +++ b/priam/src/test/resources/conf/jvm-server.options @@ -175,65 +175,4 @@ # The example below assumes a modern 8-core+ machine for decent # times. If in doubt, and if you do not particularly want to tweak, go # 100 MB per physical CPU core. -#-Xmn800M - -################# -# GC SETTINGS # -################# - -### CMS Settings - --XX:+UseParNewGC --XX:+UseConcMarkSweepGC --XX:+CMSParallelRemarkEnabled --XX:SurvivorRatio=8 --XX:MaxTenuringThreshold=1 --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly --XX:CMSWaitDuration=10000 --XX:+CMSParallelInitialMarkEnabled --XX:+CMSEdenChunksRecordAlways -# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 --XX:+CMSClassUnloadingEnabled - -### G1 Settings (experimental, comment previous section and uncomment section below to enable) - -## Use the Hotspot garbage-first collector. -#-XX:+UseG1GC -# -## Have the JVM do less remembered set work during STW, instead -## preferring concurrent GC. Reduces p99.9 latency. -#-XX:G1RSetUpdatingPauseTimePercent=5 -# -## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. -## 200ms is the JVM default and lowest viable setting -## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. -#-XX:MaxGCPauseMillis=500 - -## Optional G1 Settings - -# Save CPU time on large (>= 16GB) heaps by delaying region scanning -# until the heap is 70% full. The default in Hotspot 8u40 is 40%. -#-XX:InitiatingHeapOccupancyPercent=70 - -# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. -# Otherwise equal to the number of cores when 8 or less. -# Machines with > 10 cores should try setting these to <= full cores. -#-XX:ParallelGCThreads=16 -# By default, ConcGCThreads is 1/4 of ParallelGCThreads. -# Setting both to the same value can reduce STW durations. -#-XX:ConcGCThreads=16 - -### GC logging options -- uncomment to enable - --XX:+PrintGCDetails --XX:+PrintGCDateStamps --XX:+PrintHeapAtGC --XX:+PrintTenuringDistribution --XX:+PrintGCApplicationStoppedTime --XX:+PrintPromotionFailure -#-XX:PrintFLSStatistics=1 -#-Xloggc:/var/log/cassandra/gc.log --XX:+UseGCLogFileRotation --XX:NumberOfGCLogFiles=10 --XX:GCLogFileSize=10M +#-Xmn800M \ No newline at end of file diff --git a/priam/src/test/resources/conf/jvm8-server.options b/priam/src/test/resources/conf/jvm8-server.options new file mode 100644 index 000000000..74b99194b --- /dev/null +++ b/priam/src/test/resources/conf/jvm8-server.options @@ -0,0 +1,76 @@ +########################################################################### +# jvm8-server.options # +# # +# See jvm-server.options. This file is specific for Java 8 and newer. # +########################################################################### + +######################## +# GENERAL JVM SETTINGS # +######################## + +# allows lowering thread priority without being root on linux - probably +# not necessary on Windows but doesn't harm anything. +# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workaround.html +-XX:ThreadPriorityPolicy=42 + +################# +# GC SETTINGS # +################# + +### CMS Settings +-XX:+UseParNewGC +-XX:+UseConcMarkSweepGC +-XX:+CMSParallelRemarkEnabled +-XX:SurvivorRatio=8 +-XX:MaxTenuringThreshold=1 +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly +-XX:CMSWaitDuration=10000 +-XX:+CMSParallelInitialMarkEnabled +-XX:+CMSEdenChunksRecordAlways +## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 +-XX:+CMSClassUnloadingEnabled + +### G1 Settings +## Use the Hotspot garbage-first collector. +#-XX:+UseG1GC +#-XX:+ParallelRefProcEnabled + +# +## Have the JVM do less remembered set work during STW, instead +## preferring concurrent GC. Reduces p99.9 latency. +#-XX:G1RSetUpdatingPauseTimePercent=5 +# +## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. +## 200ms is the JVM default and lowest viable setting +## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. +#-XX:MaxGCPauseMillis=500 + +## Optional G1 Settings +# Save CPU time on large (>= 16GB) heaps by delaying region scanning +# until the heap is 70% full. The default in Hotspot 8u40 is 40%. +#-XX:InitiatingHeapOccupancyPercent=70 + +# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. +# Otherwise equal to the number of cores when 8 or less. +# Machines with > 10 cores should try setting these to <= full cores. +#-XX:ParallelGCThreads=16 +# By default, ConcGCThreads is 1/4 of ParallelGCThreads. +# Setting both to the same value can reduce STW durations. +#-XX:ConcGCThreads=16 + +### GC logging options -- uncomment to enable + +-XX:+PrintGCDetails +-XX:+PrintGCDateStamps +-XX:+PrintHeapAtGC +-XX:+PrintTenuringDistribution +-XX:+PrintGCApplicationStoppedTime +-XX:+PrintPromotionFailure +#-XX:PrintFLSStatistics=1 +#-Xloggc:/var/log/cassandra/gc.log +-XX:+UseGCLogFileRotation +-XX:NumberOfGCLogFiles=10 +-XX:GCLogFileSize=10M + +# The newline in the end of file is intentional \ No newline at end of file