Merge pull request #12 from Netflix-Skunkworks/feature/jolynch/add-ja…

…va-17 Adding Java 17 and 21
Netflix-Skunkworks · Apr 12, 2024 · 8fc3471 · 8fc3471
2 parents fc9f7b5 + 5d8b5d0
commit 8fc3471
Show file tree

Hide file tree

Showing 7 changed files with 169 additions and 27 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -23,7 +23,8 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        java: [ '8', '11']
+        dist: ['zulu', 'temurin', 'corretto']
+        java: ['8', '11', '17', '21']
     steps:
     - uses: actions/checkout@v2
     - name: "Set nproc limits to control threads"
@@ -38,7 +39,39 @@ jobs:
       run: pip install tox
     - uses: actions/setup-java@v4
       with:
-        distribution: 'zulu'
+        distribution: ${{ matrix.dist }}
+        java-version: |
+          8
+          ${{ matrix.java }}
+    - name: "Build jvmquake .so with Java 8"
+      run: JAVA_HOME=$JAVA_HOME_8_X64 make -C src
+    - name: Build jvmquake tests against Java ${{ matrix.java }}
+      run: env | grep JAVA && make -C tests
+    - name: Run jvmquake tests against Java ${{ matrix.java }}
+      env:
+        JAVA_MAJOR_VERSION: ${{ matrix.java }}
+      run: tox -e test,test_jvm
+  ubuntu-bionic:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        dist: ['zulu', 'temurin', 'corretto']
+        java: ['8', '11', '17', '21']
+    steps:
+    - uses: actions/checkout@v2
+    - name: "Set nproc limits to control threads"
+      run: |
+        sudo prlimit --pid $$ --nofile=32768:32768 --nproc=32768:32768
+        ulimit -n -u
+    - name: "Setup python for tox"
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+    - name: "Install tox for test suite"
+      run: pip install tox
+    - uses: actions/setup-java@v4
+      with:
+        distribution: ${{ matrix.dist }}
         java-version: |
           8
           ${{ matrix.java }}

diff --git a/Makefile b/Makefile
@@ -78,6 +78,17 @@ test_jammy_openjdk11: JAVA_VERSION=11
 test_jammy_openjdk11: TEST_NAME=test_jammy_openjdk11
 test_jammy_openjdk11: test_ubuntu_with_openjdk
 
+test_jammy_openjdk17: UBUNTU_VERSION=22.04
+test_jammy_openjdk17: JAVA_VERSION=17
+test_jammy_openjdk17: TEST_NAME=test_jammy_openjdk17
+test_jammy_openjdk17: test_ubuntu_with_openjdk
+
+test_jammy_openjdk21: UBUNTU_VERSION=22.04
+test_jammy_openjdk21: JAVA_VERSION=17
+test_jammy_openjdk21: TEST_NAME=test_jammy_openjdk17
+test_jammy_openjdk21: test_ubuntu_with_openjdk
+
+
 test_ubuntu_with_openjdk: build_deb_in_docker
 	docker build -f dockerfiles/test/Dockerfile.ubuntu --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg JAVA_VERSION=$(JAVA_VERSION) . -t jolynch/jvmquake:$(TEST_NAME)
 	docker build -f dockerfiles/test/Dockerfile.ubuntu.minimal --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg JAVA_VERSION=$(JAVA_VERSION) . -t jolynch/jvmquake:$(TEST_NAME)_minimal

diff --git a/README.md b/README.md
@@ -255,7 +255,7 @@ threads, gcing too much, etc) which you can run if you have a `jdk`, `tox` and
 ```bash
 # Run the test suite which uses tox, pytest, and plumbum under the hood
 # to run jvmquake through numerous difficult failure modes
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ make test
+JAVA_MAJOR_VERSION=8 JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ make test
 ```
 
 If you have docker you can also run specific environment tests that bundle all
@@ -273,15 +273,19 @@ handles.
 ## Automated Tests
 
 We currently [test](.github/workflows/ci.yml) every commit and the released
-`.so` generated with OpenJDK 8 against the following platforms:
-
-* Ubuntu Xenial with OpenJDK8
-* Ubuntu Bionic with OpenJDK8
-* Ubuntu Bionic with OpenJDK11
-* Ubuntu Bionic with Zulu8
-* Ubuntu Bionic with Zulu11
-* Ubuntu Focal with OpenJDK8
-* Ubuntu Focal with OpenJDK11
-* Centos7 with OpenJDK8
+`.so` generated with Java 8 against the following platforms and distributions:
+
+### Ubuntu Jammy(22.04)
+Verified for Java 8, 11, 17, 21 against openjdk (temurin), zulu and corretto
+
+### Ubuntu Bionic (20.04)
+Verified for Java 8, 11, 17, 21 against openjdk (temurin), zulu and corretto
+
+### Centos7
+Verified for Java 8 against openjdk
+
+### Other Architectures
+We do not test against ARM or Windows. `jvmquake` probably still works fine
+if compiled on the same platform, but it is not tested.
 
 ![Build Status](https://github.com/Netflix-Skunkworks/jvmquake/actions/workflows/ci.yml/badge.svg)
diff --git a/tests/SlowDeathOOM.java b/tests/SlowDeathOOM.java
@@ -15,28 +15,28 @@
  */
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
+
 
 public final class SlowDeathOOM
 {
     @SuppressWarnings("InfiniteLoopStatement")
     public static void main(String[] args)
     {
+        Random random = new Random();
         List<Object> list = new ArrayList<>();
         double cmfZone = 0.90 * Runtime.getRuntime().maxMemory();
-        System.out.println(String.format(
-            "triggering OutOfMemory by allocating %.2f bytes",
-            cmfZone
-        ));
+        System.out.printf("triggering OutOfMemory by allocating %.2f bytes\n", cmfZone);
         try {
             while (true) {
                 byte[] bytes = new byte[8096];
                 list.add(bytes);
                 if ((double)(list.size() * 8096) > cmfZone)
-                    list.remove(0);
+                    list.remove(random.nextInt(list.size()));
             }
         }
         catch (Exception t) {
-            System.out.println(t.toString());
+            System.out.println(t);
         }
         System.out.println("final list size: " + list.size());
     }

diff --git a/tests/environment.py b/tests/environment.py
@@ -25,6 +25,8 @@
 assert JAVA_HOME is not None
 
 # Java 9 and 11 removed a bunch of GC flags
+# Then Java 14 removed CMS
+# Then ZGC was added in 15
 JAVA_MAJOR_VERSION = int(os.environ.get('JAVA_MAJOR_VERSION', '8'))
 
 AGENT_DIR = os.environ.get('AGENT_DIR', Path(os.getcwd(), 'build').as_posix())

diff --git a/tests/test_hard_ooms.py b/tests/test_hard_ooms.py
@@ -33,8 +33,10 @@ def test_jvmquake_cms_slow_death_oom():
     We use the zero option to indicate to jvmquake to trigger a java level
     OOM and cause a heap dump.
     """
-
-    if JAVA_MAJOR_VERSION > 8:
+    if JAVA_MAJOR_VERSION >= 14:
+        print(f"CMS is not available on {JAVA_MAJOR_VERSION} >= 14")
+        return
+    elif JAVA_MAJOR_VERSION > 8:
         cms_slow_death = java_cmd[
             '-Xmx100m',
             '-XX:+HeapDumpOnOutOfMemoryError',
@@ -88,7 +90,7 @@ def test_jvmquake_g1_slow_death_oom():
             '-Xmx100m',
             '-XX:+HeapDumpOnOutOfMemoryError',
             '-XX:+UseG1GC',
-            '-XX:+PrintGCDetails',
+            '-Xlog:gc*',
             '-Xlog:gc:gclog',
             agent_path + "=1,1,0",
             '-cp', class_path,
@@ -110,7 +112,59 @@ def test_jvmquake_g1_slow_death_oom():
         ]
     print("Executing Complex G1GC Slow Death OOM")
     print("[{0}]".format(g1_slow_death))
-    with g1_slow_death.bgrun(retcode=-9, timeout=10) as proc:
+    with g1_slow_death.bgrun(retcode=-9, timeout=30) as proc:
+        pid = proc.pid
+
+    heapdump_path = Path.cwd().joinpath("java_pid{0}.hprof".format(pid))
+    gclog_path = Path.cwd().joinpath('gclog')
+
+    files = (heapdump_path, gclog_path)
+    try:
+        assert_files(*files)
+    finally:
+        cleanup(*files)
+
+
+def test_jvmquake_zgc_slow_death_oom():
+    """
+    Executes a program which over time does way more GC than actual execution
+    We use the zero option to indicate to jvmquake to trigger a java level
+    OOM and cause a heap dump.
+    """
+    if JAVA_MAJOR_VERSION < 15:
+        print(f"ZGC is not available on {JAVA_MAJOR_VERSION} < 14")
+        return
+
+    if JAVA_MAJOR_VERSION >= 21:
+        zgc_slow_death = java_cmd[
+            '-Xmx100m',
+            '-XX:+HeapDumpOnOutOfMemoryError',
+            '-XX:+UseZGC',
+            '-XX:+ZGenerational',
+            '-Xlog:safepoint',
+            '-Xlog:gc*',
+            '-Xlog:::time,level,tags',
+            '-Xlog:gc:gclog',
+            agent_path + "=1,1,0",
+            '-cp', class_path,
+            'SlowDeathOOM'
+        ]
+    else:
+        zgc_slow_death = java_cmd[
+            '-Xmx100m',
+            '-XX:+HeapDumpOnOutOfMemoryError',
+            '-XX:+UseZGC',
+            '-Xlog:safepoint',
+            '-Xlog:gc*',
+            '-Xlog:::time,level,tags',
+            '-Xlog:gc:gclog',
+            agent_path + "=1,1,0",
+            '-cp', class_path,
+            'SlowDeathOOM'
+        ]
+    print("Executing Complex ZGC Slow Death OOM")
+    print("[{0}]".format(zgc_slow_death))
+    with zgc_slow_death .bgrun(retcode=-9, timeout=30) as proc:
         pid = proc.pid
 
     heapdump_path = Path.cwd().joinpath("java_pid{0}.hprof".format(pid))
@@ -123,11 +177,16 @@ def test_jvmquake_g1_slow_death_oom():
         cleanup(*files)
 
 
+
+
 def test_jvmquake_cms_slow_death_core(core_ulimit):
     """
     Executes a program which over time does way more GC than actual execution
     """
-    if JAVA_MAJOR_VERSION > 8:
+    if JAVA_MAJOR_VERSION >= 14:
+        print(f"CMS is not available on {JAVA_MAJOR_VERSION} >= 14")
+        return
+    elif JAVA_MAJOR_VERSION > 8:
         cms_slow_death = java_cmd[
             '-Xmx100m',
             '-XX:+HeapDumpOnOutOfMemoryError',
@@ -190,7 +249,10 @@ def test_jvmquake_cms_touch_warning():
 
     start_time = time.time()
 
-    if JAVA_MAJOR_VERSION > 8:
+    if JAVA_MAJOR_VERSION >= 14:
+        print(f"CMS is not available on {JAVA_MAJOR_VERSION} >= 14")
+        return
+    elif JAVA_MAJOR_VERSION > 8:
         cms_slow_death_warn_only = java_cmd[
             '-Xmx100m',
             '-XX:+UseConcMarkSweepGC',
@@ -235,6 +297,9 @@ def test_jvmquake_cms_touch_warning_custom_path():
 
     start_time = time.time()
 
+    if JAVA_MAJOR_VERSION >= 14:
+        print(f"CMS is not available on {JAVA_MAJOR_VERSION} >= 14")
+        return
     if JAVA_MAJOR_VERSION > 8:
         cms_slow_death_warn_only = java_cmd[
             '-Xmx100m',

diff --git a/tests/test_java_opts.py b/tests/test_java_opts.py
@@ -90,8 +90,10 @@ def test_jvm_cms_slow_death_oom():
 
     In this case -XX:GCTimeLimit and -XX:GCHeapFreeLimit do squat
     """
-
-    if JAVA_MAJOR_VERSION > 8:
+    if JAVA_MAJOR_VERSION >= 14:
+        print(f"CMS is not available on {JAVA_MAJOR_VERSION} >= 14")
+        return
+    elif JAVA_MAJOR_VERSION > 8:
         cms_slow_death = java_cmd[
             '-Xmx100m',
             '-XX:+UseConcMarkSweepGC',
@@ -136,3 +138,28 @@ def test_jvm_g1_slow_death_oom():
     print("[{0}]".format(g1_slow_death))
     with (pytest.raises(plumbum.commands.processes.ProcessTimedOut)):
         g1_slow_death.run(retcode=3, timeout=10)
+
+@pytest.mark.skip(reason="ZGC just ooms immediately")
+def test_jvm_zgc_slow_death_oom():
+    """
+    Executes a program which over time does way more GC than actual execution
+
+    In this case -XX:GCTimeLimit and -XX:GCHeapFreeLimit do squat for ZGC
+    """
+    if JAVA_MAJOR_VERSION < 21:
+        print(f"ZGC+generation is not available on {JAVA_MAJOR_VERSION} < 21")
+        return
+
+    zgc_slow_death = java_cmd[
+        '-Xmx100m',
+        '-XX:+UseZGC',
+        '-XX:+ZGenerational',
+        '-XX:GCTimeLimit=20',
+        '-XX:GCHeapFreeLimit=80',
+        '-cp', class_path,
+        'SlowDeathOOM'
+    ]
+    print("Executing Complex ZGC Slow Death OOM")
+    print("[{0}]".format(zgc_slow_death))
+    with (pytest.raises(plumbum.commands.processes.ProcessTimedOut)):
+        zgc_slow_death.run(retcode=3, timeout=10)