8329013: StackOverflowError when starting Apache Tomcat with signed jar

Reviewed-by: mbaesken
Backport-of: 925d82931c09dc11ea5a3bc410ea5cfd67ee14aa

src/java.base/share/classes/jdk/internal/event/EventHelper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
 
 import jdk.internal.misc.JavaUtilJarAccess;
 import jdk.internal.misc.SharedSecrets;
+import jdk.internal.misc.ThreadTracker;
 
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
@@ -130,6 +131,18 @@ private static String getDurationString(Instant start) {
         }
     }
 
+    private static class ThreadTrackHolder {
+        static final ThreadTracker TRACKER = new ThreadTracker();
+    }
+
+    private static Object tryBeginLookup() {
+        return ThreadTrackHolder.TRACKER.tryBegin();
+    }
+
+    private static void endLookup(Object key) {
+        ThreadTrackHolder.TRACKER.end(key);
+    }
+
     /**
      * Helper to determine if security events are being logged
      * at a preconfigured logging level. The configuration value
@@ -138,14 +151,20 @@ private static String getDurationString(Instant start) {
      * @return boolean indicating whether an event should be logged
      */
     public static boolean isLoggingSecurity() {
-        // Avoid a bootstrap issue where the commitEvent attempts to
-        // trigger early loading of System Logger but where
-        // the verification process still has JarFiles locked
-        if (securityLogger == null && !JUJA.isInitializing()) {
-            LOGGER_HANDLE.compareAndSet( null, System.getLogger(SECURITY_LOGGER_NAME));
-            loggingSecurity = securityLogger.isLoggable(LOG_LEVEL);
+        Object key;
+        // Avoid bootstrap issues where
+        // * commitEvent triggers early loading of System Logger but where
+        //   the verification process still has JarFiles locked
+        // * the loading of the logging libraries involves recursive
+        //   calls to security libraries triggering recursion
+        if (securityLogger == null && !JUJA.isInitializing() && (key = tryBeginLookup()) != null) {
+            try {
+                LOGGER_HANDLE.compareAndSet(null, System.getLogger(SECURITY_LOGGER_NAME));
+                loggingSecurity = securityLogger.isLoggable(LOG_LEVEL);
+            } finally {
+                endLookup(key);
+            }
         }
         return loggingSecurity;
     }
-
 }

src/java.base/share/classes/jdk/internal/misc/ThreadTracker.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package jdk.internal.misc;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Tracks threads to help detect reentrancy without using ThreadLocal variables.
+ * A thread invokes the {@code begin} or {@code tryBegin} methods at the start
+ * of a block, and the {@code end} method at the end of a block.
+ */
+public class ThreadTracker {
+
+    /**
+     * A reference to a Thread that is suitable for use as a key in a collection.
+     * The hashCode/equals methods do not invoke the Thread hashCode/equals method
+     * as they may run arbitrary code and/or leak references to Thread objects.
+     */
+    private final class ThreadRef {
+        private final Thread thread;
+
+        ThreadRef(Thread thread) {
+            this.thread = thread;
+        }
+
+        Thread thread() {
+            return this.thread;
+        }
+
+        @Override
+        public int hashCode() {
+            return Long.hashCode(thread.getId());
+        }
+        @Override
+        public boolean equals(Object obj) {
+            return (obj instanceof ThreadRef)
+                    && this.thread == ((ThreadRef)obj).thread;
+        }
+    }
+
+    private final Set<ThreadRef> threads = ConcurrentHashMap.newKeySet();
+
+    /**
+     * Adds the current thread to thread set if not already in the set.
+     * Returns a key to remove the thread or {@code null} if already in the set.
+     */
+    public Object tryBegin() {
+        var threadRef = new ThreadRef(Thread.currentThread());
+        return threads.add(threadRef) ? threadRef : null;
+    }
+
+    /**
+     * Adds the current thread to thread set if not already in the set.
+     * Returns a key to remove the thread.
+     */
+    public Object begin() {
+        var threadRef = new ThreadRef(Thread.currentThread());
+        boolean added = threads.add(threadRef);
+        assert added;
+        return threadRef;
+    }
+
+    /**
+     * Removes the thread identified by the key from the thread set.
+     */
+    public void end(Object key) {
+        var threadRef = (ThreadRef) key;
+        assert threadRef.thread() == Thread.currentThread();
+        boolean removed = threads.remove(threadRef);
+        assert removed;
+    }
+
+    /**
+     * Returns true if the given thread is tracked.
+     */
+    public boolean contains(Thread thread) {
+        var threadRef = new ThreadRef(thread);
+        return threads.contains(threadRef);
+    }
+}

test/jdk/jdk/security/logging/RecursiveEventHelper.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.util.logging.*;
+
+import jdk.internal.event.EventHelper;
+
+/*
+ * @test
+ * @bug 8329013
+ * @summary StackOverflowError when starting Apache Tomcat with signed jar
+ * @modules java.base/jdk.internal.event:+open
+ * @run main/othervm -Xmx32m -Djava.util.logging.manager=RecursiveEventHelper RecursiveEventHelper
+ */
+public class RecursiveEventHelper extends LogManager {
+    // an extra check to ensure the custom manager is in use
+    static volatile boolean customMethodCalled;
+
+    public static void main(String[] args) throws Exception {
+        String classname = System.getProperty("java.util.logging.manager");
+        if (!classname.equals("RecursiveEventHelper")) {
+            throw new RuntimeException("java.util.logging.manager not set");
+        }
+
+        // this call will trigger initialization of logging framework
+        // which will call into our custom LogManager and use the
+        // custom getProperty method below. EventHelper.isLoggingSecurity()
+        // is also on the code path of original report and triggers
+        // similar recursion.
+        System.getLogger("testLogger");
+        if (!customMethodCalled) {
+            throw new RuntimeException("Method not called");
+        }
+    }
+
+    @Override
+    public String getProperty(String p) {
+        // this call mimics issue reported in initial bug report where
+        // opening of a signed jar during System logger initialization triggered
+        // a recursive call (via EventHelper.isLoggingSecurity) back into
+        // logger API
+        EventHelper.isLoggingSecurity();
+        customMethodCalled = true;
+        return super.getProperty(p);
+    }
+}