[6.8.22] Upgrade to log4j 2.17.0 (#81906)

Backport of:

Upgrade to log4j 2.17.0 Upgrade to log4j 2.17.0 #81902
Tolerate unprivileged log4j getClassLoaders calls Tolerate unprivileged log4j getClassLoaders calls #81840
Tolerate benign log4j status messages in tests Tolerate benign log4j status messages in tests #81851

buildSrc/version.properties

@@ -12,7 +12,7 @@ snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0
 # when updating log4j, please update also docs/java-api/index.asciidoc
-log4j             = 2.11.1
+log4j             = 2.17.0
 slf4j             = 1.6.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle

plugins/repository-hdfs/licenses/log4j-slf4j-impl-2.17.0.jar.sha1

@@ -0,0 +1 @@
+1ec25ce0254749c94549ea9c3cea34bd0488c9c6

server/build.gradle

@@ -283,13 +283,11 @@ thirdPartyAudit.ignoreMissingClasses (
   'org.apache.commons.compress.utils.IOUtils',
   'org.apache.commons.csv.CSVFormat',
   'org.apache.commons.csv.QuoteMode',
-  'org.apache.kafka.clients.producer.Callback',
   'org.apache.kafka.clients.producer.KafkaProducer',
   'org.apache.kafka.clients.producer.Producer',
   'org.apache.kafka.clients.producer.ProducerRecord',
   'org.apache.kafka.clients.producer.RecordMetadata',
   'org.codehaus.stax2.XMLStreamWriter2',
-  'org.jctools.queues.MessagePassingQueue$Consumer',
   'org.jctools.queues.MpscArrayQueue',
   'org.osgi.framework.AdaptPermission',
   'org.osgi.framework.AdminPermission',

server/licenses/log4j-1.2-api-2.17.0.jar.sha1

@@ -0,0 +1 @@
+aaf998968370edf738322fb750fbda118055508b

server/licenses/log4j-api-2.17.0.jar.sha1

@@ -0,0 +1 @@
+bbd791e9c8c9421e45337c4fe0a10851c086e36c

server/licenses/log4j-core-2.17.0.jar.sha1

@@ -0,0 +1 @@
+587127e2f8c5daaef9ba3806848675a1652959d3

server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -31,8 +31,10 @@
 import java.security.Permissions;
 import java.security.Policy;
 import java.security.ProtectionDomain;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Predicate;
 
 /** custom policy for union of static and dynamic permissions */
@@ -61,6 +63,26 @@ final class ESPolicy extends Policy {
         this.plugins = plugins;
     }
 
+    private static final Predicate<StackTraceElement> JDK_BOOT = f -> f.getClassName().startsWith("java.lang.")
+        || f.getClassName().startsWith("java.security.");
+    private static final Predicate<StackTraceElement> ES_BOOTSTRAP = f -> f.getClassName().startsWith("org.elasticsearch.bootstrap");
+    private static final Predicate<StackTraceElement> IS_LOG4J = f -> "org.apache.logging.log4j.util.LoaderUtil".equals(f.getClassName())
+        && "getClassLoaders".equals(f.getMethodName());
+
+    /**
+     *  Returns true if the top of the call stack has:
+     *   1) Only frames belonging from the JDK's boot loader or org.elasticsearch.bootstrap, followed directly by
+     *   2) org.apache.logging.log4j.util.LoaderUtil.getClassLoaders
+     */
+    private static boolean isLoaderUtilGetClassLoaders() {
+        Optional<StackTraceElement> frame = Arrays.stream(Thread.currentThread().getStackTrace())
+            .filter(JDK_BOOT.or(ES_BOOTSTRAP).negate())
+            .limit(1)
+            .findFirst()
+            .filter(IS_LOG4J);
+        return frame.isPresent();
+    }
+
     @Override @SuppressForbidden(reason = "fast equals check is desired")
     public boolean implies(ProtectionDomain domain, Permission permission) {
         CodeSource codeSource = domain.getCodeSource();
@@ -98,6 +120,10 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
             }
         }
 
+        if (permission instanceof RuntimePermission && "getClassLoader".equals(permission.getName()) && isLoaderUtilGetClassLoaders()) {
+            return true;
+        }
+
         // otherwise defer to template + dynamic file permissions
         return template.implies(domain, permission) || dynamic.implies(permission) || system.implies(domain, permission);
     }

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -161,9 +161,13 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -501,6 +505,18 @@ public void log(StatusData data) {
         });
     }
 
+    // Tolerate the absence or otherwise denial of these specific lookup classes.
+    // At some future time, we should require the JDNI warning.
+    private static final List<String> LOG_4J_MSG_PREFIXES = getLog4jMsgPrefixes();
+    private static List<String> getLog4jMsgPrefixes() {
+        ArrayList<String> list = new ArrayList<>();
+        list.add("JNDI lookup class is not available because this JRE does not support JNDI. "
+            + "JNDI string lookups will not be available, continuing configuration.");
+        list.add("JMX runtime input lookup class is not available because this JRE does not support JMX. "
+            + "JMX lookups will not be available, continuing configuration. ");
+        return Collections.unmodifiableList(list);
+    }
+
     // separate method so that this can be checked again after suite scoped cluster is shut down
     protected static void checkStaticState(boolean afterClass) throws Exception {
         if (afterClass) {
@@ -516,7 +532,12 @@ protected static void checkStaticState(boolean afterClass) throws Exception {
                 // StatusData instances to Strings as otherwise their toString output is useless
                 assertThat(
                     statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()),
-                    empty());
+                    anyOf(
+                        emptyCollectionOf(String.class),
+                        contains(startsWith(LOG_4J_MSG_PREFIXES.get(0)), startsWith(LOG_4J_MSG_PREFIXES.get(1))),
+                        contains(startsWith(LOG_4J_MSG_PREFIXES.get(1)))
+                    )
+                );
             } finally {
                 // we clear the list so that status data from other tests do not interfere with tests within the same JVM
                 statusData.clear();

x-pack/plugin/security/licenses/log4j-slf4j-impl-2.17.0.jar.sha1

@@ -0,0 +1 @@
+1ec25ce0254749c94549ea9c3cea34bd0488c9c6

x-pack/plugin/sql/sql-action/build.gradle

@@ -115,13 +115,11 @@ thirdPartyAudit.ignoreMissingClasses (
         'org.apache.commons.compress.utils.IOUtils',
         'org.apache.commons.csv.CSVFormat',
         'org.apache.commons.csv.QuoteMode',
-        'org.apache.kafka.clients.producer.Callback',
         'org.apache.kafka.clients.producer.KafkaProducer',
         'org.apache.kafka.clients.producer.Producer',
         'org.apache.kafka.clients.producer.ProducerRecord',
         'org.apache.kafka.clients.producer.RecordMetadata',
         'org.codehaus.stax2.XMLStreamWriter2',
-        'org.jctools.queues.MessagePassingQueue$Consumer',
         'org.jctools.queues.MpscArrayQueue',
         'org.osgi.framework.AdaptPermission',
         'org.osgi.framework.AdminPermission',

x-pack/plugin/sql/sql-action/licenses/log4j-api-2.17.0.jar.sha1

@@ -0,0 +1 @@
+bbd791e9c8c9421e45337c4fe0a10851c086e36c

x-pack/plugin/sql/sql-action/licenses/log4j-core-2.17.0.jar.sha1

@@ -0,0 +1 @@
+fe6e7a32c1228884b9691a744f953a55d0dd8ead