From e784f432bb7970e0d32a9ab35e094e8227c96a2d Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Fri, 4 Apr 2025 17:59:57 +0200 Subject: [PATCH 1/7] Add support for a custom transport version check predicate --- .../internal/dependencies/patches/Utils.java | 74 +++++++++++++++++ .../patches/awsv2sdk/Awsv2ClassPatcher.java | 57 +++++++++++++ .../StringFormatInPathResolverPatcher.java | 83 +++++++++++++++++++ .../patches/hdfs/HdfsClassPatcher.java | 57 +------------ .../hdfs/SubjectGetSubjectPatcher.java | 4 +- plugins/discovery-ec2/build.gradle | 38 ++++++++- 6 files changed, 255 insertions(+), 58 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java new file mode 100644 index 0000000000000..e250f6ca9223d --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; + +public class Utils { + public static void patchJar(File inputFile, File outputFile, Map> patchers) { + var classPatchers = new HashMap<>(patchers); + try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + // Add the entry to the new JAR file + jos.putNextEntry(new JarEntry(entryName)); + + Function classPatcher = classPatchers.remove(entryName); + if (classPatcher != null) { + byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); + + ClassReader classReader = new ClassReader(classToPatch); + ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_MAXS | COMPUTE_FRAMES); + classReader.accept(classPatcher.apply(classWriter), 0); + jos.write(classWriter.toByteArray()); + } else { + // Read the entry's data and write it to the new JAR + try (InputStream is = jarFile.getInputStream(entry)) { + is.transferTo(jos); + } + } + jos.closeEntry(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + + if (classPatchers.isEmpty() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "error patching [%s]: the jar does not contain [%s]", + inputFile.getName(), + String.join(", ", patchers.keySet()) + ) + ); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java new file mode 100644 index 0000000000000..f505b93f6eff8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk; + +import org.elasticsearch.gradle.internal.dependencies.patches.Utils; +import org.gradle.api.artifacts.transform.CacheableTransform; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; +import org.jetbrains.annotations.NotNull; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.util.Map; +import java.util.function.Function; + +import static java.util.Map.entry; + +@CacheableTransform +public abstract class Awsv2ClassPatcher implements TransformAction { + + private static final String JAR_FILE_TO_PATCH = "aws-query-protocol"; + + private static final Map> CLASS_PATCHERS = Map.ofEntries( + entry("software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class", StringFormatInPathResolverPatcher::new) + ); + + @Classpath + @InputArtifact + public abstract Provider getInputArtifact(); + + @Override + public void transform(@NotNull TransformOutputs outputs) { + File inputFile = getInputArtifact().get().getAsFile(); + + if (inputFile.getName().startsWith(JAR_FILE_TO_PATCH) == false) { + System.out.println("Skipping " + inputFile.getName()); + outputs.file(getInputArtifact()); + } else { + System.out.println("Patching " + inputFile.getName()); + File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); + Utils.patchJar(inputFile, outputFile, CLASS_PATCHERS); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java new file mode 100644 index 0000000000000..021d921519572 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.util.Locale; + +import static org.objectweb.asm.Opcodes.ASM9; +import static org.objectweb.asm.Opcodes.GETSTATIC; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; + +class StringFormatInPathResolverPatcher extends ClassVisitor { + + StringFormatInPathResolverPatcher(ClassWriter classWriter) { + super(ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions)); + } + + /** + * Replaces calls to String.format(format, args); with calls to String.format(Locale.ROOT, format, args); + */ + private static class ReplaceCallMethodVisitor extends MethodVisitor { + private static final String CLASS_INTERNAL_NAME = Type.getInternalName(String.class); + private static final String METHOD_NAME = "format"; + private static final String OLD_METHOD_DESCRIPTOR = Type.getMethodDescriptor( + Type.getType(String.class), + Type.getType(String.class), + Type.getType(Object[].class) + ); + private static final String NEW_METHOD_DESCRIPTOR = Type.getMethodDescriptor( + Type.getType(String.class), + Type.getType(Locale.class), + Type.getType(String.class), + Type.getType(Object[].class) + ); + + private boolean foundFormatPattern = false; + + ReplaceCallMethodVisitor(MethodVisitor methodVisitor) { + super(ASM9, methodVisitor); + } + + @Override + public void visitLdcInsn(Object value) { + if (value instanceof String s && s.startsWith("%s")) { + // Push the extra arg on the stack + mv.visitFieldInsn(GETSTATIC, Type.getInternalName(Locale.class), "ROOT", Type.getDescriptor(Locale.class)); + foundFormatPattern = true; + } + super.visitLdcInsn(value); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (opcode == INVOKESTATIC + && foundFormatPattern + && CLASS_INTERNAL_NAME.equals(owner) + && METHOD_NAME.equals(name) + && OLD_METHOD_DESCRIPTOR.equals(descriptor)) { + // Replace the call with String.format(Locale.ROOT, format, args) + mv.visitMethodInsn(INVOKESTATIC, CLASS_INTERNAL_NAME, METHOD_NAME, NEW_METHOD_DESCRIPTOR, false); + foundFormatPattern = false; + } else { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java index adb39368d8d24..e6d870156ddc6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; +import org.elasticsearch.gradle.internal.dependencies.patches.Utils; import org.gradle.api.artifacts.transform.CacheableTransform; import org.gradle.api.artifacts.transform.InputArtifact; import org.gradle.api.artifacts.transform.TransformAction; @@ -20,28 +21,17 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Optional; import org.jetbrains.annotations.NotNull; -import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Enumeration; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.function.Function; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.JarOutputStream; import java.util.regex.Pattern; import static java.util.Map.entry; -import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; -import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; @CacheableTransform public abstract class HdfsClassPatcher implements TransformAction { @@ -99,51 +89,8 @@ public void transform(@NotNull TransformOutputs outputs) { Map> jarPatchers = new HashMap<>(patchers.jarPatchers()); File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); - patchJar(inputFile, outputFile, jarPatchers); - - if (jarPatchers.isEmpty() == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "error patching [%s] with [%s]: the jar does not contain [%s]", - inputFile.getName(), - patchers.artifactPattern().toString(), - String.join(", ", jarPatchers.keySet()) - ) - ); - } + Utils.patchJar(inputFile, outputFile, jarPatchers); }); } } - - private static void patchJar(File inputFile, File outputFile, Map> jarPatchers) { - try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { - Enumeration entries = jarFile.entries(); - while (entries.hasMoreElements()) { - JarEntry entry = entries.nextElement(); - String entryName = entry.getName(); - // Add the entry to the new JAR file - jos.putNextEntry(new JarEntry(entryName)); - - Function classPatcher = jarPatchers.remove(entryName); - if (classPatcher != null) { - byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); - - ClassReader classReader = new ClassReader(classToPatch); - ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); - classReader.accept(classPatcher.apply(classWriter), 0); - - jos.write(classWriter.toByteArray()); - } else { - // Read the entry's data and write it to the new JAR - try (InputStream is = jarFile.getInputStream(entry)) { - is.transferTo(jos); - } - } - jos.closeEntry(); - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java index 3b241f7001fb4..e8741311bdd44 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java @@ -25,7 +25,7 @@ class SubjectGetSubjectPatcher extends ClassVisitor { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor); + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions)); } /** @@ -35,7 +35,7 @@ private static class ReplaceCallMethodVisitor extends MethodVisitor { private static final String SUBJECT_CLASS_INTERNAL_NAME = "javax/security/auth/Subject"; private static final String METHOD_NAME = "getSubject"; - ReplaceCallMethodVisitor(MethodVisitor methodVisitor, String name, int access, String descriptor) { + ReplaceCallMethodVisitor(MethodVisitor methodVisitor) { super(ASM9, methodVisitor); } diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index f4eb1d3a90f01..237b6e0236072 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -15,6 +15,31 @@ esplugin { classname ='org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin' } +def patched = Attribute.of('patched', Boolean) + +configurations { + compileClasspath { + attributes { + attribute(patched, true) + } + } + runtimeClasspath { + attributes { + attribute(patched, true) + } + } + testCompileClasspath { + attributes { + attribute(patched, true) + } + } + testRuntimeClasspath { + attributes { + attribute(patched, true) + } + } +} + dependencies { implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}" @@ -30,7 +55,7 @@ dependencies { implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}" implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}" - runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" + implementation "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}" @@ -65,6 +90,17 @@ dependencies { testImplementation project(':test:fixtures:ec2-imds-fixture') internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture') + + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk.Awsv2ClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + } } tasks.named("dependencyLicenses").configure { From 5828031660cd5b46f1ea893d209ffc7423cbdd64 Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Fri, 4 Apr 2025 17:59:57 +0200 Subject: [PATCH 2/7] Patcher for awssdkv2 String.format --- .../internal/dependencies/patches/Utils.java | 74 +++++++++++++++++ .../patches/awsv2sdk/Awsv2ClassPatcher.java | 57 +++++++++++++ .../StringFormatInPathResolverPatcher.java | 83 +++++++++++++++++++ .../patches/hdfs/HdfsClassPatcher.java | 57 +------------ .../hdfs/SubjectGetSubjectPatcher.java | 4 +- plugins/discovery-ec2/build.gradle | 38 ++++++++- 6 files changed, 255 insertions(+), 58 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java new file mode 100644 index 0000000000000..e250f6ca9223d --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; +import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; + +public class Utils { + public static void patchJar(File inputFile, File outputFile, Map> patchers) { + var classPatchers = new HashMap<>(patchers); + try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + // Add the entry to the new JAR file + jos.putNextEntry(new JarEntry(entryName)); + + Function classPatcher = classPatchers.remove(entryName); + if (classPatcher != null) { + byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); + + ClassReader classReader = new ClassReader(classToPatch); + ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_MAXS | COMPUTE_FRAMES); + classReader.accept(classPatcher.apply(classWriter), 0); + jos.write(classWriter.toByteArray()); + } else { + // Read the entry's data and write it to the new JAR + try (InputStream is = jarFile.getInputStream(entry)) { + is.transferTo(jos); + } + } + jos.closeEntry(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + + if (classPatchers.isEmpty() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "error patching [%s]: the jar does not contain [%s]", + inputFile.getName(), + String.join(", ", patchers.keySet()) + ) + ); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java new file mode 100644 index 0000000000000..f505b93f6eff8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk; + +import org.elasticsearch.gradle.internal.dependencies.patches.Utils; +import org.gradle.api.artifacts.transform.CacheableTransform; +import org.gradle.api.artifacts.transform.InputArtifact; +import org.gradle.api.artifacts.transform.TransformAction; +import org.gradle.api.artifacts.transform.TransformOutputs; +import org.gradle.api.artifacts.transform.TransformParameters; +import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Classpath; +import org.jetbrains.annotations.NotNull; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.io.File; +import java.util.Map; +import java.util.function.Function; + +import static java.util.Map.entry; + +@CacheableTransform +public abstract class Awsv2ClassPatcher implements TransformAction { + + private static final String JAR_FILE_TO_PATCH = "aws-query-protocol"; + + private static final Map> CLASS_PATCHERS = Map.ofEntries( + entry("software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class", StringFormatInPathResolverPatcher::new) + ); + + @Classpath + @InputArtifact + public abstract Provider getInputArtifact(); + + @Override + public void transform(@NotNull TransformOutputs outputs) { + File inputFile = getInputArtifact().get().getAsFile(); + + if (inputFile.getName().startsWith(JAR_FILE_TO_PATCH) == false) { + System.out.println("Skipping " + inputFile.getName()); + outputs.file(getInputArtifact()); + } else { + System.out.println("Patching " + inputFile.getName()); + File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); + Utils.patchJar(inputFile, outputFile, CLASS_PATCHERS); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java new file mode 100644 index 0000000000000..021d921519572 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.util.Locale; + +import static org.objectweb.asm.Opcodes.ASM9; +import static org.objectweb.asm.Opcodes.GETSTATIC; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; + +class StringFormatInPathResolverPatcher extends ClassVisitor { + + StringFormatInPathResolverPatcher(ClassWriter classWriter) { + super(ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions)); + } + + /** + * Replaces calls to String.format(format, args); with calls to String.format(Locale.ROOT, format, args); + */ + private static class ReplaceCallMethodVisitor extends MethodVisitor { + private static final String CLASS_INTERNAL_NAME = Type.getInternalName(String.class); + private static final String METHOD_NAME = "format"; + private static final String OLD_METHOD_DESCRIPTOR = Type.getMethodDescriptor( + Type.getType(String.class), + Type.getType(String.class), + Type.getType(Object[].class) + ); + private static final String NEW_METHOD_DESCRIPTOR = Type.getMethodDescriptor( + Type.getType(String.class), + Type.getType(Locale.class), + Type.getType(String.class), + Type.getType(Object[].class) + ); + + private boolean foundFormatPattern = false; + + ReplaceCallMethodVisitor(MethodVisitor methodVisitor) { + super(ASM9, methodVisitor); + } + + @Override + public void visitLdcInsn(Object value) { + if (value instanceof String s && s.startsWith("%s")) { + // Push the extra arg on the stack + mv.visitFieldInsn(GETSTATIC, Type.getInternalName(Locale.class), "ROOT", Type.getDescriptor(Locale.class)); + foundFormatPattern = true; + } + super.visitLdcInsn(value); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (opcode == INVOKESTATIC + && foundFormatPattern + && CLASS_INTERNAL_NAME.equals(owner) + && METHOD_NAME.equals(name) + && OLD_METHOD_DESCRIPTOR.equals(descriptor)) { + // Replace the call with String.format(Locale.ROOT, format, args) + mv.visitMethodInsn(INVOKESTATIC, CLASS_INTERNAL_NAME, METHOD_NAME, NEW_METHOD_DESCRIPTOR, false); + foundFormatPattern = false; + } else { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java index adb39368d8d24..e6d870156ddc6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/HdfsClassPatcher.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.dependencies.patches.hdfs; +import org.elasticsearch.gradle.internal.dependencies.patches.Utils; import org.gradle.api.artifacts.transform.CacheableTransform; import org.gradle.api.artifacts.transform.InputArtifact; import org.gradle.api.artifacts.transform.TransformAction; @@ -20,28 +21,17 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Optional; import org.jetbrains.annotations.NotNull; -import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Enumeration; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.function.Function; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.JarOutputStream; import java.util.regex.Pattern; import static java.util.Map.entry; -import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; -import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; @CacheableTransform public abstract class HdfsClassPatcher implements TransformAction { @@ -99,51 +89,8 @@ public void transform(@NotNull TransformOutputs outputs) { Map> jarPatchers = new HashMap<>(patchers.jarPatchers()); File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); - patchJar(inputFile, outputFile, jarPatchers); - - if (jarPatchers.isEmpty() == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "error patching [%s] with [%s]: the jar does not contain [%s]", - inputFile.getName(), - patchers.artifactPattern().toString(), - String.join(", ", jarPatchers.keySet()) - ) - ); - } + Utils.patchJar(inputFile, outputFile, jarPatchers); }); } } - - private static void patchJar(File inputFile, File outputFile, Map> jarPatchers) { - try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { - Enumeration entries = jarFile.entries(); - while (entries.hasMoreElements()) { - JarEntry entry = entries.nextElement(); - String entryName = entry.getName(); - // Add the entry to the new JAR file - jos.putNextEntry(new JarEntry(entryName)); - - Function classPatcher = jarPatchers.remove(entryName); - if (classPatcher != null) { - byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); - - ClassReader classReader = new ClassReader(classToPatch); - ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_FRAMES | COMPUTE_MAXS); - classReader.accept(classPatcher.apply(classWriter), 0); - - jos.write(classWriter.toByteArray()); - } else { - // Read the entry's data and write it to the new JAR - try (InputStream is = jarFile.getInputStream(entry)) { - is.transferTo(jos); - } - } - jos.closeEntry(); - } - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java index 3b241f7001fb4..e8741311bdd44 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/hdfs/SubjectGetSubjectPatcher.java @@ -25,7 +25,7 @@ class SubjectGetSubjectPatcher extends ClassVisitor { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor); + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions)); } /** @@ -35,7 +35,7 @@ private static class ReplaceCallMethodVisitor extends MethodVisitor { private static final String SUBJECT_CLASS_INTERNAL_NAME = "javax/security/auth/Subject"; private static final String METHOD_NAME = "getSubject"; - ReplaceCallMethodVisitor(MethodVisitor methodVisitor, String name, int access, String descriptor) { + ReplaceCallMethodVisitor(MethodVisitor methodVisitor) { super(ASM9, methodVisitor); } diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index f4eb1d3a90f01..237b6e0236072 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -15,6 +15,31 @@ esplugin { classname ='org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin' } +def patched = Attribute.of('patched', Boolean) + +configurations { + compileClasspath { + attributes { + attribute(patched, true) + } + } + runtimeClasspath { + attributes { + attribute(patched, true) + } + } + testCompileClasspath { + attributes { + attribute(patched, true) + } + } + testRuntimeClasspath { + attributes { + attribute(patched, true) + } + } +} + dependencies { implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}" @@ -30,7 +55,7 @@ dependencies { implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}" implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}" - runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" + implementation "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}" @@ -65,6 +90,17 @@ dependencies { testImplementation project(':test:fixtures:ec2-imds-fixture') internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture') + + attributesSchema { + attribute(patched) + } + artifactTypes.getByName("jar") { + attributes.attribute(patched, false) + } + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk.Awsv2ClassPatcher) { + from.attribute(patched, false) + to.attribute(patched, true) + } } tasks.named("dependencyLicenses").configure { From 91f54ff8ee319b11730a5ab751ad1b2b7a5a0033 Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Fri, 4 Apr 2025 18:28:45 +0200 Subject: [PATCH 3/7] Added reference to AWS SDKv2 bug + additional check for safety --- .../dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java | 2 ++ .../patches/awsv2sdk/StringFormatInPathResolverPatcher.java | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java index f505b93f6eff8..ea15b49e014ce 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java @@ -34,6 +34,8 @@ public abstract class Awsv2ClassPatcher implements TransformAction> CLASS_PATCHERS = Map.ofEntries( + // This patcher is needed because of this AWS bug: https://github.com/aws/aws-sdk-java-v2/issues/5968 + // As soon as the bug is resolved and we upgrade our AWS SDK v2 libraries, we can remove this. entry("software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class", StringFormatInPathResolverPatcher::new) ); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java index 021d921519572..e38e450deb4c7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java @@ -58,6 +58,10 @@ private static class ReplaceCallMethodVisitor extends MethodVisitor { @Override public void visitLdcInsn(Object value) { if (value instanceof String s && s.startsWith("%s")) { + if (foundFormatPattern) { + throw new IllegalStateException("A previous string format constant was not paired with a String.format() call. " + + "Patching would generate an unbalances stack"); + } // Push the extra arg on the stack mv.visitFieldInsn(GETSTATIC, Type.getInternalName(Locale.class), "ROOT", Type.getDescriptor(Locale.class)); foundFormatPattern = true; From 8c89760008dd777aef80f51694602dce269b2609 Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Mon, 7 Apr 2025 14:22:21 +0200 Subject: [PATCH 4/7] spotless + revert gradle change --- .../patches/awsv2sdk/StringFormatInPathResolverPatcher.java | 6 ++++-- plugins/discovery-ec2/build.gradle | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java index e38e450deb4c7..506dab001dbe7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java @@ -59,8 +59,10 @@ private static class ReplaceCallMethodVisitor extends MethodVisitor { public void visitLdcInsn(Object value) { if (value instanceof String s && s.startsWith("%s")) { if (foundFormatPattern) { - throw new IllegalStateException("A previous string format constant was not paired with a String.format() call. " + - "Patching would generate an unbalances stack"); + throw new IllegalStateException( + "A previous string format constant was not paired with a String.format() call. " + + "Patching would generate an unbalances stack" + ); } // Push the extra arg on the stack mv.visitFieldInsn(GETSTATIC, Type.getInternalName(Locale.class), "ROOT", Type.getDescriptor(Locale.class)); diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 237b6e0236072..454508d0298f9 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -55,7 +55,7 @@ dependencies { implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}" implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}" - implementation "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" + runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}" runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}" From f9980f2de04ea56ff5c53a3d19fe3776485c8fb1 Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Mon, 7 Apr 2025 15:32:05 +0200 Subject: [PATCH 5/7] add (optional) match on class bytes SHA256 digest --- .../dependencies/patches/PatcherInfo.java | 47 +++++++++++++++++ .../internal/dependencies/patches/Utils.java | 52 ++++++++++++++++--- .../patches/awsv2sdk/Awsv2ClassPatcher.java | 24 +++++---- .../patches/hdfs/HdfsClassPatcher.java | 33 +++++------- 4 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java new file mode 100644 index 0000000000000..48f163d3fe3c0 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.dependencies.patches; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +import java.util.Arrays; +import java.util.HexFormat; +import java.util.function.Function; + +public record PatcherInfo(String jarEntryName, byte[] classSha256, Function visitorFactory) { + + /** + * Creates a patcher info entry, linking a jar entry path name to a patcher factory (a factory to create an ASM visitor) + * @param jarEntryName the jar entry path, as a string + * @param visitorFactory the factory to create an ASM visitor from a ASM writer + */ + public static PatcherInfo classPatcher(String jarEntryName, Function visitorFactory) { + return new PatcherInfo(jarEntryName, null, visitorFactory); + } + + /** + * Creates a patcher info entry, linking a jar entry path name and its SHA256 digest to a patcher factory (a factory to create an ASM + * visitor) + * @param jarEntryName the jar entry path, as a string + * @param classSha256 the SHA256 digest of the class bytes, as a HEX string + * @param visitorFactory the factory to create an ASM visitor from a ASM writer + */ + public static PatcherInfo classPatcher(String jarEntryName, String classSha256, Function visitorFactory) { + return new PatcherInfo(jarEntryName, HexFormat.of().parseHex(classSha256), visitorFactory); + } + + boolean matches(byte[] otherClassSha256) { + if (this.classSha256 == null) { + return true; + } + return Arrays.equals(this.classSha256, otherClassSha256); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java index e250f6ca9223d..d5adbe994e33f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java @@ -10,28 +10,50 @@ package org.elasticsearch.gradle.internal.dependencies.patches; import org.objectweb.asm.ClassReader; -import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; import java.util.Enumeration; -import java.util.HashMap; +import java.util.HexFormat; import java.util.Locale; -import java.util.Map; import java.util.function.Function; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.jar.JarOutputStream; +import java.util.stream.Collectors; import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES; import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS; public class Utils { - public static void patchJar(File inputFile, File outputFile, Map> patchers) { - var classPatchers = new HashMap<>(patchers); + + private static final MessageDigest SHA_256; + + static { + try { + SHA_256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Patches the classes in the input JAR file, using the collection of patchers. If the patcher info specify a SHA256 digest, and + * the class to patch does not match it, an IllegalArgumentException is thrown. + * If the input file does not contain all the classes to patch specified in the patcher info collection, an IllegalArgumentException + * is also thrown. + * @param inputFile the JAR file to patch + * @param outputFile the output (patched) JAR file + * @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them) + */ + public static void patchJar(File inputFile, File outputFile, Collection patchers) { + var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity())); try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { Enumeration entries = jarFile.entries(); while (entries.hasMoreElements()) { @@ -40,13 +62,27 @@ public static void patchJar(File inputFile, File outputFile, Map classPatcher = classPatchers.remove(entryName); + var classPatcher = classPatchers.remove(entryName); if (classPatcher != null) { byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes(); + var classSha256 = SHA_256.digest(classToPatch); + + if (classPatcher.matches(classSha256) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "error patching [%s]: jar entry [%s] digest mismatch (expected: [%s], found: [%s])", + inputFile.getName(), + classPatcher.jarEntryName(), + HexFormat.of().formatHex(classPatcher.classSha256()), + HexFormat.of().formatHex(classSha256) + ) + ); + } ClassReader classReader = new ClassReader(classToPatch); ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_MAXS | COMPUTE_FRAMES); - classReader.accept(classPatcher.apply(classWriter), 0); + classReader.accept(classPatcher.visitorFactory().apply(classWriter), 0); jos.write(classWriter.toByteArray()); } else { // Read the entry's data and write it to the new JAR @@ -66,7 +102,7 @@ public static void patchJar(File inputFile, File outputFile, Map { private static final String JAR_FILE_TO_PATCH = "aws-query-protocol"; - private static final Map> CLASS_PATCHERS = Map.ofEntries( + private static final List CLASS_PATCHERS = List.of( // This patcher is needed because of this AWS bug: https://github.com/aws/aws-sdk-java-v2/issues/5968 // As soon as the bug is resolved and we upgrade our AWS SDK v2 libraries, we can remove this. - entry("software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class", StringFormatInPathResolverPatcher::new) + classPatcher( + "software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class", + "213e84d9a745bdae4b844334d17aecdd6499b36df32aa73f82dc114b35043009", + StringFormatInPathResolverPatcher::new + ) ); @Classpath @@ -47,13 +49,13 @@ public abstract class Awsv2ClassPatcher implements TransformAction { - record JarPatchers(String artifactTag, Pattern artifactPattern, Map> jarPatchers) {} + record JarPatchers(String artifactTag, Pattern artifactPattern, List jarPatchers) {} static final List allPatchers = List.of( new JarPatchers( "hadoop-common", Pattern.compile("hadoop-common-(?!.*tests)"), - Map.ofEntries( - entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), - entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), - entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new) + List.of( + classPatcher("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + classPatcher("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + classPatcher("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new) ) ), new JarPatchers( "hadoop-client-api", Pattern.compile("hadoop-client-api.*"), - Map.ofEntries( - entry("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), - entry("org/apache/hadoop/util/Shell.class", ShellPatcher::new), - entry("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new), - entry("org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", SubjectGetSubjectPatcher::new) + List.of( + classPatcher("org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new), + classPatcher("org/apache/hadoop/util/Shell.class", ShellPatcher::new), + classPatcher("org/apache/hadoop/security/UserGroupInformation.class", SubjectGetSubjectPatcher::new), + classPatcher("org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", SubjectGetSubjectPatcher::new) ) ) ); @@ -85,11 +81,8 @@ public void transform(@NotNull TransformOutputs outputs) { } else { patchersToApply.forEach(patchers -> { System.out.println("Patching " + inputFile.getName()); - - Map> jarPatchers = new HashMap<>(patchers.jarPatchers()); File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar")); - - Utils.patchJar(inputFile, outputFile, jarPatchers); + Utils.patchJar(inputFile, outputFile, patchers.jarPatchers()); }); } } From 83b80cd48050ea5acb05d9a0b09ac32da1099afa Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Tue, 8 Apr 2025 15:06:44 +0200 Subject: [PATCH 6/7] differentiate by hdfs version --- .../dependencies/patches/PatcherInfo.java | 3 -- .../internal/dependencies/patches/Utils.java | 9 +++--- .../patches/hdfs/HdfsClassPatcher.java | 31 ++++++++++++++++--- test/fixtures/hdfs-fixture/build.gradle | 27 +++++++++++----- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java index eb0a97b63774b..6fbcde93bad6a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java @@ -29,9 +29,6 @@ public static PatcherInfo classPatcher(String jarEntryName, String classSha256, } boolean matches(byte[] otherClassSha256) { - if (this.classSha256 == null) { - return true; - } return Arrays.equals(this.classSha256, otherClassSha256); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java index 7265ab7954777..ea8c89a77aacb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java @@ -71,10 +71,11 @@ public static void patchJar(File inputFile, File outputFile, Collection allPatchers = List.of( new JarPatchers( - "hadoop-common", - Pattern.compile("hadoop-common-(?!.*tests)"), + "hadoop2-common", + Pattern.compile("hadoop-common-2(?!.*tests)"), List.of( classPatcher( "org/apache/hadoop/util/ShutdownHookManager.class", - "90641e0726fc9372479728ef9b7ae2be20fb7ab4cddd4938e55ffecadddd4d94", + "3912451f02da9199dae7dba3f1420e0d951067addabbb235e7551de52234a0ef", ShutdownHookManagerPatcher::new ), classPatcher( "org/apache/hadoop/util/Shell.class", - "8837c7f3eeda3f658fc3d6595f18e77a4558220ff0becdf3e175fa4397a6fd0c", + "60400dc800e7c3e1a5fc499793033d877f5319bbd7633fee05d5a1d96b947bbd", ShellPatcher::new ), classPatcher( "org/apache/hadoop/security/UserGroupInformation.class", - "3c34bbc2716a6c8f4e356e78550599b0a4f01882712b4f7787d032fb10527212", + "218078b8c77838f93d015c843775985a71f3c7a8128e2a9394410f0cd1da5f53", + SubjectGetSubjectPatcher::new + ) + ) + ), + new JarPatchers( + "hadoop3-common", + Pattern.compile("hadoop-common-3(?!.*tests)"), + List.of( + classPatcher( + "org/apache/hadoop/util/ShutdownHookManager.class", + "7720e8545a02de6fd03f4170f0e471d1301ef73d7d6a09097bad361f9e31f819", + ShutdownHookManagerPatcher::new + ), + classPatcher( + "org/apache/hadoop/util/Shell.class", + "856d0b829cf550df826387af15fa1c772bc7d26d6461535b17b9d5114d308dc4", + ShellPatcher::new + ), + classPatcher( + "org/apache/hadoop/security/UserGroupInformation.class", + "52f5973f35a282908d48a573a03c04f240a22c9f6007d7c5e7852aff1c641420", SubjectGetSubjectPatcher::new ) ) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 6360ead8126fa..a273f5d6656dd 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -13,16 +13,17 @@ apply plugin: 'com.gradleup.shadow' import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar -def patched = Attribute.of('patched', Boolean) +def hdfs2patched = Attribute.of('hdfs2-patched', Boolean) +def hdfs3patched = Attribute.of('hdfs3-patched', Boolean) configurations { hdfs2 { attributes { - attribute(patched, true) + attribute(hdfs2patched, true) } } hdfs3 { attributes { - attribute(patched, true) + attribute(hdfs3patched, true) } } consumable("shadowedHdfs2") @@ -30,16 +31,26 @@ configurations { dependencies { attributesSchema { - attribute(patched) + attribute(hdfs2patched) + attribute(hdfs3patched) } artifactTypes.getByName("jar") { - attributes.attribute(patched, false) + attributes.attribute(hdfs2patched, false) + attributes.attribute(hdfs3patched, false) } registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { - from.attribute(patched, false) - to.attribute(patched, true) + from.attribute(hdfs2patched, false) + to.attribute(hdfs2patched, true) parameters { - matchingArtifacts = ["hadoop-common"] + matchingArtifacts = ["hadoop2-common"] + } + } + + registerTransform(org.elasticsearch.gradle.internal.dependencies.patches.hdfs.HdfsClassPatcher) { + from.attribute(hdfs3patched, false) + to.attribute(hdfs3patched, true) + parameters { + matchingArtifacts = ["hadoop3-common"] } } From 5a3f7184255b4413596547ca75842fda5c4f4bd9 Mon Sep 17 00:00:00 2001 From: Lorenzo Dematte Date: Fri, 11 Apr 2025 15:09:27 +0200 Subject: [PATCH 7/7] PR feedback --- .../dependencies/patches/PatcherInfo.java | 29 ++++++++- .../internal/dependencies/patches/Utils.java | 60 +++++++++++++------ 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java index 6fbcde93bad6a..5647315bae7a4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java @@ -16,12 +16,23 @@ import java.util.HexFormat; import java.util.function.Function; -public record PatcherInfo(String jarEntryName, byte[] classSha256, Function visitorFactory) { +public final class PatcherInfo { + private final String jarEntryName; + private final byte[] classSha256; + private final Function visitorFactory; + + private PatcherInfo(String jarEntryName, byte[] classSha256, Function visitorFactory) { + this.jarEntryName = jarEntryName; + this.classSha256 = classSha256; + this.visitorFactory = visitorFactory; + } + /** * Creates a patcher info entry, linking a jar entry path name and its SHA256 digest to a patcher factory (a factory to create an ASM * visitor) - * @param jarEntryName the jar entry path, as a string - * @param classSha256 the SHA256 digest of the class bytes, as a HEX string + * + * @param jarEntryName the jar entry path, as a string + * @param classSha256 the SHA256 digest of the class bytes, as a HEX string * @param visitorFactory the factory to create an ASM visitor from a ASM writer */ public static PatcherInfo classPatcher(String jarEntryName, String classSha256, Function visitorFactory) { @@ -31,4 +42,16 @@ public static PatcherInfo classPatcher(String jarEntryName, String classSha256, boolean matches(byte[] otherClassSha256) { return Arrays.equals(this.classSha256, otherClassSha256); } + + public String jarEntryName() { + return jarEntryName; + } + + public byte[] classSha256() { + return classSha256; + } + + public ClassVisitor createVisitor(ClassWriter classWriter) { + return visitorFactory.apply(classWriter); + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java index ea8c89a77aacb..63831b6f062ce 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Collection; import java.util.Enumeration; import java.util.HexFormat; @@ -43,9 +44,26 @@ public class Utils { } } + private record MismatchInfo(String jarEntryName, String expectedClassSha256, String foundClassSha256) { + @Override + public String toString() { + return "[class='" + + jarEntryName + + '\'' + + ", expected='" + + expectedClassSha256 + + '\'' + + ", found='" + + foundClassSha256 + + '\'' + + ']'; + } + } + /** - * Patches the classes in the input JAR file, using the collection of patchers. If the patcher info specify a SHA256 digest, and - * the class to patch does not match it, an IllegalArgumentException is thrown. + * Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry + * name) and the SHA256 digest on the class bytes. + * This digest is checked against the class bytes in the JAR, and if it does not match, an IllegalArgumentException is thrown. * If the input file does not contain all the classes to patch specified in the patcher info collection, an IllegalArgumentException * is also thrown. * @param inputFile the JAR file to patch @@ -54,6 +72,7 @@ public class Utils { */ public static void patchJar(File inputFile, File outputFile, Collection patchers) { var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity())); + var mismatchedClasses = new ArrayList(); try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) { Enumeration entries = jarFile.entries(); while (entries.hasMoreElements()) { @@ -67,27 +86,20 @@ public static void patchJar(File inputFile, File outputFile, Collection