Check for code usage fluctuations in native images

This aims to aid in detecting code usage fluctuations as early as possible. I set the threshold to 3% which might be a bit tight. Ideally most (if not all) ITs being tested in native mode should include such checks. This way when a dependency gets updated and brings in more bloat, or a GraalVM change results in more code becoming reachable we will be able to notice. If significant fluctuations between GraalVM/Mandrel versions are detected, the annotations @DisableIfBuiltWithGraalVMOlderThan and @DisableIfBuiltWithGraalVMNewerThan may be used to run different tests with different expected values for each version. The `EXPECTED_*` constant values part of the tests can be generated by running the following script (passing the build output json file as input to it) after a failed run. This way enabling the check for a new integration test is a matter of: 1. copying `ImageMetricsITCase.java` in the corresponding folder 2. running a test that is expected to fail (unless to tests happen to have pretty similar expected values) 3. running the following script 4. copy-pasting the script output in the pasted `ImageMetricsITCase.java` file ```bash echo " private static final int EXPECTED_IMAGE_SIZE = $(jq .image_details.total_bytes $1);" echo " private static final int EXPECTED_REACHABLE_TYPES = $(jq .analysis_results.types.reachable $1);" echo " private static final int EXPECTED_REACHABLE_METHODS = $(jq .analysis_results.methods.reachable $1);" echo " private static final int EXPECTED_REACHABLE_FIELDS = $(jq .analysis_results.fields.reachable $1);" echo " private static final int EXPECTED_REFLECTION_TYPES = $(jq .analysis_results.types.reflection $1);" echo " private static final int EXPECTED_REFLECTION_METHODS = $(jq .analysis_results.methods.reflection $1);" echo " private static final int EXPECTED_REFLECTION_FIELDS = $(jq .analysis_results.fields.reflection $1);" echo " private static final int EXPECTED_JNI_TYPES = $(jq .analysis_results.types.jni $1);" echo " private static final int EXPECTED_JNI_METHODS = $(jq .analysis_results.methods.jni $1);" echo " private static final int EXPECTED_JNI_FIELDS = $(jq .analysis_results.fields.jni $1);" ```
quarkusio · Sep 22, 2023 · 2118e75 · 2118e75
1 parent af4208a
commit 2118e75
Show file tree

Hide file tree

Showing 4 changed files with 317 additions and 0 deletions.
diff --git a/...jpa-postgresql-withxml/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java b/...jpa-postgresql-withxml/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java
@@ -0,0 +1,80 @@
+package io.quarkus.it.jpa.postgresql;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.nativeimage.BuildOutput;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@QuarkusIntegrationTest
+public class ImageMetricsITCase {
+    private static final int TOLERANCE_PERCENTAGE = 3;
+
+    private static final int EXPECTED_IMAGE_SIZE = 84501512;
+    private static final int EXPECTED_REACHABLE_TYPES = 19103;
+    private static final int EXPECTED_REACHABLE_METHODS = 94740;
+    private static final int EXPECTED_REACHABLE_FIELDS = 28075;
+    private static final int EXPECTED_REFLECTION_TYPES = 6093;
+    private static final int EXPECTED_REFLECTION_METHODS = 4197;
+    private static final int EXPECTED_REFLECTION_FIELDS = 150;
+    private static final int EXPECTED_JNI_TYPES = 61;
+    private static final int EXPECTED_JNI_METHODS = 55;
+    private static final int EXPECTED_JNI_FIELDS = 59;
+
+    private static BuildOutput buildOutput;
+
+    @BeforeAll
+    public static void beforeAll() {
+        buildOutput = new BuildOutput();
+    }
+
+    @Test
+    public void verifyCodeSizeIsWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_IMAGE_SIZE, TOLERANCE_PERCENTAGE, "image_details", "total_bytes");
+    }
+
+    // TODO: Switch to using analysis_results.types.total key once we drop support for GraalVM 22.3.0
+    @Test
+    public void verifyEmbeddedTypesAreWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedMethodsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedFieldsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reachable");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "jni");
+    }
+}
diff --git a/...n-tests/jpa-postgresql/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java b/...n-tests/jpa-postgresql/src/test/java/io/quarkus/it/jpa/postgresql/ImageMetricsITCase.java
@@ -0,0 +1,80 @@
+package io.quarkus.it.jpa.postgresql;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.nativeimage.BuildOutput;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@QuarkusIntegrationTest
+public class ImageMetricsITCase {
+    private static final int TOLERANCE_PERCENTAGE = 3;
+
+    private static final int EXPECTED_IMAGE_SIZE = 76616712;
+    private static final int EXPECTED_REACHABLE_TYPES = 18140;
+    private static final int EXPECTED_REACHABLE_METHODS = 89015;
+    private static final int EXPECTED_REACHABLE_FIELDS = 25221;
+    private static final int EXPECTED_REFLECTION_TYPES = 5764;
+    private static final int EXPECTED_REFLECTION_METHODS = 4290;
+    private static final int EXPECTED_REFLECTION_FIELDS = 177;
+    private static final int EXPECTED_JNI_TYPES = 61;
+    private static final int EXPECTED_JNI_METHODS = 55;
+    private static final int EXPECTED_JNI_FIELDS = 59;
+
+    private static BuildOutput buildOutput;
+
+    @BeforeAll
+    public static void beforeAll() {
+        buildOutput = new BuildOutput();
+    }
+
+    @Test
+    public void verifyCodeSizeIsWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_IMAGE_SIZE, TOLERANCE_PERCENTAGE, "image_details", "total_bytes");
+    }
+
+    // TODO: Switch to using analysis_results.types.total key once we drop support for GraalVM 22.3.0
+    @Test
+    public void verifyEmbeddedTypesAreWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedMethodsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedFieldsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reachable");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "jni");
+    }
+}
diff --git a/integration-tests/main/src/test/java/io/quarkus/it/main/ImageMetricsITCase.java b/integration-tests/main/src/test/java/io/quarkus/it/main/ImageMetricsITCase.java
@@ -0,0 +1,82 @@
+package io.quarkus.it.main;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.nativeimage.BuildOutput;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@QuarkusIntegrationTest
+public class ImageMetricsITCase {
+    private static final int TOLERANCE_PERCENTAGE = 3;
+
+    // ThTq
+
+    private static final int EXPECTED_IMAGE_SIZE = 138778008;
+    private static final int EXPECTED_REACHABLE_TYPES = 30005;
+    private static final int EXPECTED_REACHABLE_METHODS = 149440;
+    private static final int EXPECTED_REACHABLE_FIELDS = 44161;
+    private static final int EXPECTED_REFLECTION_TYPES = 8966;
+    private static final int EXPECTED_REFLECTION_METHODS = 7346;
+    private static final int EXPECTED_REFLECTION_FIELDS = 438;
+    private static final int EXPECTED_JNI_TYPES = 64;
+    private static final int EXPECTED_JNI_METHODS = 55;
+    private static final int EXPECTED_JNI_FIELDS = 70;
+
+    private static BuildOutput buildOutput;
+
+    @BeforeAll
+    public static void beforeAll() {
+        buildOutput = new BuildOutput();
+    }
+
+    @Test
+    public void verifyCodeSizeIsWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_IMAGE_SIZE, TOLERANCE_PERCENTAGE, "image_details", "total_bytes");
+    }
+
+    // TODO: Switch to using analysis_results.types.total key once we drop support for GraalVM 22.3.0
+    @Test
+    public void verifyEmbeddedTypesAreWithinRangeOld() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedMethodsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reachable");
+    }
+
+    @Test
+    public void verifyEmbeddedFieldsAreWithinRange() {
+        buildOutput.assertValueWithinRange(EXPECTED_REACHABLE_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reachable");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForReflection() {
+        buildOutput.assertValueWithinRange(EXPECTED_REFLECTION_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "reflection");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredTypesForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_TYPES, TOLERANCE_PERCENTAGE, "analysis_results", "classes", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredMethodsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_METHODS, TOLERANCE_PERCENTAGE, "analysis_results", "methods", "jni");
+    }
+
+    @Test
+    public void verifyNumberOfRegisteredFieldsForJNI() {
+        buildOutput.assertValueWithinRange(EXPECTED_JNI_FIELDS, TOLERANCE_PERCENTAGE, "analysis_results", "fields", "jni");
+    }
+}
diff --git a/test-framework/junit5/src/main/java/io/quarkus/test/junit/nativeimage/BuildOutput.java b/test-framework/junit5/src/main/java/io/quarkus/test/junit/nativeimage/BuildOutput.java
@@ -0,0 +1,75 @@
+package io.quarkus.test.junit.nativeimage;
+
+import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Locale;
+
+import jakarta.json.Json;
+import jakarta.json.JsonObject;
+
+import org.junit.jupiter.api.Assertions;
+
+/**
+ * This is a general utility to assert via
+ * unit testing how many classes, methods, objects etc. have been included in a native-image.
+ * <p>
+ * For detailed information and explanations on the build output, visit
+ * <a href="https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/BuildOutput.md">the upstream GraalVM
+ * documentation</a>.
+ */
+public class BuildOutput {
+
+    private final JsonObject buildOutput;
+
+    public BuildOutput() {
+        this.buildOutput = getBuildOutput();
+    }
+
+    public void assertValueWithinRange(int expectedValue, int tolerancePercentage, String... key) {
+        JsonObject currentObject = buildOutput;
+        for (int i = 0; i < key.length - 1; i++) {
+            currentObject = currentObject.getJsonObject(key[i]);
+        }
+        String lastKey = key[key.length - 1];
+        int actualValue = currentObject.getInt(lastKey);
+        Assertions.assertTrue(isNumberWithinRange(expectedValue, actualValue, tolerancePercentage),
+                "Expected " + String.join(".", key) + " to be within range [" + expectedValue + " +- " + tolerancePercentage
+                        + "%] but was " + actualValue);
+    }
+
+    private boolean isNumberWithinRange(int expectedNumberOfClasses, int actualNumberOfClasses, int tolerancePercentage) {
+        final int lowerBound = expectedNumberOfClasses - (expectedNumberOfClasses * tolerancePercentage / 100);
+        final int upperBound = expectedNumberOfClasses + (expectedNumberOfClasses * tolerancePercentage / 100);
+        return actualNumberOfClasses >= lowerBound && actualNumberOfClasses <= upperBound;
+    }
+
+    private static JsonObject getBuildOutput() {
+        final Path buildOutputPath = getBuildOutputPath();
+        try (InputStream inputStream = Files.newInputStream(buildOutputPath)) {
+            return Json.createReader(inputStream).readObject();
+        } catch (Exception e) {
+            throw new RuntimeException("Could not load build output", e);
+        }
+    }
+
+    private static Path getBuildOutputPath() {
+        final Path buildDirectory = locateNativeImageBuildDirectory();
+        final File[] buildOutput = buildDirectory.toFile().listFiles((dir, name) -> name.toLowerCase(Locale.ROOT)
+                .endsWith("-build-output-stats.json"));
+        Assertions.assertNotNull(buildOutput, "Could not identify the native image build output");
+        Assertions.assertEquals(1, buildOutput.length, "Could not identify the native image build output");
+        return buildOutput[0].toPath();
+    }
+
+    private static Path locateNativeImageBuildDirectory() {
+        Path buildPath = Paths.get("target");
+        final File[] files = buildPath.toFile().listFiles((dir, name) -> name.toLowerCase(Locale.ROOT)
+                .endsWith("-native-image-source-jar"));
+        Assertions.assertNotNull(files, "Could not identify the native image build directory");
+        Assertions.assertEquals(1, files.length, "Could not identify the native image build directory");
+        return files[0].toPath();
+    }
+}