Update Maven Integration Tests to use JVM Test Suites

.github/workflows/preview-publish-web-s3.yml

@@ -106,7 +106,7 @@ jobs:
         with:
           gradle-home-cache-cleanup: true
       - name: Document biojava-core
-        run: ./gradlew :dokka-integration-tests:maven:integrationTest --tests org.jetbrains.dokka.it.maven.BiojavaIntegrationTest --stacktrace
+        run: ./gradlew :dokka-integration-tests:maven:testExternalProjectBioJava --stacktrace
         env:
           DOKKA_TEST_OUTPUT_PATH: /home/runner/work/dokka/biojava
       - name: Configure AWS credentials for S3 access

build-logic/src/main/kotlin/dokkabuild.test-integration.gradle.kts

@@ -1,72 +1,123 @@
 /*
  * Copyright 2014-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
+@file:Suppress("UnstableApiUsage")
 
-import org.gradle.api.tasks.testing.logging.TestExceptionFormat
-import org.gradle.api.tasks.testing.logging.TestLogEvent
+import dokkabuild.utils.systemProperty
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
+import org.gradle.api.tasks.testing.logging.TestLogEvent.FAILED
+import org.gradle.api.tasks.testing.logging.TestLogEvent.SKIPPED
+import org.gradle.language.base.plugins.LifecycleBasePlugin.VERIFICATION_GROUP
 
 plugins {
-    id("dokkabuild.kotlin-jvm")
+    id("dokkabuild.base")
+    `jvm-test-suite`
 }
 
-val integrationTestSourceSet: SourceSet = sourceSets.create("integrationTest") {
-    compileClasspath += sourceSets.main.get().output
-    runtimeClasspath += sourceSets.main.get().output
+val integrationTest by tasks.registering {
+    description = "Lifecycle task for running all integration tests."
+    group = VERIFICATION_GROUP
 }
 
-val integrationTestImplementation: Configuration by configurations.getting {
-    extendsFrom(configurations.implementation.get())
-}
-
-val integrationTestRuntimeOnly: Configuration by configurations.getting {
-    extendsFrom(configurations.runtimeOnly.get())
-}
-
-/**
- * Dokka's integration test task is not cacheable because the HTML outputs
- * it produces when running the tests are used for showcasing resulting documentation,
- * which does not work well with caching.
- *
- * At the moment there are two problems that do not allow to make it cacheable:
- *
- * 1. The task's inputs are such that changes in Dokka's code do not invalidate the cache,
- *    because it is run with the same version of Dokka (`"DOKKA_VERSION"`) on the same
- *    test project inputs.
- * 2. The tests generate HTML output which is then used to showcase documentation.
- *    The outputs are usually copied to a location from which it will be served.
- *    However, if the test is cacheable, it produces no outputs, so no documentation
- *    to showcase. It needs to be broken into two separate tasks: one cacheable for running
- *    the tests and producing HTML output, and another non-cacheable for copying the output.
- *
- * @see [org.jetbrains.dokka.it.TestOutputCopier] for more details on showcasing documentation
- */
-@DisableCachingByDefault(because = "Contains incorrect inputs/outputs configuration, see the KDoc for details")
-abstract class NonCacheableIntegrationTest : Test()
-
-val integrationTest by tasks.registering(NonCacheableIntegrationTest::class) {
+tasks.withType<Test>().configureEach {
+    setForkEvery(1)
     maxHeapSize = "2G"
-    description = "Runs integration tests."
-    group = "verification"
-    testClassesDirs = integrationTestSourceSet.output.classesDirs
-    classpath = integrationTestSourceSet.runtimeClasspath
-
-    useJUnitPlatform {
-        if (dokkaBuild.integrationTestUseK2.get()) excludeTags("onlyDescriptors", "onlyDescriptorsMPP")
+    dokkaBuild.integrationTestParallelism.orNull?.let { parallelism ->
+        maxParallelForks = parallelism
     }
 
-    systemProperty("org.jetbrains.dokka.experimental.tryK2", dokkaBuild.integrationTestUseK2.get())
+    systemProperty.inputProperty("dokkaVersion", provider { project.version.toString() })
+    systemProperty.inputProperty("dokkaVersionOverride", dokkaBuild.integrationTestDokkaVersionOverride)
+        .optional(true)
 
-    dokkaBuild.integrationTestParallelism.orNull?.let { parallelism ->
-        maxParallelForks = parallelism
+    val useK2 = dokkaBuild.integrationTestUseK2
+    systemProperty.inputProperty("org.jetbrains.dokka.experimental.tryK2", useK2)
+        .optional(true)
+    useJUnitPlatform {
+        if (useK2.get()) excludeTags("onlyDescriptors", "onlyDescriptorsMPP")
     }
 
-    environment("isExhaustive", dokkaBuild.integrationTestExhaustive.get())
+    systemProperty.inputProperty("isExhaustive", dokkaBuild.integrationTestExhaustive)
+
+    // allow inspecting projects in temporary dirs after a test fails
+    systemProperty.inputProperty(
+        "junit.jupiter.tempdir.cleanup.mode.default",
+        dokkaBuild.isCI.map { if (it) "ALWAYS" else "ON_SUCCESS" }
+    )
 
     testLogging {
-        exceptionFormat = TestExceptionFormat.FULL
-        events(TestLogEvent.SKIPPED, TestLogEvent.FAILED)
+        exceptionFormat = FULL
+        events(SKIPPED, FAILED)
         showExceptions = true
         showCauses = true
         showStackTraces = true
     }
+
+    // For validation, on CI the generated output is uploaded, so the test must produce output in
+    // DOKKA_TEST_OUTPUT_PATH. For Gradle up-to-date checks the output dir must be specified.
+    val testOutputPath = System.getenv("DOKKA_TEST_OUTPUT_PATH")
+    inputs.property("testOutputPath", testOutputPath).optional(true)
+    if (testOutputPath != null) {
+        outputs.dir(testOutputPath).withPropertyName("testOutput")
+    }
+
+    // The tests produce report data and generated Dokka output.
+    // Always cache them so Gradle can skip running integration tests if nothing has changed.
+    outputs.cacheIf("always cache") { true }
+}
+
+testing {
+    suites {
+        withType<JvmTestSuite>().configureEach {
+            useJUnitJupiter()
+
+            targets.configureEach {
+                testTask.configure {
+                    doFirst {
+                        logger.info("running $path with javaLauncher:${javaLauncher.orNull?.metadata?.javaRuntimeVersion}")
+                    }
+                }
+            }
+        }
+    }
+}
+
+integrationTest.configure {
+    dependsOn(testing.suites)
+}
+
+tasks.check {
+    dependsOn(integrationTest)
+}
+
+//region project tests management
+
+// set up task ordering - template projects (which are generally faster) should be tested before external projects
+val jvmTestTask = tasks.withType<Test>().matching { it.name == "test" }
+val testTemplateProjectsTasks = tasks.withType<Test>().matching { it.name.startsWith("testTemplateProject") }
+val testExternalProjectsTasks = tasks.withType<Test>().matching { it.name.startsWith("testExternalProject") }
+
+testTemplateProjectsTasks.configureEach {
+    shouldRunAfter(jvmTestTask)
+}
+testExternalProjectsTasks.configureEach {
+    shouldRunAfter(jvmTestTask)
+    shouldRunAfter(testTemplateProjectsTasks)
+}
+
+// define lifecycle tasks for project tests
+val testAllTemplateProjects by tasks.registering {
+    description = "Lifecycle task for running all template-project tests"
+    group = VERIFICATION_GROUP
+    dependsOn(testTemplateProjectsTasks)
+    doNotTrackState("lifecycle task, should always run")
+}
+
+val testAllExternalProjects by tasks.registering {
+    description = "Lifecycle task for running all external-project tests"
+    group = VERIFICATION_GROUP
+    shouldRunAfter(testAllTemplateProjects)
+    dependsOn(testExternalProjectsTasks)
+    doNotTrackState("lifecycle task, should always run")
 }
+//endregion

build-logic/src/main/kotlin/dokkabuild/DevMavenPublishExtension.kt

@@ -3,11 +3,13 @@
  */
 package dokkabuild
 
+import dokkabuild.utils.systemProperty
 import org.gradle.api.Task
 import org.gradle.api.file.FileCollection
-import org.gradle.api.file.FileTree
+import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.PathSensitivity.RELATIVE
 import org.gradle.process.JavaForkOptions
+import java.io.File
 
 abstract class DevMavenPublishExtension(
     /**
@@ -21,19 +23,23 @@ abstract class DevMavenPublishExtension(
     /**
      * Files suitable for registering as a task input (as in, the files are reproducible-build compatible).
      */
-    private val devMavenRepositoriesInputFiles: FileTree = devMavenRepositories
-        .asFileTree
-        .matching {
+    private val devMavenRepositoriesInputFiles: Provider<List<File>> =
+        devMavenRepositories
+            // Convert to a FileTree, which converts directories to all files, so we can filter on specific files.
+            .asFileTree
             // Exclude Maven Metadata files because they contain timestamps, meaning tasks that use
             // devMavenRepositories as an input will never be up-to-date.
             // The Gradle Module Metadata contains the same information (and more),
             // so the Maven metadata is redundant.
-            exclude("**/maven-metadata*.xml")
-        }
+            .matching { exclude("**/maven-metadata*.xml") }
+            // FileTrees have an unstable order (even on the same machine), which means Gradle up-to-date checks fail.
+            // So, manually sort the files so that Gradle can cache the task.
+            .elements
+            .map { files -> files.map { it.asFile }.sorted() }
 
     /**
      * Configures [task] to register [devMavenRepositories] as a task input,
-     * and (if possible) adds `devMavenRepository` as a [JavaForkOptions.systemProperty].
+     * and (if possible) adds `devMavenRepositories` as a [JavaForkOptions.systemProperty].
      */
     fun configureTask(task: Task) {
         task.inputs.files(devMavenRepositoriesInputFiles)
@@ -43,13 +49,12 @@ abstract class DevMavenPublishExtension(
         task.dependsOn(devMavenRepositories)
 
         if (task is JavaForkOptions) {
-            task.doFirst("devMavenRepositories systemProperty") {
-                // workaround https://github.com/gradle/gradle/issues/24267
-                task.systemProperty(
-                    "devMavenRepositories",
-                    devMavenRepositories.joinToString(",") { it.canonicalFile.invariantSeparatorsPath }
-                )
-            }
+            task.jvmArgumentProviders.systemProperty(
+                "devMavenRepositories",
+                devMavenRepositories.elements.map { paths ->
+                    paths.joinToString(",") { it.asFile.canonicalFile.invariantSeparatorsPath }
+                }
+            )
         }
     }
 

build-logic/src/main/kotlin/dokkabuild/utils/SystemPropertyAdder.kt

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2014-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+package dokkabuild.utils
+
+import org.gradle.api.file.Directory
+import org.gradle.api.file.FileCollection
+import org.gradle.api.file.RegularFile
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskInputFilePropertyBuilder
+import org.gradle.api.tasks.TaskInputPropertyBuilder
+import org.gradle.api.tasks.testing.Test
+import org.gradle.kotlin.dsl.create
+import org.gradle.kotlin.dsl.findByType
+import org.gradle.process.CommandLineArgumentProvider
+import java.io.File
+import javax.inject.Inject
+
+
+/**
+ * Utility for adding a System Property command line arguments to this [Test] task,
+ * and correctly registering the values as task inputs (for Gradle up-to-date checks).
+ */
+// https://github.com/gradle/gradle/issues/11534
+// https://github.com/gradle/gradle/issues/12247
+val Test.systemProperty: SystemPropertyAdder
+    get() {
+        val spa = extensions.findByType<SystemPropertyAdder>()
+            ?: extensions.create<SystemPropertyAdder>("SystemPropertyAdder", this)
+        return spa
+    }
+
+
+abstract class SystemPropertyAdder @Inject internal constructor(
+    private val task: Test,
+) {
+    fun inputDirectory(
+        key: String,
+        value: Directory,
+    ): TaskInputFilePropertyBuilder {
+        task.jvmArgumentProviders.add(
+            SystemPropertyArgumentProvider(key, value) {
+                it.asFile.invariantSeparatorsPath
+            }
+        )
+        return task.inputs.dir(value)
+            .withPropertyName("SystemProperty input directory $key")
+    }
+
+    fun inputFile(
+        key: String,
+        file: RegularFile,
+    ): TaskInputFilePropertyBuilder {
+        task.jvmArgumentProviders.add(
+            SystemPropertyArgumentProvider(key, file) {
+                it.asFile.invariantSeparatorsPath
+            }
+        )
+        return task.inputs.file(file)
+            .withPropertyName("SystemProperty input file $key")
+    }
+
+    fun inputFile(
+        key: String,
+        file: Provider<out RegularFile>,
+    ): TaskInputFilePropertyBuilder {
+        task.jvmArgumentProviders.add(
+            SystemPropertyArgumentProvider(key, file) {
+                it.orNull?.asFile?.invariantSeparatorsPath
+            }
+        )
+        return task.inputs.file(file)
+            .withPropertyName("SystemProperty input file $key")
+    }
+
+    fun inputFiles(
+        key: String,
+        files: Provider<out FileCollection>,
+    ): TaskInputFilePropertyBuilder {
+        task.jvmArgumentProviders.add(
+            SystemPropertyArgumentProvider(key, files) { it.orNull?.asPath }
+        )
+        return task.inputs.files(files)
+            .withPropertyName("SystemProperty input files $key")
+    }
+
+    fun inputProperty(
+        key: String,
+        value: Provider<out String>,
+    ): TaskInputPropertyBuilder {
+        task.jvmArgumentProviders.add(
+            SystemPropertyArgumentProvider(key, value) { it.orNull }
+        )
+        return task.inputs.property("SystemProperty input property $key", value)
+    }
+
+    @JvmName("inputBooleanProperty")
+    fun inputProperty(
+        key: String,
+        value: Provider<out Boolean>,
+    ): TaskInputPropertyBuilder = inputProperty(key, value.map { it.toString() })
+}
+
+private class SystemPropertyArgumentProvider<T : Any>(
+    @get:Input
+    val key: String,
+    private val value: T,
+    private val transformer: (value: T) -> String?,
+) : CommandLineArgumentProvider {
+    override fun asArguments(): Iterable<String> {
+        val value = transformer(value) ?: return emptyList()
+        return listOf("-D$key=$value")
+    }
+}
+
+/**
+ * Add a System Property (in the format `-D$key=$value`).
+ *
+ * [value] will _not_ be registered as a Gradle [org.gradle.api.Task] input.
+ * (Which might be beneficial in cases where the property is optional, or not reproducible,
+ * as such a property might disrupt task caching.)
+ *
+ * If you want to register the property as a Task input, use the
+ * [Test.systemProperty][dokkabuild.utils.systemProperty] above instead.
+ */
+fun MutableList<CommandLineArgumentProvider>.systemProperty(
+    key: String,
+    value: Provider<String>,
+) {
+    add(
+        SystemPropertyArgumentProvider(
+            key = key,
+            value = value,
+            transformer = { it.orNull },
+        )
+    )
+}