diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt index 96d02a3b8..d9a0de73a 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt @@ -51,9 +51,10 @@ constructor( private val consoleWriter: Writer = System.out.writer(), ) : CliCommand(options.base) { /** - * Output files for the modules to be evaluated. Returns `null` if `options.outputPath` is `null`. - * Multiple modules may be mapped to the same output file, in which case their outputs are - * concatenated with [CliEvaluatorOptions.moduleOutputSeparator]. + * Output files for the modules to be evaluated. Returns `null` if `options.outputPath` is `null` + * or if `options.multipleFileOutputPath` is not `null`. Multiple modules may be mapped to the + * same output file, in which case their outputs are concatenated with + * [CliEvaluatorOptions.moduleOutputSeparator]. */ @Suppress("MemberVisibilityCanBePrivate") val outputFiles: Set? by lazy { diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java index 4cf47e5d0..5cbb0e9dd 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/BasePklTask.java @@ -28,15 +28,18 @@ import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.inject.Inject; import org.gradle.api.DefaultTask; import org.gradle.api.InvalidUserDataException; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileSystemLocation; +import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.MapProperty; import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; @@ -142,6 +145,7 @@ public void runTask() { @LateInit protected CliBaseOptions cachedOptions; + // Must be called during task execution time only. @Internal protected CliBaseOptions getCliBaseOptions() { if (cachedOptions == null) { @@ -176,6 +180,12 @@ protected List getSourceModulesAsUris() { return Collections.emptyList(); } + @Inject + protected abstract ObjectFactory getObjects(); + + @Inject + protected abstract ProviderFactory getProviders(); + protected List parseModulePath() { return getModulePath().getFiles().stream().map(File::toPath).collect(Collectors.toList()); } diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/EvalTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/EvalTask.java index f680bee97..30a8e5e41 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/EvalTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/EvalTask.java @@ -16,50 +16,82 @@ package org.pkl.gradle.task; import java.io.File; +import java.util.Collections; +import java.util.Set; +import javax.annotation.Nullable; import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.Property; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Optional; -import org.gradle.api.tasks.OutputDirectory; -import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.OutputDirectories; +import org.gradle.api.tasks.OutputFiles; import org.pkl.cli.CliEvaluator; import org.pkl.cli.CliEvaluatorOptions; public abstract class EvalTask extends ModulesTask { - @OutputFile - @Optional + + // not tracked because it might contain placeholders + // required + @Internal public abstract RegularFileProperty getOutputFile(); + // not tracked because it might contain placeholders + // optional + @Internal + public abstract DirectoryProperty getMultipleFileOutputDir(); + @Input - @Optional public abstract Property getOutputFormat(); @Input - @Optional public abstract Property getModuleOutputSeparator(); - @OutputDirectory + @Input + public abstract Property getExpression(); + + private final Provider cliEvaluator = + getProviders() + .provider( + () -> + new CliEvaluator( + new CliEvaluatorOptions( + getCliBaseOptions(), + getOutputFile().get().getAsFile().getAbsolutePath(), + getOutputFormat().get(), + getModuleOutputSeparator().get(), + mapAndGetOrNull( + getMultipleFileOutputDir(), it -> it.getAsFile().getAbsolutePath()), + getExpression().get()))); + + @SuppressWarnings("unused") + @OutputFiles @Optional - public abstract DirectoryProperty getMultipleFileOutputDir(); + public FileCollection getEffectiveOutputFiles() { + return getObjects() + .fileCollection() + .from(cliEvaluator.map(e -> nullToEmpty(e.getOutputFiles()))); + } - @Input + @OutputDirectories @Optional - public abstract Property getExpression(); + public FileCollection getEffectiveOutputDirs() { + return getObjects() + .fileCollection() + .from(cliEvaluator.map(e -> nullToEmpty(e.getOutputDirectories()))); + } + + private static Set nullToEmpty(@Nullable Set set) { + return set == null ? Collections.emptySet() : set; + } @Override protected void doRunTask() { //noinspection ResultOfMethodCallIgnored getOutputs().getPreviousOutputFiles().forEach(File::delete); - - new CliEvaluator( - new CliEvaluatorOptions( - getCliBaseOptions(), - getOutputFile().get().getAsFile().getAbsolutePath(), - getOutputFormat().get(), - getModuleOutputSeparator().get(), - mapAndGetOrNull(getMultipleFileOutputDir(), it -> it.getAsFile().getAbsolutePath()), - getExpression().get())) - .run(); + cliEvaluator.get().run(); } } diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java index 97421cfc8..2ac3ea9ed 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java @@ -128,12 +128,15 @@ private Pair, List> splitFilesAndUris(List modules) { } /** - * Converts either a file or a URI to a URI. We convert a file to a URI via the {@link + * Converts either a file or a URI to a URI. We convert a relative file to a URI via the {@link * IoUtils#createUri(String)} because other ways of conversion can make relative paths into * absolute URIs, which may break module loading. */ private URI parsedModuleNotationToUri(Object notation) { if (notation instanceof File file) { + if (file.isAbsolute()) { + return file.toPath().toUri(); + } return IoUtils.createUri(IoUtils.toNormalizedPathString(file.toPath())); } else if (notation instanceof URI uri) { return uri; diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt index e13688ac6..98aee9451 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt @@ -16,7 +16,9 @@ package org.pkl.gradle import java.nio.file.Path +import kotlin.io.path.readText import org.assertj.core.api.Assertions.assertThat +import org.gradle.testkit.runner.TaskOutcome import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import org.pkl.commons.readString @@ -549,12 +551,301 @@ class EvaluatorsTest : AbstractTest() { assertThat(testProjectDir.resolve("proj1/foo.pcf")).exists() } + @Test + fun `implicit dependency tracking for effective output files`() { + writeFile("file1.pkl", "foo = 1") + + writeFile("file2.pkl", "bar = 1") + + writeFile( + "build.gradle.kts", + """ + import org.pkl.gradle.task.EvalTask + + plugins { + id("org.pkl-lang") + } + + pkl { + evaluators { + register("doEval") { + sourceModules = files("file1.pkl", "file2.pkl") + outputFile = layout.projectDirectory.file("%{moduleName}.%{outputFormat}") + outputFormat = "yaml" + } + } + } + + val doEval by tasks.existing(EvalTask::class) { + doLast { + file("evalCounter.txt").appendText("doEval executed\n") + } + } + + val printEvalFiles by tasks.registering { + inputs.files(doEval) + doLast { + println("evalCounter.txt") + println(file("evalCounter.txt").readText()) + + inputs.files.forEach { + println(it.name) + println(it.readText()) + } + } + } + """ + .trimIndent() + ) + + val result1 = runTask("printEvalFiles") + + // `doEval` task is invoked transitively. + assertThat(result1.task(":doEval")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + + assertThat(result1.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + doEval executed + + file1.yaml + foo: 1 + + file2.yaml + bar: 1 + """ + .trimIndent() + ) + + // Run the task again to check that it is cached. + val result2 = runTask("printEvalFiles") + + assertThat(result2.task(":doEval")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + + // evalCounter.txt content is the same as before. + assertThat(result2.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + doEval executed + + file1.yaml + foo: 1 + + file2.yaml + bar: 1 + """ + .trimIndent() + ) + + // Modify the input file. + writeFile("file1.pkl", "foo = 7") + + // Run the build again - the evaluation task will not be cached. + val result3 = runTask("printEvalFiles") + + assertThat(result3.task(":doEval")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + + // evalCounter.txt content is updated too. + assertThat(result3.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + doEval executed + doEval executed + + file1.yaml + foo: 7 + + file2.yaml + bar: 1 + """ + .trimIndent() + ) + } + + @Test + fun `implicit dependency tracking for multiple output directory`() { + writePklFile( + """ + pigeon { + name = "Pigeon" + diet = "Seeds" + } + + parrot { + name = "Parrot" + diet = "Seeds" + } + + output { + files { + ["birds/pigeon.json"] { + value = pigeon + renderer = new JsonRenderer {} + } + ["birds/parrot.pcf"] { + value = parrot + renderer = new PcfRenderer {} + } + } + } + """ + .trimIndent() + ) + + writeBuildFile( + "yaml", + additionalContents = + """ + multipleFileOutputDir = layout.projectDirectory.dir("%{moduleDir}/%{moduleName}-%{outputFormat}") + """ + .trimIndent(), + additionalBuildScript = + """ + tasks.named('evalTest') { + doLast { + file("evalCounter.txt").append("evalTest executed\n") + } + } + + abstract class PrintTask extends DefaultTask { + @InputFiles + public abstract ConfigurableFileCollection getInputDirs(); + } + + // ensure that iteration order is the same across environments + def sortByTypeThenName = { a, b -> + a.isFile() != b.isFile() ? a.isFile() <=> b.isFile() : a.name <=> b.name + } + + tasks.register('printEvalDirs', PrintTask) { + inputDirs.from(tasks.named('evalTest')) + + doLast { + println "evalCounter.txt" + println file("evalCounter.txt").text + + inputDirs.forEach { f -> + f.traverse(visitRoot: true, sort: sortByTypeThenName) { + if (it.isDirectory()) { + println layout.projectDirectory.asFile.relativePath(it) + '/' + println() + } else { + println layout.projectDirectory.asFile.relativePath(it) + println it.text + } + } + } + } + } + """ + .trimIndent() + ) + + val result1 = runTask("printEvalDirs") + + // `doEval` task is invoked transitively. + assertThat(result1.task(":evalTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + + // NB: Configured output format, 'yaml', is only used to replace the placeholder in the path; + // the output files themselves are formatted according to configuration + // in the rendered module. + + assertThat(result1.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + evalTest executed + + test-yaml/birds/ + + test-yaml/birds/parrot.pcf + name = "Parrot" + diet = "Seeds" + + test-yaml/birds/pigeon.json + { + "name": "Pigeon", + "diet": "Seeds" + } + + test-yaml/ + """ + .trimIndent() + ) + + // Run the task again to check that it is cached. + val result2 = runTask("printEvalDirs") + + assertThat(result2.task(":evalTest")?.outcome).isEqualTo(TaskOutcome.UP_TO_DATE) + + // evalCounter.txt content is the same as before. + assertThat(result2.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + evalTest executed + + test-yaml/birds/ + + test-yaml/birds/parrot.pcf + name = "Parrot" + diet = "Seeds" + + test-yaml/birds/pigeon.json + { + "name": "Pigeon", + "diet": "Seeds" + } + + test-yaml/ + """ + .trimIndent() + ) + + // Modify the input file. + writePklFile(testProjectDir.resolve("test.pkl").readText().replace("Parrot", "Macaw")) + + // Run the build again - the evaluation task will not be cached. + val result3 = runTask("printEvalDirs") + + assertThat(result3.task(":evalTest")?.outcome).isEqualTo(TaskOutcome.SUCCESS) + + // evalCounter.txt content is updated too. + assertThat(result3.output) + .containsIgnoringNewLines( + """ + evalCounter.txt + evalTest executed + evalTest executed + + test-yaml/birds/ + + test-yaml/birds/parrot.pcf + name = "Macaw" + diet = "Seeds" + + test-yaml/birds/pigeon.json + { + "name": "Pigeon", + "diet": "Seeds" + } + + test-yaml/ + """ + .trimIndent() + ) + } + private fun writeBuildFile( // don't use `org.pkl.core.OutputFormat` // because test compile class path doesn't contain pkl-core outputFormat: String, additionalContents: String = "", - sourceModules: List = listOf("test.pkl") + sourceModules: List = listOf("test.pkl"), + additionalBuildScript: String = "" ) { writeFile( "build.gradle", @@ -573,6 +864,8 @@ class EvaluatorsTest : AbstractTest() { } } } + + $additionalBuildScript """ ) }