-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Cleanup stale output files during execution (#2572)
We now keep a registry of all the outputs generated by Gradle which will be reset on each version change. If Gradle encounters an existing output file, then it will remove it if is not part of the registered outputs and owned by Gradle/the build. We do also not remove directories containing outputs from different tasks. The check to delete the stale outputs happens now just before the task executes and not up-front as it did before. The `build` directory and all delete targets of the `clean` task are registered as owned by Gradle and are considered safe to delete. Currently, the set of recorded task output files is only growing if we do not change the Gradle version. In the future we can improve on this by also detecting that some directories where removed (e.g. by running a clean task) and reflect this in the registry. Fixes #1168 Fixes #973 +review REVIEW-6557
- Loading branch information
Showing
32 changed files
with
957 additions
and
752 deletions.
There are no files selected for viewing
153 changes: 75 additions & 78 deletions
153
...cts/core/src/integTest/groovy/org/gradle/api/tasks/IncrementalBuildIntegrationTest.groovy
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
.../java/org/gradle/api/internal/changedetection/state/DefaultTaskOutputFilesRepository.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| /* | ||
| * Copyright 2017 the original author or authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.gradle.api.internal.changedetection.state; | ||
|
|
||
| import org.gradle.cache.PersistentCache; | ||
| import org.gradle.cache.PersistentIndexedCache; | ||
| import org.gradle.cache.PersistentIndexedCacheParameters; | ||
| import org.gradle.internal.nativeintegration.filesystem.FileType; | ||
|
|
||
| import java.io.Closeable; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
|
|
||
| public class DefaultTaskOutputFilesRepository implements TaskOutputFilesRepository, Closeable { | ||
|
|
||
| private final PersistentCache cacheAccess; | ||
| private final FileSystemMirror fileSystemMirror; | ||
| private final PersistentIndexedCache<String, Boolean> outputFiles; // The value is true if it is an output file, false if it is a parent of an output file | ||
|
|
||
| public DefaultTaskOutputFilesRepository(PersistentCache cacheAccess, FileSystemMirror fileSystemMirror, InMemoryCacheDecoratorFactory inMemoryCacheDecoratorFactory) { | ||
| this.cacheAccess = cacheAccess; | ||
| this.fileSystemMirror = fileSystemMirror; | ||
| this.outputFiles = cacheAccess.createCache(cacheParameters(inMemoryCacheDecoratorFactory)); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isGeneratedByGradle(File file) { | ||
| File absoluteFile = file.getAbsoluteFile(); | ||
| return containsFilesGeneratedByGradle(absoluteFile) || isContainedInAnOutput(absoluteFile); | ||
| } | ||
|
|
||
| private Boolean isContainedInAnOutput(File file) { | ||
| File currentFile = file; | ||
| while (currentFile != null) { | ||
| if (outputFiles.get(currentFile.getPath()) == Boolean.TRUE) { | ||
| return true; | ||
| } | ||
| currentFile = currentFile.getParentFile(); | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| private boolean containsFilesGeneratedByGradle(File file) { | ||
| return outputFiles.get(file.getPath()) != null; | ||
| } | ||
|
|
||
| @Override | ||
| public void recordOutputs(TaskExecution taskExecution) { | ||
| for (String outputFilePath : taskExecution.getDeclaredOutputFilePaths()) { | ||
| FileSnapshot fileSnapshot = fileSystemMirror.getFile(outputFilePath); | ||
| File outputFile = new File(outputFilePath); | ||
| boolean exists = fileSnapshot == null ? outputFile.exists() : fileSnapshot.getType() != FileType.Missing; | ||
| if (exists) { | ||
| outputFiles.put(outputFilePath, Boolean.TRUE); | ||
| File outputFileParent = outputFile.getParentFile(); | ||
| while (outputFileParent != null) { | ||
| String parentPath = outputFileParent.getPath(); | ||
| if (outputFiles.get(parentPath) != null) { | ||
| break; | ||
| } | ||
| outputFiles.put(parentPath, Boolean.FALSE); | ||
| outputFileParent = outputFileParent.getParentFile(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static PersistentIndexedCacheParameters<String, Boolean> cacheParameters(InMemoryCacheDecoratorFactory inMemoryCacheDecoratorFactory) { | ||
| return new PersistentIndexedCacheParameters<String, Boolean>("outputFiles", String.class, Boolean.class) | ||
| .cacheDecorator(inMemoryCacheDecoratorFactory.decorator(100000, true)); | ||
| } | ||
|
|
||
| @Override | ||
| public void close() throws IOException { | ||
| cacheAccess.close(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
...re/src/main/java/org/gradle/api/internal/tasks/execution/CleanupStaleOutputsExecuter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* | ||
| * Copyright 2017 the original author or authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.gradle.api.internal.tasks.execution; | ||
|
|
||
| import org.gradle.api.file.FileCollection; | ||
| import org.gradle.api.internal.TaskInternal; | ||
| import org.gradle.api.internal.changedetection.state.TaskOutputFilesRepository; | ||
| import org.gradle.api.internal.tasks.TaskExecuter; | ||
| import org.gradle.api.internal.tasks.TaskExecutionContext; | ||
| import org.gradle.api.internal.tasks.TaskOutputFilePropertySpec; | ||
| import org.gradle.api.internal.tasks.TaskStateInternal; | ||
| import org.gradle.api.logging.Logger; | ||
| import org.gradle.api.logging.Logging; | ||
| import org.gradle.internal.cleanup.BuildOutputCleanupRegistry; | ||
| import org.gradle.internal.operations.BuildOperationContext; | ||
| import org.gradle.internal.operations.BuildOperationExecutor; | ||
| import org.gradle.internal.operations.RunnableBuildOperation; | ||
| import org.gradle.internal.progress.BuildOperationDescriptor; | ||
| import org.gradle.util.GFileUtils; | ||
|
|
||
| import java.io.File; | ||
|
|
||
| public class CleanupStaleOutputsExecuter implements TaskExecuter { | ||
|
|
||
| public static final String CLEAN_STALE_OUTPUTS_DISPLAY_NAME = "Clean stale outputs"; | ||
|
|
||
| private final Logger logger = Logging.getLogger(CleanupStaleOutputsExecuter.class); | ||
| private final BuildOperationExecutor buildOperationExecutor; | ||
| private final TaskExecuter executer; | ||
| private final TaskOutputFilesRepository taskOutputFilesRepository; | ||
| private final BuildOutputCleanupRegistry cleanupRegistry; | ||
|
|
||
| public CleanupStaleOutputsExecuter(BuildOutputCleanupRegistry cleanupRegistry, TaskOutputFilesRepository taskOutputFilesRepository, BuildOperationExecutor buildOperationExecutor, TaskExecuter executer) { | ||
| this.cleanupRegistry = cleanupRegistry; | ||
| this.buildOperationExecutor = buildOperationExecutor; | ||
| this.executer = executer; | ||
| this.taskOutputFilesRepository = taskOutputFilesRepository; | ||
| } | ||
|
|
||
| @Override | ||
| public void execute(final TaskInternal task, TaskStateInternal state, TaskExecutionContext context) { | ||
| buildOperationExecutor.run(new RunnableBuildOperation() { | ||
| @Override | ||
| public void run(BuildOperationContext context) { | ||
| for (TaskOutputFilePropertySpec outputFileSpec : task.getOutputs().getFileProperties()) { | ||
| FileCollection files = outputFileSpec.getPropertyFiles(); | ||
| for (File file : files) { | ||
| if (cleanupRegistry.isOutputOwnedByBuild(file) && !taskOutputFilesRepository.isGeneratedByGradle(file) && file.exists()) { | ||
| logger.info("Deleting stale output file: {}", file.getAbsolutePath()); | ||
| GFileUtils.forceDelete(file); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public BuildOperationDescriptor.Builder description() { | ||
| return BuildOperationDescriptor.displayName(CLEAN_STALE_OUTPUTS_DISPLAY_NAME).progressDisplayName("Cleaning stale outputs"); | ||
| } | ||
| }); | ||
| executer.execute(task, state, context); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 0 additions & 54 deletions
54
.../src/main/java/org/gradle/internal/cleanup/BuildOperationBuildOutputDeleterDecorator.java
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.