Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MBUILDCACHE-64] Exclusion mechanism bugfix #91

Merged
merged 7 commits into from
Oct 12, 2023
Merged

[MBUILDCACHE-64] Exclusion mechanism bugfix #91

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
556965c
[MBUILDCACHE-64] Fix exclusions not working properly (specifically wi…
kbuntrock Jul 31, 2023
95c483a
[MBUILDCACHE-64] Added an integration test for the include/exclude fu…
kbuntrock Jul 31, 2023
70b1992
[MBUILDCACHE-64] Add documentation about the include / exclude mechanism
kbuntrock Aug 1, 2023
a0af3df
[MBUILDCACHE-64] Exclude mecanism based on glob patterns
kbuntrock Aug 7, 2023
f1c3c6a
[MBUILDCACHE-64] Update IT test and documentation of the include/excl…
kbuntrock Aug 7, 2023
45f2e1a
[MBUILDCACHE-64] Code review comments
kbuntrock Aug 23, 2023
eeaa49f
[MBUILDCACHE-64] Rework path + pattern exclusion
kbuntrock Sep 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 49 additions & 28 deletions src/main/java/org/apache/maven/buildcache/checksum/MavenProjectInput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -136,7 +136,8 @@ public class MavenProjectInput {
private final RepositorySystem repoSystem;
private final CacheConfig config;
private final PathIgnoringCaseComparator fileComparator;
private final List<Path> filteredOutPaths;
private final Set<String> filenameExclusions = new LinkedHashSet<>();
private final Set<String> filePathExclusions = new LinkedHashSet<>();
private final NormalizedModelProvider normalizedModelProvider;
private final MultiModuleSupport multiModuleSupport;
private final ProjectInputCalculator projectInputCalculator;
Expand Down Expand Up @@ -171,39 +172,49 @@ public MavenProjectInput(
this.tmpDir = System.getProperty("java.io.tmpdir");

org.apache.maven.model.Build build = project.getBuild();
filteredOutPaths = new ArrayList<>(Arrays.asList(
normalizedPath(build.getDirectory()), // target by default
normalizedPath(build.getOutputDirectory()),
normalizedPath(build.getTestOutputDirectory())));
filePathExclusions.add(normalizedPath(build.getDirectory()).toString()); // target by default
filePathExclusions.add(normalizedPath(build.getOutputDirectory()).toString()); // target/classes by default
filePathExclusions.add(
normalizedPath(build.getTestOutputDirectory()).toString()); // target/test-classes by default

List<Exclude> excludes = config.getGlobalExcludePaths();
for (Exclude excludePath : excludes) {
filteredOutPaths.add(Paths.get(excludePath.getValue()));
addToExcludedSection(excludePath.getValue());
}

for (String propertyName : properties.stringPropertyNames()) {
if (propertyName.startsWith(CACHE_EXCLUDE_NAME)) {
String propertyValue = properties.getProperty(propertyName);
Path path = Paths.get(propertyValue);
filteredOutPaths.add(path);
addToExcludedSection(propertyValue);

if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
"Adding an excludePath from property '{}', values is '{}', path is '{}' ",
propertyName,
propertyValue,
path);
"Adding an excludePath from property '{}', value is '{}'", propertyName, propertyValue);
}
}
}
CacheUtils.debugPrintCollection(
LOGGER,
filteredOutPaths,
"List of excluded paths (checked either by fileName or by startsWith prefix)",
"Path entry");
CacheUtils.debugPrintCollection(LOGGER, filePathExclusions, "List of excluded paths", "Path entry");

CacheUtils.debugPrintCollection(LOGGER, filenameExclusions, "List of excluded files", "File entry");

this.fileComparator = new PathIgnoringCaseComparator();
}

/**
* Add a value from the excluded section list to the directories and/or the filenames ban list.
* @param excludedValue a value from the exclude list
*/
private void addToExcludedSection(String excludedValue) {
Path excluded = Paths.get(excludedValue);
// If the string can be a filename, we add it to the filename exclusion
if (excluded.equals(excluded.getFileName())) {
filenameExclusions.add(excludedValue);
}
// And we always add it to the filePath exclusion, since files and directories names can't be differentiated on
// some OS.
filePathExclusions.add(convertToAbsolutePath(excluded).toString());
}

public ProjectsInputInfo calculateChecksum() throws IOException {
final long t0 = System.currentTimeMillis();

Expand Down Expand Up @@ -410,13 +421,17 @@ private SortedSet<Path> getInputFiles() {
return sorted;
}

private Path convertToAbsolutePath(Path path) {
Path resolvedPath = path.isAbsolute() ? path : baseDirPath.resolve(path);
return resolvedPath.toAbsolutePath().normalize();
}

/**
* entry point for directory walk
*/
private void startWalk(
Path candidate, String glob, boolean recursive, List<Path> collectedFiles, Set<WalkKey> visitedDirs) {
Path normalized = candidate.isAbsolute() ? candidate : baseDirPath.resolve(candidate);
normalized = normalized.toAbsolutePath().normalize();
Path normalized = convertToAbsolutePath(candidate);
WalkKey key = new WalkKey(normalized, glob, recursive);
if (visitedDirs.contains(key) || !Files.exists(normalized)) {
return;
Expand Down Expand Up @@ -506,8 +521,7 @@ public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFil
return FileVisitResult.SKIP_SUBTREE;
}

walkDirectoryFiles(path, collectedFiles, key.getGlob(), entry -> filteredOutPaths.stream()
.anyMatch(it -> it.getFileName().equals(entry.getFileName())));
walkDirectoryFiles(path, collectedFiles, key.getGlob(), entry -> entryMustBeSkipped(entry));

if (!key.isRecursive()) {
LOGGER.debug("Skipping subtree (non recursive): {}", path);
Expand All @@ -526,6 +540,18 @@ public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOExce
});
}

private boolean entryMustBeSkipped(Path entry) {
if (filePathExclusions.stream()
.anyMatch(it -> entry.toAbsolutePath().toString().startsWith(it))) {
// skip file entry defined by a path + a full or prefixed filename (exclusion only via path is handled at a
// higher level)
return true;
}
// skip file entry defined without a path (full or prefixed filename)
return filenameExclusions.stream()
.anyMatch(it -> entry.getFileName().toString().startsWith(it));
}

private void addInputsFromPluginConfigs(
Object[] configurationChildren,
PluginScanConfig scanConfig,
Expand Down Expand Up @@ -634,12 +660,7 @@ private static boolean isReadable(Path entry) throws IOException {

private boolean isFilteredOutSubpath(Path path) {
Path normalized = path.normalize();
for (Path filteredOutDir : filteredOutPaths) {
if (normalized.startsWith(filteredOutDir)) {
return true;
}
}
return false;
return filePathExclusions.stream().anyMatch(it -> normalized.toString().startsWith(it));
}

private SortedMap<String, String> getMutableDependencies() throws IOException {
Expand Down
46 changes: 46 additions & 0 deletions src/main/mdo/build-cache-config.mdo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ under the License.
-->
<class>
<name>Input</name>
<description>Configuration for source code input files participating in checksum calculation</description>
<fields>
<field>
<name>global</name>
Expand Down Expand Up @@ -786,21 +787,26 @@ under the License.
-->
<class>
<name>PathSet</name>
<description>Global input calculation rules applicable to all projects and plugins in the build</description>
<fields>
<field>
<name>glob</name>
<type>String</type>
<description><![CDATA[Type of files to globally scan. All types are scanned by default. Example to scan only java and xml files : <code>{*.java,*.xml}</code>
]]></description>
<defaultValue>*</defaultValue>
</field>
<field>
<name>includes</name>
<description>Extra paths to scan in order to compute the checksum (Project and test - sources/resources are already in the scan list)</description>
<association>
<type>Include</type>
<multiplicity>*</multiplicity>
</association>
</field>
<field>
<name>excludes</name>
<description>Paths and files to exclude from checksum computation</description>
<association>
<type>Exclude</type>
<multiplicity>*</multiplicity>
Expand All @@ -810,28 +816,68 @@ under the License.
</class>
<class>
<name>Exclude</name>
<description><![CDATA[A path or a file to exclude from checksum computation. <b>Paths are relative to each module basedir.</b><br/>
Absolute path are allowed but should be use for debug purposes only. Please note that a path starting with <code>/</code> is considered as absolute.<br/>
Remember that "exclude" elements can also be added per project with the use of maven properties.
]]></description>
<fields>
<field xml.content="true">
<name>value</name>
<type>String</type>
<description><![CDATA[Complete or uncomplete path or filename<br/><br/>
Example : A module has some files located in the default resource directory (<code>src/main/resources</code>). By default, all these files are included in the checksum computation.

<ul>
<li>readme.md</li>
<li>my-folder-1</li>
<li>
<ul>
<li>my-file-001.txt</li>
<li>my-file-002.txt</li>
</ul>
</li>
<li>my-folder-2</li>
<li>
<ul>
<li>my-file-001.txt</li>
<li>my-file-002.txt</li>
</ul>
</ul>

Here are some possible "exclude" values with their respective effects :
<ul>
<li><code>src/main/resources/my-folder-1</code> : excludes all the files located under this path</li>
<li><code>src/main/resources/my-folder</code> : excludes all the files located under <code>my-folder-1</code> and <code>my-folder-2</code></li>
<li><code>src/main/resources/my-folder-1/my-file-001.txt</code> : exclude this precise file</li>
<li><code>my-file-001.txt</code> : excludes the files with this name under <code>my-folder-1</code> and <code>my-folder-2</code></li>
<li><code>my-file</code> : excludes all the files under <code>my-folder-1</code> and <code>my-folder-2</code>. Side effect : if a folder named "my-file" exists at the module base directory, it would be excluded too.</li>
</ul>

]]></description>
</field>
</fields>
</class>
<class>
<name>Include</name>
<description><![CDATA[A path or a file to include to checksum computation. <b>Paths are relative to each module basedir.</b><br/>
Remember that "include" elements can also be added per project with the use of maven properties.
]]></description>
<fields>
<field xml.content="true">
<name>value</name>
<type>String</type>
<description>Path is relative to each visited project base directory</description>
</field>
<field xml.attribute="true">
<name>recursive</name>
<type>boolean</type>
<description>Will search in sub-folders if set to true</description>
<defaultValue>true</defaultValue>
</field>
<field xml.attribute="true">
<name>glob</name>
<type>String</type>
<description>Type of files to scan in this path. If not set, the global glob value is used.</description>
</field>
</fields>
</class>
Expand Down
16 changes: 8 additions & 8 deletions src/site/markdown/parameters.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,11 @@ project properties:
</pom>
```

| Parameter | Description |
|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `maven.build.cache.input.glob` | Project specific glob to select sources. Overrides the global glob. |
| `maven.build.cache.input` | Additional source code locations. Relative paths calculated from the current project/module root |
| `maven.build.cache.exclude` | Paths to exclude from source code search. Relative paths calculated from the current project/module root |
| `maven.build.cache.processPlugins` | Introspect plugins to find inputs or not. The default value is true. |
| `maven.build.cache.skipCache` | Skip looking up artifacts for a particular project in caches. The default value is false. |
| `maven.build.cache.restoreGeneratedSources` | Restore generated sources and directly attached files. The default value is true. |
| Parameter | Description |
|---------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `maven.build.cache.input.glob` | Project specific glob to select sources. Overrides the global glob. |
| `maven.build.cache.input` | Additional source code locations. Relative paths calculated from the current project/module root<br/>Example :<br/>```<maven.build.cache.input.1>src/main/scala<maven.build.cache.input.1>```<br/>```<maven.build.cache.input.2>assembly-conf<maven.build.cache.input.2>``` |
| `maven.build.cache.exclude` | Paths to exclude from source code search. Relative paths calculated from the current project/module root<br/>Example :<br/>```<maven.build.cache.exclude.1>dist<maven.build.cache.exclude.1>```<br/>```<maven.build.cache.exclude.2>src/main/javagen<maven.build.cache.exclude.2>``` |
| `maven.build.cache.processPlugins` | Introspect plugins to find inputs or not. The default value is true. |
| `maven.build.cache.skipCache` | Skip looking up artifacts for a particular project in caches. The default value is false. |
| `maven.build.cache.restoreGeneratedSources` | Restore generated sources and directly attached files. The default value is true. |
Loading