From 08593ad28f0f749500d4f20c1cb5e02a2c441db2 Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Mon, 24 Nov 2025 09:38:04 +0000 Subject: [PATCH 1/5] Fix asset detection when item is not a sibling of main script Signed-off-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --- modules/nf-commons/src/main/nextflow/util/HashBuilder.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index 3b90d1d48c..791fc66676 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -16,6 +16,8 @@ package nextflow.util; +import static nextflow.Const; + import java.io.IOException; import java.io.OutputStream; import java.nio.file.FileVisitResult; @@ -522,7 +524,7 @@ static protected boolean isAssetFile(Path path) { if( session.getBaseDir().getFileSystem()!=path.getFileSystem() ) return false; // if the file is in the same directory as the base dir it's a asset by definition - return path.startsWith(session.getBaseDir()); + return path.startsWith(session.getBaseDir()) || path.startsWith(DEFAULT_ROOT.toPath()); } } From 1e1349e886dd2cd3bf5ad785c089186b259d534f Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Mon, 24 Nov 2025 09:42:30 +0000 Subject: [PATCH 2/5] Fix import Signed-off-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --- modules/nf-commons/src/main/nextflow/util/HashBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index 791fc66676..b7d7d23649 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -16,7 +16,7 @@ package nextflow.util; -import static nextflow.Const; +import static nextflow.Const.*; import java.io.IOException; import java.io.OutputStream; From 2a0c367b3c60da9d44a5a1ae2dae54c4c17c6040 Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Mon, 24 Nov 2025 13:52:03 +0000 Subject: [PATCH 3/5] Add tests Signed-off-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --- .../src/main/nextflow/util/HashBuilder.java | 11 +++++-- .../test/nextflow/util/HashBuilderTest.groovy | 32 +++++++++++++++++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index b7d7d23649..a9594d8f51 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.io.File; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -264,7 +265,7 @@ static private Hasher hashFile( Hasher hasher, Path path, HashMode mode, Path ba log.warn("Unable to get file attributes file: {} -- Cause: {}", FilesEx.toUriString(path), e.toString()); } - if( (mode==HashMode.STANDARD || mode==HashMode.LENIENT) && isAssetFile(path) ) { + if( (mode==HashMode.STANDARD || mode==HashMode.LENIENT) && isAssetFile(path, DEFAULT_ROOT) ) { if( attrs==null ) { // when file attributes are not avail, or it's a directory // hash the file using the file name path and the repository @@ -511,9 +512,13 @@ static private byte[] sumBytes(byte[] resultBytes, byte[] nextBytes) { * pipeline Git repository * * @param path + * The item to check. + * @param asset_root + * Location where assets are being stored. * @return + * Whether or not `path` is included in the pipeline Git repository. */ - static protected boolean isAssetFile(Path path) { + static protected boolean isAssetFile(Path path, File asset_root) { final ISession session = Global.getSession(); if( session==null ) return false; @@ -524,7 +529,7 @@ static protected boolean isAssetFile(Path path) { if( session.getBaseDir().getFileSystem()!=path.getFileSystem() ) return false; // if the file is in the same directory as the base dir it's a asset by definition - return path.startsWith(session.getBaseDir()) || path.startsWith(DEFAULT_ROOT.toPath()); + return path.startsWith(session.getBaseDir()) || path.startsWith(asset_root.toPath()); } } diff --git a/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy b/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy index 12a5f4a2e8..01dba6c5ef 100644 --- a/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy +++ b/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy @@ -20,6 +20,7 @@ import java.nio.file.Files import java.nio.file.Paths import com.google.common.hash.Hashing +import nextflow.Const import nextflow.Global import nextflow.Session import org.apache.commons.codec.digest.DigestUtils @@ -73,10 +74,11 @@ class HashBuilderTest extends Specification { def 'should validate is asset file'() { when: def BASE = Paths.get("/some/pipeline/dir") + def ROOT = new File("/some/pipeline/") and: Global.session = Mock(Session) { getBaseDir() >> BASE } then: - !HashBuilder.isAssetFile(BASE.resolve('foo')) + !HashBuilder.isAssetFile(BASE.resolve('foo'), ROOT) when: @@ -85,9 +87,33 @@ class HashBuilderTest extends Specification { getCommitId() >> '123456' } then: - HashBuilder.isAssetFile(BASE.resolve('foo')) + HashBuilder.isAssetFile(BASE.resolve('foo'), ROOT) and: - !HashBuilder.isAssetFile(Paths.get('/other/dir')) + !HashBuilder.isAssetFile(Paths.get('/other/dir'), ROOT) + } + + def 'should validate is asset file when not part of base directory'() { + when: + def BASE = Paths.get("/some/pipeline/dir") + def ROOT = new File("/some/pipeline/") + + and: + Global.session = Mock(Session) { getBaseDir() >> BASE } + + then: + !HashBuilder.isAssetFile(BASE.resolve('foo'), ROOT) + + when: + Global.session = Mock(Session) { + getBaseDir() >> BASE + getCommitId() >> '123456' + } + + then: + HashBuilder.isAssetFile(Paths.get('/some/pipeline/foo'), ROOT) + and: + !HashBuilder.isAssetFile(Paths.get('/other/dir'), ROOT) + } def 'should hash file content'() { From 399b713942b83478bfe5836840484ca6ae94a6a1 Mon Sep 17 00:00:00 2001 From: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:47:10 +0000 Subject: [PATCH 4/5] Rename asset_root to assetRoot Signed-off-by: Dries Schaumont <5946712+DriesSchaumont@users.noreply.github.com> --- modules/nf-commons/src/main/nextflow/util/HashBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index a9594d8f51..4288e33277 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -513,12 +513,12 @@ static private byte[] sumBytes(byte[] resultBytes, byte[] nextBytes) { * * @param path * The item to check. - * @param asset_root + * @param assetRoot * Location where assets are being stored. * @return * Whether or not `path` is included in the pipeline Git repository. */ - static protected boolean isAssetFile(Path path, File asset_root) { + static protected boolean isAssetFile(Path path, File assetRoot) { final ISession session = Global.getSession(); if( session==null ) return false; @@ -529,7 +529,7 @@ static protected boolean isAssetFile(Path path, File asset_root) { if( session.getBaseDir().getFileSystem()!=path.getFileSystem() ) return false; // if the file is in the same directory as the base dir it's a asset by definition - return path.startsWith(session.getBaseDir()) || path.startsWith(asset_root.toPath()); + return path.startsWith(session.getBaseDir()) || path.startsWith(assetRoot.toPath()); } } From 5e274ae51c4803c8c01f35de64a1ba66dc9829a6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 25 Nov 2025 11:26:57 +0100 Subject: [PATCH 5/5] Improve documentation for isAssetFile method [ci fast] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enhanced JavaDoc explaining asset file hashing strategy - Added context about SHA-256 content hashing for cache validity - Documented dual-check logic for baseDir and assetRoot - Improved test using Spock where block for better readability - Added inline comments explaining main-script parameter use case Related to #6604 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Signed-off-by: Paolo Di Tommaso --- .../src/main/nextflow/util/HashBuilder.java | 35 ++++++++++++++----- .../test/nextflow/util/HashBuilderTest.groovy | 32 ++++++++--------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index 4288e33277..a753f368e4 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -16,11 +16,9 @@ package nextflow.util; -import static nextflow.Const.*; - +import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.io.File; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -51,7 +49,7 @@ import nextflow.script.types.Bag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import static nextflow.Const.DEFAULT_ROOT; import static nextflow.util.CacheHelper.HashMode; @@ -509,14 +507,32 @@ static private byte[] sumBytes(byte[] resultBytes, byte[] nextBytes) { /** * Check if the argument is an asset file i.e. a file that makes part of the - * pipeline Git repository + * pipeline Git repository. + * + *

Asset files are hashed using their content (SHA-256) rather than metadata + * to maintain cache validity across different clones where timestamps may differ + * on remote executors like batch processing systems. + * + *

This method checks two locations: + *

    + *
  1. Files under {@code session.getBaseDir()} - the script's working directory
    2. + *
  2. Files under {@code assetRoot} - the repository root (typically ~/.nextflow/assets)
    3. + *
+ * + * The distinction is important when executing workflows from subdirectories using + * the main-script parameter, as repository assets may exist outside the script's + * directory but still be part of the repository. * * @param path * The item to check. * @param assetRoot - * Location where assets are being stored. + * Location where assets are being stored (the repository root). * @return - * Whether or not `path` is included in the pipeline Git repository. + * {@code true} if the path is included in the pipeline Git repository, + * {@code false} otherwise. + * + * @see Issue #6604 + * @see PR #6605 */ static protected boolean isAssetFile(Path path, File assetRoot) { final ISession session = Global.getSession(); @@ -528,7 +544,10 @@ static protected boolean isAssetFile(Path path, File assetRoot) { // if the file belong to different file system, cannot be a file belonging to the repo if( session.getBaseDir().getFileSystem()!=path.getFileSystem() ) return false; - // if the file is in the same directory as the base dir it's a asset by definition + // Check both the script's base directory and the repository root. + // This handles cases where a workflow is executed from a subdirectory + // (using the main-script parameter) but references assets elsewhere in the repo. + // The assetRoot check ensures these non-sibling assets are still recognized. return path.startsWith(session.getBaseDir()) || path.startsWith(assetRoot.toPath()); } diff --git a/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy b/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy index 01dba6c5ef..40be6c799c 100644 --- a/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy +++ b/modules/nf-commons/src/test/nextflow/util/HashBuilderTest.groovy @@ -20,7 +20,6 @@ import java.nio.file.Files import java.nio.file.Paths import com.google.common.hash.Hashing -import nextflow.Const import nextflow.Global import nextflow.Session import org.apache.commons.codec.digest.DigestUtils @@ -93,27 +92,24 @@ class HashBuilderTest extends Specification { } def 'should validate is asset file when not part of base directory'() { - when: - def BASE = Paths.get("/some/pipeline/dir") - def ROOT = new File("/some/pipeline/") - - and: - Global.session = Mock(Session) { getBaseDir() >> BASE } - - then: - !HashBuilder.isAssetFile(BASE.resolve('foo'), ROOT) - - when: + given: Global.session = Mock(Session) { - getBaseDir() >> BASE - getCommitId() >> '123456' + getBaseDir() >> Paths.get(BASE) + getCommitId() >> COMMIT_ID } - then: - HashBuilder.isAssetFile(Paths.get('/some/pipeline/foo'), ROOT) - and: - !HashBuilder.isAssetFile(Paths.get('/other/dir'), ROOT) + expect: + HashBuilder.isAssetFile(Paths.get(PATH), new File(ROOT)) == EXPECTED + where: + BASE | ROOT | COMMIT_ID | PATH | EXPECTED + "/some/pipeline/dir" | "/some/pipeline/" | null | "/some/pipeline/dir/foo" | false + "/some/pipeline/dir" | "/some/pipeline/" | '123456' | '/other/dir' | false + "/some/pipeline/dir" | "/some/pipeline/" | '123456' | '/some/pipeline/foo' | true + and: + "/this/pipeline" | "/that/pipeline/" | '123456' | '/other/pipeline/foo' | false + "/this/pipeline" | "/that/pipeline/" | '123456' | '/this/pipeline/foo' | true + "/this/pipeline" | "/that/pipeline/" | '123456' | '/that/pipeline/foo' | true } def 'should hash file content'() {