Merge pull request #104 from mirpedrol/no-validate-s3

do not validate S3 URL paths by default
nextflow-io · Oct 10, 2023 · 97df1cf · 97df1cf
2 parents 7ca22a4 + e578dee
commit 97df1cf
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@
 - Add parameters defined on the top level of the schema and within the definitions section as expected params ([#79](https://github.com/nextflow-io/nf-validation/pull/79))
 - Fix error when a parameter is not present in the schema and evaluates to false ([#89](https://github.com/nextflow-io/nf-validation/pull/89))
 - Changed the `schema_filename` option of `fromSamplesheet` to `parameters_schema` to make this option more clear to the user ([#91](https://github.com/nextflow-io/nf-validation/pull/91))
+- Do not check if S3 URL paths exists to avoid AWS errors, and add a new parameter `validationS3PathCheck` ([#104](https://github.com/nextflow-io/nf-validation/pull/104))
 
 ## Version 0.3.1
 

diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md
@@ -370,7 +370,12 @@ Example usage is as follows:
 
 !!! note
 
-    If the parameter is set to `null`, `false` or an empty string, this validation is ignored. Does not check if the path exists.
+    If the parameter is set to `null`, `false` or an empty string, this validation is ignored. It does not check if the path exists.
+
+!!! note
+
+    If the parameter is an S3 URL path, this validation is ignored.
+    Use `--validationS3PathCheck` or set `params.validationS3PathCheck = true` to validate them.
 
 ### `mimetype`
 

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -185,6 +185,7 @@ class SchemaValidator extends PluginExtensionPoint {
         def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
         def List<Map<String,String>> fileContent
         def List<Map<String,String>> fileContentCasted = []
+        def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false
         if(fileType == "yaml"){
             fileContent = new Yaml().load((samplesheetFile.text))
         }
@@ -193,7 +194,7 @@ class SchemaValidator extends PluginExtensionPoint {
             fileContent = samplesheetFile.splitCsv(header:true, strip:true, sep:delimiter)
             fileContentCasted = castToType(fileContent, types)
         }
-        if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir)) {
+        if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) {
             log.debug "Validation passed: '$samplesheetFile' with '$schemaFile'"
         }
 
@@ -235,6 +236,9 @@ class SchemaValidator extends PluginExtensionPoint {
         if( !params.containsKey("validationSkipDuplicateCheck") ) {
             params.validationSkipDuplicateCheck = false
         }
+        if( !params.containsKey("validationS3PathCheck") ) {
+            params.validationS3PathCheck = false
+        }
 
         return params
     }
@@ -251,7 +255,8 @@ class SchemaValidator extends PluginExtensionPoint {
             "help",
             "validationShowHiddenParams",
             "validationSchemaIgnoreParams",
-            "validationSkipDuplicateCheck"
+            "validationSkipDuplicateCheck",
+            "validationS3PathCheck",
         ]
 
         return expectedParams
@@ -267,6 +272,7 @@ class SchemaValidator extends PluginExtensionPoint {
 
         def Map params = initialiseExpectedParams(session.params)
         def String baseDir = session.baseDir
+        def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false
         log.debug "Starting parameters validation"
 
         // Clean the parameters
@@ -297,7 +303,7 @@ class SchemaValidator extends PluginExtensionPoint {
         def List<String> pathsToCheck = (List) collectExists(schemaParams)
         pathsToCheck.each {
             if (params[it]) {
-                pathExists(params[it].toString(), it.toString())
+                pathExists(params[it].toString(), it.toString(), s3PathCheck)
             }
         }
 
@@ -406,7 +412,7 @@ class SchemaValidator extends PluginExtensionPoint {
                         fileContent = file_path.splitCsv(header:true, strip:true, sep:delimiter)
                         fileContentCasted = castToType(fileContent, types)
                     }
-                    if (validateFile(monochrome_logs, key, fileContentCasted, schema_name, baseDir)) {
+                    if (validateFile(monochrome_logs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {
                         log.debug "Validation passed: '$key': '$file_path' with '$schema_name'"
                     }
                 }
@@ -498,7 +504,9 @@ class SchemaValidator extends PluginExtensionPoint {
     // Function to validate a file by its schema
     //
     /* groovylint-disable-next-line UnusedPrivateMethodParameter */
-    boolean validateFile(Boolean monochrome_logs, String paramName, Object fileContent, String schema_filename, String baseDir) {
+    boolean validateFile(
+        Boolean monochrome_logs, String paramName, Object fileContent, String schema_filename, String baseDir, Boolean s3PathCheck = false
+    ) {
 
         // Load the schema
         def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schema_filename)) )
@@ -538,7 +546,7 @@ class SchemaValidator extends PluginExtensionPoint {
             for (int i=0; i < arrayJSON.size(); i++) {
                 def JSONObject entry = arrayJSON.getJSONObject(i)
                 if ( entry.has(fieldName) ) {
-                    pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}")
+                    pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck)
                 }
             }
         }
@@ -576,10 +584,14 @@ class SchemaValidator extends PluginExtensionPoint {
     //
     // Function to check if a file or directory exists
     //
-    List pathExists(String path, String paramName) {
-        def Path file = Nextflow.file(path) as Path
-        if (!file.exists()) {
-            errors << "* --${paramName}: the file or directory '${path}' does not exist.".toString()
+    List pathExists(String path, String paramName, Boolean s3PathCheck) {
+        if (path.startsWith('s3://') && !s3PathCheck) {
+            log.debug "Ignoring validation of S3 URL path '${path}'".toString()
+        } else {
+            def Path file = Nextflow.file(path) as Path
+            if (!file.exists()) {
+                errors << "* --${paramName}: the file or directory '${path}' does not exist.".toString()
+            }
         }
     }