Merge pull request #124 from nvnieuwk/fix-yaml-validation

Fix YAML validation + prepare for patch release

CHANGELOG.md

@@ -1,5 +1,12 @@
 # nextflow-io/nf-validation: Changelog
 
+# Version 1.1.1 - Shoyu
+
+## Bug fixes
+
+- Fixed an issue where samplesheet with a lot of null values would take forever to validate ([#120](https://github.com/nextflow-io/nf-validation/pull/120)) => Thanks @awgymer for fixing this!
+- Now YAML files are actually validated instead of skipped ([#124](https://github.com/nextflow-io/nf-validation/pull/120))
+
 # Version 1.1.0 - Miso
 
 ## Features

plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy

@@ -184,16 +184,25 @@ class SchemaValidator extends PluginExtensionPoint {
         def String fileType = SamplesheetConverter.getFileType(samplesheetFile)
         def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
         def List<Map<String,String>> fileContent
-        def List<Map<String,String>> fileContentCasted = []
         def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false
+        def Map types = variableTypes(schemaFile.toString(), baseDir)
+        def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
+
+        if(!containsHeader){
+            types = ["empty": types[""]]
+        }
         if(fileType == "yaml"){
-            fileContent = new Yaml().load((samplesheetFile.text))
+            fileContent = new Yaml().load((samplesheetFile.text)).collect {
+                if(containsHeader) {
+                    return it as Map
+                }
+                return ["empty": it] as Map
+            }
         }
         else {
-            Map types = variableTypes(schemaFile.toString(), baseDir)
-            fileContent = samplesheetFile.splitCsv(header:true, strip:true, sep:delimiter)
-            fileContentCasted = castToType(fileContent, types)
+            fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
         }
+        def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
         if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) {
             log.debug "Validation passed: '$samplesheetFile' with '$schemaFile'"
         }
@@ -413,15 +422,25 @@ class SchemaValidator extends PluginExtensionPoint {
                     def String fileType = SamplesheetConverter.getFileType(file_path)
                     def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
                     def List<Map<String,String>> fileContent
-                    def List<Map<String,String>> fileContentCasted = []
+                    def Map types = variableTypes(schema_name, baseDir)
+                    def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
+
+                    if(!containsHeader){
+                        types = ["empty": types[""]]
+                    }
+
                     if(fileType == "yaml"){
-                        fileContent = new Yaml().load((file_path.text))
+                        fileContent = new Yaml().load(file_path.text).collect {
+                            if(containsHeader) {
+                                return it as Map
+                            }
+                            return ["empty": it] as Map
+                        }
                     }
                     else {
-                        Map types = variableTypes(schema_name, baseDir)
-                        fileContent = file_path.splitCsv(header:true, strip:true, sep:delimiter)
-                        fileContentCasted = castToType(fileContent, types)
+                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
                     }
+                    def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
                     if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {
                         log.debug "Validation passed: '$key': '$file_path' with '$schema_name'"
                     }

plugins/nf-validation/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Id: nf-validation
-Plugin-Version: 1.1.0
+Plugin-Version: 1.1.1
 Plugin-Class: nextflow.validation.ValidationPlugin
 Plugin-Provider: nextflow
 Plugin-Requires: >=22.10.0

plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy

@@ -194,6 +194,35 @@ class PluginExtensionMethodsTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'should validate a schema yaml with failures' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.input = 'src/testResources/wrong.yaml'
+            params.outdir = 'src/testResources/testDir'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        def error = thrown(SchemaValidationException)
+        def errorMessages = error.message.readLines()
+        errorMessages[0] == "\033[0;31mThe following errors have been detected:"
+        errorMessages[2] == "* -- Entry 1: Missing required value: sample"
+        errorMessages[3] == "* -- Entry 1 - strandedness: Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded' (weird)"
+        errorMessages[4] == "* -- Entry 1 - fastq_2: FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq2.fasta)"
+        errorMessages[5] == "* -- Entry 2 - sample: Sample name must be provided and cannot contain spaces (test 2)"
+        !stdout
+    }
+
     def 'should find unexpected params' () {
         given:
         def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString()

plugins/nf-validation/src/testResources/wrong.yaml

@@ -0,0 +1,6 @@
+- fastq_1: test1_fastq1.fastq.gz
+  fastq_2: test1_fastq2.fasta
+  strandedness: weird
+- sample: test 2
+  fastq_1: test2_fastq1.fastq.gz
+  strandedness: forward