TST Update tests to newer deps

Megahit in particular has changed its outputs significantly. This should
be detected and reported as the version line is checked.

.github/workflows/build_w_nix.yml

@@ -24,6 +24,8 @@ jobs:
     - run: nix build -f release.nix NGLess.components.tests.nglesstest --out-link ngless-test-dyn
     - run: ./ngless-test-dyn/bin/nglesstest
 
+    - run: nix-shell --pure --command ./run-tests.sh
+
     - run: nix build -f release.nix projectCross.musl64.hsPkgs.NGLess.components.exes.ngless --out-link ngless-static
 
     - name: Upload

ChangeLog

@@ -5,6 +5,9 @@ Unreleased
 	* write() uses multiple threads to write multiple FQ files
 	* Use binaries from path (not self installed)
 	* Better error message when parsing misformed FastQ files
+	* Overhall building infrastructure to use haskell.nix and build
+	dependencies with nix
+	* Update megahit version
 
 Version 1.3.0 2021-01-28 by luispedro
 	* Validate count() headers on --validate-only

shell.nix

@@ -11,6 +11,7 @@ in pkgs.mkShell {
     pkgs.prodigal
     pkgs.samtools
     pkgs.megahit
+    pkgs.python3 # required for megahit (dependency should be upstreamed)
     pkgs.minimap2
     pkgs.bwa
 

tests/compress_sam/check.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 for out in output.from.bam.sam.gz output.sam.gz; do
-    if ! diff input.sam <(zcat ${out}); then
+    if ! diff <(grep -v '^@PG' input.sam) <(zcat ${out} | grep -v '^@PG'); then
         echo "Difference between ${out} and input.sam" >&2
         exit 1
     fi

tests/map-minimap2/expected.sam

@@ -2,7 +2,7 @@
 @SQ	SN:B	LN:3437
 @SQ	SN:C	LN:11798
 @SQ	SN:D	LN:3620
-@PG	ID:minimap2	PN:minimap2	VN:2.16-r922	CL:minimap2 -t 1 -a ref.fna -
+@PG	ID:minimap2	PN:minimap2	VN:2.24-r1122	CL:minimap2 -t 1 -a ref.fna -
 ReadP	0	A	189	38	80M	*	0	0	CTCGAACCTCCAATCTTCGGATCCGAAGTCCGACGCCCCCGCGTCGGATGCGTTGTTACCACTGCTTTTGGTATCATTGA	HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH	NM:i:0	ms:i:160	AS:i:160	nn:i:0	tp:A:P	cm:i:8	s1:i:62	s2:i:0	de:f:0	rl:i:0
 ReadP	0	A	69	49	80M	*	0	0	TATATATATATTTCTTGTAATTTGTTGGAATACGAGAACATCGTCAATAATATATCGTATGAATTGAACCACACGGCACA	HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH	NM:i:0	ms:i:160	AS:i:160	nn:i:0	tp:A:P	cm:i:9	s1:i:68	s2:i:0	de:f:0	rl:i:0
 readS	0	C	35	50	80M	*	0	0	TCTTGGGAACCTACAGAACTGGTTATTGGTGTCGTGGAACCTCTTACTGCTTTCAATACACGATTAGTAATCAACTGTTT	HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH	NM:i:0	ms:i:160	AS:i:160	nn:i:0	tp:A:P	cm:i:12	s1:i:69	s2:i:0	de:f:0	rl:i:0

tests/map_sort_stream/texpected.sam

@@ -1,4 +1,4 @@
-@HD	VN:1.3	SO:queryname
+@HD	VN:1.6	SO:queryname
 @SQ	SN:A	LN:2556
 @SQ	SN:B	LN:3437
 @SQ	SN:C	LN:11798

tests/regression-cigar-filter/check.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+if ! diff <(grep -v '^@PG' output.sam) <(grep -v '^@PG' texpected.sam); then
+    exit 1
+fi

tests/regression-cigar-filter/select_sort.ngl

@@ -7,4 +7,4 @@ mapped = select(mapped) using |mr|:
 
 mapped = samtools_sort(mapped, by={name})
 
-write(mapped, ofile='output.result.sam')
+write(mapped, ofile='output.sam')

tests/regression-cigar-filter/expected.result.sam → tests/regression-cigar-filter/texpected.sam

@@ -1,4 +1,4 @@
-@HD	VN:1.3	SO:queryname
+@HD	VN:1.6	SO:queryname
 @SQ	SN:JPJG01000058	LN:30739
 @SQ	SN:JPJG01000046	LN:31516
 @SQ	SN:JPJG01000064	LN:32447

tests/reuse/check.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+if ! diff <(grep -v '^@PG' output.sam) <(grep -v '^@PG' texpected.sam); then
+    exit 1
+fi
+
+if ! diff <(grep -v '^@PG' output.reused.sam) <(grep -v '^@PG' texpected.reused.sam); then
+    exit 1
+fi

tests/samfile-select-sort/check.sh

@@ -10,18 +10,21 @@ else
     SAMTOOLS="$(which samtools)"
 fi
 
-if ! diff <($SAMTOOLS view -h output.unsorted.bam) texpected.unsorted.sam ; then
-    exit 1
+failed=0
+if ! diff <($SAMTOOLS view -h output.unsorted.bam | grep -v '^@PG') texpected.unsorted.sam ; then
+    failed=1
 fi
 
-if ! diff <($SAMTOOLS view -h output.pos_sorted.bam) texpected.pos_sorted.sam ; then
-    exit 1
+if ! diff <($SAMTOOLS view -h output.pos_sorted.bam | grep -v '^@PG') texpected.pos_sorted.sam ; then
+    failed=1
 fi
 
-if ! diff <($SAMTOOLS view -h output.pos_sorted2.bam) texpected.pos_sorted.sam ; then
-    exit 1
+if ! diff <($SAMTOOLS view -h output.pos_sorted2.bam | grep -v '^@PG') texpected.pos_sorted.sam ; then
+    failed=1
 fi
 
-if ! diff <($SAMTOOLS view -h output.name_sorted.bam) texpected.name_sorted.sam ; then
-    exit 1
+if ! diff <($SAMTOOLS view -h output.name_sorted.bam | grep -v '^@PG') texpected.name_sorted.sam ; then
+    failed=1
 fi
+
+exit $failed

tests/samfile-select-sort/texpected.name_sorted.sam

@@ -1,4 +1,4 @@
-@HD	VN:1.3	SO:queryname
+@HD	VN:1.6	SO:queryname
 @SQ	SN:122_DCM_0d2-0d45_scaffold97490_1_gene122702	LN:30803
 H8:D1BN1ACXX:7:1103:3626:94786	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	17908	60	98M	=	17966	156	TGTGTCTAATGTAAAAAATATTTATCCTGGTGGTCAAAGTGATTTTGCAATCACTTTAAACGATGATACAGTTGAATATTGGAGTGCAATTAACCAAA	DDFEFHHHGHHFHIIJJIIJJJJHICGIJJIJIJIJIJIHIJJFHIIJGIHHIIJJHGGHII@FHCGGEGGGEHCACEDB;?;BCE@@CECCDD;>A9	NM:i:5	MD:Z:6C28T5G7G34C13	AS:i:73	XS:i:50
 H8:D1BN1ACXX:7:1104:15929:33048	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	268	11	60S38M	*	609	0	TGGATCCATTGTAGCTTGGGGTGATTCGACTACTGGTGGTTCTACAACCAATGGTGGTGGTTATACTGGTGCCCCTAGTAGTGTGACAAATGCAAGTA	ACCCCDCDACCDDDDB?>;DCCCEBCDDCFFHHHHHGGCIHJIHHGHJJIJJJIIHJIJIJJJJIIIJIJJJIHJJJJJJJIJJJIJJHHHHHFFFFF	NM:i:1	MD:Z:28G9	AS:i:33	XS:i:30

tests/samfile-select-sort/texpected.pos_sorted.sam

@@ -1,4 +1,4 @@
-@HD	VN:1.3	SO:coordinate
+@HD	VN:1.6	SO:coordinate
 @SQ	SN:122_DCM_0d2-0d45_scaffold97490_1_gene122702	LN:30803
 H8:D1BN1ACXX:7:1104:15929:33048	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	268	11	60S38M	*	609	0	TGGATCCATTGTAGCTTGGGGTGATTCGACTACTGGTGGTTCTACAACCAATGGTGGTGGTTATACTGGTGCCCCTAGTAGTGTGACAAATGCAAGTA	ACCCCDCDACCDDDDB?>;DCCCEBCDDCFFHHHHHGGCIHJIHHGHJJIJJJIIHJIJIJJJJIIIJIJJJIHJJJJJJJIJJJIJJHHHHHFFFFF	NM:i:1	MD:Z:28G9	AS:i:33	XS:i:30
 H8:D1BN1ACXX:7:1103:3626:94786	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	17908	60	98M	=	17966	156	TGTGTCTAATGTAAAAAATATTTATCCTGGTGGTCAAAGTGATTTTGCAATCACTTTAAACGATGATACAGTTGAATATTGGAGTGCAATTAACCAAA	DDFEFHHHGHHFHIIJJIIJJJJHICGIJJIJIJIJIJIHIJJFHIIJGIHHIIJJHGGHII@FHCGGEGGGEHCACEDB;?;BCE@@CECCDD;>A9	NM:i:5	MD:Z:6C28T5G7G34C13	AS:i:73	XS:i:50

tests/samfile-select-view/check.sh

@@ -10,6 +10,6 @@ else
     SAMTOOLS="$(which samtools)"
 fi
 
-if ! diff <($SAMTOOLS view -h output.bam) <(zcat texpected.sam.gz) ; then
+if ! diff <($SAMTOOLS view -h output.bam | grep -v '^@PG') <(zcat texpected.sam.gz | grep -v '^@PG') ; then
     exit 1
 fi

tests/samfile-write/check.sh

@@ -10,17 +10,19 @@ else
     SAMTOOLS="$(which samtools)"
 fi
 
-if ! diff <($SAMTOOLS view -h output.1.bam) texpected.sam ; then
-    exit 1
+failed=0
+if ! diff <($SAMTOOLS view -h output.1.bam | grep -v '^@PG') texpected.sam ; then
+    failed=1
 fi
 
 
-if ! diff <(zcat output.2.sam.gz) texpected.sam ; then
-    exit 1
+if ! diff <(zcat output.2.sam.gz | grep -v '^@PG') texpected.sam ; then
+    failed=1
 fi
 
-if ! diff <(zcat output.3.sam.gz) texpected.sam ; then
-    exit 1
+if ! diff <(zcat output.3.sam.gz | grep -v '^@PG') texpected.sam ; then
+    failed=1
 fi
 
+exit $failed
 

tests/samfile-write/texpected.sam

@@ -1,4 +1,4 @@
-@HD	VN:1.3	SO:coordinate
+@HD	VN:1.6	SO:coordinate
 @SQ	SN:122_DCM_0d2-0d45_scaffold97490_1_gene122702	LN:30803
 H8:D1BN1ACXX:7:1104:15929:33048	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	268	11	60S38M	*	609	0	TGGATCCATTGTAGCTTGGGGTGATTCGACTACTGGTGGTTCTACAACCAATGGTGGTGGTTATACTGGTGCCCCTAGTAGTGTGACAAATGCAAGTA	ACCCCDCDACCDDDDB?>;DCCCEBCDDCFFHHHHHGGCIHJIHHGHJJIJJJIIHJIJIJJJJIIIJIJJJIHJJJJJJJIJJJIJJHHHHHFFFFF	NM:i:1	MD:Z:28G9	AS:i:33	XS:i:30
 H8:D1BN1ACXX:7:1103:3626:94786	1	122_DCM_0d2-0d45_scaffold97490_1_gene122702	17908	60	98M	=	17966	156	TGTGTCTAATGTAAAAAATATTTATCCTGGTGGTCAAAGTGATTTTGCAATCACTTTAAACGATGATACAGTTGAATATTGGAGTGCAATTAACCAAA	DDFEFHHHGHHFHIIJJIIJJJJHICGIJJIJIJIJIJIHIJJFHIIJGIHHIIJJHGGHII@FHCGGEGGGEHCACEDB;?;BCE@@CECCDD;>A9	NM:i:5	MD:Z:6C28T5G7G34C13	AS:i:73	XS:i:50