Release 1.16.1

NEWS

@@ -1,3 +1,17 @@
+Release 1.16.1 (2nd September 2022)
+-----------------------------------
+
+Bug fixes:
+
+ * Fixed a bug with the template-coordinate sort which caused incorrect
+   ordering when using threads, or processing large files that don't
+   fit completely in memory.
+   (PR#1703, thanks to Nils Homer)
+
+ * Fixed a crash that occurred when trying to use `samtools merge` in
+   template-coordinate mode.
+   (PR#1705, thanks to Nils Homer)
+
 Release 1.16 (18th August 2022)
 -------------------------------
 
@@ -7,7 +21,7 @@ New work and changes:
    reference out of a CRAM file.
    (PR#1649, addresses #723.  Requested by Torsten Seemann)
 
-  * samtools import now adds grouped by query-name to the header.
+ * samtools import now adds grouped by query-name to the header.
    (PR#1633, thanks to Nils Homer)
 
  * Made samtools view read error messages more generic.  Former error message

README

@@ -9,7 +9,7 @@ Building samtools
 The typical simple case of building Samtools using the HTSlib bundled within
 this Samtools release tarball is done as follows:
 
-    cd .../samtools-1.16 # Within the unpacked release directory
+    cd .../samtools-1.16.1 # Within the unpacked release directory
     ./configure
     make
 
@@ -21,7 +21,7 @@ install samtools etc properly into a directory of your choosing.  Building for
 installation using the HTSlib bundled within this Samtools release tarball,
 and building the various HTSlib utilities such as bgzip is done as follows:
 
-    cd .../samtools-1.16 # Within the unpacked release directory
+    cd .../samtools-1.16.1 # Within the unpacked release directory
     ./configure --prefix=/path/to/location
     make all all-htslib
     make install install-htslib
@@ -48,7 +48,7 @@ There are two advantages to this:
 To build with plug-ins, you need to use the --enable-plugins configure option
 as follows:
 
-    cd .../samtools-1.16 # Within the unpacked release directory
+    cd .../samtools-1.16.1 # Within the unpacked release directory
     ./configure --enable-plugins --prefix=/path/to/location
     make all all-htslib
     make install install-htslib
@@ -66,7 +66,7 @@ Setting --with-plugin-path is useful if you want to run directly from
 the source distribution instead of installing the package.  In that case
 you can use:
 
-    cd .../samtools-1.16 # Within the unpacked release directory
+    cd .../samtools-1.16.1 # Within the unpacked release directory
     ./configure --enable-plugins --with-plugin-path=$PWD/htslib-1.16
     make all all-htslib
 

bam_sort.c

@@ -155,6 +155,8 @@ KHASH_MAP_INIT_STR(const_c2c, char *)
 #define hdrln_free_char(p)
 KLIST_INIT(hdrln, char*, hdrln_free_char)
 
+static template_coordinate_key_t* template_coordinate_key(bam1_t *b, template_coordinate_key_t *key, sam_hdr_t *hdr, khash_t(const_c2c) *lib_lookup);
+
 typedef enum {Coordinate, QueryName, TagCoordinate, TagQueryName, MinHash, TemplateCoordinate} SamOrder;
 static SamOrder g_sam_order = Coordinate;
 static char g_sort_tag[2] = {0,0};
@@ -195,6 +197,8 @@ typedef struct {
 static inline int bam1_cmp_by_tag(const bam1_tag a, const bam1_tag b);
 static inline int bam1_cmp_by_minhash(const bam1_tag a, const bam1_tag b);
 static inline int bam1_cmp_template_coordinate(const bam1_tag a, const bam1_tag b);
+static khash_t(const_c2c) * lookup_libraries(sam_hdr_t *header);
+static void lib_lookup_destroy(khash_t(const_c2c) *lib_lookup);
 
 // Function to compare reads in the heap and determine which one is < the other
 // Note, unlike the bam1_cmp_by_X functions which return <0, 0, >0 this
@@ -1088,6 +1092,8 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
     merged_header_t *merged_hdr = init_merged_header();
     if (!merged_hdr) return -1;
     refs_t *refs = NULL;
+    template_coordinate_keys_t *keys = NULL;
+    khash_t(const_c2c) *lib_lookup = NULL;
 
     // Is there a specified pre-prepared header to use for output?
     if (headers) {
@@ -1294,6 +1300,26 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
         rtrans = NULL;
     }
 
+    // Make sure that there's enough memory for template coordinate keys, one per file to read
+    if (sam_order == TemplateCoordinate) {
+        if ((keys = malloc(sizeof(template_coordinate_keys_t))) == NULL) {
+            print_error("sort", "could not allocate memory for the top-level keys");
+            goto mem_fail;
+        }
+        keys->n = 0;
+        keys->m = 0;
+        keys->buffer_size = 0x10000;
+        keys->buffers = NULL;
+        // Make sure that there's enough memory for template coordinate keys, one per file to read
+        if (keys->n + n >= keys->m * keys->buffer_size) {
+            if (template_coordinate_keys_realloc(keys, keys->n + n) < 0) goto mem_fail;
+        }
+        lib_lookup = lookup_libraries(hout);
+        if (!lib_lookup) {
+            goto mem_fail;
+        }
+    }
+
     // Load the first read from each file into the heap
     for (i = 0; i < n; ++i) {
         heap1_t *h = heap + i;
@@ -1311,6 +1337,10 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
             h->idx = idx++;
             if (g_sam_order == TagQueryName || g_sam_order == TagCoordinate) {
                 h->entry.u.tag = bam_aux_get(h->entry.bam_record, g_sort_tag);
+            } else if (g_sam_order == TemplateCoordinate) {
+                template_coordinate_key_t *key = template_coordinate_keys_get(keys, i); // get the next key to use
+                h->entry.u.key = template_coordinate_key(heap->entry.bam_record, key, hout, lib_lookup); // update the key
+                if (heap->entry.u.key == NULL) goto mem_fail; // key could not be created, error out
             } else {
                 h->entry.u.tag = NULL;
             }
@@ -1320,6 +1350,7 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
             bam_destroy1(h->entry.bam_record);
             h->entry.bam_record = NULL;
             h->entry.u.tag = NULL;
+            h->entry.u.key = NULL;
         } else {
             print_error(cmd, "failed to read first record from \"%s\"", fn[i]);
             goto fail;
@@ -1379,6 +1410,10 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
             heap->idx = idx++;
             if (g_sam_order == TagQueryName || g_sam_order == TagCoordinate) {
                 heap->entry.u.tag = bam_aux_get(heap->entry.bam_record, g_sort_tag);
+            } else if (g_sam_order == TemplateCoordinate) {
+                template_coordinate_key_t *key = template_coordinate_keys_get(keys, heap->i); // get the next key to use
+                heap->entry.u.key = template_coordinate_key(heap->entry.bam_record, key, hout, lib_lookup); // update the key
+                if (heap->entry.u.key == NULL) goto mem_fail; // key could not be created, error out
             } else {
                 heap->entry.u.tag = NULL;
             }
@@ -1453,6 +1488,14 @@ int bam_merge_core2(SamOrder sam_order, char* sort_tag, const char *out, const c
     free(fp);
     free(rtrans);
     free(out_idx_fn);
+    if (keys != NULL) {
+        for (i = 0; i < keys->m; ++i) {
+            free(keys->buffers[i]);
+        }
+        free(keys->buffers);
+        free(keys);
+    }
+    lib_lookup_destroy(lib_lookup);
     return -1;
 }
 
@@ -1657,7 +1700,6 @@ int bam_merge(int argc, char *argv[])
  * BAM sorting *
  ***************/
 
-static template_coordinate_key_t* template_coordinate_key(bam1_t *b, template_coordinate_key_t *key, sam_hdr_t *hdr, khash_t(const_c2c) *lib_lookup);
 
 typedef struct {
     size_t from;
@@ -1686,7 +1728,7 @@ static inline int heap_add_read(heap1_t *heap, int nfiles, samFile **fp,
         if (in_mem[i - nfiles].from < in_mem[i - nfiles].to) {
             size_t from = in_mem[i - nfiles].from;
             heap->entry.bam_record = buf[from].bam_record;
-            if (g_sam_order == TemplateCoordinate) heap->entry.u.key = template_coordinate_keys_get(keys, from);
+            if (g_sam_order == TemplateCoordinate) heap->entry.u.key = buf[from].u.key;
             in_mem[i - nfiles].from++;
             res = 0;
         } else {
@@ -1742,7 +1784,7 @@ static int bam_merge_simple(SamOrder sam_order, char *sort_tag, const char *out,
     heap = (heap1_t*)calloc(heap_size, sizeof(heap1_t));
     if (!heap) goto mem_fail;
 
-    // Make sure that there's enough memory for template coordinate keys
+    // Make sure that there's enough memory for template coordinate keys, one per file to read
     if (keys && keys->n + n >= keys->m * keys->buffer_size) {
         if (template_coordinate_keys_realloc(keys, keys->n + n) < 0) goto mem_fail;
     }

doc/samtools-addreplacerg.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-addreplacerg 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-addreplacerg 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools addreplacerg \- adds or replaces read group tags
 .\"

doc/samtools-ampliconclip.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-ampliconclip 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-ampliconclip 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools ampliconclip \- clip reads using a BED file
 .\"

doc/samtools-ampliconstats.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-ampliconstats 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-ampliconstats 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools ampliconstats \- produces statistics from amplicon sequencing alignment file
 .\"

doc/samtools-bedcov.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-bedcov 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-bedcov 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools bedcov \- reports coverage over regions in a supplied BED file
 .\"

doc/samtools-calmd.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-calmd 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-calmd 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools calmd \- calculates MD and NM tags
 .\"

doc/samtools-cat.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-cat 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-cat 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools cat \- concatenate files together
 .\"

doc/samtools-collate.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-collate 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-collate 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools collate \- shuffles and groups reads together by their names
 .\"

doc/samtools-consensus.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-consensus 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-consensus 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools consensus \- produces produce a consensus FASTA/FASTQ/PILEUP
 .\"

doc/samtools-coverage.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-coverage 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-coverage 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools coverage \- produces a histogram or table of coverage per chromosome
 .\"

doc/samtools-depad.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-depad 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-depad 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools depad \- convert padded BAM to unpadded BAM
 .\"

doc/samtools-depth.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-depth 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-depth 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools depth \- computes the read depth at each position or region
 .\"

doc/samtools-dict.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-dict 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-dict 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools dict \- create a sequence dictionary file from a fasta file
 .\"

doc/samtools-faidx.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-faidx 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-faidx 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools faidx \- indexes or queries regions from a fasta file
 .\"

doc/samtools-fasta.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-fasta 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-fasta 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools fasta / fastq \- converts a SAM/BAM/CRAM file to FASTA or FASTQ
 .\"

doc/samtools-fixmate.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-fixmate 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-fixmate 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools fixmate \- fills in mate coordinates and insert size fields.
 .\"

doc/samtools-flags.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-flags 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-flags 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools flags \- convert between textual and numeric flag representation.
 .\"

doc/samtools-flagstat.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-flagstat 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-flagstat 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools flagstat \- counts the number of alignments for each FLAG type
 .\"

doc/samtools-fqidx.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-fqidx 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-fqidx 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools fqidx \- Indexes or queries regions from a fastq file
 .\"

doc/samtools-head.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-head 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-head 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools head \- view SAM/BAM/CRAM file headers
 .\"

doc/samtools-idxstats.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-idxstats 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-idxstats 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools idxstats \- reports alignment summary statistics
 .\"

doc/samtools-import.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-import 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-import 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools import \- converts FASTQ files to unmapped SAM/BAM/CRAM
 .\"

doc/samtools-index.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-index 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-index 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools index \- indexes SAM/BAM/CRAM files
 .\"

doc/samtools-markdup.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-markdup 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-markdup 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools markdup \- mark duplicate alignments in a coordinate sorted file
 .\"

doc/samtools-merge.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-merge 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-merge 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools merge \- merges multiple sorted files into a single file
 .\"

doc/samtools-mpileup.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-mpileup 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-mpileup 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools mpileup \- produces "pileup" textual format from an alignment
 .\"

doc/samtools-phase.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-phase 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-phase 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools phase \- call and phase heterozygous SNPs
 .\"

doc/samtools-quickcheck.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-quickcheck 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-quickcheck 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools quickcheck \- a rapid sanity check on input files
 .\"

doc/samtools-reference.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-reference 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-reference 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools reference \- extracts an embedded reference from a CRAM file
 .\"
@@ -91,7 +91,7 @@ specified, an index file must be present.
 Write the FASTA records to \fIFILE\fR.  By default this is sent to stdout.
 
 .TP 8
--BI "-@ " INT
+.BI "-@ " INT
 The number of BAM/CRAM decompression threads to use in addition to the
 main thread [0].
 

doc/samtools-reheader.1

@@ -1,5 +1,5 @@
 '\" t
-.TH samtools-reheader 1 "18 August 2022" "samtools-1.16" "Bioinformatics tools"
+.TH samtools-reheader 1 "2 September 2022" "samtools-1.16.1" "Bioinformatics tools"
 .SH NAME
 samtools reheader \- replaces the header in the input file
 .\"