Unify kroundup macros and fix gcc warnings

Replace several kroundup32 and kroundup_size_t macro definitions
with a new unified implementation.  The new one will work
correctly for all integer types (both signed and unsigned) up to
64 bits.  An attempt to round up that would overflow results
in the maximum value that can be stored in the given type.

The implementation avoids conditionals involving large values
in an attempt to fix some gcc warnings in samtools and bcftools.
They appeared after commit 29c294e which fixed a bug where kroundup
would wrap around to zero on large values.  The warnings are
caused by jump threading optimisations that create code paths
for the large value cases, which gcc then warns about. See:
https://developers.redhat.com/blog/2019/03/13/understanding-gcc-warnings-part-2/

Includes extra tests to ensure the new kroundup works as desired.

bgzf.c

@@ -2152,10 +2152,6 @@ int bgzf_getc(BGZF *fp)
     return c;
 }
 
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 int bgzf_getline(BGZF *fp, int delim, kstring_t *str)
 {
     int l, state = 0;

faidx.c

@@ -60,10 +60,6 @@ struct __faidx_t {
     enum fai_format_options format;
 };
 
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 static int fai_name2id(void *v, const char *ref)
 {
     faidx_t *fai = (faidx_t *)v;

htslib/hts.h

@@ -34,6 +34,8 @@ DEALINGS IN THE SOFTWARE.  */
 
 #include "hts_defs.h"
 #include "hts_log.h"
+#include "kstring.h"
+#include "kroundup.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -57,18 +59,6 @@ struct hFILE;
 struct hts_tpool;
 struct sam_hdr_t;
 
-#ifndef KSTRING_T
-#define KSTRING_T kstring_t
-typedef struct kstring_t {
-    size_t l, m;
-    char *s;
-} kstring_t;
-#endif
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 /**
  * @hideinitializer
  * Deprecated macro to expand a dynamic array of a given type

htslib/khash.h

@@ -130,6 +130,7 @@ int main() {
 #include <string.h>
 #include <limits.h>
 #include <htslib/kstring.h>
+#include <htslib/kroundup.h>
 
 /* compiler specific configuration */
 

htslib/kroundup.h

@@ -0,0 +1,76 @@
+/* The MIT License
+
+   Copyright (C) 2020 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KROUNDUP_H
+#define KROUNDUP_H
+
+// Value of this macro is 1 if x is a signed type; 0 if unsigned
+#define k_signed_type(x) (!(-((x) * 0 + 1) > 0))
+
+/*
+  Macro with value 1 if the highest bit in x is set for any integer type
+
+  This is written avoiding conditionals (?: operator) to reduce the liklihood
+  of gcc attempting jump thread optimisations for code paths where (x) is
+  large.  These optimisations can cause gcc to issue warnings about excessively
+  large memory allocations when the kroundup64() macro below is used with
+  malloc().  Such warnings can be misleading as they imply only the large
+  allocation happens when it's actually working fine for normal values of (x).
+
+  See https://developers.redhat.com/blog/2019/03/13/understanding-gcc-warnings-part-2/
+*/
+#define k_high_bit_set(x) ((((x) >> (sizeof(x) * 8 - 1 - k_signed_type(x))) & 1))
+
+/*! @hideinitializer
+  @abstract  Round up to next power of two
+  @discussion
+  This macro will work for unsigned types up to uint64_t.
+
+  If the next power of two does not fit in the given type, it will set
+  the largest value that does.
+ */
+#define kroundup64(x) ((x) > 0 ?                                        \
+                       (--(x),                                          \
+                        (x)|=(x)>>(sizeof(x)/8),                        \
+                        (x)|=(x)>>(sizeof(x)/4),                        \
+                        (x)|=(x)>>(sizeof(x)/2),                        \
+                        (x)|=(x)>>(sizeof(x)),                          \
+                        (x)|=(x)>>(sizeof(x)*2),                        \
+                        (x)|=(x)>>(sizeof(x)*4),                        \
+                        (x) += !k_high_bit_set(x),                      \
+                        (x))                                            \
+                       : 0)
+
+// Historic interfaces for 32-bit and size_t values.  The macro above
+// works for both (as long as size_t is no more than 64 bits).
+
+#ifndef kroundup32
+#define kroundup32(x) kroundup64(x)
+#endif
+#ifndef kroundup_size_t
+#define kroundup_size_t(x) kroundup64(x)
+#endif
+
+#endif

htslib/kseq.h

@@ -33,6 +33,8 @@
 #include <string.h>
 #include <stdlib.h>
 
+#include "kstring.h"
+
 #ifndef klib_unused
 #if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
 #define klib_unused __attribute__ ((__unused__))
@@ -88,18 +90,6 @@
 	static inline klib_unused int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \
 	{ return ks_getuntil2(ks, delimiter, str, dret, 0); }
 
-#ifndef KSTRING_T
-#define KSTRING_T kstring_t
-typedef struct kstring_t {
-	size_t l, m;
-	char *s;
-} kstring_t;
-#endif
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
-#endif
-
 #define __KS_GETUNTIL(SCOPE, __read) \
 	SCOPE int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append)  \
 	{ \

htslib/kstring.h

@@ -33,24 +33,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <limits.h>
+#include <errno.h>
 #include <sys/types.h>
 
 #include "hts_defs.h"
-
-#ifndef kroundup32
-#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x),(x)=(x)?(x):(uint32_t)-1)
-#endif
-
-#ifndef kroundup_size_t
-#define kroundup_size_t(x) (--(x),                                       \
-                            (x)|=(x)>>(sizeof(size_t)/8), /*  0 or  1 */ \
-                            (x)|=(x)>>(sizeof(size_t)/4), /*  1 or  2 */ \
-                            (x)|=(x)>>(sizeof(size_t)/2), /*  2 or  4 */ \
-                            (x)|=(x)>>(sizeof(size_t)),   /*  4 or  8 */ \
-                            (x)|=(x)>>(sizeof(size_t)*2), /*  8 or 16 */ \
-                            (x)|=(x)>>(sizeof(size_t)*4), /* 16 or 32 */ \
-                            ++(x),(x)=(x)?(x):(size_t)-1)
-#endif
+#include "kroundup.h"
 
 #if defined __GNUC__ && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4))
 #ifdef __MINGW_PRINTF_FORMAT

htslib_vars.mk

@@ -29,20 +29,21 @@ htslib_bgzf_h = $(HTSPREFIX)htslib/bgzf.h $(htslib_hts_defs_h)
 htslib_cram_h = $(HTSPREFIX)htslib/cram.h $(htslib_hts_defs_h) $(htslib_hts_h) $(htslib_sam_h)
 htslib_faidx_h = $(HTSPREFIX)htslib/faidx.h $(htslib_hts_defs_h) $(htslib_hts_h)
 htslib_hfile_h = $(HTSPREFIX)htslib/hfile.h $(htslib_hts_defs_h)
-htslib_hts_h = $(HTSPREFIX)htslib/hts.h $(htslib_hts_defs_h) $(htslib_hts_log_h)
+htslib_hts_h = $(HTSPREFIX)htslib/hts.h $(htslib_hts_defs_h) $(htslib_hts_log_h) $(htslib_kstring_h) $(htslib_kroundup_h)
 htslib_hts_defs_h = $(HTSPREFIX)htslib/hts_defs.h
 htslib_hts_endian_h = $(HTSPREFIX)htslib/hts_endian.h
 htslib_hts_log_h = $(HTSPREFIX)htslib/hts_log.h $(htslib_hts_defs_h)
 htslib_hts_os_h = $(HTSPREFIX)htslib/hts_os.h $(htslib_hts_defs_h)
 htslib_kbitset_h = $(HTSPREFIX)htslib/kbitset.h
 htslib_kfunc_h = $(HTSPREFIX)htslib/kfunc.h $(htslib_hts_defs_h)
-htslib_khash_h = $(HTSPREFIX)htslib/khash.h $(htslib_kstring_h)
+htslib_khash_h = $(HTSPREFIX)htslib/khash.h $(htslib_kstring_h) $(htslib_kroundup_h)
 htslib_khash_str2int_h = $(HTSPREFIX)htslib/khash_str2int.h $(htslib_khash_h)
 htslib_klist_h = $(HTSPREFIX)htslib/klist.h
 htslib_knetfile_h = $(HTSPREFIX)htslib/knetfile.h $(htslib_hts_defs_h)
+htslib_kroundup_h = $(HTSPREFIX)htslib/kroundup.h
 htslib_kseq_h = $(HTSPREFIX)htslib/kseq.h
 htslib_ksort_h = $(HTSPREFIX)htslib/ksort.h
-htslib_kstring_h = $(HTSPREFIX)htslib/kstring.h $(htslib_hts_defs_h)
+htslib_kstring_h = $(HTSPREFIX)htslib/kstring.h $(htslib_hts_defs_h) $(htslib_kroundup_h)
 htslib_regidx_h = $(HTSPREFIX)htslib/regidx.h $(htslib_hts_h)
 htslib_sam_h = $(HTSPREFIX)htslib/sam.h $(htslib_hts_h)
 htslib_synced_bcf_reader_h = $(HTSPREFIX)htslib/synced_bcf_reader.h $(htslib_hts_h) $(htslib_vcf_h) $(htslib_tbx_h)

test/test_kstring.c

@@ -38,6 +38,88 @@ static inline void clamp(int64_t *val, int64_t min, int64_t max) {
     if (*val > max) *val = max;
 }
 
+static int test_kroundup_size_t(int verbose) {
+    size_t val, exp;
+    int ret = 0;
+
+    val = 0;
+    kroundup_size_t(val);
+    if (verbose) {
+        printf("kroundup_size_t(0) = 0x%zx\n", val);
+    }
+    if (val != 0) {
+        fprintf(stderr, "kroundup_size_t(0) produced 0x%zx, expected 0\n", val);
+        ret = -1;
+    }
+
+    for (exp = 0; exp < sizeof(val) * 8; exp++) {
+        size_t expected = ((size_t) 1) << exp;
+        ssize_t delta;
+        for (delta = exp > 1 ? -1 : 0; delta <= (exp < 2 ? 0 : 1); delta++) {
+            size_t val_in = expected + delta;
+            val = val_in;
+            kroundup_size_t(val);
+            if (verbose) {
+                printf("kroundup_size_t(0x%zx) = 0x%zx\n", val_in, val);
+            }
+            if (delta <= 0) {
+                if (val != expected) {
+                    fprintf(stderr, "kroundup_size_t(0x%zx) produced 0x%zx, "
+                            "expected 0x%zx\n",
+                            val_in, val, expected);
+                    ret = -1;
+                }
+            } else {
+                expected *= 2;
+                if (!expected) --expected;
+                if (val != expected) {
+                    fprintf(stderr, "kroundup_size_t(0x%zx) produced 0x%zx, "
+                            "expected 0x%zx\n",
+                            val_in, val, expected);
+                    ret = -1;
+                }
+            }
+        }
+    }
+    return ret;
+}
+
+static int test_kroundup_signed(int verbose) {
+    int32_t val, ret = 0;
+    size_t exp;
+    for (exp = 0; exp < sizeof(val) * 8 - 1; exp++) {
+        uint32_t expected = ((uint32_t) 1) << exp;
+        ssize_t delta;
+        for (delta = exp > 1 ? -1 : 0; delta <= (exp < 2 ? 0 : 1); delta++) {
+            int32_t val_in = expected + delta;
+            val = val_in;
+            kroundup32(val);
+            if (verbose) {
+                printf("kroundup32(%d) = %d\n", val_in, val);
+            }
+            if (delta <= 0) {
+                if ((uint32_t) val != expected) {
+                    fprintf(stderr, "kroundup32(%d) produced %d, expected %u\n",
+                            val_in, val, expected);
+                    ret = -1;
+                }
+            } else {
+                if (exp < sizeof(val) * 8 - 2) {
+                    expected *= 2;
+                } else {
+                    expected = ((expected - 1) << 1 | 1);
+                }
+                if ((uint32_t) val != expected) {
+                    fprintf(stderr, "kroundup32(%d) produced %d, expected %u\n",
+                            val_in, val, expected);
+                    ret = -1;
+                }
+            }
+        }
+    }
+    return ret;
+}
+
 static int test_kputuw_from_to(kstring_t *str, unsigned int s, unsigned int e) {
     unsigned int i = s;
 
@@ -184,8 +266,9 @@ int main(int argc, char **argv) {
     int64_t start = 0;
     int64_t end   = 0;
     char *test    = NULL;
+    int verbose = 0;
 
-    while ((opt = getopt(argc, argv, "e:s:t:")) != -1) {
+    while ((opt = getopt(argc, argv, "e:s:t:v")) != -1) {
         switch (opt) {
         case 's':
             start = strtoll(optarg, NULL, 0);
@@ -196,13 +279,22 @@ int main(int argc, char **argv) {
         case 't':
             test = optarg;
             break;
+        case 'v':
+            verbose++;
+            break;
         default:
             fprintf(stderr, "Usage : %s [-s <num>] [-e <num>] [-t <test>]\n",
                     argv[0]);
             return EXIT_FAILURE;
         }
     }
 
+    if (!test || strcmp(test, "kroundup_size_t") == 0)
+        if (test_kroundup_size_t(verbose) != 0) res = EXIT_FAILURE;
+
+    if (!test || strcmp(test, "kroundup_signed") == 0)
+        if (test_kroundup_signed(verbose) != 0) res = EXIT_FAILURE;
+
     if (!test || strcmp(test, "kputuw") == 0)
         if (test_kputuw(start, end) != 0) res = EXIT_FAILURE;