Musl support for htslib 1.10.2

.cargo/config

@@ -1,12 +1,2 @@
-[target.x86_64-apple-darwin]
-rustflags = [
-  "-C", "link-arg=-undefined",
-  "-C", "link-arg=dynamic_lookup",
-]
-
 [target.x86_64-unknown-linux-musl]
-rustflags = [
-  "-C", "link-arg=-undefined",
-  "-C", "link-arg=dynamic_lookup",
-]
-linker = "x86_64-linux-musl-gcc"
+linker = "x86_64-linux-musl-gcc"

.github/workflows/rust.yml

@@ -76,16 +76,21 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ./lcov.info
 
-#      - name: Test musl build without default features
-#        uses: actions-rs/cargo@v1
-#        with:
-#          use-cross: true
-#          command: build
-#          args: --target x86_64-unknown-linux-musl --no-default-features
-#
-#      - name: Test musl build with all features
-#        uses: actions-rs/cargo@v1
-#        with:
-#          use-cross: true  
-#          command: build
-#          args: --target x86_64-unknown-linux-musl --all-features
+      - name: Test musl build without default features
+        env:
+           CFLAGS: -I/usr/local/musl/include
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --target x86_64-unknown-linux-musl --no-default-features
+
+      - name: Test musl build with all features
+        env:
+           CFLAGS: -I/usr/local/musl/include
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: true  
+          command: build
+          args: --target x86_64-unknown-linux-musl --all-features
+

Cargo.toml

@@ -17,7 +17,7 @@ tag-message = "Version {{version}} of Rust-HTSlib."
 
 [dependencies]
 libc = "0.2"
-itertools = "0.8"
+itertools = "0.9.0"
 newtype_derive = "0.1"
 custom_derive = "0.1"
 url = "2.1"
@@ -29,16 +29,17 @@ linear-map = "1.2"
 serde_base = { version = "^1", optional = true, package = "serde" }
 serde_bytes = { version = "0.11", optional = true }
 bio-types = ">=0.6"
-snafu = ">= 0.5.0, <= 0.6.0"
+snafu = "0.6.8"
 hts-sys = { version = "^1.10", path = "hts-sys", default-features = false }
 
 [features]
 default = ["bzip2", "lzma", "curl"]
 bzip2 = ["hts-sys/bzip2"]
 lzma = ["hts-sys/lzma"]
 curl = ["hts-sys/curl"]
-openssl = ["hts-sys/openssl"]
+#openssl = ["hts-sys/openssl"]
 serde = ["serde_base", "serde_bytes"]
+static = []
 
 [dev-dependencies]
 tempdir = "0.3"

Cross.toml

@@ -1,4 +1,12 @@
+[build.env]
+passthrough = [
+    "RUST_DEBUG",
+    "RUST_BACKTRACE",
+    "CFLAGS"
+]
+
+
 [target.x86_64-unknown-linux-musl]
-image = "brainstorm/rust_musl_docker:stable-latest-libcurl"
+image = "brainstorm/cross-x86_64-unknown-linux-musl:latest"
 [target.x86_64-unknown-linux-gnu]
 image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl"

README.md

@@ -10,16 +10,30 @@ This library provides HTSlib bindings and a high level Rust API for reading and
 
 To clone this repository, issue
 
-```
-git clone --recursive https://github.com/rust-bio/rust-htslib.git
+```shell
+$ git clone --recursive https://github.com/rust-bio/rust-htslib.git
 ```
 
 ensuring that the HTSlib submodule is fetched, too.
 If you only want to use the library, there is no need to clone the repository. Go on to the **Usage** section in this case.
 
 ## Requirements
 
-To compile this crate you need the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies:
+To compile this crate you need docker and cross:
+
+```shell
+$ cargo install cross
+$ cross build 				              # will build with GNU toolchain
+```
+
+If you want to run rust-htslib code on AWS lambda, you'll need to statically compile it with MUSL as follows:
+
+```shell
+$ export CFLAGS="-I/usr/local/musl/include"
+$ cross build --target x86_64-unknown-linux-musl      # will build with MUSL toolchain
+```
+
+Alternatively, you can also install it locally by installing the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies:
 
 ```shell
 $ sudo apt-get install zlib1g-dev libbz2-dev liblzma-dev clang pkg-config

docker/Dockerfile.gnu

@@ -1,8 +1,9 @@
 FROM rustembedded/cross:x86_64-unknown-linux-gnu
 
-ENV LIBCLANG_PATH /usr/lib/x86_64-linux-gnu
+#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl
+ENV LIBCLANG_PATH /usr/lib/llvm-10/lib
 ENV LLVM_CONFIG_PATH /usr/bin
-RUN apt-get update && \
-    apt-get install -y libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev clang-8 && \
-    ln -sf /usr/bin/llvm-config-8 /usr/bin/llvm-config && \
-    ln -sf /usr/lib/x86_64-linux-gnu/libclang-8.so.1 /usr/lib/x86_64-linux-gnu/libclang.so.1
+RUN apt-get update
+RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails
+RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps

docker/Dockerfile.musl

@@ -1,7 +1,121 @@
 FROM rustembedded/cross:x86_64-unknown-linux-musl
 
+ENV MUSL_CROSS_MAKE_VERSION 0.9.9
 ENV PKG_CONFIG_ALLOW_CROSS 1
-ENV OPENSSL_LIB_DIR /usr/lib/x86_64-linux-gnu
-ENV OPENSSL_INCLUDE_DIR /usr/include/openssl
+ENV LZMA_VERSION 5.2.5
+ENV ZLIB_VERSION 1.2.11
+ENV OPENSSL_VERSION 1_1_1g
+ENV CURL_VERSION 7.70.0
+
+ENV LIBCLANG_PATH /usr/lib/llvm-10/lib
+ENV LLVM_CONFIG_PATH /usr/bin
+
+
+# The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see:
+#   https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651
+# Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/
+#    apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic
+
+# Install basics to locally compile htslib dependencies
 RUN apt-get update && \
-    apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic
+            apt-get install -y build-essential autoconf automake autotools-dev git wget
+
+# Otherwise LLVM bump below fails
+RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates
+
+# Autodetect and fetch latest LLVM repos for the current distro, avoids LLVM warnings and other issues, might generate slower builds for now though, see:
+# https://www.phoronix.com/scan.php?page=news_item&px=Rust-Hurt-On-LLVM-10
+RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps
+
+# Remove pre-installed musl, to avoid cross-musl-make interference
+RUN apt-get remove -y musl
+
+# For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too...
+# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207
+WORKDIR /root
+RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz \
+    && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz
+WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION
+COPY config-musl-cross-make.mak config.mak
+RUN make -j40 install
+
+# Now we assume we have a properly configured musl-cross...
+ENV PATH "/usr/local/musl/bin:$PATH"
+ENV CFLAGS "-fPIC"
+ENV CROSS_COMPILE x86_64-linux-musl-
+ENV CC ${CROSS_COMPILE}cc
+ENV AR ${CROSS_COMPILE}ar
+ENV RANLIB ${CROSS_COMPILE}ranlib
+ENV CXX ${CROSS_COMPILE}g++
+ENV CPPFLAGS "-I/usr/local/musl/include -I/usr/local/include"
+ENV LDFLAGS "-L/usr/local/musl/lib -L/usr/local/lib"
+
+# .. and carry on with the htslib deps
+RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \
+    && tar xvfz zlib-$ZLIB_VERSION.tar.gz \
+    && cd zlib-$ZLIB_VERSION \
+    && ./configure --static --prefix=/usr/local/musl \
+    && make -j40 install
+WORKDIR /root
+RUN git clone git://sourceware.org/git/bzip2 \
+    && cd bzip2 \
+    #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \
+    && make PREFIX=/usr/local/musl -j40 bzip2 \
+    && make -j40 install
+WORKDIR /root
+RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \
+    && tar xvfj xz-$LZMA_VERSION.tar.bz2 \
+    && cd xz-$LZMA_VERSION \
+    && ./configure --prefix=/usr/local/musl --enable-static --disable-shared --host x86_64-unknown-linux-musl \
+    && make -j40 install
+WORKDIR /root
+
+# A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation
+# "OpenSSL has been around a long time, and it carries around a lot of cruft"
+# "SSLv2 is completely broken, and you should disable it during configuration"
+# "You should specify both --prefix and --openssldir to ensure make install works as expected."
+#
+# And also this:
+# https://github.com/openssl/openssl/issues/11362
+#
+# And having to redefine AR,CC & RANLIB because otherwise, this:
+#
+# make[1]: x86_64-linux-musl-x86_64-linux-musl-ar: Command not found
+RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz \
+    && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION \
+    && ./Configure --prefix=/usr/local/musl --openssldir=/usr/local/musl \
+                --with-zlib-lib=/usr/local/lib/zlib-$ZLIB_VERSION linux-x86_64 \
+                no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \
+                no-weak-ssl-ciphers \
+    && make AR=x86_64-linux-musl-ar \
+            CC=x86_64-linux-musl-cc \
+            -j40 \
+    && make RANLIB=x86_64-linux-musl-ranlib -j40 install
+
+WORKDIR /root
+RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \
+    && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \
+    && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \
+        --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \
+        --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \
+        --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \
+        --disable-tftp --disable-ntlm --disable-ldap \
+        && make -j40 install
+
+# To cater Rust's openssl-sys needs...
+#ENV OPENSSL_DIR /usr/local/openssl
+ENV OPENSSL_DIR /usr/local/musl
+
+# Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise:
+#   = note: /usr/bin/ld: /target/debug/deps/liblibloading-689161fea10b6234.rlib(global_static.o): unable to initialize decompress status for section .debug_info
+RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld
+
+# Prepare rustup and toolchain locally for easy manual intervention in this container
+#RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \
+#    && chmod +x rustup-init \
+#    && ./rustup-init -y \
+#    && . $HOME/.cargo/env \
+#    && rustup target add x86_64-unknown-linux-musl
+
+CMD ["bash"]

docker/README.md

@@ -10,7 +10,7 @@ $ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . -
 $ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu
 ```
 
-Then to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e:
+Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e:
 
 ```shell
 $ cross build --target x86_64-unknown-linux-musl 

docker/config-musl-cross-make.mak

@@ -0,0 +1,4 @@
+TARGET = x86_64-linux-musl
+OUTPUT = /usr/local/musl
+GCC_VER = 9.2.0
+BINUTILS_VER = 2.33.1

hts-sys/Cargo.toml

@@ -17,22 +17,22 @@ pre-release-commit-message = "release version {{version}}"
 tag-message = "Version {{version}} of Rust-HTSlib."
 
 [dependencies]
-libc = "0.2"
-libz-sys = "1.0"
-bzip2-sys = { version = "0.1", optional = true }
-lzma-sys = { version = "0.1", optional = true }
-curl-sys = { version = "0.4.26", optional = true }
-openssl-sys = { version = "0.9.54", optional = true }
+libz-sys = { version = "1.0.25", features = ["static"] }
+# https://github.com/alexcrichton/bzip2-rs/issues/56
+bzip2-sys = { version = "0.1.8", optional = true }
+lzma-sys = { version = "0.1.16", optional = true, features = ["static"] }
+curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] }
 
 [features]
-default = ["bzip2", "lzma", "curl"]
+default = ["bzip2", "lzma"]
 bzip2 = ["bzip2-sys"]
 lzma = ["lzma-sys"]
-openssl = ["openssl-sys"]
+#openssl = ["openssl-sys"]
 curl = ["curl-sys"]
+static = []
 
 [build-dependencies]
 fs-utils = "1.1"
-bindgen = { version = "0.53.1", default-features = false, features = ["runtime"] }
+bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] }
 cc = "1.0"
-glob = "0.3.0"
+glob = "0.3.0"