Permalink
Browse files

Merge libcurl backend and hFILE plugin infrastructure

[NEWS]
* Files can now be accessed via HTTPS and Amazon S3 in addition to HTTP
  and FTP, when HTSlib is configured to use libcurl for network file access
  rather than the included basic knetfile networking.
  • Loading branch information...
jmarshall committed Oct 29, 2015
2 parents 0e2bc68 + 07d3be7 commit fe1f08a3a80b8a5a17fa56b3fc1808ab2ac25d63
Showing with 1,475 additions and 59 deletions.
  1. +3 −0 .gitignore
  2. +23 −0 INSTALL
  3. +41 −19 Makefile
  4. +15 −0 config.h.in
  5. +45 −5 config.mk.in
  6. +82 −0 configure.ac
  7. +190 −22 hfile.c
  8. +61 −6 hfile_internal.h
  9. +12 −6 hfile_irods.c
  10. +798 −0 hfile_libcurl.c
  11. +12 −1 hfile_net.c
  12. +20 −0 hts_internal.h
  13. +2 −0 htslib.mk
  14. +171 −0 plugin.c
View
@@ -11,6 +11,9 @@ config.mk
config.status
configure
hfile_*.bundle
hfile_*.so
lib*.a
lib*.dylib
lib*.so
View
23 INSTALL
@@ -34,6 +34,24 @@ requirements such as the zlib development files, and arranges for a plain
HTSlib build. The following configure options can be used to enable
various features and specify further optional external requirements:
--enable-plugins
Use plugins to implement exotic file access protocols and other
specialised facilities. This enables such facilities to be developed
and packaged outwith HTSlib, and somewhat isolates HTSlib-using programs
from their library dependencies. By default, any enabled pluggable
facilities (such as iRODS and libcurl file access) are built directly
within HTSlib.
--with-plugin-dir=DIR
Specifies the directory into which plugins built while building HTSlib
should be installed; by default, LIBEXECDIR/htslib.
--with-plugin-path=DIR:DIR:DIR...
Specifies the list of directories that HTSlib will search for plugins.
By default, only the directory specified via --with-plugin-dir will be
searched; you can use --with-plugin-path='DIR:$(plugindir):DIR' and so
on to cause additional directories to be searched.
--with-irods[=DIR]
Specifies the location of the iRODS client library to use to enable
access to data objects stored in iRODS (<http://irods.org/>) via file
@@ -42,6 +60,11 @@ various features and specify further optional external requirements:
headers are present under DIR/lib/api/include and so on. If '=DIR' is
omitted, $IRODS_HOME will be used as a base directory.
--enable-libcurl
Use libcurl (<http://curl.haxx.se/>) to implement network access to
remote files via FTP, HTTP, HTTPS, S3, etc. By default, HTSlib uses
its own simple networking code to provide access via FTP and HTTP only.
The configure script also accepts the usual options and environment variables
for tuning installation locations and compilers: type './configure --help'
for details. For example,
View
@@ -56,6 +56,7 @@ exec_prefix = $(prefix)
bindir = $(exec_prefix)/bin
includedir = $(prefix)/include
libdir = $(exec_prefix)/lib
libexecdir = $(exec_prefix)/libexec
datarootdir = $(prefix)/share
mandir = $(datarootdir)/man
man1dir = $(mandir)/man1
@@ -68,6 +69,9 @@ INSTALL_PROGRAM = $(INSTALL)
INSTALL_DATA = $(INSTALL) -m 644
INSTALL_DIR = $(MKDIR_P) -m 755
# Set by config.mk if plugins are enabled
plugindir =
BUILT_PROGRAMS = \
bgzip \
htsfile \
@@ -82,24 +86,11 @@ BUILT_TEST_PROGRAMS = \
test/test-vcf-api \
test/test-vcf-sweep
all: lib-static lib-shared $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
all: lib-static lib-shared $(BUILT_PROGRAMS) plugins $(BUILT_TEST_PROGRAMS)
HTSPREFIX =
include htslib_vars.mk
lib-static: libhts.a
# $(shell), :=, and ifeq/.../endif are GNU Make-specific. If you don't have
# GNU Make, comment out the parts of this conditional that don't apply.
PLATFORM := $(shell uname -s)
ifeq "$(PLATFORM)" "Darwin"
SHLIB_FLAVOUR = dylib
lib-shared: libhts.dylib
else
SHLIB_FLAVOUR = so
lib-shared: libhts.so
endif
PACKAGE_VERSION = 1.2.1
LIBHTS_SOVERSION = 1
@@ -137,7 +128,7 @@ print-version:
@echo $(PACKAGE_VERSION)
.SUFFIXES: .c .o .pico
.SUFFIXES: .bundle .c .o .pico .so
.c.o:
$(CC) $(CFLAGS) -I. $(CPPFLAGS) -c -o $@ $<
@@ -182,6 +173,8 @@ LIBHTS_OBJS = \
cram/vlen.o \
cram/zfio.o
PLUGIN_OBJS =
cram_h = cram/cram.h $(cram_samtools_h) $(cram_sam_header_h) $(cram_structs_h) $(cram_io_h) cram/cram_encode.h cram/cram_decode.h cram/cram_stats.h cram/cram_codecs.h cram/cram_index.h $(htslib_cram_h)
cram_io_h = cram/cram_io.h $(cram_misc_h)
cram_misc_h = cram/misc.h cram/os.h
@@ -208,6 +201,24 @@ config.h:
echo '/* Empty config.h generated by Makefile */' > $@
lib-static: libhts.a
# $(shell), :=, and ifeq/.../endif are GNU Make-specific. If you don't have
# GNU Make, comment out the parts of this conditional that don't apply.
PLATFORM := $(shell uname -s)
ifeq "$(PLATFORM)" "Darwin"
SHLIB_FLAVOUR = dylib
lib-shared: libhts.dylib
BUILT_PLUGINS = $(PLUGIN_OBJS:.o=.bundle)
else
SHLIB_FLAVOUR = so
lib-shared: libhts.so
BUILT_PLUGINS = $(PLUGIN_OBJS:.o=.so)
endif
plugins: $(BUILT_PLUGINS)
libhts.a: $(LIBHTS_OBJS)
@-rm -f $@
$(AR) -rc $@ $(LIBHTS_OBJS)
@@ -232,11 +243,19 @@ libhts.dylib: $(LIBHTS_OBJS)
ln -sf $@ libhts.$(LIBHTS_SOVERSION).dylib
.pico.so:
$(CC) -shared -Wl,-E -pthread $(LDFLAGS) -o $@ $< $(LDLIBS)
.o.bundle:
$(CC) -bundle -Wl,-undefined,dynamic_lookup $(LDFLAGS) -o $@ $< $(LDLIBS)
bgzf.o bgzf.pico: bgzf.c config.h $(htslib_hts_h) $(htslib_bgzf_h) $(htslib_hfile_h) $(htslib_khash_h)
kstring.o kstring.pico: kstring.c $(htslib_kstring_h)
knetfile.o knetfile.pico: knetfile.c $(htslib_knetfile_h)
hfile.o hfile.pico: hfile.c config.h $(htslib_hfile_h) $(hfile_internal_h)
hfile.o hfile.pico: hfile.c config.h $(htslib_hfile_h) $(hfile_internal_h) $(hts_internal_h) $(htslib_khash_h)
hfile_irods.o hfile_irods.pico: hfile_irods.c config.h $(hfile_internal_h)
hfile_libcurl.o hfile_libcurl.pico: hfile_libcurl.c config.h $(hfile_internal_h) $(htslib_kstring_h)
hfile_net.o hfile_net.pico: hfile_net.c config.h $(hfile_internal_h) $(htslib_knetfile_h)
hts.o hts.pico: hts.c config.h version.h $(htslib_hts_h) $(htslib_bgzf_h) $(cram_h) $(htslib_hfile_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_ksort_h) $(hts_internal_h)
vcf.o vcf.pico: vcf.c config.h $(htslib_vcf_h) $(htslib_bgzf_h) $(htslib_tbx_h) $(htslib_hfile_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_kstring_h) $(htslib_khash_str2int_h)
@@ -249,6 +268,7 @@ vcfutils.o vcfutils.pico: vcfutils.c config.h $(htslib_vcfutils_h) $(htslib_kbit
kfunc.o kfunc.pico: kfunc.c $(htslib_kfunc_h)
regidx.o regidx.pico: regidx.c config.h $(htslib_hts_h) $(htslib_kstring_h) $(htslib_kseq_h) $(htslib_khash_str2int_h) $(htslib_regidx_h)
md5.o md5.pico: md5.c config.h $(htslib_hts_h)
plugin.o plugin.pico: plugin.c config.h $(hts_internal_h) $(htslib_kstring_h)
cram/cram_codecs.o cram/cram_codecs.pico: cram/cram_codecs.c config.h $(cram_h)
cram/cram_decode.o cram/cram_decode.pico: cram/cram_decode.c config.h $(cram_h) cram/os.h $(htslib_hts_h)
@@ -324,15 +344,17 @@ test/test-vcf-api.o: test/test-vcf-api.c $(htslib_hts_h) $(htslib_vcf_h) $(htsli
test/test-vcf-sweep.o: test/test-vcf-sweep.c $(htslib_vcf_sweep_h)
install: libhts.a $(BUILT_PROGRAMS) installdirs install-$(SHLIB_FLAVOUR) install-pkgconfig
install: libhts.a $(BUILT_PROGRAMS) $(BUILT_PLUGINS) installdirs install-$(SHLIB_FLAVOUR) install-pkgconfig
$(INSTALL_PROGRAM) $(BUILT_PROGRAMS) $(DESTDIR)$(bindir)
test -n "$(BUILT_PLUGINS)" && $(INSTALL_PROGRAM) $(BUILT_PLUGINS) $(DESTDIR)$(plugindir)
$(INSTALL_DATA) htslib/*.h $(DESTDIR)$(includedir)/htslib
$(INSTALL_DATA) libhts.a $(DESTDIR)$(libdir)/libhts.a
$(INSTALL_DATA) htsfile.1 tabix.1 $(DESTDIR)$(man1dir)
$(INSTALL_DATA) faidx.5 sam.5 vcf.5 $(DESTDIR)$(man5dir)
installdirs:
$(INSTALL_DIR) $(DESTDIR)$(bindir) $(DESTDIR)$(includedir) $(DESTDIR)$(includedir)/htslib $(DESTDIR)$(libdir) $(DESTDIR)$(man1dir) $(DESTDIR)$(man5dir) $(DESTDIR)$(pkgconfigdir)
test -n "$(plugindir)" && $(INSTALL_DIR) $(DESTDIR)$(plugindir)
# After installation, the real file in $(libdir) will be libhts.so.X.Y.Z,
# with symlinks libhts.so (used via -lhts during linking of client programs)
@@ -367,7 +389,7 @@ mostlyclean: testclean
-rm -f *.o *.pico cram/*.o cram/*.pico test/*.o test/*.dSYM version.h
clean: mostlyclean clean-$(SHLIB_FLAVOUR)
-rm -f libhts.a $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
-rm -f libhts.a $(BUILT_PROGRAMS) $(BUILT_PLUGINS) $(BUILT_TEST_PROGRAMS)
distclean maintainer-clean: clean
-rm -f config.cache config.h config.log config.mk config.status
@@ -399,6 +421,6 @@ force:
.PHONY: all check clean distclean distdir force
.PHONY: install install-pkgconfig installdirs lib-shared lib-static
.PHONY: maintainer-clean mostlyclean print-version tags test testclean
.PHONY: maintainer-clean mostlyclean plugins print-version tags test testclean
.PHONY: clean-so install-so
.PHONY: clean-dylib install-dylib
View
@@ -5,8 +5,23 @@
This template file can be updated with autoheader, but do so carefully
as autoheader adds #defines such as PACKAGE_* that we don't want. */
/* Define if HTSlib should enable plugins. */
#undef ENABLE_PLUGINS
/* Define if you have the Common Crypto library. */
#undef HAVE_COMMONCRYPTO
/* Define to 1 if you have the `gmtime_r' function. */
#undef HAVE_GMTIME_R
/* Define to 1 if iRODS file access is enabled. */
#undef HAVE_IRODS
/* Define to 1 if you have the `crypto' library (-lcrypto). */
#undef HAVE_LIBCRYPTO
/* Define if libcurl file access is enabled. */
#undef HAVE_LIBCURL
/* Define to 1 if you have a working `mmap' system call. */
#undef HAVE_MMAP
View
@@ -33,6 +33,7 @@ exec_prefix = @exec_prefix@
bindir = @bindir@
includedir = @includedir@
libdir = @libdir@
libexecdir = @libexecdir@
datarootdir = @datarootdir@
mandir = @mandir@
@@ -44,6 +45,10 @@ CFLAGS = @CFLAGS@
LDFLAGS = @LDFLAGS@
LDLIBS = @LIBS@
# Lowercase here indicates these are "local" to config.mk
plugin_OBJS =
noplugin_LDFLAGS =
noplugin_LIBS =
# ifeq/.../endif, +=, and target-specific variables are GNU Make-specific.
# If you don't have GNU Make, comment out this conditional and note that
@@ -52,7 +57,7 @@ ifeq "iRODS-@irods@" "iRODS-enabled"
@define_IRODS_HOME@
EXTRA_CPPFLAGS_IRODS = \
IRODS_CPPFLAGS = \
-I$(IRODS_HOME)/lib/api/include \
-I$(IRODS_HOME)/lib/core/include \
-I$(IRODS_HOME)/lib/md5/include \
@@ -61,11 +66,46 @@ EXTRA_CPPFLAGS_IRODS = \
-I$(IRODS_HOME)/server/drivers/include \
-I$(IRODS_HOME)/server/icat/include
LDFLAGS += -L$(IRODS_HOME)/lib/core/obj
LDLIBS += -lRodsAPIs -lgssapi_krb5
IRODS_LDFLAGS = -L$(IRODS_HOME)/lib/core/obj
IRODS_LIBS = -lRodsAPIs -lgssapi_krb5
LIBHTS_OBJS += hfile_irods.o
plugin_OBJS += hfile_irods.o
hfile_irods.o hfile_irods.pico: CPPFLAGS += $(EXTRA_CPPFLAGS_IRODS)
hfile_irods.o hfile_irods.pico: CPPFLAGS += $(IRODS_CPPFLAGS)
hfile_irods.so hfile_irods.bundle: LDFLAGS += $(IRODS_LDFLAGS)
hfile_irods.so hfile_irods.bundle: LDLIBS += $(IRODS_LIBS)
noplugin_LDFLAGS += $(IRODS_LDFLAGS)
noplugin_LIBS += $(IRODS_LIBS)
endif
ifeq "libcurl-@libcurl@" "libcurl-enabled"
LIBCURL_LIBS = -lcurl @CRYPTO_LIBS@
plugin_OBJS += hfile_libcurl.o
hfile_libcurl.so hfile_libcurl.bundle: LDLIBS += $(LIBCURL_LIBS)
noplugin_LIBS += $(LIBCURL_LIBS)
endif
ifeq "plugins-@enable_plugins@" "plugins-yes"
plugindir = @plugindir@
pluginpath = @pluginpath@
LIBHTS_OBJS += plugin.o
PLUGIN_OBJS += $(plugin_OBJS)
plugin.o plugin.pico: CPPFLAGS += -DPLUGINPATH=\"$(pluginpath)\"
else
LIBHTS_OBJS += $(plugin_OBJS)
LDFLAGS += $(noplugin_LDFLAGS)
LDLIBS += $(noplugin_LIBS)
endif
Oops, something went wrong.

0 comments on commit fe1f08a

Please sign in to comment.