…_Path fails

CHANGES.md

@@ -1,6 +1,36 @@
 # NCBI External Developer Release:
 
 
+## SRA Toolkit 2.8.2
+**March 6, 2017**
+
+  **blast**: Updated blast library to be able to process runs having empty rows  
+  **blast, build**: removed library dependencies that were preventing users from launching these tools  
+  **blast, sra-tools**: Prepared completely static build of blast tools for windows with HTTPS support  
+  **build**: **bam-load**: changed memcpy to memmove when regions overlap - seems to corrupt data on centos7  
+  **build**: Added ability to specify ncbi-vdb/configure --with-magic-prefix. Look for libraries in (lib lib64) when running "configure --with-...-prefix"  
+  **build**: configure detects location of ngs libraries  
+  **build**: configure was fixed to skip options unrecognized by gcc 4.4.7  
+  **build**: created sra-toolkit Debian package  
+  **build**: fixed a bug in 'configure' when in could not find source files in repository saved with non-standard name  
+  **build, ncbi-vdb, sra-tools**: installation will back up old configuration files if they differ from the ones being installed  
+  **cg-load**: added loading of CG File format v2.4  
+  **kns**: SRA tools respect standard set of environment variables for proxy specification  
+  **kns**: updated mbedtls library to version 2.4.1  
+  **ncbi-vdb, ngs, ngs-tools, sra-tools**: eliminated memcpy from sources due to potential for overlap  
+  **ngs, sra-search**: now supports search on reference  
+  **ngs-tools**: updated the NCBI download page to incorporate ngs versions into 3rd party package names  
+  **prefetch**: Fixed error message "path excessive while opening directory" when prefetch is trying to get vdbcache  
+  **prefetch**: Fixed regression in prefetch-2.8.1 when downloading dbGaP files via HTTP  
+  **prefetch**: Fixed regression in prefetch-2.8.1 when downloading vdbcache files from dbGaP via HTTP  
+  **sam-dump**: consistency of sam-dump in fastq-mod improved  
+  **sam-dump**: consistency of sam-dump in fastq-mode improved  
+  **sra-stat**: sra-stat does XML-escaping when printing spot-groups  
+  **test-sra**: extended test-sra to debug user problems with https connections to NCBI  
+  **test-sra**: test-sra print amount of available disk space in user repositories  
+  **vdb-config**: vdb-config correctly works when there is non-canonical path in configuration  
+
+
 ## SRA Toolkit 2.8.1-2
 **January 19, 2017**
 

build/Makefile.env

@@ -200,7 +200,7 @@ endif
 .PHONY: stdclean removelinks makedirs vers-includes rebuild-dirlinks
 .PHONY: stdjclean makejdirs
 
-# configuration targets 
+# configuration targets
 out:
 	@ echo $(OUTDIR) > $(TOP)/build/OUTDIR.$(BUILD_OS)
 	@ $(MAKE) TOP=$(TOP) -f $(TOP)/build/Makefile.env rebuild-dirlinks
@@ -362,11 +362,11 @@ LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
 
 #-------------------------------------------------------------------------------
 # runtests
-# 
+#
 # MallocScribble=1 is for catching allocation problems on Mac
 #
 ifeq ($(RUNTESTS_OVERRIDE),)
-runtests: std $(TEST_TOOLS)
+runtests: all $(TEST_TOOLS)
 	@ export VDB_CONFIG=$(VDB_CONFIG);export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
 	for i in $(TEST_TOOLS);\
 	do\
@@ -380,11 +380,11 @@ endif
 
 #-------------------------------------------------------------------------------
 # slowtests
-# 
+#
 # $(SLOWTESTSDATADIR) should be used to create temporary test files
 SLOWTESTSDATADIR ?= /panfs/pan1.be-md.ncbi.nlm.nih.gov/sra-test/slowtests/$(shell whoami)
 
-slowtests: std $(SLOW_TEST_TOOLS)
+slowtests: all $(SLOW_TEST_TOOLS)
 	@ export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;\
 	for i in $(SLOW_TEST_TOOLS);\
 	do\

test/Makefile

@@ -35,6 +35,7 @@ include $(TOP)/build/Makefile.config
 # default
 #
 SUBDIRS =    \
+	vdb-config      \
 	fastq-loader    \
 	vcf-loader      \
 	kget            \

test/vdb-config/Makefile

@@ -0,0 +1,35 @@
+# ===========================================================================
+#
+#                            PUBLIC DOMAIN NOTICE
+#               National Center for Biotechnology Information
+#
+#  This software/database is a "United States Government Work" under the
+#  terms of the United States Copyright Act.  It was written as part of
+#  the author's official duties as a United States Government employee and
+#  thus cannot be copyrighted.  This software/database is freely available
+#  to the public for use. The National Library of Medicine and the U.S.
+#  Government have not placed any restriction on its use or reproduction.
+#
+#  Although all reasonable efforts have been taken to ensure the accuracy
+#  and reliability of the software and data, the NLM and the U.S.
+#  Government do not and cannot warrant the performance or results that
+#  may be obtained by using this software or data. The NLM and the U.S.
+#  Government disclaim all warranties, express or implied, including
+#  warranties of performance, merchantability or fitness for any particular
+#  purpose.
+#
+#  Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+include $(TOP)/build/Makefile.env # BINDIR
+
+runtests: test-vdb-config
+
+test-vdb-config:
+	@ printf "Testing exit code vdb-config of vdb-config... "
+	@ PATH=$(BINDIR):$(PATH) ; ./test-vdb-config.pl
+	@ echo OK

test/vdb-config/test-vdb-config.pl

@@ -0,0 +1,12 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+use File::Temp   "tempdir";
+
+my $tmp = tempdir ( "phgvXXXX", CLEANUP => 1 );
+$ENV{VDB_CONFIG}=$tmp;
+$ENV{NCBI_SETTINGS}="$tmp/u.mkfg";
+
+`vdb-config -s foo=bar`;
+die "vdb-config exited with " . ( $? >> 8 ) if ( $? );

test/vdb-dump/Makefile

@@ -53,6 +53,14 @@ $(ALL_TOOLS): makedirs
 .PHONY: all std $(ALL_TOOLS)
 
 #-------------------------------------------------------------------------------
+# all
+#
+$(TARGDIR)/all: \
+	$(addprefix $(BINDIR)/,$(ALL_TOOLS))
+
+.PHONY: $(TARGDIR)/all
+
+#-------------------------------------------------------------------------------
 # std
 #
 $(TARGDIR)/std: \

tools/bam-loader/loader-imp.c

@@ -1548,6 +1548,7 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
     KDataBuffer fragBuf;
     KDataBuffer cigBuf;
     rc_t rc;
+    const BAMRefSeq *refSeq = NULL;
     int32_t lastRefSeqId = -1;
     bool wasRenamed = false;
     size_t rsize;
@@ -1650,7 +1651,6 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
         uint16_t flags;
         int64_t rpos=0;
         char *seqDNA;
-        const BAMRefSeq *refSeq;
         ctx_value_t *value;
         bool wasInserted;
         int32_t refSeqId=-1;

tools/sra-dump/fastq.c

@@ -3694,7 +3694,7 @@ rc_t SRADumper_Init( SRADumperFmt* fmt )
             /* DO NOT ADD IN THE MIDDLE ORDER IS IMPORTANT IN USAGE FUNCTION ABOVE!!! */
             {NULL, "split-spot", NULL, {"Split spots into individual reads", NULL}},            /* H_splip_sot = 0 */
 
-            {"W", "clip", NULL, {"Clip adapter sequences", NULL}},                          /* H_clip = 1 */
+            {"W", "clip", NULL, {"Remove adapter sequences from reads", NULL}},                          /* H_clip = 1 */
 
             {"M", "minReadLen", "len", {"Filter by sequence length >= <len>", NULL}},           /* H_minReadLen = 2 */
             {"E", "qual-filter", NULL, {"Filter used in early 1000 Genomes data:",              /* H_qual_filter = 3 */

tools/sra-sort/Makefile

@@ -36,7 +36,7 @@ INT_TOOLS = \
 EXT_TOOLS = \
 
 ifneq (win,$(OS))
-EXT_TOOLS += sra-sort
+EXT_TOOLS += sra-sort sra-sort-cg
 endif	
 
 ALL_TOOLS = \
@@ -65,40 +65,53 @@ clean: stdclean
 .PHONY: clean
 
 #-------------------------------------------------------------------------------
+# special rules for enabling a define
+#
+%.cg.$(OBJX): %.c
+	$(CC) -o $@ -DSEQUENCE_BEFORE_SECONDARY=0 $< $(LOC_INFO) -D__file_ext__=c
+
+#-------------------------------------------------------------------------------
 # sra-sort
 #
-SRA_SORT_SRC = \
-	caps \
-	mem \
-	membank \
-	paged-membank \
-	paged-mmapbank \
-	except \
-	idx-mapping \
-	map-file \
-	col-pair \
-	row-set \
-	simple-row-set \
-	mapping-row-set \
-	sorting-row-set \
-	meta-pair \
-	dir-pair \
-	tbl-pair \
-	db-pair \
-	glob-poslen \
-	poslen-col-pair \
-	ref-alignid-col \
-	buff-writer \
-	id-mapper-col \
+SRA_SORT_SRC =                 \
+	caps                       \
+	mem                        \
+	membank                    \
+	paged-membank              \
+	paged-mmapbank             \
+	except                     \
+	idx-mapping                \
+	map-file                   \
+	col-pair                   \
+	row-set                    \
+	simple-row-set             \
+	mapping-row-set            \
+	sorting-row-set            \
+	meta-pair                  \
+	dir-pair                   \
+	tbl-pair                   \
+	db-pair                    \
+	glob-poslen                \
+	poslen-col-pair            \
+	ref-alignid-col            \
+	buff-writer                \
+	id-mapper-col              \
 	capture-first-half-aligned \
-	csra-tbl \
-	csra-pair \
-	run \
-	sra-sort \
+	run                        \
+	sra-sort                   \
 	xcheck-ref-align
 
-SRA_SORT_OBJ = \
-	$(addsuffix .$(OBJX),$(SRA_SORT_SRC))
+SRA_SORT_SPECIAL_SRC =         \
+	csra-tbl                   \
+	csra-pair
+
+SRA_SORT_OBJ =                                         \
+	$(addsuffix .$(OBJX),$(SRA_SORT_SRC))              \
+	$(addsuffix .$(OBJX),$(SRA_SORT_SPECIAL_SRC))
+
+SRA_SORT_CG_OBJ =                                      \
+	$(addsuffix .$(OBJX),$(SRA_SORT_SRC))              \
+	$(addsuffix .cg.$(OBJX),$(SRA_SORT_SPECIAL_SRC))
 
 SRA_SORT_LIB = \
 	-lkapp \
@@ -109,6 +122,9 @@ SRA_SORT_LIB = \
 $(BINDIR)/sra-sort: $(SRA_SORT_OBJ)
 	$(LD) --exe --vers $(SRCDIR)/../../shared/toolkit.vers -o $@ $^ $(SRA_SORT_LIB)
 
+$(BINDIR)/sra-sort-cg: $(SRA_SORT_CG_OBJ)
+	$(LD) --exe --vers $(SRCDIR)/../../shared/toolkit.vers -o $@ $^ $(SRA_SORT_LIB)
+
 #-------------------------------------------------------------------------------
 # dump-blob-boundaries
 #

tools/sra-sort/buff-writer.c

@@ -761,7 +761,8 @@ ColumnWriter *cSRATblPairMakeBufferedIdRemapColumnWriter ( cSRATblPair *self,
 
     TRY ( buff = MemAlloc ( ctx, sizeof * buff, true ) )
     {
-        TRY ( ColumnWriterInit ( & buff -> dad, ctx, & MappedBufferedPairColWriter_vt, idx != NULL ) )
+        TRY ( ColumnWriterInit ( & buff -> dad, ctx,
+            idx != NULL ? & MappedBufferedPairColWriter_vt : & UnmappedBufferedPairColWriter_vt, idx != NULL ) )
         {
             /* duplicate our friend */
             TRY ( buff -> cw = ColumnWriterDuplicate ( writer, ctx ) )

tools/sra-sort/csra-pair.c

@@ -214,7 +214,8 @@ DbPair *cSRAPairMake ( const ctx_t *ctx,
                 "PRIMARY_ALIGNMENT",
                 "REFERENCE",
                 "SECONDARY_ALIGNMENT",
-                "SEQUENCE"
+                "SEQUENCE",
+                NULL
             };
             db -> dad . exclude_tbls = exclude_tbls;
 

tools/sra-sort/csra-pair.h

@@ -32,8 +32,9 @@
 #endif
 
 
+#ifndef SEQUENCE_BEFORE_SECONDARY
 #define SEQUENCE_BEFORE_SECONDARY 1
-
+#endif
 
 /*--------------------------------------------------------------------------
  * forwards

tools/util/test-sra.c

@@ -3438,11 +3438,11 @@ static rc_t MainFreeSpace ( const Main * self, const KDirectory * dir ) {
 
     if ( self -> xml )
         OUTMSG ( (
-            "      <Space free=\"%d\" total=\"%d\" units=\"KBytes\"/>\n",
+            "      <Space free=\"%lu\" total=\"%lu\" units=\"KBytes\"/>\n",
             free_bytes_available, total_number_of_bytes ) );
     else
         OUTMSG ( (
-            "    Space free=\"%d\" total=\"%d\" units=\"KBytes\"\n",
+            "    Space free=\"%lu\" total=\"%lu\" units=\"KBytes\"\n",
             free_bytes_available, total_number_of_bytes ) );
 
     return rc;

tools/vdb-config/interactive.cpp

@@ -883,7 +883,7 @@ bool vdbconf_controller::on_set_location_error( Dlg &dlg, ESetRootState s )
         case eSetRootState_NewDirNotEmpty   : vdbconf_msg( dlg, r, "the given location is not empty" ); break;
         case eSetRootState_NewNotDir        : vdbconf_msg( dlg, r, "new location is not a directory" ); break;
         case eSetRootState_Error            : vdbconf_msg( dlg, r, "error changing location" ); break;
-        default                             : vdbconf_msg( dlg, r, "unknow enum" ); break;
+        default                             : vdbconf_msg( dlg, r, "unknown enum" ); break;
     }
     return result;
 }

tools/vdb-config/vdb-config.c

@@ -1672,6 +1672,8 @@ rc_t CC KMain(int argc, char* argv[]) {
                     }
                 }
             }
+            else
+                rc = 0;
         }
 
         RELEASE ( KDirectory, d );