…_Path fails

CHANGES.md

@@ -1,6 +1,36 @@
 # NCBI External Developer Release:
 
 
+## SRA Toolkit 2.8.2
+**March 6, 2017**
+
+  **blast**: Updated blast library to be able to process runs having empty rows  
+  **blast, build**: removed library dependencies that were preventing users from launching these tools  
+  **blast, sra-tools**: Prepared completely static build of blast tools for windows with HTTPS support  
+  **build**: **bam-load**: changed memcpy to memmove when regions overlap - seems to corrupt data on centos7  
+  **build**: Added ability to specify ncbi-vdb/configure --with-magic-prefix. Look for libraries in (lib lib64) when running "configure --with-...-prefix"  
+  **build**: configure detects location of ngs libraries  
+  **build**: configure was fixed to skip options unrecognized by gcc 4.4.7  
+  **build**: created sra-toolkit Debian package  
+  **build**: fixed a bug in 'configure' when in could not find source files in repository saved with non-standard name  
+  **build, ncbi-vdb, sra-tools**: installation will back up old configuration files if they differ from the ones being installed  
+  **cg-load**: added loading of CG File format v2.4  
+  **kns**: SRA tools respect standard set of environment variables for proxy specification  
+  **kns**: updated mbedtls library to version 2.4.1  
+  **ncbi-vdb, ngs, ngs-tools, sra-tools**: eliminated memcpy from sources due to potential for overlap  
+  **ngs, sra-search**: now supports search on reference  
+  **ngs-tools**: updated the NCBI download page to incorporate ngs versions into 3rd party package names  
+  **prefetch**: Fixed error message "path excessive while opening directory" when prefetch is trying to get vdbcache  
+  **prefetch**: Fixed regression in prefetch-2.8.1 when downloading dbGaP files via HTTP  
+  **prefetch**: Fixed regression in prefetch-2.8.1 when downloading vdbcache files from dbGaP via HTTP  
+  **sam-dump**: consistency of sam-dump in fastq-mod improved  
+  **sam-dump**: consistency of sam-dump in fastq-mode improved  
+  **sra-stat**: sra-stat does XML-escaping when printing spot-groups  
+  **test-sra**: extended test-sra to debug user problems with https connections to NCBI  
+  **test-sra**: test-sra print amount of available disk space in user repositories  
+  **vdb-config**: vdb-config correctly works when there is non-canonical path in configuration  
+
+
 ## SRA Toolkit 2.8.1-2
 **January 19, 2017**
 

build/Makefile.env

@@ -200,7 +200,7 @@ endif
 .PHONY: stdclean removelinks makedirs vers-includes rebuild-dirlinks
 .PHONY: stdjclean makejdirs
 
-# configuration targets 
+# configuration targets
 out:
 	@ echo $(OUTDIR) > $(TOP)/build/OUTDIR.$(BUILD_OS)
 	@ $(MAKE) TOP=$(TOP) -f $(TOP)/build/Makefile.env rebuild-dirlinks
@@ -362,11 +362,11 @@ LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
 
 #-------------------------------------------------------------------------------
 # runtests
-# 
+#
 # MallocScribble=1 is for catching allocation problems on Mac
 #
 ifeq ($(RUNTESTS_OVERRIDE),)
-runtests: std $(TEST_TOOLS)
+runtests: all $(TEST_TOOLS)
 	@ export VDB_CONFIG=$(VDB_CONFIG);export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
 	for i in $(TEST_TOOLS);\
 	do\
@@ -380,11 +380,11 @@ endif
 
 #-------------------------------------------------------------------------------
 # slowtests
-# 
+#
 # $(SLOWTESTSDATADIR) should be used to create temporary test files
 SLOWTESTSDATADIR ?= /panfs/pan1.be-md.ncbi.nlm.nih.gov/sra-test/slowtests/$(shell whoami)
 
-slowtests: std $(SLOW_TEST_TOOLS)
+slowtests: all $(SLOW_TEST_TOOLS)
 	@ export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;\
 	for i in $(SLOW_TEST_TOOLS);\
 	do\

test/Makefile

@@ -35,6 +35,7 @@ include $(TOP)/build/Makefile.config
 # default
 #
 SUBDIRS =    \
+	vdb-config      \
 	fastq-loader    \
 	vcf-loader      \
 	kget            \

test/vdb-config/Makefile

@@ -0,0 +1,35 @@
+# ===========================================================================
+#
+#                            PUBLIC DOMAIN NOTICE
+#               National Center for Biotechnology Information
+#
+#  This software/database is a "United States Government Work" under the
+#  terms of the United States Copyright Act.  It was written as part of
+#  the author's official duties as a United States Government employee and
+#  thus cannot be copyrighted.  This software/database is freely available
+#  to the public for use. The National Library of Medicine and the U.S.
+#  Government have not placed any restriction on its use or reproduction.
+#
+#  Although all reasonable efforts have been taken to ensure the accuracy
+#  and reliability of the software and data, the NLM and the U.S.
+#  Government do not and cannot warrant the performance or results that
+#  may be obtained by using this software or data. The NLM and the U.S.
+#  Government disclaim all warranties, express or implied, including
+#  warranties of performance, merchantability or fitness for any particular
+#  purpose.
+#
+#  Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+include $(TOP)/build/Makefile.env # BINDIR
+
+runtests: test-vdb-config
+
+test-vdb-config:
+	@ printf "Testing exit code vdb-config of vdb-config... "
+	@ PATH=$(BINDIR):$(PATH) ; ./test-vdb-config.pl
+	@ echo OK

test/vdb-config/test-vdb-config.pl

@@ -0,0 +1,12 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+use File::Temp   "tempdir";
+
+my $tmp = tempdir ( "phgvXXXX", CLEANUP => 1 );
+$ENV{VDB_CONFIG}=$tmp;
+$ENV{NCBI_SETTINGS}="$tmp/u.mkfg";
+
+`vdb-config -s foo=bar`;
+die "vdb-config exited with " . ( $? >> 8 ) if ( $? );

test/vdb-dump/Makefile

@@ -53,6 +53,14 @@ $(ALL_TOOLS): makedirs
 .PHONY: all std $(ALL_TOOLS)
 
 #-------------------------------------------------------------------------------
+# all
+#
+$(TARGDIR)/all: \
+	$(addprefix $(BINDIR)/,$(ALL_TOOLS))
+
+.PHONY: $(TARGDIR)/all
+
+#-------------------------------------------------------------------------------
 # std
 #
 $(TARGDIR)/std: \

tools/bam-loader/bam.c

@@ -3997,7 +3997,7 @@ static unsigned insert_B(unsigned const T, unsigned const G, unsigned const n, u
     return n;
 }
 
-static unsigned canonicalize(uint32_t cigar[], unsigned n)
+static unsigned canonicalizeCIGAR(uint32_t cigar[], unsigned n)
 {
     unsigned i;
     
@@ -4042,7 +4042,7 @@ static unsigned GetCGCigar(BAM_Alignment const *self, unsigned const N, uint32_t
     memmove(cigar, getCigarBase(self), n * 4);
 
     if (n > 1)
-        n = canonicalize(cigar, n); /* just in case */
+        n = canonicalizeCIGAR(cigar, n); /* just in case */
     
     for (i = 0, S = 0; i < gaps; ++i) {
         unsigned const s = seg[2 * i + 0];

tools/bam-loader/loader-imp.c

@@ -1548,6 +1548,7 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
     KDataBuffer fragBuf;
     KDataBuffer cigBuf;
     rc_t rc;
+    const BAMRefSeq *refSeq = NULL;
     int32_t lastRefSeqId = -1;
     bool wasRenamed = false;
     size_t rsize;
@@ -1650,7 +1651,6 @@ static rc_t ProcessBAM(char const bamFile[], context_t *ctx, VDatabase *db,
         uint16_t flags;
         int64_t rpos=0;
         char *seqDNA;
-        const BAMRefSeq *refSeq;
         ctx_value_t *value;
         bool wasInserted;
         int32_t refSeqId=-1;

tools/sra-dump/fastq.c

@@ -3694,7 +3694,7 @@ rc_t SRADumper_Init( SRADumperFmt* fmt )
             /* DO NOT ADD IN THE MIDDLE ORDER IS IMPORTANT IN USAGE FUNCTION ABOVE!!! */
             {NULL, "split-spot", NULL, {"Split spots into individual reads", NULL}},            /* H_splip_sot = 0 */
 
-            {"W", "clip", NULL, {"Clip adapter sequences", NULL}},                          /* H_clip = 1 */
+            {"W", "clip", NULL, {"Remove adapter sequences from reads", NULL}},                          /* H_clip = 1 */
 
             {"M", "minReadLen", "len", {"Filter by sequence length >= <len>", NULL}},           /* H_minReadLen = 2 */
             {"E", "qual-filter", NULL, {"Filter used in early 1000 Genomes data:",              /* H_qual_filter = 3 */

tools/util/test-sra.c

@@ -3438,11 +3438,11 @@ static rc_t MainFreeSpace ( const Main * self, const KDirectory * dir ) {
 
     if ( self -> xml )
         OUTMSG ( (
-            "      <Space free=\"%d\" total=\"%d\" units=\"KBytes\"/>\n",
+            "      <Space free=\"%lu\" total=\"%lu\" units=\"KBytes\"/>\n",
             free_bytes_available, total_number_of_bytes ) );
     else
         OUTMSG ( (
-            "    Space free=\"%d\" total=\"%d\" units=\"KBytes\"\n",
+            "    Space free=\"%lu\" total=\"%lu\" units=\"KBytes\"\n",
             free_bytes_available, total_number_of_bytes ) );
 
     return rc;

tools/vdb-config/vdb-config.c

@@ -1672,6 +1672,8 @@ rc_t CC KMain(int argc, char* argv[]) {
                     }
                 }
             }
+            else
+                rc = 0;
         }
 
         RELEASE ( KDirectory, d );

tools/vdb-dump/vdb-dump-coldefs.c

@@ -844,3 +844,77 @@ rc_t vdcd_collect_spread( const struct num_gen * row_set, col_defs * cols, const
 	}
 	return rc;
 }
+
+static uint32_t same_values( const VCursor * curs, uint32_t col_idx, int64_t first, uint32_t test_rows )
+{
+    uint32_t res = 0;
+    const void * base;
+    uint32_t elem_bits, boff, row_len;
+    rc_t rc = VCursorCellDataDirect( curs, first, col_idx, &elem_bits, &base, &boff, &row_len );
+    while ( rc == 0 && res < test_rows && rc == 0 )
+    {
+        const void * base_1;
+        uint32_t elem_bits_1, boff_1, row_len_1;
+        rc = VCursorCellDataDirect( curs, first + res + 1, col_idx, &elem_bits_1, &base_1, &boff_1, &row_len_1 );
+        if ( rc == 0 )
+        {
+            if ( elem_bits != elem_bits_1 ) return res;
+            if ( boff != boff_1 ) return res;
+            if ( row_len != row_len_1 ) return res;
+            if ( base != base_1 ) return res;
+        }
+        res += 1;
+    }
+    return res;
+}
+
+static bool vdcd_is_static_column( const VTable *my_table, col_def * col )
+{
+    bool res = false;
+    const VCursor * curs;
+    rc_t rc = VTableCreateCursorRead( my_table, &curs );
+    if ( rc == 0 )
+    {
+        uint32_t idx;
+        rc = VCursorAddColumn( curs, &idx, "%s", col->name );
+        if ( rc == 0 )
+        {
+            rc = VCursorOpen( curs );
+            if ( rc == 0 )
+            {
+                int64_t first;
+                uint64_t count;
+                rc = VCursorIdRange( curs, idx, &first, &count );
+                if ( rc == 0 && count == 0 )
+                {
+                    res = ( same_values( curs, idx, first, 100 ) == 100 );
+                }
+            }
+        }
+        VCursorRelease( curs );
+    }
+    return res;
+}
+
+
+bool vdcd_extract_static_columns( col_defs* defs, const VTable *my_table, const size_t str_limit )
+{
+    col_defs * temp_defs;
+    bool res = vdcd_init( &temp_defs, str_limit );
+    if ( res )
+    {
+        uint32_t count = vdcd_extract_from_table( temp_defs, my_table );
+        uint32_t idx;
+        for ( idx = 0; idx < count; ++idx )
+        {
+            col_def * col = VectorGet( &(temp_defs->cols), idx );
+            if ( col != NULL )
+            {
+                if ( vdcd_is_static_column( my_table, col ) )
+                    vdcd_append_col( defs, col->name  );
+            }
+        }
+        vdcd_destroy( temp_defs );
+    }
+    return res;
+}

tools/vdb-dump/vdb-dump-coldefs.h

@@ -101,6 +101,8 @@ void vdcd_ins_trans_fkt( col_defs* defs, const VSchema *my_schema );
 void vdcd_exclude_these_columns( col_defs* defs, const char* column_names );
 bool vdcd_get_first_none_static_column_idx( col_defs* defs, const VCursor * cur, uint32_t * idx );
 
+bool vdcd_extract_static_columns( col_defs* defs, const VTable *my_table, const size_t str_limit );
+
 rc_t vdcd_collect_spread( const struct num_gen * row_set, col_defs * cols, const VCursor * cursor );
 
 #ifdef __cplusplus

tools/vdb-dump/vdb-dump-context.c

@@ -597,6 +597,7 @@ static void vdco_evaluate_options( const Args *my_args,
     ctx->show_blobbing = vdco_get_bool_option( my_args, OPTION_SHOW_BLOBBING, false );
     ctx->enum_phys = vdco_get_bool_option( my_args, OPTION_ENUM_PHYS, false );
     ctx->enum_readable = vdco_get_bool_option( my_args, OPTION_ENUM_READABLE, false );
+    ctx->enum_static = vdco_get_bool_option( my_args, OPTION_ENUM_STATIC, false );
     ctx->idx_enum_requested = vdco_get_bool_option( my_args, OPTION_IDX_ENUM, false );
     ctx->disable_multithreading = vdco_get_bool_option( my_args, OPTION_NO_MULTITHREAD, false );
     ctx->print_info = vdco_get_bool_option( my_args, OPTION_INFO, false );

tools/vdb-dump/vdb-dump-context.h

@@ -68,6 +68,7 @@ extern "C" {
 #define OPTION_SHOW_BLOBBING     "blobbing"
 #define OPTION_ENUM_PHYS         "phys"
 #define OPTION_ENUM_READABLE     "readable"
+#define OPTION_ENUM_STATIC       "static"
 #define OPTION_IDX_ENUM          "idx-report"
 #define OPTION_IDX_RANGE         "idx-range"
 #define OPTION_CUR_CACHE         "cur-cache"
@@ -186,6 +187,7 @@ typedef struct dump_context
     bool show_blobbing;
     bool enum_phys;
     bool enum_readable;
+    bool enum_static;
     bool idx_enum_requested;
     bool idx_range_requested;
     bool disable_multithreading;

tools/vdb-dump/vdb-dump.c

@@ -104,6 +104,7 @@ static const char * numelemsum_usage[]          = { "sum element-count",
 static const char * show_blobbing_usage[]       = { "show blobbing",                                NULL };
 static const char * enum_phys_usage[]           = { "enumerate physical columns",                   NULL };
 static const char * enum_readable_usage[]       = { "enumerate readable columns",                   NULL };
+static const char * enum_static_usage[]         = { "enumerate static columns",                     NULL };
 static const char * objtype_usage[]             = { "report type of object",                        NULL };
 static const char * idx_enum_usage[]            = { "enumerate all available index",                NULL };
 static const char * idx_range_usage[]           = { "enumerate values and row-ranges of one index", NULL };
@@ -149,6 +150,7 @@ OptDef DumpOptions[] =
     { OPTION_SHOW_BLOBBING,         NULL,                     NULL, show_blobbing_usage,     1, false,  false },
     { OPTION_ENUM_PHYS,             NULL,                     NULL, enum_phys_usage,         1, false,  false },
     { OPTION_ENUM_READABLE,         NULL,                     NULL, enum_readable_usage,     1, false,  false },
+    { OPTION_ENUM_STATIC,           NULL,                     NULL, enum_static_usage,       1, false,  false },    
     { OPTION_OBJVER,                ALIAS_OBJVER,             NULL, objver_usage,            1, false,  false },
     { OPTION_OBJTS,                 NULL,                     NULL, objts_usage,             1, false,  false },
     { OPTION_OBJTYPE,               ALIAS_OBJTYPE,            NULL, objtype_usage,           1, false,  false },
@@ -558,6 +560,24 @@ static bool vdm_extract_or_parse_phys_columns( const p_dump_context ctx,
     return res;
 }
 
+
+static bool vdm_extract_or_parse_static_columns( const p_dump_context ctx,
+                                               const VTable *my_table,
+                                               p_col_defs my_col_defs )
+{
+    bool res = false;
+    if ( ctx != NULL && my_col_defs != NULL )
+    {
+            /* the user does not know the column-names or wants all of them */
+        res = vdcd_extract_static_columns( my_col_defs, my_table, ctx->max_line_len );
+
+        if ( ctx->excluded_columns != NULL )
+            vdcd_exclude_these_columns( my_col_defs, ctx->excluded_columns );
+    }
+    return res;
+
+}
+
 /*************************************************************************************
     dump_tab_table:
     * called by "dump_db_table()" and "dump_tab()" as a fkt-pointer
@@ -1299,6 +1319,7 @@ static rc_t vdm_enum_readable_columns( const VTable *my_table )
     return rc;
 }
 
+
 /*************************************************************************************
     enum_tab_columns:
     * called by "enum_db_columns()" and "dump_table()" as fkt-pointer
@@ -1344,6 +1365,13 @@ static rc_t vdm_enum_tab_columns( const p_dump_context ctx, const VTable *my_tab
                 rc = VTableOpenKTableRead( my_table, &ci_ctx.my_ktable );
                 DISP_RC( rc, "VTableOpenKTableRead() failed" );
             }
+            if ( ctx->enum_static )
+            {
+                extracted = vdm_extract_or_parse_static_columns( ctx, my_table, my_col_defs );
+                rc = VTableOpenKTableRead( my_table, &ci_ctx.my_ktable );
+                DISP_RC( rc, "VTableOpenKTableRead() failed" );
+            
+            }
             else
             {
                 extracted = vdm_extract_or_parse_columns( ctx, my_table, my_col_defs );