…ce encountering a 0 length read

clipper/Makefile

@@ -42,7 +42,7 @@ getrel:
 $(PKG).spec:
 	perl -pe 's/%RELEASE%/${REL}/' $(PKG).spex > $(PKG).spec
 
-$(PKG).tar.gz: Makefile $(TOOLS) $(SRC) $(PKG).spec fastq-lib.cpp fastq-lib.h sam-stats.cpp fastq-stats.cpp gcModel.cpp gcModel.h varcall.cpp utils.h README CHANGES sparsehash-2.0.2 samtools/*.c t
+$(PKG).tar.gz: Makefile $(TOOLS) $(SRC) $(PKG).spec fastq-lib.cpp fastq-lib.h sam-stats.cpp fastq-stats.cpp gcModel.cpp gcModel.h varcall.cpp utils.h README CHANGES sparsehash-2.0.3 samtools/*.c t
 	rm -rf $(PKG).${VER}-${REL}
 	mkdir $(PKG).${VER}-${REL}
 	mkdir $(PKG).${VER}-${REL}/tidx
@@ -68,10 +68,10 @@ $(PKG).${VER}-${REL}.tar.gz: $(PKG).tar.gz
 %: %.cpp fastq-lib.cpp fastq-lib.h sparsehash
 	$(CC) $(CFLAGS) $< fastq-lib.cpp -o $@
 
-sparsehash: sparsehash-2.0.2
-	cd sparsehash-2.0.2; ./configure; make
+sparsehash: sparsehash-2.0.3
+	cd sparsehash-2.0.3; ./configure; make
 	mkdir sparsehash
-	cp -r sparsehash-2.0.2/src/sparsehash/* sparsehash/
+	cp -r sparsehash-2.0.3/src/sparsehash/* sparsehash/
 
 # why the libbam.a doesn't work?  not sure... *.o works
 sam-stats: sam-stats.cpp samtools/libbam.a samtools/bam.h fastq-lib.h sparsehash

clipper/ea-bcl2fastq.cpp

@@ -290,7 +290,7 @@ int main (int argc, char **argv) {
             ++output_fnum;                        // output file number is sequential
             masks[i].rnum=output_fnum;            // save file number as "read number"
             if (usegz) {
-                outtmp = string_format("gzip -2 --rsyncable -c > %s.%d.fq.gz",out.c_str(),output_fnum); 
+                outtmp = string_format("gzip -2  -c > %s.%d.fq.gz",out.c_str(),output_fnum); 
                 fo=popenordie(outtmp.c_str(),"w");
             } else {
                 outtmp = string_format("%s.%d.fq",out.c_str(),output_fnum); 

clipper/fastq-lib.cpp

@@ -160,7 +160,7 @@ FILE *gzopen(const char *f, const char *m, bool*isgz) {
         if (!strcmp(ext,".gz")) {
             char *tmp=(char *)malloc(strlen(f)+100);
             if (strchr(m,'w')) {
-                    strcpy(tmp, "gzip -3 --rsyncable > '");
+                    strcpy(tmp, "gzip -3  > '");
                     strcat(tmp, f);
                     strcat(tmp, "'");
             } else {
@@ -361,7 +361,7 @@ int getstr (char ** lineptr, size_t *n, FILE * stream, char terminator, int offs
 	 NUL-terminate the line buffer.  */
 
       assert(*n - nchars_avail == read_pos - *lineptr);
-      if (nchars_avail < 1)
+      if (nchars_avail < 2)
 	{
 	  if (*n > 64)
 	    *n *= 2;

clipper/fastq-mcf.cpp

@@ -31,7 +31,7 @@ See "void usage" below for usage.
 
 #include "fastq-lib.h"
 
-#define VERSION "1.04.807"
+#define VERSION "1.05"
 
 #define MAX_ADAPTER_NUM 1000
 #define SCANLEN 15
@@ -174,7 +174,7 @@ class inbuffer {
                 fq->qual.s[--fq->qual.n] = '\0';
             }
  
-            return fq->qual.n > 0;
+            return fq->qual.n >= 0;  // github issue 46, 53
         } else {
             return ::read_fq(fin, rno, fq, name);
         }
@@ -220,6 +220,8 @@ int main (int argc, char **argv) {
 	int ilv3 = -1;
 	int duplen = 0;
 	int dupskip = 0;
+    int min_start_trim = 0;
+    int min_end_trim = 0;
     bool noexec = 0;
     bool hompol_filter = 0;
     bool lowcom_filter = 0;
@@ -258,6 +260,8 @@ int main (int argc, char **argv) {
        {"mate-min-len", 1, 0, 0},
        {"homopolymer-pct", 1, 0, 0},
        {"lowcomplex-pct", 1, 0, 0},
+       {"min-start-trim", 1, 0, 0},
+       {"min-end-trim", 1, 0, 0},
        {0, 0, 0, 0}
     };
 
@@ -275,6 +279,10 @@ int main (int argc, char **argv) {
                         keeponlyclip=1;
                     } else if(!strcmp(oname, "mate-qual-mean")) {
                         qf2_mean=atoi(optarg);
+                    } else if (!strcmp(oname, "min-start-trim")) {
+        		min_start_trim = atoi(optarg);
+                    } else if (!strcmp(oname, "min-end-trim")) {
+        		min_end_trim = atoi(optarg);
                     } else if(!strcmp(oname, "homopolymer-pct")) {
                         hompol_pct=atof(optarg)/100.0;
                         hompol_filter=1;
@@ -631,6 +639,10 @@ int main (int argc, char **argv) {
 				--nq; --ns;				// don't count newline for read len
 
 				// skip poor quals/lots of N's when doing sampling (otherwise you'll miss some)
+                if (ns == 0) {  // github issue 46, 53
+                    ++skipped;
+                    continue;
+                }
 				if ((st.st_size > (sampcnt * 500)) && (skipped < sampcnt) && poorqual(i, ns, s, q)) {
                     ++skipped;
 					continue;
@@ -970,6 +982,15 @@ int main (int argc, char **argv) {
 			}
 		}
 		++nrec;
+        if (fq[0].qual.n == 0) { // github issue 46, 53
+            ++nfiltered;
+            continue;
+        } else if (i_n > 1) {
+            if (fq[1].qual.n == 0) {
+                ++nfiltered;
+                continue;
+            }
+        }
 		if (read_ok < 0) {
 			++nerr;
 			continue;
@@ -1003,6 +1024,9 @@ int main (int argc, char **argv) {
 		for (f=0;f<i_n;++f) {
 			dotrim[f][0] = sktrim[f][0];					// default, trim to detected skew levels
 			dotrim[f][1] = sktrim[f][1];
+			// trim to minimum, if specified
+			dotrim[f][0] = max(dotrim[f][0], min_start_trim);
+			dotrim[f][1] = max(dotrim[f][1], min_end_trim);
 			if (avgns[f] < 11)  
 				// reads of avg length < 11 ? barcode lane, skip it
 				continue;
@@ -1493,6 +1517,10 @@ void usage(FILE *f, const char *msg) {
 "    -C N     Number of reads to use for subsampling (300k)\n"
 "    -d       Output lots of random debugging stuff\n"
 "\n"
+"Minimum trimming options:\n"
+"    --min-start-trim    NUM       Always trim at least NUM bases from start\n"
+"    --min-end-trim      NUM       Always trim at least NUM bases from end\n"
+"\n"
 "Quality adjustment options:\n"
 "    --cycle-adjust      CYC,AMT   Adjust cycle CYC (negative = offset from end) by amount AMT\n"
 "    --phred-adjust      SCORE,AMT Adjust score SCORE by amount AMT\n"

clipper/fastq-multx.cpp

@@ -34,7 +34,7 @@ See "void usage" below for usage.
 #define THFIXFACTOR 20
 #define endstr(e) (e=='e'?"end":e=='b'?"start":"n/a")
 
-const char * VERSION = "1.02.772";
+const char * VERSION = "1.03";
 
 // barcode
 struct bc {
@@ -43,7 +43,7 @@ struct bc {
 	char *out[6];			// one output per input
 	FILE *fout[6];
 	bool gzout[6];
-	int cnt;			// count found
+	uint64_t cnt;			// count found
 	bool shifted;			// count found in 1-shifted position
 	char * dual;			// is this a dual-indexed barcode?  if so, this points to the second index.
 	int dual_n;			// length of dual
@@ -116,6 +116,7 @@ int main (int argc, char **argv) {
 	const char* list=NULL;		// use a barcode master list
 	char verify='\0';
 	bool noexec = false;
+    bool seqnames = false;
 	const char *group = NULL;
     bool usefile1 = false;
     int phred = 33;
@@ -125,7 +126,7 @@ int main (int argc, char **argv) {
 	int i;
 	bool omode = false;	
 	char *bfil = NULL;
-	while (	(c = getopt (argc, argv, "-DzxnHhbeov:m:B:g:L:l:G:q:d:t:")) != -1) {
+	while (	(c = getopt (argc, argv, "-DzxnHhbeosv:m:B:g:L:l:G:q:d:t:")) != -1) {
 		switch (c) t:{
 		case '\1': 
                        	if (omode) {
@@ -143,6 +144,7 @@ int main (int argc, char **argv) {
 			}
 			break;
                 case 'o': omode=true; break;
+                case 's': seqnames=true; break;
                 case 'v': 
 			if (strlen(optarg)>1) {
 				fprintf(stderr, "Option -v requires a single character argument");
@@ -801,18 +803,34 @@ int main (int argc, char **argv) {
 	// TODO: output barcode read ...but only for unmatched?
 	int b;
     for (b=0;b<=bcnt;++b) {
+		size_t nameseq_len = strlen(bc[b].id.s);
+		if ((b < bcnt) && seqnames) {
+                  nameseq_len = strlen(bc[b].seq.s);
+                  if (bc[b].dual)
+                    nameseq_len += bc[b].dual_n + 1;
+		}
+
 		for (i=0;i<f_n;++i) {
 			if (!strcasecmp(out[i],"n/a") || !strcasecmp(out[i],"/dev/null")) {
 				bc[b].out[i] = NULL;
 				bc[b].fout[i] = NULL;
 				continue;
 			}
 			const char *p=strchr(out[i],'%');
-			if (!p) fail("Each output file name must contain a '%%' sign, which is replaced by the barcode id\n");
-			bc[b].out[i]=(char *) malloc(strlen(out[i])+strlen(bc[b].id.s)+100);
+			if (!p) fail("Each output file name must contain a '%%' sign, which is replaced by the barcode id or sequence\n");
+			bc[b].out[i]=(char *) malloc(strlen(out[i])+nameseq_len+100);
 			strncpy(bc[b].out[i], out[i], p-out[i]);
 			bc[b].out[i][p-out[i]]='\0';
-			strcat(bc[b].out[i], bc[b].id.s);
+			if (seqnames && (b < bcnt)) {
+			  strcat(bc[b].out[i], bc[b].seq.s);
+                          if (bc[b].dual) {
+                            strcat(bc[b].out[i], "-");
+                            strcat(bc[b].out[i], bc[b].dual);
+                          }
+                        }
+			else {
+			  strcat(bc[b].out[i], bc[b].id.s);
+                        }
 			strcat(bc[b].out[i], p+1);
 			if (!(bc[b].fout[i]=gzopen(bc[b].out[i], "w", &bc[b].gzout[i]))) {
 				fprintf(stderr, "Error opening output file '%s': %s\n",bc[b].out[i], strerror(errno));
@@ -1011,7 +1029,7 @@ int main (int argc, char **argv) {
 			if (!f) continue;
             if (!trimmed) {
 			    // todo: capture always, not just when trim is off
-                *strrchr(fq[i].id.s, '\n') = '\0';
+                if (!debug) *strrchr(fq[i].id.s, '\n') = '\0';
                 fputs(fq[i].id.s,f);
                 fputc(' ', f);
                 fputs(fq[0].seq.s,f);
@@ -1054,17 +1072,17 @@ int main (int argc, char **argv) {
 
 	int j;
 	printf("Id\tCount\tFile(s)\n");
-	int tot=0;
+	uint64_t tot=0;
 	for (i=0;i<=bcnt;++i) {
-		printf("%s\t%d", bc[i].id.s, bc[i].cnt);
+		printf("%s\t%lu", bc[i].id.s, bc[i].cnt);
 		tot+=bc[i].cnt;
 		for (j=0;j<f_n;++j) {
 			if (bc[i].out[j])
 				printf("\t%s", bc[i].out[j]);
 		}
 		printf("\n");
 	}
-	printf("total\t%d\n", tot);
+	printf("total\t%lu\n", tot);
 
     if (!io_ok)
         return 3;
@@ -1145,6 +1163,7 @@ void usage(FILE *f) {
 "-L BCFIL    Determine barcodes from <read1.fq>, using BCFIL as a master list\n"
 "-B BCFIL    Use barcodes from BCFIL, no determination step, codes in <read1.fq>\n"
 "-H          Use barcodes from illumina's header, instead of a read\n"
+"-s          Substitute barcode sequence instead of barcode label into output file names\n"
 "-b          Force beginning of line (5') for barcode matching\n"
 "-e          Force end of line (3') for batcode matching\n"
 "-t NUM      Divide threshold for auto-determine by factor NUM (1), > 1 = more sensitive\n"

clipper/fastq-stats.cpp

@@ -153,7 +153,7 @@ int window = 2000000;
 int cyclemax = 35; 
 int gcCyclemax = 100; // to compare with fastqc, seq is rounded to nearest 100 to reduce # of gc models; for < 200 length, this is teh same as max=100
 float gcSum;
-int gcTotal;
+uint64_t gcTotal;
 
 int show_max = 10;
 bool debug = 0;
@@ -565,6 +565,8 @@ int main( int argc, char**argv ) {
 	
 	if(gc) {
         // put these where they belong
+		if (debug)
+			printf("gcTotal\t%lu\tgcSum\t%f\n\n", gcTotal, gcSum);
         printf("pct-gc cycle-max\t%d\n", gcCyclemax);
         printf("pct-gc mean\t%.2f\n", 100.0 * gcSum / gcTotal);
     }

clipper/mirna-quant.cpp

@@ -681,7 +681,7 @@ FILE *gzopen(const char *f, const char *m, bool*isgz) {
         if (!strcmp(fext(f),".gz")) {
                 char *tmp=(char *)malloc(strlen(f)+100);
                 if (strchr(m,'w')) {
-                        strcpy(tmp, "gzip --rsyncable > '");
+                        strcpy(tmp, "gzip  > '");
                         strcat(tmp, f);
                         strcat(tmp, "'");
                 } else {

clipper/sparsehash-2.0.2/ChangeLog → clipper/sparsehash-2.0.3/ChangeLog

@@ -1,3 +1,8 @@
+Mon Oct 12 21:00:00 2015 Google Inc. <google-sparsehash@googlegroups.com>
+
+	* sparsehash: version 2.0.3
+	* Fix compilation on modern compilers and operating systems
+
 Thu Feb 23 23:47:18 2012 Google Inc. <google-sparsehash@googlegroups.com>
 
 	* sparsehash: version 2.0.2

clipper/sparsehash-2.0.2/NEWS → clipper/sparsehash-2.0.3/NEWS

@@ -1,4 +1,9 @@
-== 23 Ferbruary 2012 ==
+== 12 October 2015 ==
+
+Various small fixes to ensure compilation on modern compilers and operating 
+systems. Tagged as 2.0.3
+
+== 23 February 2012 ==
 
 A backwards incompatibility arose from flattening the include headers
 structure for the <google> folder. 

clipper/sparsehash-2.0.2/README_windows.txt → clipper/sparsehash-2.0.3/README_windows.txt

@@ -1,25 +1,25 @@
-This project has been ported to Windows.  A working solution file
-exists in this directory:
-    sparsehash.sln
-
-You can load this solution file into either VC++ 7.1 (Visual Studio
-2003) or VC++ 8.0 (Visual Studio 2005) -- in the latter case, it will
-automatically convert the files to the latest format for you.
-
-When you build the solution, it will create a number of
-unittests,which you can run by hand (or, more easily, under the Visual
-Studio debugger) to make sure everything is working properly on your
-system.  The binaries will end up in a directory called "debug" or
-"release" in the top-level directory (next to the .sln file).
-
-Note that these systems are set to build in Debug mode by default.
-You may want to change them to Release mode.
-
-I have little experience with Windows programming, so there may be
-better ways to set this up than I've done!  If you run across any
-problems, please post to the google-sparsehash Google Group, or report
-them on the sparsehash Google Code site:
-   http://groups.google.com/group/google-sparsehash
-   http://code.google.com/p/sparsehash/issues/list
-
--- craig
+This project has been ported to Windows.  A working solution file
+exists in this directory:
+    sparsehash.sln
+
+You can load this solution file into either VC++ 7.1 (Visual Studio
+2003) or VC++ 8.0 (Visual Studio 2005) -- in the latter case, it will
+automatically convert the files to the latest format for you.
+
+When you build the solution, it will create a number of
+unittests,which you can run by hand (or, more easily, under the Visual
+Studio debugger) to make sure everything is working properly on your
+system.  The binaries will end up in a directory called "debug" or
+"release" in the top-level directory (next to the .sln file).
+
+Note that these systems are set to build in Debug mode by default.
+You may want to change them to Release mode.
+
+I have little experience with Windows programming, so there may be
+better ways to set this up than I've done!  If you run across any
+problems, please post to the google-sparsehash Google Group, or report
+them on the sparsehash Google Code site:
+   http://groups.google.com/group/google-sparsehash
+   http://code.google.com/p/sparsehash/issues/list
+
+-- craig

clipper/sparsehash-2.0.3/autogen.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# Before using, you should figure out all the .m4 macros that your
+# configure.m4 script needs and make sure they exist in the m4/
+# directory.
+#
+# These are the files that this script might edit:
+#    aclocal.m4 configure Makefile.in src/config.h.in \
+#    compile config.guess config.sub depcomp install-sh missing mkinstalldirs
+#
+# Here's a command you can run to see what files aclocal will import:
+#  aclocal -I ../autoconf --output=- | sed -n 's/^m4_include..\([^]]*\).*/\1/p'
+
+set -ex
+rm -rf autom4te.cache
+
+aclocal --force -I m4
+autoconf -f -W all,no-obsolete
+autoheader -f -W all
+automake -a -c -f -W all
+
+rm -rf autom4te.cache
+exit 0

clipper/sparsehash-2.0.3/experimental/Makefile

@@ -0,0 +1,9 @@
+example: example.o libchash.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
+
+.SUFFIXES: .c .o .h
+.c.o:
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $<
+
+example.o: example.c libchash.h
+libchash.o: libchash.c libchash.h