Add files via upload

Readme.md

@@ -0,0 +1 @@
+Readme HPup

config/environment.ini

@@ -0,0 +1,40 @@
+; the central environment config file will be used as default
+; for the pipeline runner (Pied Piper), so think twice before
+; blindly changing entries
+
+[Assemblies]
+prefix=/TL/deep-share/nobackup/deep_svn/Deep/pipelines/trunk/configs/assemblies
+; please make sure to include only compatible reference files/values in the
+; respective INI files - think about "chr" vs "non-chr" assemblies!
+hg19= ${prefix}/hg19.ini
+hs37d5= ${prefix}/hs37d5.ini
+grcm38= ${prefix}/grcm38.ini
+
+[RegExps]
+prefix= /TL/deep-share/nobackup/deep_svn/Deep/pipelines/trunk/configs/regexps
+deep_generic= ${prefix}/deep_generic_re.ini
+
+[Misc]
+prefix= /TL/deep-share/nobackup/deep_svn/Deep/pipelines/trunk/configs/misc
+color_codes= ${prefix}/color_codes.ini
+
+[LibPython3]
+pottyplotty= /home/pebert/work/code/sandbox/tools/trunk
+statelabeller= /home/pebert/work/code/sandbox/tools/trunk
+
+[LibPython2]
+deeptools= /TL/deep-share/archive00/software/packages/deeptools/v1.5.9.1/install/lib/python2.7/site-packages
+macs2= /TL/deep-share/archive00/software/packages/macs2/MACS/install/lib/python2.7/site-packages
+bxpython= /TL/deep-share/archive00/software/packages/bxpython/v20140415/install/lib/python2.7/site-packages
+cutadapt= /TL/deep-share/archive00/software/lib/python2.7/site-packages
+ld_library_path= /usr/local/lib
+
+[BinPaths]
+common= /bin:/usr/bin:/usr/local/bin:/TL/deep-share/archive00/software/bin
+cluster=
+#cluster= #/TL/deep-share/archive00/software/bin
+petools= /home/pebert/work/code/sandbox/tools/trunk
+
+[Software]
+trim_galore= /TL/deep-share/archive00/software/bin/trim_galore
+

config/hpSeq/hpSeq.analysis.ini

@@ -0,0 +1,35 @@
+[Analysis]
+#things to adjust for each run
+sampleid= SAMPLEID
+mainFolder= /current/Folder/
+input= ${mainFolder}/data/${sampleid}
+output= ${mainFolder}/out/${sampleid}/${date}
+date= DATE
+fastqid=@HISEQ
+
+#reference genome
+#genome_reference_gem= /projects/student/Andreas/reference/Mus_musculus.GRCm38.dna.chromosome.1.gem
+genome_reference_gem= /DEEP_fhgfs/data/external/references/DEEP/genomes/mouse/Methylation/indices/gem/GRCm38mm10_PhiX_Lambda.gem
+
+#mostly fixed parameters
+#/DEEP_fhgfs/projects/karln/hpseq/150929/pipelines
+installFolder= /DEEP_fhgfs/projects/karln/hpseq/pipeline/local_pipelines
+name= ${sampleid}.${date}
+dir_work_root= ${output}/wd
+dir_work_temp=${dir_work_root}/tmp
+dir_work_log=${dir_work_root}/log
+
+#hairpin linker information
+linker= GGGTTT[AGT][AGT][AGT]T[AGT][AGT][AGT]AGGTTT
+linkerGrep= GGG[CT]{2}T...T...AGG[CT]{3}           
+linkerSed= GGG\(..\)T\(...\)T\(...\)AGG\(...\)
+#size of added wildcard nucleatides(N) to prepare spike-in sequences
+padSize=100
+#spikein sequences
+spikein_seq= /DEEP_fhgfs/projects/karln/hpseq/pipeline/data/Spikes.ref.upper.fa
+q1hmc_upp= TACGATCACGGCGAATCCGATCGAATCACAGTGGCGCTTTACGAAGTGCGACAGCCTTAG
+q3hmc_upp= TACGATCACGGCGAATCCGATCGAATCCTTGTAGCGCTTTACGAAGTGCGACAGCCTTAG
+q6hmc_upp= TACGATCACGGCGAATCCGATCGAATCAGTCAAGCGCTTTACGAAGTGCGACAGCCTTAG
+qfc_upp= TACGATCACGGCGAATCCGATCGAATCGTTTCGGCGCTTTACGAAGTGCGACAGCCTTAG
+qmc_upp= TACGATCACGGCGAATCCGATCGAATCTAGCTTGCGCTTTACGAAGTGCGACAGCCTTAG
+sqc_upp= TACGATCACGGCGAATCCGATCGAATCCAGATCGCGCTTTACGAAGTGCGACAGCCTTAG

config/hpSeq/hpSeq.process.ini

@@ -0,0 +1,89 @@
+[Process]
+process= hpSeqV1
+fastqPattern= *q.gz
+
+[hpSeqV1]
+# count reads from fasta files
+countReads= bash ${System:installFolder}/wrappers/hpSeq/countReads.sh -n {description} -i {{inputfile}} -o {{outputfile}}
+
+# count reads from BAM files
+countBAMReads= bash ${System:installFolder}/wrappers/hpSeq/countBAMReads.sh -n {description} -i {{inputfile}} -o {{outputfile}}
+
+# count reads from output file
+countOUTReads= bash ${System:installFolder}/wrappers/hpSeq/countOUTReads.sh -n {description} -i {{inputfile}} -o {{outputfile}}
+
+#merge reads
+mergeReads= cat {inputfile} > {outputfile}
+
+#defining the process
+joinReads= bash ${System:installFolder}/wrappers/hpSeq/preProcess.sh -L "${Analysis:linker}" -a {inputfile[0]} -b {inputfile[1]} -x {outputfile[0]} -y {outputfile[1]}
+
+#clean up step of raw files
+cleanup= > {outputfile[0]} 
+
+#replace= ${Commands:java8} -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpReplaceSeq {inputfile} > {outputfile}
+
+linkanalyze=${Commands:java8} -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpSeqBamAnalyze {inputfile[0]} {inputfile[1]} | sort -k1,1 -k2,2n > {outputfile[0]}
+
+mergelink= bash ${System:installFolder}/wrappers/hpSeq/mergeLink.sh -i {inputfile} -o {outputfile}
+
+#${syncReads_boolean}
+#joinReads= bash preProcess.sh -L "${Analysis:linker}" -a {inputfile[0]} -b {inputfile[1]} -x {outputfile[0]} -y {outputfile[1]}
+syncReads_boolean=false
+syncReads_maxShift=12
+syncReads_minLength=16
+syncReads_maxError=7
+syncReads= ${Commands:java8} -Xmx32G -jar ${System:jarFolder}/fastqUtils.jar hpSeqSync ${syncReads_boolean}  ${syncReads_maxShift} ${syncReads_minLength} ${syncReads_maxError} {inputfile[0]} {inputfile[1]}
+
+#syncReads= hpSync.sh -Xmx4G ${System:jarFolder}/fastqUtils.jar hpSeqSync ${syncReads_maxShift} ${syncReads_minLength} ${syncReads_maxError} {inputfile[0]} {inputfile[1]}
+
+#map R1 synchronized fastq files with reference library to output bam files
+mapReads= ${Commands:mapWithGem} -a {inputfile} -o {outputfile} -r ${Analysis:genome_reference_gem}
+
+#map wobble position with mapped reads
+wobblemapRead= bash ${System:installFolder}/wrappers/hpSeq/wobblemapRead.sh -i {inputfile[0]} -j {inputfile[1]} -o {outputfile[0]}
+
+#replace sequence with sequence including methylation information (problems with bam format)
+replace= bash ${System:installFolder}/wrappers/hpSeq/replaceSeqmarkDup.sh -i {inputfile[0]} -j {inputfile[1]} -o {outputfile[0]}
+
+#analyze statistics of duplicats
+markduplicates= bash ${System:installFolder}/wrappers/hpSeq/markduplicates.sh  -i {inputfile} -o {outputfile} 
+
+
+#mapReads= mapWithGem.sh -a {inputfile} -o {outputfile} -r ${Analysis:genome_reference_gem}
+
+pileup= ${Commands:java8} -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpSeqPileup {inputfile} | sort -k1,1 -k2,2n > {outputfile}
+
+pileup2= ${Commands:java8} -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpSeqPileup2 {inputfile} | sort -k1,1 -k2,2n > {outputfile}
+
+#pileup= pileup.sh -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpSeqPileup {inputfile[1]} {inputfile[0]} > {outputfile}
+
+# the mergeCall command is cheating... but it will have to do for now
+mergeCall= sort -m -k1,1 -k2,2n {inputfolder}/seq_????_pileup.txt |mawk -f ${System:installFolder}/wrappers/hpSeq/mergeCall.awk | ${Commands:java8} -cp .:${System:jarFolder}/htsjdk-1.130.jar:${System:jarFolder}/bamUtils.jar tools.bam.bamUtils hpSeqPileupCall - |${Commands:bgzip} -c  > {{outputfile}}
+
+#call tabix for igv browser
+tabixCall= ${Commands:tabix} -s1 -b2 -e2 -f {inputfile} > {outputfile}
+
+
+# generate file with sequence identifier and sequence of wobble position
+# calculate the bisulfite conversion rate of the hairpin linker
+
+conversionHAIRPIN= bash ${System:installFolder}/wrappers/hpSeq/conversionHAIRPIN.sh  -i {inputfile[0]} -y {outputfile[0]} -x {outputfile[1]} -c ${Analysis:output}/config.ini -I ${Analysis:installFolder}
+
+# generate wobble position
+
+wobbleHAIRPIN=bash ${System:installFolder}/wrappers/hpSeq/wobbleHairpin.sh -i {inputfile} -o {outputfile}
+
+#pad spikes
+padReferenceSpikes_paddingSize=100
+padReferenceSpikes= ${Commands:java8} -jar ${System:jarFolder}/fastaUtils.jar pad {inputfile} ${padReferenceSpikes_paddingSize} > {outputfile}
+
+createReferenceSpikes= ${Commands:gmap_build} -D {spikeReferenceFolder} -d repeats {{inputfile}} && ${Commands:cmetindex} -F {spikeReferenceFolder} -d repeats &> {{outputfile}}
+
+filterSpikes= ${Commands:gsnap} --nthreads=8 --npaths=1 --db=repeats --dir={spikeReferenceFolder} --sam-use-0M  --format=sam --gunzip --mode=cmet-stranded {{inputfile}} |${Commands:samtools} view -u -F 260 - | ${Commands:samtools} sort -T - > {{outputfile}}
+
+forwardmergeSpikes= ${Commands:samtools} merge -u - {{inputfile}}| ${Commands:samtools} view -F16 -u - | ${Commands:samtools} mpileup -f {paddedReference} - > {{outputfile}}
+
+reversemergeSpikes= ${Commands:samtools} merge -u - {{inputfile}}| ${Commands:samtools} view -f16 -u - | ${Commands:samtools} mpileup -f {paddedReference} - > {{outputfile}}
+
+finalSpikes= bash ${System:installFolder}/wrappers/hpSeq/finalSpikes.sh -a {inputfile[0]} -b {inputfile[1]} -o {outputfile[0]} -c ${Analysis:output}/config.ini -I ${Analysis:installFolder}

config/system.ini

@@ -0,0 +1,87 @@
+[System]
+installFolder= ${Analysis:installFolder}
+jarFolder= ${Analysis:installFolder}/java/jar
+tmpdir=/tmp
+annotation= ${Analysis:installFolder}/annotation.bed
+
+[DeeptoolsJobSingle]
+jobname= ruffus
+workdir= ${Analysis:dir_work_root}
+outpath= ${Analysis:dir_work_log}
+errpath= ${Analysis:dir_work_log}
+#native_spec= -l h=deep0[23456789]*|deep1[0123456]*
+#native_spec= -l mem_free=16G,h_vmem=16G,slots_free=1,h=deep0[23456789]*|deep1[0123456]*
+native_spec= -l mem_free=66G,h_vmem=66G,slots_free=1,h=deep0[23456789]*|deep1[0123456]*
+scriptdir= ${Analysis:dir_work_log}
+keepscripts= 1
+
+[DeeptoolsJobDouble]
+jobname= ruffusDouble
+workdir= ${Analysis:dir_work_root}
+outpath= ${Analysis:dir_work_log}
+errpath= ${Analysis:dir_work_log}
+#native_spec= -l h=deep0[23456789]*|deep1[0123456]*
+native_spec= -l mem_free=15G,h_vmem=15G,slots_free=2,h=deep0[23456789]*|deep1[0123456]*
+scriptdir= ${Analysis:dir_work_log}
+keepscripts= 1
+
+[DeeptoolsJobQuarter]
+jobname= ruffusQuarter
+workdir= ${Analysis:dir_work_root}
+outpath= ${Analysis:dir_work_log}
+errpath= ${Analysis:dir_work_log}
+#native_spec= -l h=deep0[23456789]*|deep1[0123456]*
+native_spec= -l mem_free=31G,h_vmem=31G,slots_free=4,h=deep0[23456789]*|deep1[0123456]*
+scriptdir= ${Analysis:dir_work_log}
+keepscripts= 1
+
+[DeeptoolsJobHalf]
+jobname= ruffus
+workdir= ${Analysis:dir_work_root}
+outpath= ${Analysis:dir_work_log}
+errpath= ${Analysis:dir_work_log}
+#native_spec= -l h=deep0[23456789]*|deep1[0123456]*
+native_spec= -l mem_free=62G,h_vmem=62G,slots_free=8,h=deep0[23456789]*|deep1[0123456]*
+scriptdir= ${Analysis:dir_work_log}
+keepscripts= 1
+
+[DeeptoolsJobFull]
+jobname= ruffus
+workdir= ${Analysis:dir_work_root}
+outpath= ${Analysis:dir_work_log}
+errpath= ${Analysis:dir_work_log}
+#native_spec= -l h=deep0[23456789]*|deep1[0123456]*
+native_spec= -l mem_free=124G,h_vmem=124G,slots_free=16,h=deep0[23456789]*|deep1[0123456]*
+scriptdir= ${Analysis:dir_work_log}
+keepscripts= 1
+
+[EnvConfig]
+path= ${BinPaths:cluster}:${BinPaths:common}
+
+[EnvPy2Config]
+path= ${BinPaths:cluster}:${BinPaths:common}
+pythonpath= ${LibPython2:deeptools}:${LibPython2:macs2}:${LibPython2:bxpython}:${LibPython2:cutadapt}:${System:installFolder}
+ld_library_path= ${LibPython2:ld_library_path}
+#path= /local/projects/pipelines/wd/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/home/karln/.local/bin:/home/karln/bin
+#pythonpath= .
+
+
+[EnvPy3Config]
+path= ${BinPaths:cluster}:${BinPaths:petools}:${BinPaths:common}
+pythonpath= ${LibPython3:pottyplotty}
+
+
+[Commands]
+#trimGalorePaired= ${Software:trim_galore} --suppress_warn -q 20 --phred33 -o {outputFolder} --no_report_file --paired {{inputfile[0]}} {{inputfile[1]}}
+trimGalorePaired= ${Software:trim_galore} --suppress_warn -q 20 --phred33 -o {outputFolder} --no_report_file --paired {{inputfile[0]}} {{inputfile[1]}}  2> /dev/null
+mapWithGem= bash ${System:installFolder}/wrappers/mappers/mapWithGEM.sh
+java6= /usr/lib/jvm/java-6-openjdk-amd64/bin/java
+java8= /usr/lib/jvm/java-8-oracle/jre/bin/java
+python3= /usr/bin/python3
+gsnap= /TL/deep-share/archive00/software/bin/gsnap
+bgzip= /TL/deep-share/archive00/software/bin/bgzip
+tabix= /TL/deep-share/archive00/software/bin/tabix
+samtools= /TL/deep-share/archive00/software/bin/samtools
+picardjar= ${System:jarFolder}/picard.jar
+cmetindex= /TL/deep-share/archive00/software/bin/cmetindex
+gmap_build= /TL/deep-share/archive00/software/bin/gmap_build

java/manifest/bamUtils.manifest

@@ -0,0 +1,4 @@
+Manifest-Version: 1.0
+Created-By: 1.7.0_99 (Oracle Corporation)
+Main-Class: tools.bam.bamUtils
+

java/manifest/fastqUtils.manifest

@@ -0,0 +1,4 @@
+Manifest-Version: 1.0
+Created-By: 1.7.0_99 (Oracle Corporation)
+Main-Class: tools.fastq.fastqUtils
+

java/tools/bam/bamUtilToJar.jardesc

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<jardesc>
+    <jar path="/home/karln/jars/bamUtils.jar"/>
+    <options buildIfNeeded="true" compress="true" descriptionLocation="/tools/src/tools/bam/bamUtilToJar.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="false" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
+    <storedRefactorings deprecationInfo="true" structuralOnly="false"/>
+    <selectedProjects/>
+    <manifest generateManifest="true" mainClassHandleIdentifier="=tools/src&lt;tools.bam{bamUtils.java[bamUtils" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
+        <sealing sealJar="false">
+            <packagesToSeal/>
+            <packagesToUnSeal/>
+        </sealing>
+    </manifest>
+    <selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
+        <javaElement handleIdentifier="=tools/src&lt;tools.fastq{FastqSeq.java"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.utils{general.java"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.bam"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.fastq{fastqParser.java"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.sequences"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.fasta{FastaSeq.java"/>
+        <javaElement handleIdentifier="=tools/src&lt;tools.fasta{fastaParser.java"/>
+    </selectedElements>
+</jardesc>