/OSG-GEM

Permalink
OSG-GEM/submit
Newer
Older
Raw Normal view History
100755 91 lines (69 sloc) 2.5 KB
Mar 21, 2016 @wpoehlm added files
1 #!/bin/bash
2
3 set -e
4
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
5 # are we on OSG Connect?
6 if [ -e /stash2 ]; then
Aug 8, 2016 @rynge Added support for multiple inputs and a GEM merge at the end. Simplie…
7
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
8 # needed for stashcp to be picked up the site catalog for the local site
9 module load xrootd
10 module load stashcp
11
Mar 13, 2017 @wpoehlm use new Pegasus OASIS module
12 module load pegasus
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
13
14 TOPDIR=`pwd`
15
16 export RUN_ID=osg-gem-`date +'%s'`
17
18 export RUN_DIR=/local-scratch/$USER/workflows/$RUN_ID
19 mkdir -p $RUN_DIR/scratch/$RUN_ID/level-2
20 # make the data availabile over http
21 mkdir -p /stash2/user/$USER/public/$RUN_ID/data
22 ln -s /stash2/user/$USER/public/$RUN_ID/data $RUN_DIR/data
23
24 # generate the site catalog
25 SC=$RUN_DIR/sites.xml
26 envsubst <sites.xml.template >$SC
27
28 # generate the dax
29 export PYTHONPATH=`pegasus-config --python`
30 ./tools/dax-level-1 $RUN_ID $RUN_DIR $RUN_DIR/data
31
32 echo
33 echo "An 'Output's directory will be created within the base of the workflow directory."
34 echo "This directory, $RUN_DIR/outputs"
35 echo "will have a 'merged_GEM.tab' file, an expression vector for each individual file,"
Feb 2, 2017 @wpoehlm Added Log Merge Script to workflow. In addition to all standard outpu…
36 echo "and a report of trimming and alignment statistics, QC_Report.tab"
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
37
38 # plan and submit the workflow
39 echo
40 pegasus-plan \
41 -Dpegasus.catalog.site.file=$SC \
42 --conf pegasus.conf \
43 --relative-dir $RUN_ID \
44 --sites condorpool \
45 --staging-site stash \
46 --output-site local \
47 --dir $RUN_DIR/workflow \
48 --dax dax.xml \
49 --submit
Aug 9, 2016 @wpoehlm Update submit
50
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
51 else
52 # jetstream
Aug 8, 2016 @rynge Added support for multiple inputs and a GEM merge at the end. Simplie…
53
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
54 TOPDIR=`pwd`
55
56 export RUN_ID=osg-gem-`date +'%s'`
57
Dec 16, 2016 Improved file system layout when running on Jetstream
58 export RUN_DIR=$PWD/runs/$RUN_ID
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
59 mkdir -p $RUN_DIR/scratch/$RUN_ID/level-2
60 mkdir -p $RUN_DIR/data
61
62 # generate the site catalog
63 SC=$RUN_DIR/sites.xml
64 envsubst <sites.xml.template >$SC
65
66 # generate the dax
67 export PYTHONPATH=`pegasus-config --python`
68 ./tools/dax-level-1 $RUN_ID $RUN_DIR $RUN_DIR/data
69
70 echo
71 echo "An 'Output's directory will be created within the base of the workflow directory."
72 echo "This directory, $RUN_DIR/outputs"
73 echo "will have a 'merged_GEM.tab' file, an expression vector for each individual file,"
74 echo "and all standard output files from trimmomatic/hisat2 jobs."
75
76 # plan and submit the workflow
77 echo
78 pegasus-plan \
79 -Dpegasus.catalog.site.file=$SC \
80 --conf pegasus.conf \
81 --relative-dir $RUN_ID \
82 --sites jetstream-condorpool \
83 --staging-site jetstream-staging \
84 --output-site local \
85 --dir $RUN_DIR/workflow \
86 --dax dax.xml \
87 --submit
Mar 21, 2016 @wpoehlm added files
88
Dec 13, 2016 @rynge Enabled the workflow to be executed on Jetstream
89 fi
Apr 22, 2016 @wpoehlm fixed hisat2 merging support
90