Permalink
Browse files
Make sure sites.xml is generated once and only once with the correct …
- Loading branch information...
Showing
with
12 additions
and
9 deletions.
-
+0
−2
pegasus.conf
-
+6
−6
sites.xml → sites.xml.template
-
+5
−0
submit
-
+1
−1
tools/dax-level-1
|
|
@@ -1,7 +1,5 @@ |
|
|
pegasus.metrics.app = OSG-GEM
|
|
|
|
|
|
-pegasus.catalog.site.file = sites.xml
|
|
|
-
|
|
|
pegasus.dir.useTimestamp = true
|
|
|
pegasus.dir.storage.deep = false
|
|
|
pegasus.dir.storage.mapper = Flat
|
|
|
|
|
|
@@ -16,23 +16,23 @@ |
|
|
|
|
|
<site handle="stash" arch="x86_64" os="LINUX">
|
|
|
<directory type="shared-scratch" path="/stash2/user/${USER}/public">
|
|
|
- <file-server operation="get" url="http://stash.osgconnect.net/~${USER}"/>
|
|
|
- <!-- file-server operation="get" url="stash:///user/${USER}/public"/ -->
|
|
|
+ <!-- file-server operation="get" url="http://stash.osgconnect.net/~${USER}"/ -->
|
|
|
+ <file-server operation="get" url="stash:///user/${USER}/public"/>
|
|
|
<file-server operation="put" url="scp://${USER}@login02.osgconnect.net/stash2/user/${USER}/public"/>
|
|
|
</directory>
|
|
|
</site>
|
|
|
|
|
|
<site handle="condorpool" arch="x86_64" os="LINUX">
|
|
|
<profile namespace="pegasus" key="style" >condor</profile>
|
|
|
<profile namespace="condor" key="universe" >vanilla</profile>
|
|
|
- <profile namespace="condor" key="requirements" >OSGVO_OS_STRING == "RHEL 6" && HAS_MODULES == True && HAS_SCP == True && GLIDEIN_ResourceName != "Hyak" && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName1 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName2 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName3 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName4</profile>
|
|
|
- <profile namespace="condor" key="request_memory" >5 GB</profile>
|
|
|
+ <profile namespace="condor" key="requirements" >OSGVO_OS_STRING == "RHEL 6" && HAS_MODULES == True && HAS_SCP == True && GLIDEIN_ResourceName != "Hyak" && GLIDEIN_Site != "UUCHPC" && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName1 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName2 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName3 && TARGET.GLIDEIN_ResourceName =!= MY.MachineAttrGLIDEIN_ResourceName4</profile>
|
|
|
+ <profile namespace="condor" key="request_memory" >3 GB</profile>
|
|
|
<profile namespace="condor" key="request_disk" >30 GB</profile>
|
|
|
<profile namespace="condor" key="+WantsStashCache" >True</profile>
|
|
|
|
|
|
<!-- disable OSG squid caching for now -->
|
|
|
- <profile namespace="env" key="http_proxy" ></profile>
|
|
|
- <profile namespace="env" key="OSG_SQUID_LOCATION" ></profile>
|
|
|
+ <!-- profile namespace="env" key="http_proxy" ></profile>
|
|
|
+ <profile namespace="env" key="OSG_SQUID_LOCATION" ></profile -->
|
|
|
</site>
|
|
|
|
|
|
</sitecatalog>
|
|
|
@@ -20,6 +20,10 @@ mkdir -p $RUN_DIR/scratch/$RUN_ID/level-2 |
|
|
mkdir -p /stash2/user/$USER/public/$RUN_ID/data
|
|
|
ln -s /stash2/user/$USER/public/$RUN_ID/data $RUN_DIR/data
|
|
|
|
|
|
+# generate the site catalog
|
|
|
+SC=$RUN_DIR/sites.xml
|
|
|
+envsubst <sites.xml.template >$SC
|
|
|
+
|
|
|
# generate the dax
|
|
|
export PYTHONPATH=`pegasus-config --python`
|
|
|
./tools/dax-level-1 $RUN_ID $RUN_DIR $RUN_DIR/data
|
|
|
@@ -30,6 +34,7 @@ echo "NOTE: at the end of the workflow run, you will find the outputs in $RUN_DI |
|
|
# plan and submit the workflow
|
|
|
echo
|
|
|
pegasus-plan \
|
|
|
+ -Dpegasus.catalog.site.file=$SC \
|
|
|
--conf pegasus.conf \
|
|
|
--relative-dir $RUN_ID \
|
|
|
--sites condorpool \
|
|
|
|
|
|
@@ -142,7 +142,7 @@ for job in prepare_jobs: |
|
|
|
|
|
# sub workflow job
|
|
|
j3 = DAX("level-2.dax")
|
|
|
-j3.addArguments("-Dpegasus.catalog.site.file=%s/sites.xml" % (base_dir),
|
|
|
+j3.addArguments("-Dpegasus.catalog.site.file=%s/sites.xml" % (run_dir),
|
|
|
"--sites", "condorpool",
|
|
|
"--staging-site", "stash",
|
|
|
"--output-site", "local",
|
|
|
|
0 comments on commit
ef70e8a