-
Notifications
You must be signed in to change notification settings - Fork 0
/
wwwmetis.py
executable file
·108 lines (88 loc) · 3.73 KB
/
wwwmetis.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/bin/env python
################################
# Job tag and output hadoop path
################################
# NOTE: If you want to resubmit the skimming job, you need to delete $ANALYSIS_BASE/tasks and hadoop_path output path
job_tag = "v16_skim_v2_5"
hadoop_path = "metis/wwwlooper/{}".format(job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
###################################################################################################################
###################################################################################################################
# Below are the Metis submission code that users do not have to care about.
###################################################################################################################
###################################################################################################################
import time
import json
import metis
from time import sleep
from metis.Sample import DirectorySample
from metis.CondorTask import CondorTask
from metis.StatsParser import StatsParser
import sys
import os
import glob
import subprocess
# file/dir paths
wwwdir = os.path.dirname(os.path.realpath(__file__))
anadir = os.getenv("ANALYSIS_BASE")
scriptsdir = os.path.join(os.getenv("ANALYSIS_BASE"), "scripts")
tar_path = os.path.join(wwwdir, "package.tar")
targzpath = tar_path + ".gz"
metisdashboardpath = os.path.join(os.path.dirname(os.path.dirname(metis.__file__)), "dashboard")
# Create tarball
os.system("tar -cf {} *.C *.h".format(tar_path))
os.chdir(anadir)
os.system("tar -rf {} rooutil/rooutil.so WWW_CORE/WWW_CORE.so rooutil/*.h WWW_CORE/*.h".format(tar_path))
os.chdir(scriptsdir)
os.system("tar -rf {} *.sh *.C ".format(tar_path))
os.chdir(wwwdir)
os.system("gzip -f {}".format(tar_path))
# Configurations
baby_version = "16"
exec_path = os.path.join(scriptsdir, "run.sh")
args = "WWW_ScanChain.C output.root t -1 doskim"
total_summary = {}
while True:
task = CondorTask(
sample = DirectorySample(
dataset="/WWW_v0_1_{}".format(baby_version),
location="/hadoop/cms/store/user/bhashemi/AutoTwopler_babies/merged/VVV/WWW_v0.1.16/skim/",
globber="*.root"
),
open_dataset = False,
flush = True,
files_per_output = 1,
output_name = "merged.root",
tag = job_tag,
arguments = args,
executable = exec_path,
tarfile = targzpath,
special_dir = hadoop_path,
condor_submit_params = {"sites" : "UAF,T2_US_UCSD"}
)
task.process()
# save some information for the dashboard
total_summary["WWW_v0_1_{}_{}".format(baby_version, job_tag)] = task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metisdashboardpath).do()
os.system("msummary | tee summary.txt")
os.system("chmod -R 755 {}".format(metisdashboardpath))
if task.complete():
print ""
print "Job={} finished".format(job_tag)
print ""
break
print 'Press Ctrl-C to force update, otherwise will sleep for 30 seconds'
try:
for i in range(0,30):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
print "Renaming merged file names ..."
if not os.path.isfile("WWW_CORE/rename"):
os.chdir("WWW_CORE/")
os.system("make rename")
os.chdir("../")
import getpass
username = getpass.getuser()
os.system("./WWW_CORE/rename mapping.txt /hadoop/cms/store/user/{}/{}/WWW_v0_1_{}_{}/".format(username, hadoop_path, baby_version, job_tag))