Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

RelMon updates and fixes #582

Merged
merged 2 commits into from Sep 10, 2013
Merged

RelMon updates and fixes #582

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
29cbabb
Removed Phycis and Info directories from black list in Validation matrix
Aug 16, 2013
8434e6e
Added:
Aug 20, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Empty file modified Utilities/RelMon/python/authentication.py 100644 → 100755
View file
Open in desktop
Empty file.
Empty file modified Utilities/RelMon/python/definitions.py 100644 → 100755
View file
Open in desktop
Empty file.
103 changes: 75 additions & 28 deletions Utilities/RelMon/python/directories2html.py 100644 → 100755
View file
Open in desktop
Expand Up @@ -3,8 +3,8 @@
# https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
#
# $Author: anorkus $
# $Date: 2012/11/21 15:22:14 $
# $Revision: 1.5 $
# $Date: 2013/07/09 14:00:21 $
# $Revision: 1.10 $

#
#
Expand Down Expand Up @@ -57,10 +57,10 @@ def build_obj(run,sample,version,plot_path,tier):
obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
return encode_obj_url(obj_url)

def fairy_url(run,sample,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
fairy_url+= build_obj(run,sample,version1,plot_path,tier1)
fairy_url+= build_obj(run,sample,version2,plot_path,tier2)
fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
if len(draw_opts)>0:
fairy_url+="drawopts=%s;" %draw_opts
fairy_url+= plot_size(h,w)
Expand All @@ -76,7 +76,10 @@ def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200)

#-------------------------------------------------------------------------------
style_location="/cms-service-reldqm"
def get_page_header(directory=None,additional_header=""):
def get_page_header(directory=None, standalone=False, additional_header=""):
style_location="/cms-service-reldqm"
if standalone:
style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
javascripts=''
style=''
if directory!=None and len(directory.comparisons)>0:
Expand Down Expand Up @@ -117,25 +120,30 @@ def get_page_footer():

#-------------------------------------------------------------------------------

def get_title_section(directory, hashing_flag, depth=2):
def get_title_section(directory, hashing_flag, standalone, depth=2):
if standalone:
cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
else:
cms_logo_url = "cms-service-reldqm/style/CMS.gif"
mother_name=basename(directory.mother_dir)
mother_file_name=""
if depth==1:
mother_file_name="../RelMonSummary.html"
if mother_name!="":
mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
elif depth==2:
#mother_file_name="RelMonSummary.html"
mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
mother_file_name="RelMonSummary.html"
#mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
if mother_name!="":
mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
else:
if hashing_flag:
files = directory.mother_dir.split("/")
if len(files) != 1:
dir_name = files[-2] ##return the mother directory name only as the html file name by it
dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
else:
dir_name = files[-1]
dir_name = directory.mother_dir
mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
else:
mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
Expand All @@ -149,7 +157,7 @@ def get_title_section(directory, hashing_flag, depth=2):
'<h1>%s</h1>'%link_to_mother+\
'</div>'+\
'<div class="span-3 last">'+\
'<img src="cms-service-reldqm/style/CMS.gif" class="top right" width="54" hight="54">'+\
'<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
'</div>'+\
'<hr>'
if len(mother_name)>0:
Expand All @@ -169,6 +177,8 @@ def get_dir_stats(directory):
html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
if directory.n_skiped>0:
html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
if directory.n_missing_objs>0:
html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
html+='</ul>'
return html

Expand All @@ -187,7 +197,7 @@ def get_subdirs_section(directory, hashing_flag):
for subdir in sorted_subdirs:
name=subdir.name
if hashing_flag:
link = "%s.html" %(hash_name(name, hashing_flag))
link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
else:
link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
link=link.strip("_")
Expand Down Expand Up @@ -256,7 +266,7 @@ def get_summary_section(directory,matrix_page=True):
'<div class="span-9 last">'
if matrix_page:
html+='<h3>Sample:</h3>'+\
'<p class="caps">%s</p>'%meta.sample+\
'<p class="caps">%s</p>'%meta.sample1+\
'<h3>Run1 and Run2:</h3>'+\
'<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
html+='<h3>Releases:</h3>'+\
Expand Down Expand Up @@ -305,29 +315,33 @@ def get_comparisons(category,directory):
'<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
else:
big_fairy=fairy_url(directory.meta.run1,
directory.meta.sample,
directory.meta.run2,
directory.meta.sample1,
directory.meta.sample2,
directory.meta.release1,
directory.meta.release2,
comp_abs_path,
directory.meta.tier1,
directory.meta.tier2,
"",600,600)
small_fairy=fairy_url(directory.meta.run1,
directory.meta.sample,
directory.meta.run2,
directory.meta.sample1,
directory.meta.sample2,
directory.meta.release1,
directory.meta.release2,
comp_abs_path,
directory.meta.tier1,
directory.meta.tier2)

single_fairy1=fairy_url_single(directory.meta.run1,
directory.meta.sample,
directory.meta.sample1,
directory.meta.release1,
comp_abs_path,
directory.meta.tier1,
"",500,500)
single_fairy2=fairy_url_single(directory.meta.run2,
directory.meta.sample,
directory.meta.sample2,
directory.meta.release2,
comp_abs_path,
directory.meta.tier2,
Expand Down Expand Up @@ -401,7 +415,21 @@ def get_rank_section(directory):

#-------------------------------------------------------------------------------

def directory2html(directory, hashing, depth=0):
#-------------------------------------------------------------------------------
def get_missing_objs_section(directory):
"""Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
"""
page_html = "Unpaired in %s</br>"%(directory.filename1)
for elem in directory.different_histograms['file1']:
page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
page_html +="</br>"
page_html += "Unpaired in %s</br>"%(directory.filename2)
for elem in directory.different_histograms['file2']:
page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
return page_html
#-------------------------------------------------------------------------------

def directory2html(directory, hashing, standalone, depth=0):
"""Converts a directory tree into html pages, very nice ones.
"""
#print "d2html: depth", str(depth)," dir ",directory.name
Expand All @@ -414,10 +442,10 @@ def directory2html(directory, hashing, depth=0):
#chdir(directory.name)

for subdir in directory.subdirs:
directory2html(subdir,hashing, depth)
directory2html(subdir,hashing,standalone, depth)

page_html=get_page_header(directory)+\
get_title_section(directory,hashing, depth)+\
page_html=get_page_header(directory, standalone)+\
get_title_section(directory,hashing, standalone, depth)+\
get_summary_section(directory)+\
get_subdirs_section(directory, hashing)

Expand All @@ -428,6 +456,9 @@ def directory2html(directory, hashing, depth=0):
(directory.n_comp_skiped >0,SKIPED)):
if do_cat:
page_html+=get_comparisons(cat,directory)

if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
page_html += get_missing_objs_section(directory)

# Distribution of ranks

Expand All @@ -442,7 +473,12 @@ def directory2html(directory, hashing, depth=0):
if len(page_name)==0:
page_name="RelMonSummary"
if hashing:
ofilename = "%s.html" %(hash_name(page_name, hashing))
if page_name != "RelMonSummary":
#print " ## oFile path: %s"%(join(directory.full_path))
#print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
else:
ofilename = "RelMonSummary.html"
else:
ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
ofilename=ofilename.strip("_")
Expand Down Expand Up @@ -659,7 +695,7 @@ def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_ag

#-------------------------------------------------------------------------------

def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag):
def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
"""Create a table, with as rows the directories and as columns the samples.
Each box in the table will contain a pie chart linking to the directory.
"""
Expand Down Expand Up @@ -709,7 +745,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)

page_html = get_page_header(additional_header=directories_barchart+categories_barchart)
page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
rel1=""
rel2=""
try:
Expand All @@ -727,6 +763,8 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
all_subdirs.append(subdir_name)
all_subdirs=sorted(list(set(all_subdirs)))

#print " $$ all_subdirs: %s" %(all_subdirs)

# Get The title
page_html+= '<div class="span-20">'+\
'<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
Expand Down Expand Up @@ -811,7 +849,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
for sample in sorted_samples:
col=dir_dict[sample]
# check if the directory was a top one or not
summary_page_name=hash_name("RelMonSummary", hashing_flag)+".html"
summary_page_name="RelMonSummary.html"
if col.name!="":
summary_page_name=hash_name(col.name, hashing_flag)+".html"
img_link=col.get_summary_chart_ajax(55,55)
Expand Down Expand Up @@ -849,10 +887,18 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
subdirs_dict=directory.get_subdirs_dict()

# Check if the directory is the top one
summary_page=join(sample,"%s.html"%(hash_name(subdir_name, hashing_flag)))
if directory.name!="":
# We did not run on the topdir
summary_page=join(sample,"%s_%s.html"%(directory.name,hash_name(subdir_name,hashing_flag)))
# We did not run on the topdir
#print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
if hashing_flag:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
else:
summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
else:
#print " ## summary_page: %s"%(directory.name+subdir_name)
#print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
dir_is_there=subdirs_dict.has_key(subdir_name)

img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
Expand Down Expand Up @@ -886,6 +932,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_

#-----------UPDATES------
def hash_name(file_name, flag):
#print " HashFILE name: "+file_name
if flag: #if hashing flag is ON then return
return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
else:
Expand Down
31 changes: 23 additions & 8 deletions Utilities/RelMon/python/dirstructure.py 100644 → 100755
View file
Open in desktop
Expand Up @@ -2,9 +2,9 @@
# RelMon: a tool for automatic Release Comparison
# https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
#
# $Author: dpiparo $
# $Date: 2012/07/03 05:38:00 $
# $Revision: 1.2 $
# $Author: anorkus $
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please simply remove these.

# $Date: 2013/07/05 09:45:01 $
# $Revision: 1.5 $
#
#
# Danilo Piparo CERN - danilo.piparo@cern.ch
Expand Down Expand Up @@ -48,8 +48,9 @@ def __init__(self,name,weight=1):

#-------------------------------------------------------------------------------
class CompInfo(object):
def __init__(self,sample="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
self.sample=sample
def __init__(self,sample1="",sample2="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
self.sample1=sample1
self.sample2=sample2
self.release1=release1
self.release2=release2
self.run1=run1
Expand Down Expand Up @@ -79,7 +80,14 @@ def __init__(self,name,mother_dir="",meta=CompInfo(),draw_success=False,do_pngs=
self.do_pngs=do_pngs
self.rank_histo=TH1I("rh%s"%name,"",50,-0.01,1.001)
self.rank_histo.SetDirectory(0)

self.different_histograms = {}
self.different_histograms['file1']= {}
self.different_histograms['file2']= {}
self.filename1 = ""
self.filename2 = ""
self.n_missing_objs = 0
self.full_path = ""

def is_empty(self):
if len(self.subdirs)==0 and len(self.comparisons)==0:
return True
Expand All @@ -102,7 +110,10 @@ def calcStats(self,make_pie=True):

self.n_skiped = 0
self.n_comp_skiped = 0

self.n_missing_objs = len(self.different_histograms['file1'].keys())+len(self.different_histograms['file2'].keys())
if self.n_missing_objs != 0:
print " [*] Missing in %s: %s" %(self.filename1, self.different_histograms['file1'])
print " [*] Missing in %s: %s" %(self.filename2, self.different_histograms['file2'])
# clean from empty dirs
self.subdirs = filter(lambda subdir: not subdir.is_empty(),self.subdirs)

Expand All @@ -126,6 +137,7 @@ def calcStats(self,make_pie=True):

for subdir in self.subdirs:
subdir.mother_dir=join(self.mother_dir,self.name)
subdir.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
subdir.calcStats(make_pie)
subdir.meta=self.meta
self.weight+=subdir.weight
Expand All @@ -134,10 +146,12 @@ def calcStats(self,make_pie=True):
self.n_nulls+=subdir.n_nulls

self.n_skiped+=subdir.n_skiped
self.n_missing_objs += subdir.n_missing_objs

self.rank_histo.Add(subdir.rank_histo)

self.stats_calculated=True
self.stats_calculated=True
self.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
#if make_pie:
#self.__create_pie_image()

Expand Down Expand Up @@ -186,6 +200,7 @@ def print_report(self,indent="",verbose=False):
print " o Nulls: %.2f%% (%s/%s) " %(self.get_null_rate(),self.n_nulls,self.weight)
print " o Successes: %.2f%% (%s/%s) " %(self.get_success_rate(),self.n_successes,self.weight)
print " o Skipped: %.2f%% (%s/%s) " %(self.get_skiped_rate(),self.n_skiped,self.weight)
print " o Missing objects: %s" %(self.n_missing_objs)

def get_skiped_rate(self):
if self.weight == 0: return 0
Expand Down