Merge pull request #582 from anorkus/super-awesome-useless-branch

RelMon updates and fixes

Utilities/RelMon/python/directories2html.py

@@ -3,8 +3,8 @@
 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
 #
 # $Author: anorkus $
-# $Date: 2012/11/21 15:22:14 $
-# $Revision: 1.5 $
+# $Date: 2013/07/09 14:00:21 $
+# $Revision: 1.10 $
 
 #
 #                                                                              
@@ -57,10 +57,10 @@ def build_obj(run,sample,version,plot_path,tier):
   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
   return encode_obj_url(obj_url)
 
-def fairy_url(run,sample,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
+def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
-  fairy_url+= build_obj(run,sample,version1,plot_path,tier1)
-  fairy_url+= build_obj(run,sample,version2,plot_path,tier2)
+  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
+  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
   if len(draw_opts)>0:
     fairy_url+="drawopts=%s;" %draw_opts
   fairy_url+= plot_size(h,w)
@@ -76,7 +76,10 @@ def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200)
 
 #-------------------------------------------------------------------------------
 style_location="/cms-service-reldqm"
-def get_page_header(directory=None,additional_header=""):
+def get_page_header(directory=None, standalone=False, additional_header=""):
+  style_location="/cms-service-reldqm"
+  if standalone:
+    style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
   javascripts=''
   style=''
   if directory!=None and len(directory.comparisons)>0:
@@ -117,25 +120,30 @@ def get_page_footer():
 
 #-------------------------------------------------------------------------------
 
-def get_title_section(directory, hashing_flag, depth=2):
+def get_title_section(directory, hashing_flag, standalone, depth=2):
+  if standalone:
+      cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
+  else:
+      cms_logo_url = "cms-service-reldqm/style/CMS.gif"
   mother_name=basename(directory.mother_dir)
   mother_file_name=""
   if depth==1:
     mother_file_name="../RelMonSummary.html"
     if mother_name!="":
       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
   elif depth==2:
-    #mother_file_name="RelMonSummary.html"
-    mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
+    mother_file_name="RelMonSummary.html"
+    #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
     if mother_name!="":
       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
   else:
       if hashing_flag:
           files = directory.mother_dir.split("/")
           if len(files) != 1:
-              dir_name = files[-2] ##return the mother directory name only as the html file name by it
+              dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
           else:
               dir_name = files[-1]
+          dir_name = directory.mother_dir
           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
       else:
           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
@@ -149,7 +157,7 @@ def get_title_section(directory, hashing_flag, depth=2):
         '<h1>%s</h1>'%link_to_mother+\
         '</div>'+\
         '<div class="span-3 last">'+\
-        '<img src="cms-service-reldqm/style/CMS.gif" class="top right" width="54" hight="54">'+\
+        '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
         '</div>'+\
         '<hr>' 
   if len(mother_name)>0:
@@ -169,6 +177,8 @@ def get_dir_stats(directory):
     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
   if directory.n_skiped>0:
     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
+  if directory.n_missing_objs>0:
+    html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
   html+='</ul>'
   return html
 
@@ -187,7 +197,7 @@ def get_subdirs_section(directory, hashing_flag):
   for subdir in sorted_subdirs:
     name=subdir.name
     if hashing_flag:
-        link = "%s.html" %(hash_name(name, hashing_flag))
+        link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
     else:
         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
         link=link.strip("_")
@@ -256,7 +266,7 @@ def get_summary_section(directory,matrix_page=True):
         '<div class="span-9 last">'
   if matrix_page:
     html+='<h3>Sample:</h3>'+\
-          '<p class="caps">%s</p>'%meta.sample+\
+          '<p class="caps">%s</p>'%meta.sample1+\
           '<h3>Run1 and Run2:</h3>'+\
           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
   html+='<h3>Releases:</h3>'+\
@@ -305,29 +315,33 @@ def get_comparisons(category,directory):
                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
     else:
       big_fairy=fairy_url(directory.meta.run1,
-                        directory.meta.sample,
+                        directory.meta.run2,
+                        directory.meta.sample1,
+                        directory.meta.sample2,
                         directory.meta.release1,
                         directory.meta.release2,
                         comp_abs_path,
                         directory.meta.tier1,
                         directory.meta.tier2,
                         "",600,600)
       small_fairy=fairy_url(directory.meta.run1,
-                        directory.meta.sample,
+                        directory.meta.run2,
+                        directory.meta.sample1,
+                        directory.meta.sample2,
                         directory.meta.release1,
                         directory.meta.release2,
                         comp_abs_path,
                         directory.meta.tier1,
                         directory.meta.tier2)    
 
       single_fairy1=fairy_url_single(directory.meta.run1,
-                                   directory.meta.sample,
+                                   directory.meta.sample1,
                                    directory.meta.release1,
                                    comp_abs_path,
                                    directory.meta.tier1,
                                    "",500,500)
       single_fairy2=fairy_url_single(directory.meta.run2,
-                                   directory.meta.sample,
+                                   directory.meta.sample2,
                                    directory.meta.release2,
                                    comp_abs_path,
                                    directory.meta.tier2,
@@ -401,7 +415,21 @@ def get_rank_section(directory):
 
 #-------------------------------------------------------------------------------
 
-def directory2html(directory, hashing, depth=0):
+#-------------------------------------------------------------------------------
+def get_missing_objs_section(directory):
+    """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
+    """
+    page_html = "Unpaired in %s</br>"%(directory.filename1)
+    for elem in directory.different_histograms['file1']:
+        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
+    page_html +="</br>"
+    page_html += "Unpaired in %s</br>"%(directory.filename2)
+    for elem in directory.different_histograms['file2']:
+        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
+    return page_html
+#-------------------------------------------------------------------------------
+
+def directory2html(directory, hashing, standalone, depth=0):
   """Converts a directory tree into html pages, very nice ones.
   """
   #print "d2html: depth", str(depth)," dir ",directory.name
@@ -414,10 +442,10 @@ def directory2html(directory, hashing, depth=0):
     #chdir(directory.name)
 
   for subdir in directory.subdirs:
-    directory2html(subdir,hashing, depth)
+    directory2html(subdir,hashing,standalone, depth)
 
-  page_html=get_page_header(directory)+\
-            get_title_section(directory,hashing, depth)+\
+  page_html=get_page_header(directory, standalone)+\
+            get_title_section(directory,hashing, standalone, depth)+\
             get_summary_section(directory)+\
             get_subdirs_section(directory, hashing)
 
@@ -428,6 +456,9 @@ def directory2html(directory, hashing, depth=0):
                      (directory.n_comp_skiped >0,SKIPED)):
     if do_cat:
       page_html+=get_comparisons(cat,directory)
+
+  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
+    page_html += get_missing_objs_section(directory)
 
   # Distribution of ranks
 
@@ -442,7 +473,12 @@ def directory2html(directory, hashing, depth=0):
   if len(page_name)==0:
     page_name="RelMonSummary"
   if hashing:
-      ofilename = "%s.html" %(hash_name(page_name, hashing))
+      if page_name != "RelMonSummary":
+          #print " ## oFile path: %s"%(join(directory.full_path))
+          #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
+          ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
+      else:
+          ofilename = "RelMonSummary.html"
   else:
       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
       ofilename=ofilename.strip("_")
@@ -659,7 +695,7 @@ def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_ag
 
 #-------------------------------------------------------------------------------
 
-def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag):
+def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
   """Create a table, with as rows the directories and as columns the samples.
   Each box in the table will contain a pie chart linking to the directory.
   """  
@@ -709,7 +745,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
 
-  page_html = get_page_header(additional_header=directories_barchart+categories_barchart)
+  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
   rel1=""
   rel2=""
   try:
@@ -727,6 +763,8 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
       all_subdirs.append(subdir_name)
   all_subdirs=sorted(list(set(all_subdirs)))
 
+  #print " $$ all_subdirs: %s" %(all_subdirs)
+
   # Get The title
   page_html+= '<div class="span-20">'+\
               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
@@ -811,7 +849,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
   for sample in sorted_samples:
     col=dir_dict[sample]
     # check if the directory was a top one or not
-    summary_page_name=hash_name("RelMonSummary", hashing_flag)+".html"
+    summary_page_name="RelMonSummary.html"
     if col.name!="":
       summary_page_name=hash_name(col.name, hashing_flag)+".html"
     img_link=col.get_summary_chart_ajax(55,55)
@@ -849,10 +887,18 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
       subdirs_dict=directory.get_subdirs_dict()
 
       # Check if the directory is the top one
-      summary_page=join(sample,"%s.html"%(hash_name(subdir_name, hashing_flag)))
       if directory.name!="":
-        # We did not run on the topdir     
-        summary_page=join(sample,"%s_%s.html"%(directory.name,hash_name(subdir_name,hashing_flag)))
+        # We did not run on the topdir
+        #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
+        #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
+        if hashing_flag:
+          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
+        else:
+          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
+      else:
+        #print "   ## summary_page: %s"%(directory.name+subdir_name)
+        #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
+        summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
       dir_is_there=subdirs_dict.has_key(subdir_name)
 
       img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
@@ -886,6 +932,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
 
 #-----------UPDATES------
 def hash_name(file_name, flag):
+    #print "     HashFILE name: "+file_name
     if flag: #if hashing flag is ON then return
         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
     else:

Utilities/RelMon/python/dirstructure.py

@@ -2,9 +2,9 @@
 # RelMon: a tool for automatic Release Comparison                              
 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
 #
-# $Author: dpiparo $
-# $Date: 2012/07/03 05:38:00 $
-# $Revision: 1.2 $
+# $Author: anorkus $
+# $Date: 2013/07/05 09:45:01 $
+# $Revision: 1.5 $
 #
 #                                                                              
 # Danilo Piparo CERN - danilo.piparo@cern.ch                                   
@@ -48,8 +48,9 @@ def __init__(self,name,weight=1):
 
 #-------------------------------------------------------------------------------
 class CompInfo(object):
-  def __init__(self,sample="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
-    self.sample=sample
+  def __init__(self,sample1="",sample2="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
+    self.sample1=sample1
+    self.sample2=sample2
     self.release1=release1
     self.release2=release2
     self.run1=run1
@@ -79,7 +80,14 @@ def __init__(self,name,mother_dir="",meta=CompInfo(),draw_success=False,do_pngs=
     self.do_pngs=do_pngs
     self.rank_histo=TH1I("rh%s"%name,"",50,-0.01,1.001)
     self.rank_histo.SetDirectory(0)
-
+    self.different_histograms = {}
+    self.different_histograms['file1']= {}
+    self.different_histograms['file2']= {}
+    self.filename1 = ""
+    self.filename2 = ""
+    self.n_missing_objs = 0
+    self.full_path = ""
+
   def is_empty(self):
     if len(self.subdirs)==0 and len(self.comparisons)==0:
       return True
@@ -102,7 +110,10 @@ def calcStats(self,make_pie=True):
 
     self.n_skiped = 0
     self.n_comp_skiped = 0
-
+    self.n_missing_objs = len(self.different_histograms['file1'].keys())+len(self.different_histograms['file2'].keys())
+    if self.n_missing_objs != 0:
+      print "    [*] Missing in %s: %s" %(self.filename1, self.different_histograms['file1'])
+      print "    [*] Missing in %s: %s" %(self.filename2, self.different_histograms['file2'])
     # clean from empty dirs    
     self.subdirs = filter(lambda subdir: not subdir.is_empty(),self.subdirs)    
 
@@ -126,6 +137,7 @@ def calcStats(self,make_pie=True):
 
     for subdir in self.subdirs:
       subdir.mother_dir=join(self.mother_dir,self.name)
+      subdir.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
       subdir.calcStats(make_pie)
       subdir.meta=self.meta 
       self.weight+=subdir.weight
@@ -134,10 +146,12 @@ def calcStats(self,make_pie=True):
       self.n_nulls+=subdir.n_nulls
 
       self.n_skiped+=subdir.n_skiped
+      self.n_missing_objs += subdir.n_missing_objs
 
       self.rank_histo.Add(subdir.rank_histo)
 
-    self.stats_calculated=True 
+    self.stats_calculated=True
+    self.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
     #if make_pie:
       #self.__create_pie_image()
 
@@ -186,6 +200,7 @@ def print_report(self,indent="",verbose=False):
       print " o Nulls: %.2f%% (%s/%s) " %(self.get_null_rate(),self.n_nulls,self.weight)
       print " o Successes: %.2f%% (%s/%s) " %(self.get_success_rate(),self.n_successes,self.weight)
       print " o Skipped: %.2f%% (%s/%s) " %(self.get_skiped_rate(),self.n_skiped,self.weight)
+      print " o Missing objects: %s" %(self.n_missing_objs)
 
   def get_skiped_rate(self):
     if self.weight == 0: return 0