Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Generate csv files from condor_history execution for job length and c…

…ompletion date.
  • Loading branch information...
commit 8fbe664af9b64d109a006c9f126d58096b29cb18 1 parent e483646
authored March 16, 2012

Showing 1 changed file with 117 additions and 0 deletions. Show diff stats Hide diff stats

  1. 117  condor_csvs.py
117  condor_csvs.py
... ...
@@ -0,0 +1,117 @@
  1
+#!/bin/env python
  2
+
  3
+import csv
  4
+import commands
  5
+import datetime
  6
+import logging
  7
+import os
  8
+import sys
  9
+
  10
+class Execution( object ):
  11
+    def __init__( self ):
  12
+        self.logger = logging.getLogger("Execution")
  13
+
  14
+    def execute( self, cmd ):
  15
+        """Executes the command and prints error results"""
  16
+
  17
+        result = commands.getstatusoutput( cmd )
  18
+
  19
+        if result[0] == 0:
  20
+            return result[1]
  21
+        else:
  22
+            self.logger.error( result )
  23
+            return None
  24
+            
  25
+class Condor( object ):
  26
+    def __init__( self ):
  27
+        self.logger = logging.getLogger("Condor")
  28
+        self.data = []
  29
+        self.binned = None
  30
+
  31
+    def Count( self, cmd ):
  32
+        self.logger.debug("Starting count")
  33
+
  34
+        self.logger.debug("Getting Condor History Data")
  35
+        history_result = Execution()
  36
+        self.data.append( history_result.execute( cmd ).split() )
  37
+
  38
+    def OutputDates( self, bin_size ):
  39
+        csvfile = csv.writer( open("job_completion_count_by_date.csv", "w") )
  40
+
  41
+        for ii in sorted(self.binned.keys()):
  42
+            dt = datetime.date.fromtimestamp(ii*bin_size)
  43
+            dt_count = self.binned[ii]
  44
+            print "%s %d" % ( dt, dt_count )
  45
+            csvfile.writerow( [dt, dt_count] )
  46
+
  47
+    def Rebin( self, bin_factor ):
  48
+        self.logger.debug("Rebin")
  49
+        self.binned = {}
  50
+        for ii in self.data:
  51
+            for jj in ii:
  52
+                bin_number = int( int(jj) / bin_factor)
  53
+                if bin_number in self.binned:
  54
+                    self.binned[bin_number] = self.binned[bin_number] + 1
  55
+                else:
  56
+                    self.binned[bin_number] = 1
  57
+            
  58
+    def OutputJobLength( self ):
  59
+        csvfile = csv.writer( open("jobs_completed_count_by_execution_length.csv", "w") )
  60
+
  61
+        for ii in sorted(self.binned):
  62
+            print "%d %d" % ( ii, self.binned[ii] )
  63
+            csvfile.writerow( [ii, self.binned[ii]] )
  64
+        
  65
+class StartTime( object ):
  66
+    def __init__( self ):
  67
+        self.logger = logging.getLogger("StartTime")
  68
+
  69
+    def Get( self ): 
  70
+        date_cmd = "date -d '-365 days' +%s"
  71
+        execution = Execution()
  72
+        return execution.execute(date_cmd)
  73
+        
  74
+def main():
  75
+    logging.basicConfig()
  76
+    logging.root.setLevel( logging.DEBUG )
  77
+    logger = logging.root
  78
+
  79
+    startTime = StartTime()
  80
+    start_time = startTime.Get()
  81
+    print start_time
  82
+    logger.debug( "Start time is %d" % int(start_time) )
  83
+
  84
+    job_completion = Condor()
  85
+    job_length = Condor()
  86
+
  87
+    listing = os.listdir("./")
  88
+    for ii in listing:
  89
+        if ii.startswith( "history" ):
  90
+            logger.debug( "History file %s" % ii )
  91
+            job_completion_cmd = \
  92
+                "condor_history -f %s -constraint 'CompletionDate>=%s' -format '%%i\n' CompletionDate" \
  93
+                % ( ii, start_time )
  94
+            job_count_cmd = \
  95
+                "condor_history -f %s -constraint 'CompletionDate>=%s' -format '%%i\n' RemoteWallClockTime" \
  96
+                % ( ii, start_time )
  97
+            job_completion.Count( job_completion_cmd )
  98
+            job_length.Count( job_count_cmd )
  99
+
  100
+    # bin job completion rate by day
  101
+    #
  102
+    job_completion.Rebin( 86400 )
  103
+    #
  104
+    # bin completed job length by hour
  105
+    #
  106
+    job_length.Rebin( 3600 ) 
  107
+
  108
+    job_completion.OutputDates( 86400 )
  109
+    job_length.OutputJobLength()
  110
+
  111
+    sys.exit(0)
  112
+
  113
+
  114
+if __name__ == "__main__":
  115
+        main()
  116
+
  117
+

0 notes on commit 8fbe664

Please sign in to comment.
Something went wrong with that request. Please try again.