Merge pull request #14311 from Dr15Jones/handleUnscheduledInStallGrapher

Make edmStreamStallGrapher properly handle unscheduled processing

FWCore/Concurrency/bin/edmStreamStallGrapher.py

@@ -87,16 +87,33 @@ def readLogFile(fileName):
 
 
 #----------------------------------------------
+# Patterns:
+#
+# source: The source is identified by having a 'starting: delayed' followed immediately by a 'finished: delayed'
+#   the total time is actually the wait time plus the time the source was doing work
+# scheduled module: This is identified by no delays before call to 'starting' and then immediate call to 'finished'
+# unscheduled module: Has a 'starting: delayed' followed by a 'starting' followed by a 'finished' the a 'finished: delayed'
+#
+#
 def findStalledModules(processingSteps, numStreams):
   streamTime = [0]*(numStreams+1)
   stalledModules = {}
+  previousStartWasADelayed = [False]*(numStreams+1)
   for n,trans,s,time,delayed in processingSteps:
     waitTime = None
-    if delayed:
-      n = "source"
     if trans == kStarted:
-      waitTime = time - streamTime[s]
+      if delayed:
+        previousStartWasADelayed[s] = True
+        streamTime[s] = time
+      else:
+        previousStartWasADelayed[s] = False
+        waitTime = time - streamTime[s]
     else:
+      if delayed and previousStartWasADelayed[s]:
+        #for a source we only know the combined time for waiting and running
+        n = "source"
+        waitTime = time - streamTime[s]
+        previousStartWasADelayed[s] = False
       streamTime[s] = time
     if waitTime is not None:
       if waitTime > 0.1:
@@ -113,19 +130,33 @@ def createAsciiImage(processingSteps, numStreams, maxNameSize):
   streamTime = [0]*(numStreams+1)
   #lastTime = 0
   seenInit = False
+  previousStartWasADelayed = [False]*(numStreams+1)
   for n,trans,s,time,delayed in processingSteps:
       if n == "FINISH INIT":
           seenInit = True
           continue
       oldState = streamState[s]
       streamState[s]=trans
       waitTime = None
-      if delayed:
-          n = "source"
       if trans == kStarted:
-          waitTime = time - streamTime[s]
-          streamState[s]=2
+          if delayed:
+              previousStartWasADelayed[s] = True
+              streamTime[s] = time
+              continue
+          else:
+              previousStartWasADelayed[s] = False
+              waitTime = time - streamTime[s]
+              streamState[s]=2
       else:
+          if delayed:
+              if previousStartWasADelayed[s]:
+                  n="source"
+                  waitTime = time - streamTime[s]
+                  streamState[s]=2
+                  previousStartWasADelayed[s] = False
+              else:
+                  streamTime[s] = time
+                  continue
           if oldState != trans:
               streamState[s]=3
           streamTime[s] = time
@@ -142,7 +173,7 @@ def createAsciiImage(processingSteps, numStreams, maxNameSize):
       if waitTime is not None:
           states += " %.2f"% waitTime
           if waitTime > 0.1 and seenInit:
-              states += " STALLED"
+              states += " STALLED "+str(time)+" "+str(s)
 
       print states
       streamState[s]=trans
@@ -179,7 +210,7 @@ def sumSort(i,j):
 def createPDFImage(processingSteps, numStreams, stalledModuleInfo):
   import matplotlib.pyplot as plt
 
-  streamStartDepth = [0]*(numStreams+1)
+  previousStartWasADelayed=[False]*(numStreams+1)
   streamTime = [0]*(numStreams+1)
   streamStartTimes = [ [] for x in xrange(numStreams+1)]
   streamColors = [[] for x in xrange(numStreams+1)]
@@ -193,20 +224,23 @@ def createPDFImage(processingSteps, numStreams, stalledModuleInfo):
     if n == "FINISH INIT":
       continue
     if trans == kStarted:
-      streamStartDepth[s] +=1
+      previousStartWasADelayed[s] = delayed
       streamTime[s] = time
     else:
-      streamStartDepth[s] -=1
-      if 0 == streamStartDepth[s]:
-        streamStartTimes[s].append((streamTime[s],time-streamTime[s]))
-        c="green"
-        if delayed:
+      c="green"
+      if delayed:
+        if previousStartWasADelayed[s]:
+          #this was time for a source
           c="orange"
-        elif n in stalledModuleNames:
-          c="red"
+          previousStartWasADelayed[s] = False
+        else:
+            continue
+      streamStartTimes[s].append((streamTime[s],time-streamTime[s]))
+      if c == "green" and n in stalledModuleNames:
+        c="red"
         #elif len(streamColors[s]) %2:
         #  c="blue"
-        streamColors[s].append(c)
+      streamColors[s].append(c)
 
   #consolodate contiguous blocks with the same color
   # this drastically reduces the size of the pdf file