Merge pull request #10607 from davidlange6/dasRetries

add a simple retry logic to das queries

Configuration/Applications/python/ConfigBuilder.py

@@ -8,6 +8,7 @@
 import sys
 import re
 import collections
+from subprocess import Popen,PIPE
 import FWCore.ParameterSet.DictTypes as DictTypes
 class Options:
         pass
@@ -122,12 +123,27 @@ def filesFromList(fileName,s=None):
 	return (prim,sec)
 
 def filesFromDASQuery(query,s=None):
-	import os
+	import os,time
 	import FWCore.ParameterSet.Config as cms
 	prim=[]
 	sec=[]
 	print "the query is",query
-	for line in os.popen('das_client.py --query "%s"'%(query)):
+	eC=5
+	count=0
+	while eC!=0 and count<3:
+		if count!=0:
+			print 'Sleeping, then retrying DAS'
+			time.sleep(100)
+		p = Popen('das_client.py --query "%s"'%(query), stdout=PIPE,shell=True)
+		tupleP = os.waitpid(p.pid, 0)
+		eC=tupleP[1]
+		count=count+1
+		pipe=p.stdout.read()
+	if eC==0:
+		print "DAS succeeded after",count,"attempts",eC
+	else:
+		print "DAS failed 3 times- I give up"
+	for line in pipe.split('\n'):
 		if line.count(".root")>=2:
 			#two files solution...
 			entries=line.replace("\n","").split()

Configuration/PyReleaseValidation/python/MatrixRunner.py

@@ -60,7 +60,7 @@ def runTests(self, opt):
                 continue
 
     	    print '\nPreparing to run %s %s' % (wf.numId, item)
-
+            sys.stdout.flush()
             current = WorkFlowRunner(wf,noRun,dryRun,cafVeto, opt.dasOptions, opt.jobReports, opt.nThreads)
     	    self.threadList.append(current)
     	    current.start()
@@ -95,6 +95,7 @@ def count(collect,result):
 
         report+=' '.join(map(str,totpassed))+' tests passed, '+' '.join(map(str,totfailed))+' failed\n'
         print report
+        sys.stdout.flush()
 
         runall_report_name='runall-report-step123-.log'
         runall_report=open(runall_report_name,'w')