bug 832465: develop script/automation to replay real update requests …

…against balrog. r=nthomas

auslib/util/testing.py

@@ -2,17 +2,28 @@
 
 import requests
 
+from mozilla_buildtools.retry import retry
 
-def compare_snippets(server1, server2, paths, diff=True, retries=3, timeout=10,
-                     raise_exceptions=True):
-    cfg = {'max_retries': retries, 'danger_mode': raise_exceptions}
-    for path in paths:
-        url1 = '%s/%s' % (server1, path)
-        url2 = '%s/%s' % (server2, path)
-        xml1 = requests.get(url1, timeout=timeout, config=cfg).content.splitlines()
-        xml2 = requests.get(url2, timeout=timeout, config=cfg).content.splitlines()
-        if xml1 != xml2:
-            if diff:
-                yield (url1, xml1, url2, xml2, difflib.unified_diff(xml1, xml2, lineterm=""))
-            else:
-                yield (url1, xml1, url2, xml2)
+
+def compare_snippets(url1, url2, retries=3, timeout=10, diff=True):
+    cfg = {'danger_mode': True}
+    xml1 = retry(requests.get, sleeptime=5, attempts=retries, args=(url1,),
+                 retry_exceptions=(requests.HTTPError, requests.ConnectionError),
+                 kwargs={'timeout': timeout, 'config': cfg})
+    xml1 = xml1.content.splitlines()
+    xml2 = retry(requests.get, sleeptime=5, attempts=retries, args=(url2,),
+                 retry_exceptions=(requests.HTTPError, requests.ConnectionError),
+                 kwargs={'timeout': timeout, 'config': cfg})
+    xml2 = xml2.content.splitlines()
+    ret = [url1, xml1, url2, xml2]
+    if xml1 != xml2:
+        if diff:
+            difflines = []
+            for line in difflib.unified_diff(xml1, xml2, url1, url2, lineterm=""):
+                difflines.append(line)
+            ret.append(difflines)
+        else:
+            ret.append(True)
+    else:
+        ret.append(False)
+    return ret

scripts/compare-update-requests.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python
 
 import os.path
+from Queue import Queue
 import site
 import sys
+import traceback
+from threading import Thread
 
 site.addsitedir(os.path.join(os.path.dirname(__file__), "../lib/python"))
 site.addsitedir(os.path.join(os.path.dirname(__file__), "../lib/python/vendor"))
@@ -13,13 +16,25 @@
 from auslib.util.testing import compare_snippets
 
 
+def worker(server1, server2, callback, failure_callback):
+    while not q.empty():
+        try:
+            path = q.get()
+            url1 = '%s/%s' % (server1, path)
+            url2 = '%s/%s' % (server2, path)
+            res = compare_snippets(url1, url2)
+            callback(path, res)
+        except:
+            failure_callback(path, traceback.format_exc())
+
 if __name__ == '__main__':
     from argparse import ArgumentParser
 
     parser = ArgumentParser()
     parser.add_argument('aus_servers', metavar='aus_server', nargs=2,
         help='AUS Servers to make requests against. There must be two.')
     parser.add_argument('--paths-file', dest='paths_file', required=True)
+    parser.add_argument('-j', dest='concurrency', default=1, type=int)
 
     args = parser.parse_args()
 
@@ -31,14 +46,50 @@
     except ValueError:
         paths = open(args.paths_file).readlines()
 
-    rc = 0
-    for url1, xml1, url2, xml2, diff in compare_snippets(server1, server2, paths):
+    count = success = fail = error = rc = 0
+
+    def printer(path, res):
+        global success, fail, rc
+        url1, _, url2, _, diff = res
+        if diff:
+            fail += 1
+            rc = 1
+            print 'FAIL: Unmatched snippets on %s:' % path
+            for line in diff:
+                print line
+            print '---- end of diff'
+        else:
+            success += 1
+            print 'PASS: %s' % path
+
+    def failure(path, tb):
+        global error, rc
+        error += 1
         rc = 1
-        print 'Unmatched snippets:'
-        print url1.strip()
-        print url2.strip()
-        for line in diff:
-            print line
-        print '---- end of diff\n'
+        print 'ERROR: %s' % path
+        print tb
+
+    q = Queue()
+    for path in paths:
+        count += 1
+        q.put(path.strip())
+
+    threads = []
+    for i in range(args.concurrency):
+        t = Thread(target=worker, args=(server1, server2, printer, failure))
+        # Marking the threads as "daemon" means they can be killed. Without
+        # this, the program will not exit until all threads exit naturally.
+        # http://docs.python.org/2/library/threading.html#thread-objects
+        t.daemon = True
+        t.start()
+        threads.append(t)
+
+    for t in threads:
+        while t.isAlive():
+            t.join(1)
 
+    print "Tested %d paths." % count
+    print "Pass count: %d" % success
+    print "Fail count: %d" % fail
+    print "Error count: %d" % error
     sys.exit(rc)