reducing parallel starts for less memory consumption in example test #21

tools/extractTest.py

@@ -130,12 +130,12 @@ def main(options):
             os.makedirs(os.path.dirname(options.python_script))
         pyBatch = open(options.python_script, 'w')
         pyBatch.write('''#!/usr/bin/env python
-import subprocess, sys, os
+import subprocess, sys, os, multiprocessing
 from os.path import abspath, dirname, join
 THIS_DIR = abspath(dirname(__file__))
 SUMO_HOME = os.environ.get("SUMO_HOME", dirname(dirname(THIS_DIR)))
 os.environ["SUMO_HOME"] = SUMO_HOME
-for d, p in [
+calls = [
 ''')
     for source, target, app in targets:
         optionsFiles = defaultdict(list)
@@ -276,7 +276,7 @@ def main(options):
                     call = ['join(SUMO_HOME, "bin", "%s")' % app] + ['"%s"' % a for a in appOptions]
                 prefix = os.path.commonprefix((testPath, os.path.abspath(pyBatch.name)))
                 up = os.path.abspath(pyBatch.name)[len(prefix):].count(os.sep) * "../"
-                pyBatch.write('    (r"%s", subprocess.Popen([%s], cwd=join(THIS_DIR, r"%s%s"))),\n' %
+                pyBatch.write('    (r"%s", [%s], r"%s%s"),\n' %
                               (testPath[len(prefix):], ', '.join(call), up, testPath[len(prefix):]))
             if options.skip_configuration:
                 continue
@@ -339,10 +339,21 @@ def main(options):
         if not haveVariant:
             print("No suitable variant found for %s." % source, file=sys.stderr)
     if options.python_script:
-        pyBatch.write("""]:
-    if p.wait() != 0:
-        print("Error: '%s' failed for '%s'!" % (" ".join(getattr(p, "args", [str(p.pid)])), d))
-        sys.exit(1)\n""")
+        pyBatch.write("""]
+procs = []
+def check():
+    for d, p in procs:
+        if p.wait() != 0:
+            print("Error: '%s' failed for '%s'!" % (" ".join(getattr(p, "args", [str(p.pid)])), d))
+            sys.exit(1)
+
+for dir, call, wd in calls:
+    procs.append((dir, subprocess.Popen(call, cwd=join(THIS_DIR, wd))))
+    if len(procs) == multiprocessing.cpu_count():
+        check()
+        procs = []
+check()
+""")
         pyBatch.close()
         os.chmod(pyBatch.name, os.stat(pyBatch.name).st_mode | stat.S_IXUSR)