Parallel other tests

.circleci/config.yml

@@ -54,17 +54,12 @@ jobs:
             - .emscripten_ports/
             - .emscripten
 
-  test-other-am:
+  test-other:
     <<: *test-defaults
     environment:
-      - TEST_TARGET=other.test_a* other.test_b* other.test_c* other.test_d* other.test_e* other.test_f* other.test_g* other.test_h* other.test_i* other.test_j* other.test_k* other.test_l* other.test_m* skip:other.test_bad_triple skip:other.test_emcc_v
-      # TODO: remove skips after fastcomp update on travis for 1.37.23
-
-  test-other-nz:
-    <<: *test-defaults
-    environment:
-      - TEST_TARGET=other.test_n* other.test_o* other.test_p* other.test_q* other.test_r* other.test_s* other.test_t* other.test_u* other.test_v* other.test_w* other.test_x* other.test_y* other.test_z* skip:other.test_native_link_error_message
-      # TODO: remove skips after fastcomp update on travis for 1.37.23
+      - TEST_TARGET=other skip:other.test_bad_triple skip:other.test_native_link_error_message skip:other.test_emcc_v skip:other.test_failing_alloc
+      # some native-dependent tests fail because of the lack of native headers on emsdk-bundled clang
+      # CircleCI actively kills memory-over-consuming process
   test-browser:
     <<: *test-defaults
     environment:
@@ -134,10 +129,7 @@ workflows:
   build-test:
     jobs:
       - build
-      - test-other-am:
-          requires:
-            - build
-      - test-other-nz:
+      - test-other:
           requires:
             - build
       - test-browser:

.travis.yml

@@ -3,8 +3,7 @@ sudo: required
 language: python
 
 env:
-- TEST_TARGET="other.test_a* other.test_b* other.test_c* other.test_d* other.test_e* other.test_f* other.test_g* other.test_h* other.test_i* other.test_j* other.test_k* other.test_l* other.test_m*"
-- TEST_TARGET="other.test_n* other.test_o* other.test_p* other.test_q* other.test_r* other.test_s* other.test_t* other.test_u* other.test_v* other.test_w* other.test_x* other.test_y* other.test_z*"
+- TEST_TARGET=other
 - TEST_TARGET=browser
 - TEST_TARGET="ALL.test_a* ALL.test_b*"
 - TEST_TARGET=ALL.test_c*

emcc.py

@@ -82,7 +82,6 @@
 if DEBUG == "0":
   DEBUG = None
 
-TEMP_DIR = os.environ.get('EMCC_TEMP_DIR')
 LEAVE_INPUTS_RAW = os.environ.get('EMCC_LEAVE_INPUTS_RAW') # Do not compile .ll files into .bc, just compile them with emscripten directly
                                                            # Not recommended, this is mainly for the test runner, or if you have some other
                                                            # specific need.
@@ -573,16 +572,10 @@ def uniquename(name):
   else:
     final_suffix = ''
 
-  if TEMP_DIR:
-    temp_dir = TEMP_DIR
-    if os.path.exists(temp_dir):
-      shutil.rmtree(temp_dir) # clear it
-    os.makedirs(temp_dir)
-  else:
-    temp_root = shared.TEMP_DIR
-    if not os.path.exists(temp_root):
-      os.makedirs(temp_root)
-    temp_dir = tempfile.mkdtemp(dir=temp_root)
+  temp_root = shared.TEMP_DIR
+  if not os.path.exists(temp_root):
+    os.makedirs(temp_root)
+  temp_dir = tempfile.mkdtemp(dir=temp_root)
 
   def in_temp(name):
     return os.path.join(temp_dir, os.path.basename(name))
@@ -1920,14 +1913,10 @@ def get_eliminate():
     if DEBUG: logging.debug('total time: %.2f seconds', (time.time() - start_time))
 
   finally:
-    if not TEMP_DIR:
-      try:
-        shutil.rmtree(temp_dir)
-      except:
-        pass
-    else:
-      logging.info('emcc saved files are in:' + temp_dir)
-
+    try:
+      shutil.rmtree(temp_dir)
+    except:
+      pass
 
 def parse_args(newargs):
   options = EmccOptions()

tests/parallel_runner.py

@@ -4,16 +4,19 @@
 import subprocess
 import sys
 import unittest
+import tempfile
+import shutil
 
 try:
   import queue
 except ImportError:
   # Python 2 compatibility
   import Queue as queue
 
-def g_testing_thread(work_queue, result_queue):
+def g_testing_thread(work_queue, result_queue, temp_dir):
   for test in iter(lambda: get_from_queue(work_queue), None):
     result = BufferedParallelTestResult()
+    test.set_temp_dir(temp_dir)
     try:
       test(result)
     except Exception as e:
@@ -62,9 +65,10 @@ def reversed_tests(self):
   def init_processes(self, test_queue):
     self.processes = []
     self.result_queue = multiprocessing.Queue()
-    for i in range(num_cores()):
+    self.dedicated_temp_dirs = [tempfile.mkdtemp() for x in range(num_cores())]
+    for temp_dir in self.dedicated_temp_dirs:
       p = multiprocessing.Process(target=g_testing_thread,
-                                  args=(test_queue, self.result_queue))
+                                  args=(test_queue, self.result_queue, temp_dir))
       p.start()
       self.processes.append(p)
 
@@ -76,6 +80,8 @@ def collect_results(self):
         buffered_results.append(res)
       else:
         self.clear_finished_processes()
+    for temp_dir in self.dedicated_temp_dirs:
+      shutil.rmtree(temp_dir)
     return buffered_results
 
   def clear_finished_processes(self):

tests/runner.py

@@ -155,6 +155,7 @@ def get_browser():
 
 class RunnerCore(unittest.TestCase):
   emcc_args = None
+  temp_dir = TEMP_DIR
   save_dir = os.environ.get('EM_SAVE_DIR')
   save_JS = 0
   stderr_redirect = STDOUT # This avoids cluttering the test runner output, which is stderr too, with compiler warnings etc.
@@ -184,18 +185,24 @@ def uses_memory_init_file(self):
       # side modules handle memory differently; binaryen puts the memory in the wasm module
       return ('-O2' in self.emcc_args or '-O3' in self.emcc_args or '-Oz' in self.emcc_args) and not (Settings.SIDE_MODULE or Settings.BINARYEN)
 
+  def set_temp_dir(self, temp_dir):
+    self.temp_dir = temp_dir
+    self.canonical_temp_dir = get_canonical_temp_dir(self.temp_dir)
+    # Explicitly set dedicated temporary directory for parallel tests
+    os.environ['EMCC_TEMP_DIR'] = self.temp_dir
+
   def setUp(self):
     Settings.reset()
 
     if self.EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS:
-      for root, dirnames, filenames in os.walk(TEMP_DIR):
+      for root, dirnames, filenames in os.walk(self.temp_dir):
         for dirname in dirnames: self.temp_files_before_run.append(os.path.normpath(os.path.join(root, dirname)))
         for filename in filenames: self.temp_files_before_run.append(os.path.normpath(os.path.join(root, filename)))
 
     self.banned_js_engines = []
     self.use_all_engines = use_all_engines
     if not self.save_dir:
-      dirname = tempfile.mkdtemp(prefix='emscripten_test_' + self.__class__.__name__ + '_', dir=TEMP_DIR)
+      dirname = tempfile.mkdtemp(prefix='emscripten_test_' + self.__class__.__name__ + '_', dir=self.temp_dir)
     else:
       dirname = CANONICAL_TEMP_DIR
     if not os.path.exists(dirname):
@@ -221,7 +228,7 @@ def tearDown(self):
 
       if self.EM_TESTRUNNER_DETECT_TEMPFILE_LEAKS and not os.environ.get('EMCC_DEBUG'):
         temp_files_after_run = []
-        for root, dirnames, filenames in os.walk(TEMP_DIR):
+        for root, dirnames, filenames in os.walk(self.temp_dir):
           for dirname in dirnames: temp_files_after_run.append(os.path.normpath(os.path.join(root, dirname)))
           for filename in filenames: temp_files_after_run.append(os.path.normpath(os.path.join(root, filename)))
 
@@ -1277,7 +1284,7 @@ def flattened_tests(loaded_tests):
   return tests
 
 def suite_for_module(module, tests):
-  suite_supported = module.__name__ == 'test_core'
+  suite_supported = module.__name__ in ('test_core', 'test_other')
   has_multiple_tests = len(tests) > 1
   has_multiple_cores = parallel_runner.num_cores() > 1
   if suite_supported and has_multiple_tests and has_multiple_cores:

tests/test_other.py

@@ -30,15 +30,18 @@ def __exit__(self, type, value, traceback):
       try_delete(self.directory)
 
 class clean_write_access_to_canonical_temp_dir(object):
+  def __init__(self, dir=CANONICAL_TEMP_DIR):
+    self.canonical_temp_dir = dir
+
   def clean_emcc_files_in_temp_dir(self):
-    for x in os.listdir(CANONICAL_TEMP_DIR):
+    for x in os.listdir(self.canonical_temp_dir):
       if x.startswith('emcc-') or x.startswith('a.out'):
-        os.unlink(os.path.join(CANONICAL_TEMP_DIR, x))
+        os.unlink(os.path.join(self.canonical_temp_dir, x))
 
   def __enter__(self):
-    self.CANONICAL_TEMP_DIR_exists = os.path.exists(CANONICAL_TEMP_DIR)
+    self.CANONICAL_TEMP_DIR_exists = os.path.exists(self.canonical_temp_dir)
     if not self.CANONICAL_TEMP_DIR_exists:
-      os.makedirs(CANONICAL_TEMP_DIR)
+      os.makedirs(self.canonical_temp_dir)
     else:
       # Delete earlier files in the canonical temp directory so that
       # previous leftover files don't have a possibility of confusing
@@ -47,7 +50,7 @@ def __enter__(self):
 
   def __exit__(self, type, value, traceback):
     if not self.CANONICAL_TEMP_DIR_exists:
-      try_delete(CANONICAL_TEMP_DIR)
+      try_delete(self.canonical_temp_dir)
       pass
     else:
       self.clean_emcc_files_in_temp_dir()
@@ -391,7 +394,7 @@ def test_emcc_cache_flag(self):
 
   def test_emcc_cflags(self):
     # see we print them out
-    with clean_write_access_to_canonical_temp_dir(): # --cflags needs to set EMCC_DEBUG=1, which needs to create canonical temp directory.
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir): # --cflags needs to set EMCC_DEBUG=1, which needs to create canonical temp directory.
       output = run_process([PYTHON, EMCC, '--cflags'], stdout=PIPE, stderr=PIPE)
     flags = output.stdout.strip()
     self.assertContained(' '.join(Building.doublequote_spaces(COMPILER_OPTS)), flags)
@@ -2043,19 +2046,19 @@ def test_emcc_debug_files(self):
         print(opts, debug)
         try:
           if debug: os.environ['EMCC_DEBUG'] = debug
-          with clean_write_access_to_canonical_temp_dir():
+          with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
             check_execute([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-O'+ str(opts)], stderr=PIPE)
             if debug is None:
-              for x in os.listdir(CANONICAL_TEMP_DIR):
+              for x in os.listdir(self.canonical_temp_dir):
                 if x.startswith('emcc-'):
                   assert 0
             elif debug == '1':
-              assert os.path.exists(os.path.join(CANONICAL_TEMP_DIR, 'emcc-0-linktime.bc'))
-              assert os.path.exists(os.path.join(CANONICAL_TEMP_DIR, 'emcc-1-original.js'))
+              assert os.path.exists(os.path.join(self.canonical_temp_dir, 'emcc-0-linktime.bc'))
+              assert os.path.exists(os.path.join(self.canonical_temp_dir, 'emcc-1-original.js'))
             elif debug == '2':
-              assert os.path.exists(os.path.join(CANONICAL_TEMP_DIR, 'emcc-0-basebc.bc'))
-              assert os.path.exists(os.path.join(CANONICAL_TEMP_DIR, 'emcc-1-linktime.bc'))
-              assert os.path.exists(os.path.join(CANONICAL_TEMP_DIR, 'emcc-2-original.js'))
+              assert os.path.exists(os.path.join(self.canonical_temp_dir, 'emcc-0-basebc.bc'))
+              assert os.path.exists(os.path.join(self.canonical_temp_dir, 'emcc-1-linktime.bc'))
+              assert os.path.exists(os.path.join(self.canonical_temp_dir, 'emcc-2-original.js'))
         finally:
           if debug: del os.environ['EMCC_DEBUG']
 
@@ -2075,7 +2078,7 @@ def test_debuginfo(self):
           (['-O2', '-g4'], True, True), # drop llvm debug info as js opts kill it anyway
         ]:
         print(args, expect_llvm, expect_js)
-        with clean_write_access_to_canonical_temp_dir():
+        with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
           err = run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp')] + args, stdout=PIPE, stderr=PIPE).stderr
         assert expect_llvm == ('strip-debug' not in err)
         assert expect_js == ('registerize' not in err)
@@ -3290,7 +3293,7 @@ def test_os_oz(self):
           (['-O3'], 'LLVM opts: -O3'),
         ]:
         print(args, expect)
-        with clean_write_access_to_canonical_temp_dir():
+        with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
           err = run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp')] + args, stdout=PIPE, stderr=PIPE).stderr
         self.assertContained(expect, err)
         if '-O3' in args or '-Oz' in args or '-Os' in args:
@@ -5445,7 +5448,7 @@ def test(args, expected):
       try:
         os.environ['EMCC_DEBUG'] = '1'
         os.environ['EMCC_NATIVE_OPTIMIZER'] = '1'
-        with clean_write_access_to_canonical_temp_dir():
+        with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
           err = run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-O2',] + args, stderr=PIPE).stderr
       finally:
         if old_debug: os.environ['EMCC_DEBUG'] = old_debug
@@ -5484,7 +5487,7 @@ def test_emconfigure_js_o(self):
           del os.environ['EMCONFIGURE_JS']
 
   def test_emcc_c_multi(self):
-    with clean_write_access_to_canonical_temp_dir():
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
       def test(args, llvm_opts=None):
         print(args)
         lib = r'''
@@ -7101,7 +7104,7 @@ def test(p1, p2, p3, last, expected):
 C c;
 int main() {}
       ''')
-      with clean_write_access_to_canonical_temp_dir():
+      with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
         err = run_process([PYTHON, EMCC, 'src.cpp', '-Oz'], stderr=PIPE).stderr
       self.assertContained('___syscall54', err) # the failing call should be mentioned
       self.assertContained('ctorEval.js', err) # with a stack trace
@@ -7405,7 +7408,7 @@ def test_output_eol(self):
   def test_binaryen_opts(self):
     if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
 
-    with clean_write_access_to_canonical_temp_dir():
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
       try:
         os.environ['EMCC_DEBUG'] = '1'
         for args, expect_js_opts, expect_only_wasm in [
@@ -7469,7 +7472,7 @@ def test_binaryen_and_precise_f32(self):
         ]:
         print(args, expect)
         try_delete('a.out.js')
-        with clean_write_access_to_canonical_temp_dir():
+        with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
           err = run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.cpp'), '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'] + args, stdout=PIPE, stderr=PIPE).stderr
         assert expect == (' -emscripten-precise-f32' in err), err
         self.assertContained('hello, world!', run_js('a.out.js'))
@@ -7625,7 +7628,7 @@ def test_binaryen_ctors(self):
 
   # test debug info and debuggability of JS output
   def test_binaryen_debug(self):
-    with clean_write_access_to_canonical_temp_dir():
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
       if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
       try:
         os.environ['EMCC_DEBUG'] = '1'
@@ -7665,7 +7668,7 @@ def test_binaryen_debug(self):
         del os.environ['EMCC_DEBUG']
 
   def test_binaryen_ignore_implicit_traps(self):
-    with clean_write_access_to_canonical_temp_dir():
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
       if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
       sizes = []
       try:
@@ -7803,7 +7806,7 @@ def test(filename, expectations):
 
   # test disabling of JS FFI legalization
   def test_legalize_js_ffi(self):
-    with clean_write_access_to_canonical_temp_dir():
+    with clean_write_access_to_canonical_temp_dir(self.canonical_temp_dir):
       for (args,js_ffi) in [
           (['-s', 'LEGALIZE_JS_FFI=1', '-s', 'SIDE_MODULE=1', '-O2'], True),
           (['-s', 'LEGALIZE_JS_FFI=0', '-s', 'SIDE_MODULE=1', '-O2'], False),