Run test262 jobs only on ES5 builds (#217)

This patch fixes the 31 failures in test262 on travis which is caused by running the ES5 testsuite with ES6 build. Signed-off-by: Daniel Balla <dballa@inf.u-szeged.hu>
Samsung · May 13, 2019 · 576720c · 576720c
1 parent 94f8bfa
commit 576720c
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 18 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -21,11 +21,14 @@ matrix:
       env:
         - PKG_CONFIG_PATH=/usr/local/opt/icu4c/lib/pkgconfig
       script:
-        - cmake -H. -Bout/darwin/x64/release -DESCARGOT_HOST=darwin -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja
+        - cmake -H. -Bout/darwin/x64/release -DESCARGOT_HOST=darwin -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=1
         - ninja -Cout/darwin/x64/release
         - cp ./out/darwin/x64/release/escargot ./escargot
-        - travis_wait tools/run-tests.py --arch=x86_64 test262
         - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js internal jsc-stress regression-tests v8 es2015
+        - cmake -H. -Bout/darwin/x64/release -DESCARGOT_HOST=darwin -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=0
+        - ninja -Cout/darwin/x64/release
+        - cp ./out/darwin/x64/release/escargot ./escargot
+        - tools/run-tests.py test262
         # FIXME: jetstream-only-simple and octane takes too long running time on darwin
         # jetstream-only-simple and octane are skipped now.
         # - travis_wait 40 tools/run-tests.py --arch=x86_64 jetstream-only-simple octane
@@ -46,22 +49,29 @@ matrix:
       install:
         - npm install
       script:
-        - cmake -H. -Bout/linux/x64/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja
+        - cmake -H. -Bout/linux/x64/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=1
         - ninja -Cout/linux/x64/release
         - cp ./out/linux/x64/release/escargot ./escargot
-        - travis_wait 30 tools/run-tests.py --arch=x86_64 jetstream-only-simple test262 chakracore
-        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js internal jsc-stress v8 spidermonkey regression-tests es2015
-        - tools/run-tests.py --arch=x86_64 octane
+        - travis_wait 30 tools/run-tests.py --arch=x86_64 jetstream-only-simple chakracore
+        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js internal octane jsc-stress v8 spidermonkey regression-tests es2015
+        - cmake -H. -Bout/linux/x64/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=0
+        - ninja -Cout/linux/x64/release
+        - cp ./out/linux/x64/release/escargot ./escargot
+        - tools/run-tests.py --arch=x86_64 test262
 
     - name: "linux.x64.debug"
       addons:
         apt:
           packages: [ autoconf, automake, libtool, ninja-build, libicu-dev ]
       script:
-        - cmake -H. -Bout/linux/x64/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja
+        - cmake -H. -Bout/linux/x64/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja -DESCARGOT_ENABLE_ES2015=1
+        - ninja -Cout/linux/x64/debug
+        - cp ./out/linux/x64/debug/escargot ./escargot
+        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js internal regression-tests es2015
+        - cmake -H. -Bout/linux/x64/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja -DESCARGOT_ENABLE_ES2015=0
         - ninja -Cout/linux/x64/debug
         - cp ./out/linux/x64/debug/escargot ./escargot
-        - tools/run-tests.py --arch=x86_64 jetstream-only-cdjs sunspider-js test262 internal regression-tests es2015
+        - tools/run-tests.py --arch=x86_64 test262
 
     - name: "linux.x86.release"
       addons:
@@ -70,22 +80,29 @@ matrix:
       install:
         - npm install
       script:
-        - cmake -H. -Bout/linux/x86/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja
+        - cmake -H. -Bout/linux/x86/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=1
+        - ninja -Cout/linux/x86/release
+        - cp ./out/linux/x86/release/escargot ./escargot
+        - travis_wait 30 tools/run-tests.py --arch=x86 jetstream-only-simple chakracore
+        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js internal octane jsc-stress v8 spidermonkey regression-tests es2015
+        - cmake -H. -Bout/linux/x86/release -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=release -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=0
         - ninja -Cout/linux/x86/release
         - cp ./out/linux/x86/release/escargot ./escargot
-        - travis_wait 30 tools/run-tests.py --arch=x86 jetstream-only-simple test262 chakracore
-        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js internal jsc-stress v8 spidermonkey regression-tests es2015
-        - tools/run-tests.py --arch=x86 octane
+        - tools/run-tests.py --arch=x86 test262
 
     - name: "linux.x86.debug"
       addons:
         apt:
           packages: [ autoconf, automake, libtool, ninja-build, gcc-multilib g++-multilib, "libicu-dev:i386" ]
       script:
-        - cmake -H. -Bout/linux/x86/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja
+        - cmake -H. -Bout/linux/x86/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -GNinja -DESCARGOT_ENABLE_ES2015=1
+        - ninja -Cout/linux/x86/debug
+        - cp ./out/linux/x86/debug/escargot ./escargot
+        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js internal regression-tests es2015
+        - cmake -H. -Bout/linux/x86/debug -DESCARGOT_HOST=linux -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=debug -DESCARGOT_OUTPUT=bin -DVENDORTEST=1 -GNinja -DESCARGOT_ENABLE_ES2015=0
         - ninja -Cout/linux/x86/debug
         - cp ./out/linux/x86/debug/escargot ./escargot
-        - tools/run-tests.py --arch=x86 jetstream-only-cdjs sunspider-js test262 internal regression-tests es2015
+        - tools/run-tests.py --arch=x86 test262
 
     - name: "SonarQube"
       addons:

diff --git a/build/config.cmake b/build/config.cmake
@@ -31,9 +31,12 @@ SET (ESCARGOT_DEFINITIONS_COMMON
     -DESCARGOT_ENABLE_TYPEDARRAY
     -DESCARGOT_ENABLE_PROMISE
     -DESCARGOT_ENABLE_PROXY_REFLECT
-    -DESCARGOT_ENABLE_ES2015
 )
 
+IF (ESCARGOT_ENABLE_ES2015 OR NOT DEFINED ESCARGOT_ENABLE_ES2015)
+    SET (ESCARGOT_DEFINITIONS_COMMON ${ESCARGOT_DEFINITIONS_COMMON} -DESCARGOT_ENABLE_ES2015)
+ENDIF()
+
 SET (CXXFLAGS_FROM_ENV $ENV{CXXFLAGS})
 SEPARATE_ARGUMENTS(CXXFLAGS_FROM_ENV)
 SET (ESCARGOT_CXXFLAGS_COMMON

diff --git a/tools/run-tests.py b/tools/run-tests.py
@@ -57,7 +57,7 @@ def __call__(self, fn):
         return fn
 
 
-def run(args, cwd=None, env=None, stdout=None, checkresult=True):
+def run(args, cwd=None, env=None, stdout=None, checkresult=True, report=False):
     if cwd:
         print(COLOR_BLUE + 'cd ' + cwd + ' && \\' + COLOR_RESET)
     if env:
@@ -70,8 +70,22 @@ def run(args, cwd=None, env=None, stdout=None, checkresult=True):
         full_env.update(env)
         env = full_env
 
-    proc = Popen(args, cwd=cwd, env=env, stdout=stdout)
+    proc = Popen(args, cwd=cwd, env=env, stdout=PIPE if report else stdout)
+
+    counter = 0
+
+    while report:
+        nextline = proc.stdout.readline()
+        if nextline == '' and proc.poll() is not None:
+            break
+        stdout.write(nextline)
+        stdout.flush()
+        if counter % 250 == 0:
+            print('Ran %d tests..' % (counter))
+        counter += 1
+
     out, _ = proc.communicate()
+
     if out:
         print(out)
 
@@ -140,12 +154,26 @@ def run_test262(engine, arch):
     copy(join(TEST262_OVERRIDE_DIR, 'excludelist.orig.xml'), join(TEST262_DIR, 'test', 'config', 'excludelist.xml'))
     copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py'))
 
+    out = open('test262_out', 'w')
+
     run(['python', join('tools', 'packaging', 'test262.py'),
          '--command', engine,
          '--full-summary'],
         cwd=TEST262_DIR,
-        env={'TZ': 'US/Pacific'})
+        env={'TZ': 'US/Pacific'},
+        stdout=out,
+        report=True)
+
+    out.close()
+
+    with open('test262_out', 'r') as out:
+        full = out.read()
+        summary = full.split('=== Summary ===')[1]
+        if summary.find('- All tests succeeded') < 0:
+            print(summary)
+            raise Exception('test262 failed')
 
+        print('test262: All tests passed')
 
 @runner('test262-master')
 def run_test262_master(engine, arch):