Run OSH under "OPyPy". pgen2 not hooked up though.

- bin/opypy-osh script that runs OSH under compiler2/byterun ("OPyPy"). - Tone down byterun logging so we don't spew to stderr. - Track the delta for opypy-osh. - All tests behave the same as osh, i.e. cpython-osh! Also: - Investigated determinism problems with compile2. I think we will concentrate on functional tests for now.
oilshell · Apr 16, 2017 · 2b331d92e81eb5b3395355bb58c50aade12c7b5b · 2b331d9
1 parent bd7a886
commit 2b331d92e81eb5b3395355bb58c50aade12c7b5b
Unified Split

Showing with 119 additions and 62 deletions.

+9 −0 bin/opypy-osh

+1 −1 opy/byterun/pyvm2.py

+0 −55 opy/run.sh

+81 −1 opy/smoke.sh

+13 −1 sh_spec.py

+9 −2 spec-runner.sh

+5 −1 spec.sh

+1 −1 tests/loop.test.sh
diff --git a/bin/opypy-osh b/bin/opypy-osh
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# OSH running on OPyPy.  OPyPy is the OPy front end with byterun.
+
+readonly THIS_DIR=$(cd $(dirname $0) && pwd)
+readonly OIL_DIR=$THIS_DIR/..
+
+export PYTHONPATH=$OIL_DIR/opy/_tmp/compile2 
+$OIL_DIR/opy/byterun/__main__.py -c $OIL_DIR/opy/_tmp/osh-compile2/bin/oil.pyc osh "$@"
diff --git a/opy/byterun/pyvm2.py b/opy/byterun/pyvm2.py
@@ -238,7 +238,7 @@ def dispatch(self, byteName, arguments):
        except:
            # deal with exceptions encountered while executing the op.
            self.last_exception = sys.exc_info()[:2] + (None,)
-            log.exception("Caught exception during execution")
+            log.info("Caught exception during execution")
            why = 'exception'

        return why
diff --git a/opy/run.sh b/opy/run.sh
@@ -108,33 +108,10 @@ stdlib-determinism() {
  compare-files _tmp/det/$file.{1,2}
 }

-determinism-loop() {
-  local func=$1
-  local file=./opy_main.py
-
-  for i in $(seq 100); do
-    echo "--- $i ---"
-    $func $file _tmp/det/$file.1
-    $func $file _tmp/det/$file.2
-
-    local size1=$(stat --format '%s' _tmp/det/$file.1)
-    local size2=$(stat --format '%s' _tmp/det/$file.2)
-    if test $size1 != $size2; then
-      compare-files _tmp/det/$file.{1,2}
-      echo "Found problem after $i iterations"
-      break
-    fi
-  done
-}
-
 stdlib-determinism-loop() {
  determinism-loop _stdlib-compile-one 
 }

-compile2-determinism-loop() {
-  determinism-loop _compile2-one 
-}
-

 # We want to fix the bug here.  Hm not able to hit it?
 compile2-determinism() {
@@ -166,38 +143,6 @@ stdlib-compile2() {
 export PYTHONHASHSEED=0
 #export PYTHONHASHSEED=random

-inspect-pyc() {
-  PYTHONPATH=. misc/inspect_pyc.py "$@"
-}
-
-# For comparing different bytecode.
-compare-files() {
-  local left=$1
-  local right=$2
-
-  md5sum "$@"
-  ls -l "$@"
-
-  inspect-pyc $left > _tmp/pyc-left.txt
-  inspect-pyc $right > _tmp/pyc-right.txt
-  $DIFF _tmp/pyc-{left,right}.txt || true
-
-  return
-  strings $left > _tmp/str-left.txt
-  strings $right > _tmp/str-right.txt
-
-  # The ORDER of strings is definitely different.  But the SIZE and CONTENTS
-  # are too!
-  # Solution: can you walk your own code objects and produce custom .pyc files?
-
-  diff -u _tmp/str-{left,right}.txt || true
-
-  #xxd $left > _tmp/hexleft.txt
-  #xxd $right > _tmp/hexright.txt
-  #diff -u _tmp/hex{left,right}.txt || true
-  echo done
-}
-
 compare-opy-tree() {
  diff -u _tmp/opy-{stdlib,stdlib2}/SIZES.txt || true
  #diff -u _tmp/opy-{stdlib,stdlib2}/MD5.txt || true
diff --git a/opy/smoke.sh b/opy/smoke.sh
@@ -117,8 +117,11 @@ compile-osh-tree() {
              -name '*.py' -a -printf '%P\n') )

  _compile-tree $src _tmp/osh-ccompile/ ccompile "${files[@]}"
-  _compile-tree $src _tmp/osh-stdlib/ stdlib "${files[@]}"
  _compile-tree $src _tmp/osh-compile2/ compiler2 "${files[@]}"
+
+  # Not deterministic!
+  #_compile-tree $src _tmp/osh-compile2.gold/ compiler2 "${files[@]}"
+  #_compile-tree $src _tmp/osh-stdlib/ stdlib "${files[@]}"
 }

 fill-osh-tree() {
@@ -175,6 +178,10 @@ EOF
  time byterun -c _tmp/speed.opyc 10000
 }

+#
+# Byterun smoke tests
+#
+
 # Wow!  Runs itself to parse itself... I need some VM instrumentation to make
 # sure it's not accidentally cheating or leaking.
 opy-parse-on-byterun() {
@@ -193,4 +200,77 @@ osh-parse-on-byterun() {
  byterun -c _tmp/osh-compile2/bin/oil.pyc "${cmd[@]}"
 }

+#
+# Determinism
+#
+# There are problems here, but it's because of an underlying Python 2.7 issue.
+# For now we will do functional tests.
+#
+
+# Doesn't suffice for for compiler2 determinism.
+#export PYTHONHASHSEED=0
+
+inspect-pyc() {
+  PYTHONPATH=. misc/inspect_pyc.py "$@"
+}
+
+# For comparing different bytecode.
+compare-bytecode() {
+  local left=$1
+  local right=$2
+
+  md5sum "$@"
+  ls -l "$@"
+
+  inspect-pyc $left > _tmp/pyc-left.txt
+  inspect-pyc $right > _tmp/pyc-right.txt
+  $DIFF _tmp/pyc-{left,right}.txt || true
+
+  return
+  strings $left > _tmp/str-left.txt
+  strings $right > _tmp/str-right.txt
+
+  # The ORDER of strings is definitely different.  But the SIZE and CONTENTS
+  # are too!
+  # Solution: can you walk your own code objects and produce custom .pyc files?
+
+  diff -u _tmp/str-{left,right}.txt || true
+
+  #xxd $left > _tmp/hexleft.txt
+  #xxd $right > _tmp/hexright.txt
+  #diff -u _tmp/hex{left,right}.txt || true
+  echo done
+}
+
+# Compile opy_main a 100 times and make sure it's the same.
+#
+# NOTE: This doesn't surface all the problems.  Remember the fix was in
+# compiler2/misc.py:Set.
+determinism-loop() {
+  local func=$1
+  local file=${2:-./opy_main.py}
+
+  mkdir -p _tmp/det
+
+  local name=$(basename $file)
+  for i in $(seq 100); do
+    echo "--- $i ---"
+    $func $file _tmp/det/$name.1
+    $func $file _tmp/det/$name.2
+
+    local size1=$(stat --format '%s' _tmp/det/$name.1)
+    local size2=$(stat --format '%s' _tmp/det/$name.2)
+    if test $size1 != $size2; then
+      # TODO: Import from run.sh
+      compare-bytecode _tmp/det/$file.{1,2}
+      echo "Found problem after $i iterations"
+      break
+    fi
+  done
+}
+
+compile2-determinism-loop() {
+  determinism-loop _compile2-one ../core/lexer.py
+}
+
 "$@"
diff --git a/sh_spec.py b/sh_spec.py
@@ -374,6 +374,8 @@ def RunCases(cases, case_predicate, shells, env, out):
  stats['num_cases'] = len(cases)
  stats['osh_num_passed'] = 0
  stats['osh_num_failed'] = 0
+  # Number of opypy-osh results that differed from cpython-osh a.k.a. osh.
+  stats['opypy_delta'] = 0

  # Make an environment for each shell.  $SH is the path to the shell, so we
  # can test flags, etc.
@@ -436,7 +438,11 @@ def RunCases(cases, case_predicate, shells, env, out):
      result_row.append(cell_result)

      if cell_result == Result.FAIL:
-        stats['num_failed'] += 1
+        # Special logic: don't count opypy-osh beacuse its failures will be
+        # counted in the delta.
+        if sh_label != 'opypy-osh':
+          stats['num_failed'] += 1
+
        if sh_label == 'osh':
          stats['osh_num_failed'] += 1
      elif cell_result == Result.BUG:
@@ -452,6 +458,12 @@ def RunCases(cases, case_predicate, shells, env, out):
      else:
        raise AssertionError

+      if sh_label == 'opyopy-osh':
+        opypy_result = result_row[-1]
+        cpython_result = result_row[-2]
+        if opypy_result != cpython_result:
+          stats['opypy_delta'] += 1
+
    out.WriteRow(i, line_num, result_row, desc)

  return stats
diff --git a/spec-runner.sh b/spec-runner.sh
@@ -50,7 +50,7 @@ run-cases() {
      --format html \
      --stats-file _tmp/spec/${spec_name}.stats.txt \
      --stats-template \
-        '%(num_cases)d %(osh_num_passed)d %(osh_num_failed)d %(osh_failures_allowed)d' \
+      '%(num_cases)d %(osh_num_passed)d %(osh_num_failed)d %(osh_failures_allowed)d %(opypy_delta)d' \
    > _tmp/spec/${spec_name}.html
 }

@@ -81,6 +81,7 @@ run-task-with-status-test() {
 }

 readonly NUM_TASKS=400
+#readonly NUM_TASKS=4

 # TODO:
 #
@@ -101,7 +102,9 @@ _html-summary() {
  <thead>
    <tr>
      <td>name</td> <td>Exit Code</td> <td>Elapsed Seconds</td>
-      <td># cases</td> <td>osh # passed</td> <td>osh # failed</td> <td>osh failures allowed </d>
+      <td># cases</td> <td>osh # passed</td> <td>osh # failed</td>
+      <td>osh failures allowed</td>
+      <td>opypy-osh delta</td>
    </tr>
  </thead>
 EOF
@@ -125,13 +128,15 @@ EOF
    osh_num_passed = $2
    osh_num_failed = $3
    osh_failures_allowed = $4
+    opypy_delta = $5

    sum_status += status
    sum_wall_secs += wall_secs
    sum_num_cases += num_cases
    sum_osh_num_passed += osh_num_passed
    sum_osh_num_failed += osh_num_failed
    sum_osh_failures_allowed += osh_failures_allowed
+    sum_opypy_delta += opypy_delta
    num_rows += 1

    # For the console
@@ -159,6 +164,7 @@ EOF
    print "<td>" osh_num_passed "</td>"
    print "<td>" osh_num_failed "</td>"
    print "<td>" osh_failures_allowed "</td>"
+    print "<td>" opypy_delta "</td>"
    print "</tr>"
  }

@@ -172,6 +178,7 @@ EOF
    print "<td>" sum_osh_num_passed "</td>"
    print "<td>" sum_osh_num_failed "</td>"
    print "<td>" sum_osh_failures_allowed "</td>"
+    print "<td>" sum_opypy_delta "</td>"
    print "</tr>"
    print "</tfoot>"

diff --git a/spec.sh b/spec.sh
@@ -13,7 +13,11 @@ readonly MKSH=/bin/mksh
 readonly ZSH=/usr/bin/zsh  # Ubuntu puts it here
 readonly BUSYBOX_ASH=_tmp/shells/ash 

-readonly OSH=bin/osh
+
+# HACK that relies on word splitting
+# TODO: Use ${OSH[@]} everywhere
+readonly OSH='bin/osh bin/opypy-osh'
+#readonly OSH=bin/osh

 # ash and dash are similar, so not including ash by default.  zsh is not quite
 # POSIX.
diff --git a/tests/loop.test.sh b/tests/loop.test.sh
@@ -116,7 +116,7 @@ find bin/ -type f | ( while read path; do
  #echo $i
 done
 echo $i )
-# stdout: 1
+# stdout: 2

 ### until loop
 # This is just the opposite of while?  while ! cond?