Various fixes to release automation.

- Update HAVE_ROOT - Change the interface of test/spec.sh to accept OSH_LIST=, rather than OSH_OVM= - Change scripts/release.sh to run spec tests first with _bin/osh as a smoke test, and then with 3 binaries: CPython, OVM, and byterun. The second run will have its HTML table published. - test/sh_spec.py: - Don't assume that we're only running 2 variants of OSH. - Consolidate duplicated constants.
oilshell · Aug 15, 2018 · 53893fc93eb674ee43e77fc3b513fd863d6523c2 · 53893fc
1 parent d81643c
commit 53893fc93eb674ee43e77fc3b513fd863d6523c2
Unified Split

Showing with 78 additions and 42 deletions.

+0 −1 build/test.sh

+3 −0 opy/common.sh

+4 −3 opy/test.sh

+16 −18 scripts/release.sh

+35 −13 test/sh_spec.py

+1 −0 test/sh_spec_test.py

+19 −7 test/spec.sh
diff --git a/build/test.sh b/build/test.sh
@@ -82,5 +82,4 @@ ovm-main-func() {

 }

-
 "$@"
diff --git a/opy/common.sh b/opy/common.sh
@@ -12,6 +12,9 @@ set -o errexit

 readonly GRAMMAR=_tmp/py27.grammar.pickle

+# Used by scripts/release.sh too.
+readonly OSH_BYTERUN=opy/_tmp/repo-with-opy/bin/osh-byterun 
+
 log() {
  echo "$@" >&2
 }
diff --git a/opy/test.sh b/opy/test.sh
@@ -175,8 +175,7 @@ spec() {

  # HACK to get around __import__ problem with byterun.

-  local stub=opy/_tmp/repo-with-opy/bin/osh-byterun 
-  OSH_OVM=$stub test/spec.sh $action "$@"
+  OSH_LIST="bin/osh $OSH_BYTERUN" test/spec.sh $action "$@"
  popd
 }

@@ -292,4 +291,6 @@ fib-callgraph() {
  PYTHONPATH=.. CALLGRAPH=1 gold/fib_recursive.py
 }

-"$@"
+if test $(basename $0) = 'test.sh'; then
+  "$@"
+fi
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -5,10 +5,11 @@
 #
 # Steps:
 #   build/doc.sh update-src-versions  (optional)
-#   $0 build-and-test  (builds tarball, runs unit/spec/gold tests, etc.)
+#   $0 build-and-test  (builds tarball, runs unit,gold, initial spec tests, etc.)
 #     prereq: build/codegen.sh {download,install}-re2c
 #   test/wild.sh all (3-4 minutes on fast machine)
 #   $0 test-opy (2 minutes on fast machine)
+#   $0 spec-all  # tests 3 OSH binaries
 #   $0 metrics
 #   benchmarks:
 #     Sync up oilshell/benchmark-data repo.
@@ -42,6 +43,8 @@ readonly OIL_VERSION=$(head -n 1 oil-version.txt)
 # Dir is defined in build/test.sh.
 readonly OSH_RELEASE_BINARY=_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/osh

+source opy/common.sh  # For OSH_BYTERUN
+
 log() {
  echo "$@" 1>&2
 }
@@ -135,7 +138,7 @@ _release-build() {
  #         oil.ovm
 }

-readonly HAVE_ROOT=''
+readonly HAVE_ROOT=1

 _test-release-build() {
  # NOTE: Need test/alpine.sh download;extract;setup-dns,add-oil-build-deps,
@@ -152,10 +155,8 @@ _test-release-build() {
  test/osh2oil.sh run-for-release
  test/gold.sh run-for-release

-  # TODO: Assert that this was done with hermetic binaries?
-
-  # spec-tests-with-tar-build
-  OSH_OVM=$OSH_RELEASE_BINARY test/spec.sh all
+  # Just test the release build (not under CPython or byterun.  That comes later.)
+  OSH_LIST="$OSH_RELEASE_BINARY" test/spec.sh all
 }

 # NOTE: Following opy/README.md.  Right now this is a quick and dirty
@@ -188,22 +189,19 @@ test-opy() {
  time ./test.sh oil-unit-byterun > $out/$step.txt 2>&1
  echo $?

-  # TODO: this need its own table output.  It overwrites the output of
-  # 'test/spec.sh all' from the previous run.
-
-  step='test-spec-all'
-  echo "--- $step ---"
-  time ./test.sh spec all > $out/$step.txt 2>&1
-  echo $?
-
-  # NOTE: This is sensitive to Python 2.7.12 vs .13 vs .14, so don't bother
-  # publishing it for now.
-  #./regtest.sh compile > $out/regtest-compile.txt
-  #./regtest.sh verify-golden > $out/regtest-verify-golden.txt
+  # NOTE: This is sensitive to Python 2.7.12 vs .13 vs .14.  Ideally we would
+  # remove that.
+  ./regtest.sh compile > $out/regtest-compile.txt
+  ./regtest.sh verify-golden > $out/regtest-verify-golden.txt

  popd
 }

+spec-all() {
+  OSH_LIST="bin/osh $OSH_RELEASE_BINARY $OSH_BYTERUN" test/spec.sh all
+}
+
+
 # TODO: Log this whole thing?  Include logs with the /release/ page?
 build-and-test() {
  # 5 steps: clean, dev build, unit tests, release build, end-to-end tests.
diff --git a/test/sh_spec.py b/test/sh_spec.py
@@ -59,6 +59,15 @@
 import time


+# Magic strings for other variants of OSH.
+
+# NOTE: osh_ALT is usually _bin/osh -- the release binary.
+# It would be better to rename these osh-cpython and osh-ovm.  Have the concept
+# of a suffix?  Then we can have osh-byterun too.
+
+OTHER_OSH = ('osh_ALT', 'osh-byterun') 
+
+
 class ParseError(Exception):
  pass

@@ -357,16 +366,15 @@ def CreateAssertions(case, sh_label):
  status = False

  # So the assertion are exactly the same for osh and osh_ALT
-  if sh_label in ('osh_ALT', 'osh-byterun'):
-    sh_label = 'osh'
+  case_sh = 'osh' if sh_label in OTHER_OSH else sh_label

-  if sh_label in case:
-    q = case[sh_label]['qualifier']
-    if CreateStringAssertion(case[sh_label], 'stdout', assertions, qualifier=q):
+  if case_sh in case:
+    q = case[case_sh]['qualifier']
+    if CreateStringAssertion(case[case_sh], 'stdout', assertions, qualifier=q):
      stdout = True
-    if CreateStringAssertion(case[sh_label], 'stderr', assertions, qualifier=q):
+    if CreateStringAssertion(case[case_sh], 'stderr', assertions, qualifier=q):
      stderr = True
-    if CreateIntAssertion(case[sh_label], 'status', assertions, qualifier=q):
+    if CreateIntAssertion(case[case_sh], 'status', assertions, qualifier=q):
      status = True

  if not stdout:
@@ -451,6 +459,15 @@ def RunCases(cases, case_predicate, shells, env, out):
    e['SH'] = sh_path
    sh_env.append(e)

+  # Determine which one (if any) is osh-cpython, for comparison against other
+  # shells.
+  osh_cpython_index = -1
+  for i, (sh_label, _) in enumerate(shells):
+    if sh_label == 'osh':
+      osh_cpython_index = i
+      break
+
+  # Now run each case, and print a table.
  for i, case in enumerate(cases):
    line_num = case['line_num']
    desc = case['desc']
@@ -508,7 +525,7 @@ def RunCases(cases, case_predicate, shells, env, out):
      if cell_result == Result.FAIL:
        # Special logic: don't count osh_ALT beacuse its failures will be
        # counted in the delta.
-        if sh_label not in ('osh_ALT', 'osh-byterun'):
+        if sh_label not in OTHER_OSH:
          stats['num_failed'] += 1

        if sh_label == 'osh':
@@ -526,10 +543,14 @@ def RunCases(cases, case_predicate, shells, env, out):
      else:
        raise AssertionError

-      if sh_label in ('osh_ALT', 'osh-byterun'):
-        osh_alt_result = result_row[-1]
-        cpython_result = result_row[-2]
-        if osh_alt_result != cpython_result:
+      if sh_label in OTHER_OSH:
+        # This is only an error if we tried to run ANY OSH.
+        if osh_cpython_index == -1:
+          raise RuntimeError("Couldn't determine index of osh-cpython")
+
+        other_result = result_row[shell_index]
+        cpython_result = result_row[osh_cpython_index]
+        if other_result != cpython_result:
          stats['osh_ALT_delta'] += 1

    out.WriteRow(i, line_num, result_row, desc)
@@ -880,8 +901,9 @@ def main(argv):
    name, _ = os.path.splitext(path)
    label = os.path.basename(name)
    if label == 'osh':
+      # change the second 'osh' to 'osh_ALT' so it's distinct
      if saw_osh:
-        label = 'osh_ALT'  # distinct label
+        label = 'osh_ALT'
      else:
        saw_osh = True
    shell_pairs.append((label, path))
diff --git a/test/sh_spec_test.py b/test/sh_spec_test.py
@@ -102,6 +102,7 @@ def testRunCases(self):
      out_f = cStringIO.StringIO()
    out = AnsiOutput(out_f, False)
    RunCases([CASE1], lambda i, case: True, shells, env, out)
+    print(repr(out.f.getvalue()))


 if __name__ == '__main__':
diff --git a/test/spec.sh b/test/spec.sh
@@ -35,17 +35,29 @@ else
  readonly BUSYBOX_ASH=_tmp/shells/ash
 fi

-readonly OSH_PYTHON=${OSH_PYTHON:-bin/osh}
+# Usage: callers can override OSH_LIST to test on more than one version.
+#
+# Example:
+# OSH_LIST='bin/osh _bin/osh' test/spec.sh all
+
+readonly OSH_CPYTHON='bin/osh'
 readonly OSH_OVM=${OSH_OVM:-_bin/osh}

-if test -e $OSH_OVM; then
-  # TODO: Does it make sense to copy the binary to an unrelated to directory,
-  # like /tmp?  /tmp/{oil.ovm,osh}.
+readonly OSH_LIST=${OSH_LIST:-}  # A space-separated list.

-  # HACK that relies on word splitting.  TODO: Use ${OSH[@]} everywhere
-  readonly OSH="$OSH_PYTHON $OSH_OVM"
+# TODO: Would be nicer to use ${OSH[@]} everywhere.
+OSH=''
+if test -n "$OSH_LIST"; then
+  OSH="$OSH_LIST"
 else
-  readonly OSH="$OSH_PYTHON"
+  if test -e $OSH_OVM; then
+    # TODO: Does it make sense to copy the binary to an unrelated to directory,
+    # like /tmp?  /tmp/{oil.ovm,osh}.
+
+    OSH="$OSH_CPYTHON $OSH_OVM"
+  else
+    OSH="$OSH_CPYTHON"
+  fi
 fi