Scripts to diagnose OPy nondeterminism.

- build/metrics.sh: function to checksum all .pyc files - Add size in bytes to 'opyc dis-md5' - Make the 'opyc dis' output more deterministic, so we can diff it - Shell functions to rebuild all bytecode, copy from different hosts, and compare - Build tweaks - clean all .pyc in repo, not just those in Python-2.7.13/ - move OPy bytecode to _build/oil/bytecode-opy.zip, which is more consistent.
oilshell · Mar 17, 2018 · d657c8e0dfe62bebfae784ecb6d64ffcaeabeea1 · d657c8e
1 parent 04fedae
commit d657c8e0dfe62bebfae784ecb6d64ffcaeabeea1
Unified Split

Showing with 62 additions and 10 deletions.

+3 −1 build/actions.sh

+11 −3 build/metrics.sh

+7 −1 opy/misc/inspect_pyc.py

+1 −1 opy/opy_main.py

+39 −3 opy/smoke.sh

+1 −1 portable-rules.mk
diff --git a/build/actions.sh b/build/actions.sh
@@ -193,7 +193,9 @@ join-modules() {

 # To test building stdlib.
 clean-pyc() {
-  find $PY27/Lib -name '*.pyc' | xargs --no-run-if-empty -- rm --verbose
+  # skip _chroot, _tmp, etc.  But delete __init__.pyc
+  find . \( -type d -a -name '_*' -a -prune \) -o -name '*.pyc' -a -print |
+    xargs --no-run-if-empty -- rm --verbose
 }

 # NOTE: Not deleting _devbuild, including cpython-full.  Maybe we should, or
diff --git a/build/metrics.sh b/build/metrics.sh
@@ -32,14 +32,22 @@ linecount-pydeps() {
    tee _tmp/pydeps.txt | sort | uniq | xargs wc -l | sort -n
 }

-# Print table of [num_bytes pyc path]
-pyc-bytes() {
+pyc-files() {
  local app_name=${1:-oil}
+  awk '/\.pyc$/ { print $1 }' _build/$app_name/${BYTECODE}-manifest.txt
+}

-  awk '/\.pyc$/ { print $1 }' _build/$app_name/${BYTECODE}-manifest.txt |
+# Print table of [num_bytes pyc_path]
+pyc-bytes() {
+  pyc-files "$@" |
    tee _tmp/pycdeps.txt | sort | uniq | xargs wc --bytes | sort -n
 }

+# Print table of [md5 pyc path]
+pyc-md5() {
+  pyc-files "$@" | xargs bin/opyc dis-md5
+}
+
 _tar-lines() {
  linecount-nativedeps "$@"
  echo
diff --git a/opy/misc/inspect_pyc.py b/opy/misc/inspect_pyc.py
@@ -139,7 +139,13 @@ def out(*args, **kwargs):
          oparg_str = None

          if op in dis.hasconst:
-            oparg_str = '(%r)' % (co.co_consts[oparg],)
+            c = co.co_consts[oparg]
+            if isinstance(c, types.CodeType):
+              # %r prints a memory address, which inhibits diffing
+              oparg_str = '(code object %s %s %s)' % (
+                  c.co_name, c.co_filename, c.co_firstlineno)
+            else:
+              oparg_str = '(%r)' % (c,)

          elif op in dis.hasname:
            oparg_str = '(%s)' % (co.co_names[oparg],)
diff --git a/opy/opy_main.py b/opy/opy_main.py
@@ -295,7 +295,7 @@ def py2st(gr, raw_node):
          if not b:
            break
          h.update(b)
-      print('%s %s' % (h.hexdigest(), path))
+      print('%6d %s %s' % (os.path.getsize(path), h.hexdigest(), path))

  # NOTE: Unused
  elif action == 'old-compile':
diff --git a/opy/smoke.sh b/opy/smoke.sh
@@ -200,8 +200,8 @@ determinism-loop() {

    opyc-dis-md5 _tmp/det/$name.{1,2} | tee _tmp/det/md5.txt
    awk '
-    NR == 1 { left = $1 } 
-    NR == 2 { right = $1 } 
+    NR == 1 { left = $2 } 
+    NR == 2 { right = $2 } 
    END     { if (NR != 2) {
                print "Expected two rows, got " NR
                exit(1)
@@ -241,7 +241,8 @@ stdlib-compile() { misc/stdlib_compile.py "$@"; }
 # FAILS
 opy-determinism-loop() {
  #local file=../core/lexer.py
-  local file=../core/word_compile.py  # flanders has issue
+  #local file=../core/word_compile.py  # FIXED
+  local file=../Python-2.7.13/Lib/genericpath.py
  determinism-loop opyc-compile $file
 }

@@ -270,6 +271,41 @@ hash-determinism-loop() {
  determinism-loop hash-determinism
 }

+rebuild-and-md5() {
+  cd ..
+  make clean-repo
+  make _build/opy/py27.grammar.pickle
+  make _bin/oil.ovm-dbg
+  local out=_tmp/pyc-md5.txt
+  build/metrics.sh pyc-md5 | sort -n | tee $out
+
+  log ""
+  log "Wrote $out"
+}
+
+copy-left-right() {
+  local src=flanders.local:~/git/oilshell/oil
+  mkdir -p _tmp/flanders _tmp/lisa
+  scp $src/_build/oil/bytecode-opy.zip $src/_tmp/flanders.pyc-md5.txt _tmp/flanders
+  src=..
+  cp -v $src/_build/oil/bytecode-opy.zip $src/_tmp/lisa.pyc-md5.txt _tmp/lisa
+}
+
+unzip-left-right() {
+  for host in lisa flanders; do
+    pushd _tmp/$host
+    unzip bytecode-opy.zip core/word_compile.pyc
+    popd
+  done
+}
+
+diff-left-right() {
+  for host in lisa flanders; do
+    opyc-dis _tmp/$host/core/word_compile.pyc > _tmp/$host/word_compile.dis
+  done
+  diff -u _tmp/{lisa,flanders}/word_compile.dis
+}
+
 if test $(basename $0) = 'smoke.sh'; then
  "$@"
 fi
diff --git a/portable-rules.mk b/portable-rules.mk
@@ -66,7 +66,7 @@ _build/%/all-deps-c.txt: build/static-c-modules.txt _build/%/app-deps-c.txt
 # I should make a _build/oil/py.d file and include it?
 _build/%/opy-app-deps.txt: \
 	_build/py-to-compile.txt _build/%/py-to-compile.txt
-	sort $^ | uniq | opy/build.sh compile-manifest _build/$*-with-opy > $@
+	sort $^ | uniq | opy/build.sh compile-manifest _build/$*/bytecode-opy > $@


 PY27 := Python-2.7.13