Tweaks to prepare for a release with bytecode-opy.zip.

- Write the manifest of bytecode-*.zip to disk, and then add a pyc-size metric in build/metrics.sh. - Simplify the build slightly. Put more stuff in oil-manifest.txt. - Add osh --version to spec test HTML. - Make note of OPy divergences from Python.
oilshell · Feb 26, 2018 · 51ffcd4f3aac3ab9b122e983972ef24911a2b3aa · 51ffcd4
1 parent 4dbb64d
commit 51ffcd4f3aac3ab9b122e983972ef24911a2b3aa
Unified Split

Showing with 75 additions and 40 deletions.

+8 −3 bin/oil.py

+4 −9 build/actions.sh

+1 −1 build/make_zip.py

+14 −6 build/metrics.sh

+2 −0 build/oil-manifest.txt

+24 −7 opy/README.md

+9 −11 portable-rules.mk

+13 −3 test/spec.sh
diff --git a/bin/oil.py b/bin/oil.py
@@ -161,9 +161,14 @@ def _ShowVersion():
  finally:
    f.close()

-  f = loader.open('pyc-version.txt')
-  pyc_version = f.readline().strip()
-  f.close()
+  try:
+    f = loader.open('pyc-version.txt')
+  except IOError:
+    pyc_version = '-'  # in dev tree
+  else:
+    pyc_version = f.readline().strip()
+  finally:
+    f.close()

  # What C functions do these come from?
  print('Oil version %s' % version)
diff --git a/build/actions.sh b/build/actions.sh
@@ -81,12 +81,6 @@ py-to-compile() {
    $PREPARE_DIR/python -S build/app_deps.py py $main_module
 }

-files-manifest() {
-  for path in "$@"; do
-    echo "$path $path"
-  done
-}
-
 # For embedding in oil/bytecode.zip.
 quick-ref-manifest() {
  local dir=$1
@@ -96,12 +90,13 @@ quick-ref-manifest() {
 }

 pyc-version-manifest() {
-  local zip_path=${1:-_build/oil/bytecode-opy.zip}  # For example
+  local manifest_path=${1:-_build/oil/bytecode-opy-manifest.txt}  # For example

  # Just show a string like "bytecode-opy.zip" for now.  There is no OPy
  # version yet.
-  local user_str=$(basename $zip_path) 
-  local dir=$(dirname $zip_path)
+  local filename=$(basename $manifest_path) 
+  local user_str=${filename%-manifest.txt}
+  local dir=$(dirname $manifest_path)

  echo $user_str > $dir/pyc-version.txt

diff --git a/build/make_zip.py b/build/make_zip.py
@@ -32,7 +32,7 @@ def main(argv):
    if rel_path in seen:
      expected = seen[rel_path]
      if expected != full_path:
-        print >>sys.sterr, 'WARNING: expected %r, got %r' % (expected,
+        print >>sys.stderr, 'WARNING: expected %r, got %r' % (expected,
            full_path)
      continue

diff --git a/build/metrics.sh b/build/metrics.sh
@@ -3,7 +3,7 @@
 # Stats about build artifacts.
 #
 # Usage:
-#   ./stats.sh <function name>
+#   ./metrics.sh <function name>

 set -o nounset
 set -o pipefail
@@ -14,19 +14,27 @@ set -o errexit
 # hello: 1.41 MB native + 145 KB = 1.56 MB bundle
 # oil:   1.65 MB native + 642 KB = 2.30 MB bundle
 bundle-size() {
-  ls -l _build/*/bytecode.zip _build/*/ovm _bin/*.ovm
+  ls -l _build/*/bytecode-*.zip _build/*/ovm _bin/*.ovm
 }

 linecount-nativedeps() {
  local app_name=${1:-oil}
-  find _tmp/$app_name-tar-test -name '*.[ch]' | xargs wc -l | sort -n
+  find _tmp/${app_name}-tar-test -name '*.[ch]' | xargs wc -l | sort -n
 }

 linecount-pydeps() {
  local app_name=${1:-oil}
-  awk '/\.py$/ { print $1 }' \
-    _build/runpy-deps-py.txt _build/$app_name/app-deps-py.txt |
-  sort | uniq | xargs wc -l | sort -n
+
+  awk '/\.py$/ { print $1 }' _build/$app_name/bytecode-cpython-manifest.txt |
+    sort | uniq | xargs wc -l | sort -n
+}
+
+# Print table of [num_bytes pyc path]
+pyc-size() {
+  local app_name=${1:-oil}
+
+  awk '/\.pyc$/ { print $1 }' _build/$app_name/bytecode-cpython-manifest.txt |
+    sort | uniq | xargs wc --bytes | sort -n
 }

 _tar-lines() {
diff --git a/build/oil-manifest.txt b/build/oil-manifest.txt
@@ -1,3 +1,5 @@
 osh/osh.asdl osh/osh.asdl
 osh/types.asdl osh/types.asdl
 core/runtime.asdl core/runtime.asdl
+oil-version.txt oil-version.txt 
+doc/osh-quick-ref-toc.txt doc/osh-quick-ref-toc.txt
diff --git a/opy/README.md b/opy/README.md
@@ -23,16 +23,33 @@ Test the binary:
    ./test.sh spec smoke
    ./test.sh spec all  # Failures due to $0

-Notes
-----
+OPy Divergences from CPython
+----------------------------

- The OPy bytecode is bigger than the CPython bytecode!
-  - Why is that?
+### Lexer

- One difference: I think there are no `LOAD_FAST` bytecodes generated?
-  - TODO: Make a bytecode histogram using `opy/misc/inspect_pyc`.
+- I don't remember where exactly, but I ran into a bug lexing the CPython test
+  suite.  IIRC, CPython's lexer was more lenient about adjacent tokens without
+  spaces than `tokenize.py`.
+- `heapq.py` had `-*- coding: latin-1 -*-`, which causes problems.  OPy
+  should require `utf-8` source anyway.

+### Parser

- OSH tests don't run under byterun.  I probably don't care.
+- I ran into a bug where a file like `d = {}`, without a trailing newline,
+  gives a parse error.  Adding the newline fixes it.
+- print statements aren't allowed; we force Python 3-style `print(x, y,
+  file=sys.stderr)`.  I think this is because the parser doesn't know about
+  `__future__` statements, so it can't change the parsing mode on the fly.
+
+### Bytecode Compiler
+
+- I think there are no `LOAD_FAST` bytecodes generated?  TODO: Make a bytecode
+  histogram using `opy/misc/inspect_pyc`.
+- The OPy bytecode is bigger than the CPython bytecode!  Why is that?
+
+### Other
+
+OSH tests don't run under byterun.  I probably don't care.

    ./test.sh unit '' byterun
diff --git a/portable-rules.mk b/portable-rules.mk
@@ -125,45 +125,43 @@ _build/oil/py-to-compile.txt: _build/detected-config.sh build/app_deps.py
 _devbuild/gen/osh_help.py: doc/osh-quick-ref-pages.txt
 	build/doc.sh osh-quick-ref

+# NOTE: I should really depend on every file in build/oil-manifest.txt!
 OIL_BYTECODE_DEPS := \
 	_build/release-date.txt \
-	oil-version.txt \
-	doc/osh-quick-ref-toc.txt \
 	build/oil-manifest.txt \
 	_devbuild/gen/osh_help.py

 # NOTES:
 # - _devbuild/gen/osh_help.py is a minor hack to depend on the entire
 #   _devbuild/osh-quick-ref dir, since they both get generated by the same
-#   build action.
+#   build action.  (Hidden targets are known to cause problems with GNU Make.)
 # - release-date.txt is in different location on purpose, so we don't show it
 #   in dev mode.
 # - Do we need $(OIL_SRCS) as dependencies?

-_build/oil/bytecode-cpython.zip: $(OIL_BYTECODE_DEPS) \
+_build/oil/bytecode-cpython-manifest.txt: $(OIL_BYTECODE_DEPS) \
                         _build/oil/app-deps-cpython.txt \
                         _build/runpy-deps-cpython.txt
 	{ echo '_build/release-date.txt release-date.txt'; \
-	  $(ACTIONS_SH) files-manifest oil-version.txt \
-	                               doc/osh-quick-ref-toc.txt; \
 	  cat build/oil-manifest.txt \
 	      _build/oil/app-deps-cpython.txt \
 	      _build/runpy-deps-cpython.txt; \
 	  $(ACTIONS_SH) quick-ref-manifest _devbuild/osh-quick-ref; \
 	  $(ACTIONS_SH) pyc-version-manifest $@; \
-	} | build/make_zip.py $@ 
+	} > $@

 # NOTE: runpy deps are included in opy-app-deps.txt.
-_build/oil/bytecode-opy.zip: $(OIL_BYTECODE_DEPS) \
+_build/oil/bytecode-opy-manifest.txt: $(OIL_BYTECODE_DEPS) \
                         _build/oil/opy-app-deps.txt
 	{ echo '_build/release-date.txt release-date.txt'; \
-	  $(ACTIONS_SH) files-manifest oil-version.txt \
-	                               doc/osh-quick-ref-toc.txt; \
 	  cat build/oil-manifest.txt \
 	      _build/oil/opy-app-deps.txt; \
 	  $(ACTIONS_SH) quick-ref-manifest _devbuild/osh-quick-ref; \
 	  $(ACTIONS_SH) pyc-version-manifest $@; \
-	} | build/make_zip.py $@ 
+	} > $@
+
+_build/oil/bytecode-%.zip: _build/oil/bytecode-%-manifest.txt
+	build/make_zip.py $@ < $^

 #
 # App-Independent Pattern Rules.
diff --git a/test/spec.sh b/test/spec.sh
@@ -91,11 +91,21 @@ version-text() {
  date
  echo

-  local branch=$(git rev-parse --abbrev-ref HEAD)
-  local hash=$(git rev-parse $branch)
-  echo "oil version: $hash on branch $branch"
+  if test -d .git; then
+    local branch=$(git rev-parse --abbrev-ref HEAD)
+    local hash=$(git rev-parse $branch)
+    echo "oil repo: $hash on branch $branch"
+  else
+    echo "(not running from git repository)"
+  fi
  echo

+  for bin in $OSH; do
+    echo "\$ $bin --version"
+    $bin --version
+    echo
+  done
+
  python --version 2>&1
  echo