Building bytecode with OPy via the Makefile.

The old path also exists: - bytecode-cpython.zip vs. bytecode-opy.zip - New path: _build/py-to-compile.txt and _build/oil/py-to-compile.txt - build/runpy_deps.py gets a 'py' action just like build/app_deps.py. - _build/oil/opy-app-deps.txt vs _build/oil/app-deps-cpython.txt (the two names can't match the same pattern) Make _build/oil/bytecode-opy.zip. It's around 10K bigger! Why is that? Also make _build/hello/bytecode-opy.zip.
oilshell · Feb 25, 2018 · ca859061a04f1d742b873a015f963ed09abfab31 · ca85906
1 parent 165a5c6
commit ca859061a04f1d742b873a015f963ed09abfab31
Unified Split

Showing with 191 additions and 92 deletions.

+77 −26 Makefile

+7 −4 build/actions.sh

+4 −8 build/app_deps.py

+1 −1 build/compile.sh

+67 −36 build/runpy_deps.py

+8 −1 opy/README.md

+22 −13 opy/build.sh

+5 −3 opy/common.sh
diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@
 #   Makefile
 #   _build/                 # Intermediate files
 #     oil/                  # The app name
-#       bytecode.zip        # Arch-independent
+#       bytecode-opy.zip        # Arch-independent
 #       main_name.c
 #       module_init.c       # Python module initializer
 #       c-module-srcs.txt   # List of Modules/ etc.
@@ -36,11 +36,17 @@
 #   cpython-full/           # Full CPython build, for dynamically
 #                           # discovering Python/C dependencies
 #   c-module-toc.txt        # What files each module is in
+#   py-to-compile.txt
+#   runpy-deps-c.txt
+#   runpy-deps-cpython.txt
+#   runpy-deps-opy.txt
 #   oil/                    # App-specific dir
+#     py-to-compile.txt
 #     all-deps-c.txt        # App deps plus CPython platform deps
-#     app-deps-c.txt
-#     app-deps-py.txt
-#     bytecode.zip
+#     app-deps-cpython.txt  # compiled with CPython
+#     opy-app-deps.txt      # compiled with OPy, does NOT match app-deps-% !
+#     bytecode-cpython.zip
+#     bytecode-opy.zip
 #     c-module-srcs.txt
 #     main_name.c
 #     module_init.c
@@ -124,6 +130,9 @@ _build/c-module-toc.txt: build/c_module_toc.py
 _build/runpy-deps-%.txt: build/runpy_deps.py
 	$(ACTIONS_SH) runpy-deps _build

+_build/py-to-compile.txt: build/runpy_deps.py
+	$(ACTIONS_SH) runpy-py-to-compile > $@
+
 #
 # Hello App.  Everything below here is app-specific.
 #
@@ -137,28 +146,44 @@ _build/hello/main_name.c:

 # Dependencies calculated by importing main.  The guard is because ovm.d
 # depends on it.  Is that correct?  We'll skip it before 'make dirs'.
-_build/hello/app-deps-%.txt: $(HELLO_SRCS) _build/detected-config.sh \
-	                           build/app_deps.py
+_build/hello/app-deps-%.txt: $(HELLO_SRCS) \
+	_build/detected-config.sh build/app_deps.py
 	test -d _build/hello && \
 	  $(ACTIONS_SH) app-deps hello build/testdata hello

+_build/hello/py-to-compile.txt: \
+	_build/detected-config.sh build/app_deps.py
+	test -d _build/hello && \
+	  $(ACTIONS_SH) py-to-compile build/testdata hello > $@
+
 # NOTE: We could use src/dest paths pattern instead of _build/app?
 #
 # TODO:
 # - Deps need to be better.  Depend on .pyc and .py.    I guess
 #   app-deps hello will compile the .pyc files.  Don't need a separate action.
 #   %.pyc : %py
-_build/hello/bytecode.zip: $(HELLO_SRCS) \
-                           build/testdata/hello-version.txt \
-                           _build/release-date.txt \
-                           _build/hello/app-deps-py.txt \
-                           _build/runpy-deps-py.txt \
-                           build/testdata/hello-manifest.txt
+
+HELLO_BYTECODE_DEPS := \
+	build/testdata/hello-version.txt \
+        _build/release-date.txt \
+	build/testdata/hello-manifest.txt
+
+_build/hello/bytecode-cpython.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
+                           _build/hello/app-deps-cpython.txt \
+                           _build/runpy-deps-cpython.txt
+	{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
+	  echo '_build/release-date.txt release-date.txt'; \
+	  cat build/testdata/hello-manifest.txt \
+	      _build/hello/app-deps-cpython.txt \
+	      _build/runpy-deps-cpython.txt; \
+	} | build/make_zip.py $@
+
+_build/hello/bytecode-opy.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
+                           _build/hello/opy-app-deps.txt
 	{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
 	  echo '_build/release-date.txt release-date.txt'; \
 	  cat build/testdata/hello-manifest.txt \
-	      _build/hello/app-deps-py.txt \
-	      _build/runpy-deps-py.txt; \
+	      _build/hello/opy-app-deps.txt; \
 	} | build/make_zip.py $@

 #
@@ -191,6 +216,10 @@ _build/oil/app-deps-%.txt: _build/detected-config.sh build/app_deps.py
 	test -d _build/oil && \
 	  $(ACTIONS_SH) app-deps oil $(REPO_ROOT) bin.oil

+_build/oil/py-to-compile.txt: _build/detected-config.sh build/app_deps.py
+	test -d _build/oil && \
+	  $(ACTIONS_SH) py-to-compile $(REPO_ROOT) bin.oil > $@
+
 _build/osh_help.py: doc/osh-quick-ref-pages.txt
 	build/doc.sh osh-quick-ref

@@ -201,19 +230,34 @@ _build/osh_help.py: doc/osh-quick-ref-pages.txt
 # action.
 # - release-date is at a different location on purpose, so we don't show it in
 #   dev mode.
-_build/oil/bytecode.zip: oil-version.txt \
-                         _build/release-date.txt \
-                         _build/oil/app-deps-py.txt \
-                         _build/runpy-deps-py.txt \
-                         build/oil-manifest.txt \
-                         _build/osh_help.py \
-                         doc/osh-quick-ref-toc.txt
+
+OIL_BYTECODE_DEPS := \
+	oil-version.txt \
+	_build/release-date.txt \
+	build/oil-manifest.txt \
+	_build/osh_help.py \
+	doc/osh-quick-ref-toc.txt
+
+_build/oil/bytecode-cpython.zip: $(OIL_BYTECODE_DEPS) \
+                         _build/oil/app-deps-cpython.txt \
+                         _build/runpy-deps-cpython.txt
 	{ echo '_build/release-date.txt release-date.txt'; \
 	  $(ACTIONS_SH) files-manifest oil-version.txt \
 	                               doc/osh-quick-ref-toc.txt; \
 	  cat build/oil-manifest.txt \
-	      _build/oil/app-deps-py.txt \
-	      _build/runpy-deps-py.txt; \
+	      _build/oil/app-deps-cpython.txt \
+	      _build/runpy-deps-cpython.txt; \
+	  $(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
+	} | build/make_zip.py $@ 
+
+# NOTE: runpy deps are inncluded in opy-app-deps.txt.
+_build/oil/bytecode-opy.zip: $(OIL_BYTECODE_DEPS) \
+                         _build/oil/opy-app-deps.txt
+	{ echo '_build/release-date.txt release-date.txt'; \
+	  $(ACTIONS_SH) files-manifest oil-version.txt \
+	                               doc/osh-quick-ref-toc.txt; \
+	  cat build/oil-manifest.txt \
+	      _build/oil/opy-app-deps.txt; \
 	  $(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
 	} | build/make_zip.py $@ 

@@ -235,6 +279,13 @@ _build/%/c-module-srcs.txt: \
 _build/%/all-deps-c.txt: build/static-c-modules.txt _build/%/app-deps-c.txt
 	$(ACTIONS_SH) join-modules $^ > $@

+# NOTE: This should really depend on all the .py files.
+# I should make a _build/oil/py.d file and include it?
+_build/%/opy-app-deps.txt: \
+	_build/py-to-compile.txt _build/%/py-to-compile.txt
+	sort $^ | uniq | opy/build.sh compile-manifest _build/%-with-opy > $@
+
+
 PY27 := Python-2.7.13

 # Per-app extension module initialization.
@@ -250,7 +301,7 @@ _build/%/module_init.c: $(PY27)/Modules/config.c.in _build/%/all-deps-c.txt
 # Contain Makefile and associated shell scripts, discovered .c and .py deps,
 # app source.

-_release/%.tar: _build/%/bytecode.zip \
+_release/%.tar: _build/%/bytecode-cpython.zip \
                _build/%/module_init.c \
                _build/%/main_name.c \
                _build/%/c-module-srcs.txt
@@ -278,11 +329,11 @@ _build/%/ovm-cov: _build/%/module_init.c _build/%/main_name.c \
 	$(COMPILE_SH) build $@ $(filter-out $(COMPILE_SH),$^)

 # Make bundles quickly.
-_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/bytecode.zip
+_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/bytecode-cpython.zip
 	cat $^ > $@
 	chmod +x $@

-_bin/%.ovm: _build/%/ovm _build/%/bytecode.zip
+_bin/%.ovm: _build/%/ovm _build/%/bytecode-cpython.zip
 	cat $^ > $@
 	chmod +x $@

diff --git a/build/actions.sh b/build/actions.sh
@@ -39,7 +39,11 @@ c-module-toc() {

 # Modules needed to 'import runpy'.
 runpy-deps() {
-  $PREPARE_DIR/python -S build/runpy_deps.py "$@"
+  $PREPARE_DIR/python -S build/runpy_deps.py both "$@"
+}
+
+runpy-py-to-compile() {
+  $PREPARE_DIR/python -S build/runpy_deps.py py
 }

 # This version gets the paths out of the repo.  But it requires that we
@@ -70,9 +74,8 @@ app-deps() {

 # .py files to compile
 py-to-compile() {
-  local app_name=${1:-hello}
-  local pythonpath=${2:-build/testdata}
-  local main_module=${3:-hello}
+  local pythonpath=${1:-build/testdata}
+  local main_module=${2:-hello}

  PYTHONPATH=$pythonpath \
    $PREPARE_DIR/python -S build/app_deps.py py $main_module
diff --git a/build/app_deps.py b/build/app_deps.py
@@ -18,10 +18,6 @@
 import os  # Do it here so we don't mess up analysis


-class Error(Exception):
-  pass
-
-
 def log(msg, *args):
  if args:
    msg = msg % args
@@ -114,7 +110,7 @@ def main(argv):

  if action == 'both':  # Write files for both .py and .so dependencies
    prefix = argv[3]
-    py_out_path = prefix + '-py.txt'
+    py_out_path = prefix + '-cpython.txt'
    c_out_path = prefix + '-c.txt'

    modules = ImportMain(main_module, OLD_MODULES)
@@ -140,12 +136,12 @@ def main(argv):
        print opy_input, opy_output

  else:
-    raise AssertionError('Invalid action %r' % action)
+    raise RuntimeError('Invalid action %r' % action)


 if __name__ == '__main__':
  try:
    sys.exit(main(sys.argv))
-  except Error, e:
-    print >>sys.stderr, 'py-deps:', e.args[0]
+  except RuntimeError as e:
+    print >>sys.stderr, '%s: %s' % (sys.argv[0], e.args[0])
    sys.exit(1)
diff --git a/build/compile.sh b/build/compile.sh
@@ -333,7 +333,7 @@ make-tar() {
    build/actions.sh \
    build/common.sh \
    build/detect-*.c \
-    _build/$app_name/bytecode.zip \
+    _build/$app_name/bytecode-cpython.zip \
    _build/$app_name/*.c \
    $PY27/LICENSE \
    $PY27/Modules/ovm.c \
diff --git a/build/runpy_deps.py b/build/runpy_deps.py
@@ -8,58 +8,89 @@
 import sys  # 15 modules
 import runpy  # 34 modules

+PY_MODULE = 0
+C_MODULE = 1

-def main(argv):
-  path_prefix = argv[1]

-  py_out_path = path_prefix + '/runpy-deps-py.txt'
-  c_out_path = path_prefix + '/runpy-deps-c.txt'
+def FilterModules(modules, stdlib_dir):
+  stdlib_dir_len = len(stdlib_dir)
+
+  for name in sorted(modules):
+    mod = modules[name]
+    if name in ('__builtin__', '__main__'):
+      continue
+
+    try:
+      full_path = mod.__file__
+    except AttributeError:
+      full_path = None
+
+    # If it's cached, it will be under .pyc; otherwise under .py.
+    if full_path and full_path.endswith('.py'):
+      py_path = full_path
+      pyc_path = full_path + 'c'
+    elif full_path and full_path.endswith('.pyc'):
+      pyc_path = full_path
+      py_path = full_path[:-1]
+    else:
+      # Print a different format for C modules.
+      yield C_MODULE, name, full_path
+
+    if py_path:
+      if py_path.startswith(stdlib_dir):
+        rel_py_path = py_path[stdlib_dir_len:]
+      else:
+        rel_py_path = py_path
+
+      # .pyc file for execution
+      yield PY_MODULE, py_path, rel_py_path
+

+def main(argv):
  runpy_path = runpy.__file__
  i = runpy_path.rfind('/')
  assert i != -1, runpy_path
  stdlib_dir = runpy_path[: i + 1]  # include trailing slash
-  stdlib_dir_len = len(stdlib_dir)

-  with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
-    for name in sorted(sys.modules):
-      mod = sys.modules[name]
-      if name in ('__builtin__', '__main__'):
-        continue
-
-      try:
-        full_path = mod.__file__
-      except AttributeError:
-        full_path = None
-
-      # If it's cached, it will be under .pyc; otherwise under .py.
-      if full_path and full_path.endswith('.py'):
-        py_path = full_path
-        pyc_path = full_path + 'c'
-      elif full_path and full_path.endswith('.pyc'):
-        pyc_path = full_path
-        py_path = full_path[:-1]
-      else:
-        # Print a different format for C modules.
-        print >>c_out, name, full_path
+  action = argv[1]
+
+  if action == 'both':
+    path_prefix = argv[2]
+
+    py_out_path = path_prefix + '/runpy-deps-cpython.txt'
+    c_out_path = path_prefix + '/runpy-deps-c.txt'
+
+    # NOTE: This is very similar to build/app_deps.py.
+    with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
+      for mod_type, x, y in FilterModules(sys.modules, stdlib_dir):
+        if mod_type == PY_MODULE:
+          print >>py_out, x, y
+          print >>py_out, x + 'c', y + 'c'  # .pyc goes in bytecode.zip too
+          pass
+
+        elif mod_type == C_MODULE:
+          print >>c_out, x, y  # mod_name, full_path

-      if py_path:
-        if py_path.startswith(stdlib_dir):
-          rel_py_path = py_path[stdlib_dir_len:]
        else:
-          rel_py_path = py_path
+          raise AssertionError(mod_type)
+
+    print >>sys.stderr, '-- Wrote %s and %s' % (py_out_path, c_out_path)

-        # .pyc file for execution
-        print >>py_out, py_path + 'c', rel_py_path + 'c'
-        # .py file for tracebacks
-        print >>py_out, py_path, rel_py_path
+  elif action == 'py':
+    for mod_type, full_path, rel_path in \
+        FilterModules(sys.modules, stdlib_dir):
+      if mod_type == PY_MODULE:
+        opy_input = full_path
+        opy_output = rel_path + 'c'  # output is .pyc
+        print opy_input, opy_output

-  print >>sys.stderr, '-- Wrote %s and %s' % (py_out_path, c_out_path)
+  else:
+    raise RuntimeError('Invalid action %r' % action)


 if __name__ == '__main__':
  try:
    main(sys.argv)
  except RuntimeError as e:
-    print >>sys.stderr, 'FATAL: %s' % e
+    print >>sys.stderr, '%s: %s' % (sys.argv[0], e.args[0])
    sys.exit(1)
diff --git a/opy/README.md b/opy/README.md
@@ -26,6 +26,13 @@ Test the binary:
 Notes
 -----

-OSH tests don't run under byterun.  I probably don't care.
+- The OPy bytecode is bigger than the CPython bytecode!
+  - Why is that?
+
+- One difference: I think there are no `LOAD_FAST` bytecodes generated?
+  - TODO: Make a bytecode histogram using `opy/misc/inspect_pyc`.
+
+
+- OSH tests don't run under byterun.  I probably don't care.

    ./test.sh unit '' byterun