Makefile: Put either of bytecode-{cpython,opy}.zip in the app bundle.

Also, move most rules to portable-rules.mk, which is intentionally NOT included in the release tarball. Now we are less confused about a developer build (out of the repo) vs. an end-user build (out of the release tarball). This was motivated by a bug I wasn't able to figure out: For some reason GNU Make tried to build oil-version.txt because it's a prerequisite of _build/oil/bytecode-cpython.zip, even though the latter exists, and it didn't do this in oil-0.4.0. Also: - _build/osh_help.py moved to _devbuild/gen/osh_help.py, so _build/osh-quick-ref/ should also move to _devbuild/osh-quick-ref.
oilshell · Feb 25, 2018 · 7f405fe6172a8924568c6c70b9be6b105a728571 · 7f405fe
1 parent ca85906
commit 7f405fe6172a8924568c6c70b9be6b105a728571
Unified Split

Showing with 240 additions and 216 deletions.

+18 −208 Makefile

+2 −2 build/actions.sh

+6 −2 build/compile.sh

+1 −3 build/dev.sh

+1 −1 build/doc.sh

+211 −0 portable-rules.mk

+1 −0 scripts/release.sh
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 # Build OVM App Bundles (Python code with a statically-linked CPython
 # interpreter.)
 #
-# We can also build a tarball that allows the end user to build an app bundle.
+# We also build a tarball that allows the end user to build an app bundle.
 # They need GNU Make, bash, and a C compiler.  (And xargs, chmod, etc.)
 #
 # Tarball layout (see build/compile.sh for details):
@@ -74,11 +74,15 @@
 # handling directories but I don't know it.
 # NOTE: _devbuild is made by build/dev.sh.  That directory is NOT cleaned with
 # 'make clean'.
-$(shell mkdir -p _bin _release _tmp _build/hello _build/oil _build/gen)
+$(shell mkdir -p _bin _release _tmp _build/hello _build/oil)

 ACTIONS_SH := build/actions.sh
 COMPILE_SH := build/compile.sh

+# Change the bytecode compiler here.
+BYTECODE_ZIP := bytecode-cpython.zip
+#BYTECODE_ZIP := bytecode-opy.zip
+
 # For faster tesing of builds
 #default: _bin/oil.ovm-dbg

@@ -99,213 +103,22 @@ clean:
 clean-repo:
 	$(ACTIONS_SH) clean-repo

-.PHONY: default all clean install
-
-# NOTES:
-# - Manually rm this file to generate a new build timestamp.
-# - This messes up reproducible builds.
-# - It's not marked .PHONY because that would mess up the end user build.
-#   bytecode.zip should NOT be built by the user.
-_build/release-date.txt:
-	$(ACTIONS_SH) write-release-date
-
-# The Makesfiles generated by autoconf don't call configure, but Linux/toybox
-# config system does.  This can be overridden.
-_build/detected-config.sh:
-	./configure
-
 # .PHONY alias for compatibility
 install:
 	@./install

-# What files correspond to each C module.
-# TODO:
-# - Where to put -l z?  (Done in Modules/Setup.dist)
-_build/c-module-toc.txt: build/c_module_toc.py
-	$(ACTIONS_SH) c-module-toc > $@
-
-# Python and C dependencies of runpy.
-# NOTE: This is done with a pattern rule because of the "multiple outputs"
-# problem in Make.
-_build/runpy-deps-%.txt: build/runpy_deps.py
-	$(ACTIONS_SH) runpy-deps _build
-
-_build/py-to-compile.txt: build/runpy_deps.py
-	$(ACTIONS_SH) runpy-py-to-compile > $@
-
-#
-# Hello App.  Everything below here is app-specific.
-#
-
-# C module dependencies
-include _build/hello/ovm.d
-
-# What Python module to run.
-_build/hello/main_name.c:
-	$(ACTIONS_SH) main-name hello hello.ovm > $@
-
-# Dependencies calculated by importing main.  The guard is because ovm.d
-# depends on it.  Is that correct?  We'll skip it before 'make dirs'.
-_build/hello/app-deps-%.txt: $(HELLO_SRCS) \
-	_build/detected-config.sh build/app_deps.py
-	test -d _build/hello && \
-	  $(ACTIONS_SH) app-deps hello build/testdata hello
-
-_build/hello/py-to-compile.txt: \
-	_build/detected-config.sh build/app_deps.py
-	test -d _build/hello && \
-	  $(ACTIONS_SH) py-to-compile build/testdata hello > $@
-
-# NOTE: We could use src/dest paths pattern instead of _build/app?
-#
-# TODO:
-# - Deps need to be better.  Depend on .pyc and .py.    I guess
-#   app-deps hello will compile the .pyc files.  Don't need a separate action.
-#   %.pyc : %py
-
-HELLO_BYTECODE_DEPS := \
-	build/testdata/hello-version.txt \
-        _build/release-date.txt \
-	build/testdata/hello-manifest.txt
-
-_build/hello/bytecode-cpython.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
-                           _build/hello/app-deps-cpython.txt \
-                           _build/runpy-deps-cpython.txt
-	{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
-	  echo '_build/release-date.txt release-date.txt'; \
-	  cat build/testdata/hello-manifest.txt \
-	      _build/hello/app-deps-cpython.txt \
-	      _build/runpy-deps-cpython.txt; \
-	} | build/make_zip.py $@
-
-_build/hello/bytecode-opy.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
-                           _build/hello/opy-app-deps.txt
-	{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
-	  echo '_build/release-date.txt release-date.txt'; \
-	  cat build/testdata/hello-manifest.txt \
-	      _build/hello/opy-app-deps.txt; \
-	} | build/make_zip.py $@
-
-#
-# Oil
-#
-
-# C module dependencies
-include _build/oil/ovm.d
-
-_build/oil/main_name.c:
-	$(ACTIONS_SH) main-name bin.oil oil.ovm > $@
-
-# The root of this repo, e.g. ~/git/oil, should be our PYTHONPATH for
-# detecting dependencies.
-# 
-# From this link:
-# https://stackoverflow.com/questions/322936/common-gnu-makefile-directory-path
-# Except we're using 'firstword' instead of 'lastword', because
-# _build/oil/ovm.d is the last one.
-REPO_ROOT := $(abspath $(dir $(firstword $(MAKEFILE_LIST))))
-
-# Dependencies calculated by importing main.
-# NOTE: The list of files is used both to compile and to make a tarball.
-# - For compiling, we should respect _HAVE_READLINE in detected_config
-# - For the tarball, we should ALWAYS include readline.
-#
-# BUG: Running 'make' the first time files because it can't find the '_build'
-# package.  build/doc.sh currently makes _build/__init__.py.
-_build/oil/app-deps-%.txt: _build/detected-config.sh build/app_deps.py
-	test -d _build/oil && \
-	  $(ACTIONS_SH) app-deps oil $(REPO_ROOT) bin.oil
-
-_build/oil/py-to-compile.txt: _build/detected-config.sh build/app_deps.py
-	test -d _build/oil && \
-	  $(ACTIONS_SH) py-to-compile $(REPO_ROOT) bin.oil > $@
-
-_build/osh_help.py: doc/osh-quick-ref-pages.txt
-	build/doc.sh osh-quick-ref
-
-# TODO: Need $(OIL_SRCS) here?
-# NOTES:
-# - _build/osh_help.py is a minor hack to depend on the entire
-# _build/osh-quick-ref dir, since they both get generated by the same build
-# action.
-# - release-date is at a different location on purpose, so we don't show it in
-#   dev mode.
-
-OIL_BYTECODE_DEPS := \
-	oil-version.txt \
-	_build/release-date.txt \
-	build/oil-manifest.txt \
-	_build/osh_help.py \
-	doc/osh-quick-ref-toc.txt
-
-_build/oil/bytecode-cpython.zip: $(OIL_BYTECODE_DEPS) \
-                         _build/oil/app-deps-cpython.txt \
-                         _build/runpy-deps-cpython.txt
-	{ echo '_build/release-date.txt release-date.txt'; \
-	  $(ACTIONS_SH) files-manifest oil-version.txt \
-	                               doc/osh-quick-ref-toc.txt; \
-	  cat build/oil-manifest.txt \
-	      _build/oil/app-deps-cpython.txt \
-	      _build/runpy-deps-cpython.txt; \
-	  $(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
-	} | build/make_zip.py $@ 
-
-# NOTE: runpy deps are inncluded in opy-app-deps.txt.
-_build/oil/bytecode-opy.zip: $(OIL_BYTECODE_DEPS) \
-                         _build/oil/opy-app-deps.txt
-	{ echo '_build/release-date.txt release-date.txt'; \
-	  $(ACTIONS_SH) files-manifest oil-version.txt \
-	                               doc/osh-quick-ref-toc.txt; \
-	  cat build/oil-manifest.txt \
-	      _build/oil/opy-app-deps.txt; \
-	  $(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
-	} | build/make_zip.py $@ 
-
-#
-# App-Independent Pattern Rules.
-#
-
-# Regenerate dependencies.  But only if we made the app dirs.
-_build/%/ovm.d: _build/%/app-deps-c.txt
-	$(ACTIONS_SH) make-dotd $* $^ > $@
-
-# Source paths of all C modules the app depends on.  For the tarball.
-# A trick: remove the first dep to form the lists.  You can't just use $^
-# because './c_module_srcs.py' is rewritten to 'c_module_srcs.py'.
-_build/%/c-module-srcs.txt: \
-	build/c_module_srcs.py _build/c-module-toc.txt _build/%/app-deps-c.txt
-	build/c_module_srcs.py $(filter-out $<,$^) > $@
-
-_build/%/all-deps-c.txt: build/static-c-modules.txt _build/%/app-deps-c.txt
-	$(ACTIONS_SH) join-modules $^ > $@
-
-# NOTE: This should really depend on all the .py files.
-# I should make a _build/oil/py.d file and include it?
-_build/%/opy-app-deps.txt: \
-	_build/py-to-compile.txt _build/%/py-to-compile.txt
-	sort $^ | uniq | opy/build.sh compile-manifest _build/%-with-opy > $@
-
-
-PY27 := Python-2.7.13
-
-# Per-app extension module initialization.
-_build/%/module_init.c: $(PY27)/Modules/config.c.in _build/%/all-deps-c.txt
-	# NOTE: Using xargs < input.txt style because it will fail if input.txt
-	# doesn't exist!  'cat' errors will be swallowed.
-	xargs $(ACTIONS_SH) gen-module-init < _build/$*/all-deps-c.txt > $@
+.PHONY: default all clean clean-repo install

+# For debugging
+print-%:
+	@echo $*=$($*)

-# 
-# Tarballs
-#
-# Contain Makefile and associated shell scripts, discovered .c and .py deps,
-# app source.
+# This file is intentionally NOT included in the release tarball.  This is so
+# that we don't try to rebuild say _build/oil/bytecode-opy.zip, which is
+# already pre-built and included in the release tarball.  Portable rules can be
+# done on the developer machine rather than on the end-user machine.

-_release/%.tar: _build/%/bytecode-cpython.zip \
-                _build/%/module_init.c \
-                _build/%/main_name.c \
-                _build/%/c-module-srcs.txt
-	$(COMPILE_SH) make-tar $* $@
+-include portable-rules.mk

 #
 # Native Builds
@@ -328,15 +141,12 @@ _build/%/ovm-cov: _build/%/module_init.c _build/%/main_name.c \
                  _build/%/c-module-srcs.txt $(COMPILE_SH)
 	$(COMPILE_SH) build $@ $(filter-out $(COMPILE_SH),$^)

-# Make bundles quickly.
-_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/bytecode-cpython.zip
+# App bundles.
+_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/$(BYTECODE_ZIP)
 	cat $^ > $@
 	chmod +x $@

-_bin/%.ovm: _build/%/ovm _build/%/bytecode-cpython.zip
+_bin/%.ovm: _build/%/ovm _build/%/$(BYTECODE_ZIP)
 	cat $^ > $@
 	chmod +x $@

-# For debugging
-print-%:
-	@echo $*=$($*)
diff --git a/build/actions.sh b/build/actions.sh
@@ -187,8 +187,8 @@ clean-pyc() {
  find $PY27/Lib -name '*.pyc' | xargs --no-run-if-empty -- rm --verbose
 }

-# NOTE: Not deleting cpython-full.  Maybe we should, or we should put it in a
-# diffent directory?
+# NOTE: Not deleting _devbuild, including cpython-full.  Maybe we should, or
+# we should put it in a different directory?
 clean-repo() {
  clean-source-tarball-build
  rm -r -f _build _release
diff --git a/build/compile.sh b/build/compile.sh
@@ -293,7 +293,8 @@ python-headers() {

 make-tar() {
  local app_name=${1:-hello}
-  local out=${2:-_release/hello.tar}
+  local bytecode_zip=${2:-bytecode-cpython.zip}
+  local out=${3:-_release/hello.tar}

  local version_file
  case $app_name in
@@ -323,6 +324,9 @@ make-tar() {
  # Add oil-0.0.0/ to the beginning of every path.
  local sed_expr="s,^,${app_name}-${version}/,"

+  # TODO: Remove portable-rules.mk from Makefile.  Or maybe conditionally
+  # detect it?
+
  tar --create --transform "$sed_expr" --file $out \
    LICENSE.txt \
    INSTALL.txt \
@@ -333,7 +337,7 @@ make-tar() {
    build/actions.sh \
    build/common.sh \
    build/detect-*.c \
-    _build/$app_name/bytecode-cpython.zip \
+    _build/$app_name/$bytecode_zip \
    _build/$app_name/*.c \
    $PY27/LICENSE \
    $PY27/Modules/ovm.c \
diff --git a/build/dev.sh b/build/dev.sh
@@ -24,11 +24,9 @@ ubuntu-deps() {
  test/spec.sh install-shells
 }

-# These produce _devbuild/{osh,oil}_help.py
+# Produces _devbuild/gen/osh_help.py
 gen-help() {
  build/doc.sh osh-quick-ref
-  # I believe this will be obsolete
-  #build/doc.sh oil-quick-ref
 }

 gen-types-asdl() {
diff --git a/build/doc.sh b/build/doc.sh
@@ -63,7 +63,7 @@ x-quick-ref() {
  local out_dir=$2

  local html_out=$out_dir/doc/$prog-quick-ref.html
-  local text_out_dir=_build/$prog-quick-ref
+  local text_out_dir=_devbuild/$prog-quick-ref
  local py_out=_devbuild/gen/${prog}_help.py

  mkdir -p _build/doc $text_out_dir