Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Makefile: Put either of bytecode-{cpython,opy}.zip in the app bundle.
Also, move most rules to portable-rules.mk, which is intentionally NOT
included in the release tarball.  Now we are less confused about a
developer build (out of the repo) vs. an end-user build (out of the
release tarball).

This was motivated by a bug I wasn't able to figure out: For some
reason GNU Make tried to build oil-version.txt because it's a
prerequisite of _build/oil/bytecode-cpython.zip, even though the latter
exists, and it didn't do this in oil-0.4.0.

Also:

- _build/osh_help.py moved to _devbuild/gen/osh_help.py, so
  _build/osh-quick-ref/ should also move to _devbuild/osh-quick-ref.
  • Loading branch information
Andy Chu committed Feb 25, 2018
1 parent ca85906 commit 7f405fe
Show file tree
Hide file tree
Showing 7 changed files with 240 additions and 216 deletions.
226 changes: 18 additions & 208 deletions Makefile
@@ -1,7 +1,7 @@
# Build OVM App Bundles (Python code with a statically-linked CPython
# interpreter.)
#
# We can also build a tarball that allows the end user to build an app bundle.
# We also build a tarball that allows the end user to build an app bundle.
# They need GNU Make, bash, and a C compiler. (And xargs, chmod, etc.)
#
# Tarball layout (see build/compile.sh for details):
Expand Down Expand Up @@ -74,11 +74,15 @@
# handling directories but I don't know it.
# NOTE: _devbuild is made by build/dev.sh. That directory is NOT cleaned with
# 'make clean'.
$(shell mkdir -p _bin _release _tmp _build/hello _build/oil _build/gen)
$(shell mkdir -p _bin _release _tmp _build/hello _build/oil)

ACTIONS_SH := build/actions.sh
COMPILE_SH := build/compile.sh

# Change the bytecode compiler here.
BYTECODE_ZIP := bytecode-cpython.zip
#BYTECODE_ZIP := bytecode-opy.zip

# For faster tesing of builds
#default: _bin/oil.ovm-dbg

Expand All @@ -99,213 +103,22 @@ clean:
clean-repo:
$(ACTIONS_SH) clean-repo

.PHONY: default all clean install

# NOTES:
# - Manually rm this file to generate a new build timestamp.
# - This messes up reproducible builds.
# - It's not marked .PHONY because that would mess up the end user build.
# bytecode.zip should NOT be built by the user.
_build/release-date.txt:
$(ACTIONS_SH) write-release-date

# The Makesfiles generated by autoconf don't call configure, but Linux/toybox
# config system does. This can be overridden.
_build/detected-config.sh:
./configure

# .PHONY alias for compatibility
install:
@./install

# What files correspond to each C module.
# TODO:
# - Where to put -l z? (Done in Modules/Setup.dist)
_build/c-module-toc.txt: build/c_module_toc.py
$(ACTIONS_SH) c-module-toc > $@

# Python and C dependencies of runpy.
# NOTE: This is done with a pattern rule because of the "multiple outputs"
# problem in Make.
_build/runpy-deps-%.txt: build/runpy_deps.py
$(ACTIONS_SH) runpy-deps _build

_build/py-to-compile.txt: build/runpy_deps.py
$(ACTIONS_SH) runpy-py-to-compile > $@

#
# Hello App. Everything below here is app-specific.
#

# C module dependencies
-include _build/hello/ovm.d

# What Python module to run.
_build/hello/main_name.c:
$(ACTIONS_SH) main-name hello hello.ovm > $@

# Dependencies calculated by importing main. The guard is because ovm.d
# depends on it. Is that correct? We'll skip it before 'make dirs'.
_build/hello/app-deps-%.txt: $(HELLO_SRCS) \
_build/detected-config.sh build/app_deps.py
test -d _build/hello && \
$(ACTIONS_SH) app-deps hello build/testdata hello

_build/hello/py-to-compile.txt: \
_build/detected-config.sh build/app_deps.py
test -d _build/hello && \
$(ACTIONS_SH) py-to-compile build/testdata hello > $@

# NOTE: We could use src/dest paths pattern instead of _build/app?
#
# TODO:
# - Deps need to be better. Depend on .pyc and .py. I guess
# app-deps hello will compile the .pyc files. Don't need a separate action.
# %.pyc : %py

HELLO_BYTECODE_DEPS := \
build/testdata/hello-version.txt \
_build/release-date.txt \
build/testdata/hello-manifest.txt

_build/hello/bytecode-cpython.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
_build/hello/app-deps-cpython.txt \
_build/runpy-deps-cpython.txt
{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
echo '_build/release-date.txt release-date.txt'; \
cat build/testdata/hello-manifest.txt \
_build/hello/app-deps-cpython.txt \
_build/runpy-deps-cpython.txt; \
} | build/make_zip.py $@

_build/hello/bytecode-opy.zip: $(HELLO_SRCS) $(HELLO_BYTECODE_DEPS) \
_build/hello/opy-app-deps.txt
{ echo 'build/testdata/hello-version.txt hello-version.txt'; \
echo '_build/release-date.txt release-date.txt'; \
cat build/testdata/hello-manifest.txt \
_build/hello/opy-app-deps.txt; \
} | build/make_zip.py $@

#
# Oil
#

# C module dependencies
-include _build/oil/ovm.d

_build/oil/main_name.c:
$(ACTIONS_SH) main-name bin.oil oil.ovm > $@

# The root of this repo, e.g. ~/git/oil, should be our PYTHONPATH for
# detecting dependencies.
#
# From this link:
# https://stackoverflow.com/questions/322936/common-gnu-makefile-directory-path
# Except we're using 'firstword' instead of 'lastword', because
# _build/oil/ovm.d is the last one.
REPO_ROOT := $(abspath $(dir $(firstword $(MAKEFILE_LIST))))

# Dependencies calculated by importing main.
# NOTE: The list of files is used both to compile and to make a tarball.
# - For compiling, we should respect _HAVE_READLINE in detected_config
# - For the tarball, we should ALWAYS include readline.
#
# BUG: Running 'make' the first time files because it can't find the '_build'
# package. build/doc.sh currently makes _build/__init__.py.
_build/oil/app-deps-%.txt: _build/detected-config.sh build/app_deps.py
test -d _build/oil && \
$(ACTIONS_SH) app-deps oil $(REPO_ROOT) bin.oil

_build/oil/py-to-compile.txt: _build/detected-config.sh build/app_deps.py
test -d _build/oil && \
$(ACTIONS_SH) py-to-compile $(REPO_ROOT) bin.oil > $@

_build/osh_help.py: doc/osh-quick-ref-pages.txt
build/doc.sh osh-quick-ref

# TODO: Need $(OIL_SRCS) here?
# NOTES:
# - _build/osh_help.py is a minor hack to depend on the entire
# _build/osh-quick-ref dir, since they both get generated by the same build
# action.
# - release-date is at a different location on purpose, so we don't show it in
# dev mode.

OIL_BYTECODE_DEPS := \
oil-version.txt \
_build/release-date.txt \
build/oil-manifest.txt \
_build/osh_help.py \
doc/osh-quick-ref-toc.txt

_build/oil/bytecode-cpython.zip: $(OIL_BYTECODE_DEPS) \
_build/oil/app-deps-cpython.txt \
_build/runpy-deps-cpython.txt
{ echo '_build/release-date.txt release-date.txt'; \
$(ACTIONS_SH) files-manifest oil-version.txt \
doc/osh-quick-ref-toc.txt; \
cat build/oil-manifest.txt \
_build/oil/app-deps-cpython.txt \
_build/runpy-deps-cpython.txt; \
$(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
} | build/make_zip.py $@

# NOTE: runpy deps are inncluded in opy-app-deps.txt.
_build/oil/bytecode-opy.zip: $(OIL_BYTECODE_DEPS) \
_build/oil/opy-app-deps.txt
{ echo '_build/release-date.txt release-date.txt'; \
$(ACTIONS_SH) files-manifest oil-version.txt \
doc/osh-quick-ref-toc.txt; \
cat build/oil-manifest.txt \
_build/oil/opy-app-deps.txt; \
$(ACTIONS_SH) quick-ref-manifest _build/osh-quick-ref; \
} | build/make_zip.py $@

#
# App-Independent Pattern Rules.
#

# Regenerate dependencies. But only if we made the app dirs.
_build/%/ovm.d: _build/%/app-deps-c.txt
$(ACTIONS_SH) make-dotd $* $^ > $@

# Source paths of all C modules the app depends on. For the tarball.
# A trick: remove the first dep to form the lists. You can't just use $^
# because './c_module_srcs.py' is rewritten to 'c_module_srcs.py'.
_build/%/c-module-srcs.txt: \
build/c_module_srcs.py _build/c-module-toc.txt _build/%/app-deps-c.txt
build/c_module_srcs.py $(filter-out $<,$^) > $@

_build/%/all-deps-c.txt: build/static-c-modules.txt _build/%/app-deps-c.txt
$(ACTIONS_SH) join-modules $^ > $@

# NOTE: This should really depend on all the .py files.
# I should make a _build/oil/py.d file and include it?
_build/%/opy-app-deps.txt: \
_build/py-to-compile.txt _build/%/py-to-compile.txt
sort $^ | uniq | opy/build.sh compile-manifest _build/%-with-opy > $@


PY27 := Python-2.7.13

# Per-app extension module initialization.
_build/%/module_init.c: $(PY27)/Modules/config.c.in _build/%/all-deps-c.txt
# NOTE: Using xargs < input.txt style because it will fail if input.txt
# doesn't exist! 'cat' errors will be swallowed.
xargs $(ACTIONS_SH) gen-module-init < _build/$*/all-deps-c.txt > $@
.PHONY: default all clean clean-repo install

# For debugging
print-%:
@echo $*=$($*)

#
# Tarballs
#
# Contain Makefile and associated shell scripts, discovered .c and .py deps,
# app source.
# This file is intentionally NOT included in the release tarball. This is so
# that we don't try to rebuild say _build/oil/bytecode-opy.zip, which is
# already pre-built and included in the release tarball. Portable rules can be
# done on the developer machine rather than on the end-user machine.

_release/%.tar: _build/%/bytecode-cpython.zip \
_build/%/module_init.c \
_build/%/main_name.c \
_build/%/c-module-srcs.txt
$(COMPILE_SH) make-tar $* $@
-include portable-rules.mk

#
# Native Builds
Expand All @@ -328,15 +141,12 @@ _build/%/ovm-cov: _build/%/module_init.c _build/%/main_name.c \
_build/%/c-module-srcs.txt $(COMPILE_SH)
$(COMPILE_SH) build $@ $(filter-out $(COMPILE_SH),$^)

# Make bundles quickly.
_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/bytecode-cpython.zip
# App bundles.
_bin/%.ovm-dbg: _build/%/ovm-dbg _build/%/$(BYTECODE_ZIP)
cat $^ > $@
chmod +x $@

_bin/%.ovm: _build/%/ovm _build/%/bytecode-cpython.zip
_bin/%.ovm: _build/%/ovm _build/%/$(BYTECODE_ZIP)
cat $^ > $@
chmod +x $@

# For debugging
print-%:
@echo $*=$($*)
4 changes: 2 additions & 2 deletions build/actions.sh
Expand Up @@ -187,8 +187,8 @@ clean-pyc() {
find $PY27/Lib -name '*.pyc' | xargs --no-run-if-empty -- rm --verbose
}

# NOTE: Not deleting cpython-full. Maybe we should, or we should put it in a
# diffent directory?
# NOTE: Not deleting _devbuild, including cpython-full. Maybe we should, or
# we should put it in a different directory?
clean-repo() {
clean-source-tarball-build
rm -r -f _build _release
Expand Down
8 changes: 6 additions & 2 deletions build/compile.sh
Expand Up @@ -293,7 +293,8 @@ python-headers() {

make-tar() {
local app_name=${1:-hello}
local out=${2:-_release/hello.tar}
local bytecode_zip=${2:-bytecode-cpython.zip}
local out=${3:-_release/hello.tar}

local version_file
case $app_name in
Expand Down Expand Up @@ -323,6 +324,9 @@ make-tar() {
# Add oil-0.0.0/ to the beginning of every path.
local sed_expr="s,^,${app_name}-${version}/,"

# TODO: Remove portable-rules.mk from Makefile. Or maybe conditionally
# detect it?

tar --create --transform "$sed_expr" --file $out \
LICENSE.txt \
INSTALL.txt \
Expand All @@ -333,7 +337,7 @@ make-tar() {
build/actions.sh \
build/common.sh \
build/detect-*.c \
_build/$app_name/bytecode-cpython.zip \
_build/$app_name/$bytecode_zip \
_build/$app_name/*.c \
$PY27/LICENSE \
$PY27/Modules/ovm.c \
Expand Down
4 changes: 1 addition & 3 deletions build/dev.sh
Expand Up @@ -24,11 +24,9 @@ ubuntu-deps() {
test/spec.sh install-shells
}

# These produce _devbuild/{osh,oil}_help.py
# Produces _devbuild/gen/osh_help.py
gen-help() {
build/doc.sh osh-quick-ref
# I believe this will be obsolete
#build/doc.sh oil-quick-ref
}

gen-types-asdl() {
Expand Down
2 changes: 1 addition & 1 deletion build/doc.sh
Expand Up @@ -63,7 +63,7 @@ x-quick-ref() {
local out_dir=$2

local html_out=$out_dir/doc/$prog-quick-ref.html
local text_out_dir=_build/$prog-quick-ref
local text_out_dir=_devbuild/$prog-quick-ref
local py_out=_devbuild/gen/${prog}_help.py

mkdir -p _build/doc $text_out_dir
Expand Down

0 comments on commit 7f405fe

Please sign in to comment.