From 79d0e041c3f5a63690946c54acbb5e0496f0102c Mon Sep 17 00:00:00 2001 From: Andy C Date: Thu, 12 May 2022 01:15:31 -0400 Subject: [PATCH] [soil] Run test/py3_parse in a new Soil task 'pea' With Dockerfile.pea. We need Python 3.10 for the 'match' statement and type checking of it. Our 'soil' pattern is holding up! That wasn't too hard. --- soil/Dockerfile.pea | 26 +++++++++++++++++++++++++ soil/deps-apt.sh | 4 ++++ soil/deps-py.sh | 4 ++++ soil/deps-tar.sh | 46 ++++++++++++++++++++++++++++++++++++++++++--- soil/worker.sh | 17 ++++++++++++++++- test/py3_parse.py | 1 + test/py3_parse.sh | 30 ++++++++++++++++++++++++++++- types/run.sh | 7 +++---- 8 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 soil/Dockerfile.pea diff --git a/soil/Dockerfile.pea b/soil/Dockerfile.pea new file mode 100644 index 0000000000..40469bbbf7 --- /dev/null +++ b/soil/Dockerfile.pea @@ -0,0 +1,26 @@ +FROM debian:buster-slim + +RUN apt-get update + +WORKDIR /home/uke/tmp + +# Copy build scripts into the container and run them + +COPY soil/deps-apt.sh /home/uke/tmp/soil/deps-apt.sh +RUN soil/deps-apt.sh layer-for-soil +RUN soil/deps-apt.sh pea + +# deps-tar.sh has a 'wget' step which we're skipping +COPY _cache/Python-3.10.4.tar.xz /home/uke/tmp/_cache/Python-3.10.4.tar.xz + +COPY build/common.sh /home/uke/tmp/build/common.sh +COPY soil/deps-tar.sh /home/uke/tmp/soil/deps-tar.sh +RUN soil/deps-tar.sh layer-py3 + +RUN useradd --create-home uke && chown -R uke /home/uke +USER uke + +COPY soil/deps-py.sh /home/uke/tmp/soil/deps-py.sh +RUN soil/deps-py.sh pea + +CMD ["sh", "-c", "echo 'hello from oilshell/soil-pea'"] diff --git a/soil/deps-apt.sh b/soil/deps-apt.sh index 9216c00a70..4d9124b134 100755 --- a/soil/deps-apt.sh +++ b/soil/deps-apt.sh @@ -46,6 +46,10 @@ dev-minimal() { } +pea() { + apt-get install -y python3-pip +} + other-tests() { local -a packages=( libreadline-dev diff --git a/soil/deps-py.sh b/soil/deps-py.sh index 1255837a0e..d788c31f61 100755 --- a/soil/deps-py.sh +++ b/soil/deps-py.sh @@ -17,4 +17,8 @@ dev-minimal() { pip3 install --user mypy pexpect } +pea() { + pip3 install --user mypy +} + "$@" diff --git a/soil/deps-tar.sh b/soil/deps-tar.sh index 1b125acf1e..e33e8ce652 100755 --- a/soil/deps-tar.sh +++ b/soil/deps-tar.sh @@ -92,17 +92,43 @@ symlink-cmark() { } # -# CPython dependency for 'make' +# CPython 3.10 dependency for Pea +# + +readonly PY3_VERSION=3.10.4 +readonly PY3_URL="https://www.python.org/ftp/python/3.10.4/Python-$PY3_VERSION.tar.xz" + +download-py3() { + mkdir -p $REPO_ROOT/_cache + wget --no-clobber --directory $REPO_ROOT/_cache $PY3_URL +} + +extract-py3() { + pushd $REPO_ROOT/_cache + tar -x --xz < $(basename $PY3_URL) + popd +} + +symlink-py3() { + ln -s -f -v $DEPS_DIR/py3/python $DEPS_DIR/python3 + ls -l $DEPS_DIR/python3 +} + +test-py3() { + $DEPS_DIR/python3 -c 'import sys; print(sys.version)' +} + +# +# OVM slice -- CPython dependency for 'make' # configure-python() { local dir=${1:-$PREPARE_DIR} + local conf=${2:-$PWD/$PY27/configure} rm -r -f $dir mkdir -p $dir - local conf=$PWD/$PY27/configure - pushd $dir time $conf popd @@ -154,4 +180,18 @@ layer-cpython() { build-python } +# For pea and type checking +layer-py3() { + # Dockerfile.pea copies it + # download-py3 + + extract-py3 + + local dir=$DEPS_DIR/py3 + configure-python $dir $REPO_ROOT/_cache/Python-$PY3_VERSION/configure + build-python $dir + + symlink-py3 +} + "$@" diff --git a/soil/worker.sh b/soil/worker.sh index a1a9f63fb7..559d59c0fd 100755 --- a/soil/worker.sh +++ b/soil/worker.sh @@ -95,6 +95,20 @@ dump-hardware soil/worker.sh dump-hardware - EOF } +pea-tasks() { + ### Print tasks for the 'pea' build + + # We need a later version of Python 3 / MyPy both to type check and + # to parse + + # (task_name, script, action, result_html) + cat < None: def main(argv): ast_dump = os.getenv('AST_DUMP') diff --git a/test/py3_parse.sh b/test/py3_parse.sh index ecf1c7c17e..2adeac7cf0 100755 --- a/test/py3_parse.sh +++ b/test/py3_parse.sh @@ -11,6 +11,9 @@ set -o errexit source types/common.sh +# not using build/dev-shell.sh for now +readonly PY3=../oil_DEPS/python3 + all-files() { # Files that are reported as part of the source code. metrics/source-code.sh osh-files @@ -25,7 +28,8 @@ all-files() { } parse-one() { - test/py3_parse.py "$@" + # Use PY3 because Python 3.8 and above has type comments + $PY3 test/py3_parse.py "$@" } parse-all() { @@ -34,4 +38,28 @@ parse-all() { all-files | egrep '\.py$|\.pyi$' | xargs --verbose -- $0 parse-one } +dump-sys-path() { + ### Dump for debugging + python3 -c 'import sys; print(sys.path)' +} + +pip3-lib-path() { + ### python3.6 on Ubuntu; python3.7 in debian:buster-slim in the container + shopt -s failglob + echo ~/.local/lib/python3.?/site-packages +} + +check-types() { + #mypy_ test/py3_parse.py + + local pip3_lib_path + pip3_lib_path=$(pip3-lib-path) + + PYTHONPATH=$pip3_lib_path ../oil_DEPS/python3 ~/.local/bin/mypy test/py3_parse.py +} + +soil-run() { + check-types +} + "$@" diff --git a/types/run.sh b/types/run.sh index 2b37999255..6f5fa5d90d 100755 --- a/types/run.sh +++ b/types/run.sh @@ -161,10 +161,9 @@ soil-run() { typecheck-more-oil } -# Alias for convenience -check-osh-parse() { - types/osh-parse.sh check-some -} +# +# PyAnnotate +# collect-types() { export PYTHONPATH=".:$PYANN_REPO"