[soil] Run test/py3_parse in a new Soil task 'pea'

With Dockerfile.pea.

We need Python 3.10 for the 'match' statement and type checking of it.

Our 'soil' pattern is holding up!  That wasn't too hard.

soil/Dockerfile.pea

@@ -0,0 +1,26 @@
+FROM debian:buster-slim
+
+RUN apt-get update
+
+WORKDIR /home/uke/tmp
+
+# Copy build scripts into the container and run them
+
+COPY soil/deps-apt.sh /home/uke/tmp/soil/deps-apt.sh
+RUN soil/deps-apt.sh layer-for-soil
+RUN soil/deps-apt.sh pea
+
+# deps-tar.sh has a 'wget' step which we're skipping
+COPY _cache/Python-3.10.4.tar.xz /home/uke/tmp/_cache/Python-3.10.4.tar.xz
+
+COPY build/common.sh /home/uke/tmp/build/common.sh
+COPY soil/deps-tar.sh /home/uke/tmp/soil/deps-tar.sh
+RUN soil/deps-tar.sh layer-py3
+
+RUN useradd --create-home uke && chown -R uke /home/uke
+USER uke
+
+COPY soil/deps-py.sh /home/uke/tmp/soil/deps-py.sh
+RUN soil/deps-py.sh pea
+
+CMD ["sh", "-c", "echo 'hello from oilshell/soil-pea'"]

soil/deps-apt.sh

@@ -46,6 +46,10 @@ dev-minimal() {
 
 }
 
+pea() {
+  apt-get install -y python3-pip
+}
+
 other-tests() {
   local -a packages=(
     libreadline-dev

soil/deps-py.sh

@@ -17,4 +17,8 @@ dev-minimal() {
   pip3 install --user mypy pexpect
 }
 
+pea() {
+  pip3 install --user mypy
+}
+
 "$@"

soil/deps-tar.sh

@@ -92,17 +92,43 @@ symlink-cmark() {
 }
 
 #
-# CPython dependency for 'make'
+# CPython 3.10 dependency for Pea
+#
+
+readonly PY3_VERSION=3.10.4
+readonly PY3_URL="https://www.python.org/ftp/python/3.10.4/Python-$PY3_VERSION.tar.xz"
+
+download-py3() {
+  mkdir -p $REPO_ROOT/_cache
+  wget --no-clobber --directory $REPO_ROOT/_cache $PY3_URL
+}
+
+extract-py3() {
+  pushd $REPO_ROOT/_cache
+  tar -x --xz < $(basename $PY3_URL)
+  popd
+}
+
+symlink-py3() {
+  ln -s -f -v $DEPS_DIR/py3/python $DEPS_DIR/python3
+  ls -l $DEPS_DIR/python3
+}
+
+test-py3() {
+  $DEPS_DIR/python3 -c 'import sys; print(sys.version)'
+}
+
+#
+# OVM slice -- CPython dependency for 'make'
 #
 
 configure-python() {
   local dir=${1:-$PREPARE_DIR}
+  local conf=${2:-$PWD/$PY27/configure}
 
   rm -r -f $dir
   mkdir -p $dir
 
-  local conf=$PWD/$PY27/configure 
-
   pushd $dir 
   time $conf
   popd
@@ -154,4 +180,18 @@ layer-cpython() {
   build-python
 }
 
+# For pea and type checking
+layer-py3() {
+  # Dockerfile.pea copies it
+  # download-py3
+
+  extract-py3
+
+  local dir=$DEPS_DIR/py3
+  configure-python $dir $REPO_ROOT/_cache/Python-$PY3_VERSION/configure
+  build-python $dir
+
+  symlink-py3
+}
+
 "$@"

soil/worker.sh

@@ -95,6 +95,20 @@ dump-hardware    soil/worker.sh dump-hardware   -
 EOF
 }
 
+pea-tasks() {
+  ### Print tasks for the 'pea' build
+
+  # We need a later version of Python 3 / MyPy both to type check and
+  # to parse
+
+  # (task_name, script, action, result_html)
+  cat <<EOF
+dump-user-host      soil/worker.sh dump-user-host           -
+check-types         test/py3_parse.sh check-types           -
+parse-all           test/py3_parse.sh parse-all             -
+EOF
+}
+
 dev-minimal-tasks() {
   ### Print tasks for the 'dev-minimal' build
 
@@ -236,7 +250,6 @@ osh2oil           test/osh2oil.sh soil-run                -
 R-test            devtools/R-test.sh soil-run             -
 xargs-test        test/other.sh xargs-test                -
 csv2html-test     test/other.sh csv2html-test             -
-py3-parse         test/py3_parse.sh parse-all             -
 EOF
 }
 
@@ -390,6 +403,8 @@ run-other-tests() { job-main 'other-tests'; }
 
 run-ovm-tarball() { job-main 'ovm-tarball'; }
 
+run-pea() { job-main 'pea'; }
+
 run-app-tests() { job-main 'app-tests'; }
 
 run-cpp() { job-main 'cpp'; }

test/py3_parse.py

@@ -9,6 +9,7 @@
 import sys
 
 
+#def main(argv) -> None:
 def main(argv):
   ast_dump = os.getenv('AST_DUMP')
 

test/py3_parse.sh

@@ -11,6 +11,9 @@ set -o errexit
 
 source types/common.sh
 
+# not using build/dev-shell.sh for now
+readonly PY3=../oil_DEPS/python3
+
 all-files() {
   # Files that are reported as part of the source code.
   metrics/source-code.sh osh-files
@@ -25,7 +28,8 @@ all-files() {
 }
 
 parse-one() {
-  test/py3_parse.py "$@"
+  # Use PY3 because Python 3.8 and above has type comments
+  $PY3 test/py3_parse.py "$@"
 }
 
 parse-all() {
@@ -34,4 +38,28 @@ parse-all() {
   all-files | egrep '\.py$|\.pyi$' | xargs --verbose -- $0 parse-one
 }
 
+dump-sys-path() {
+  ### Dump for debugging
+  python3 -c 'import sys; print(sys.path)'
+}
+
+pip3-lib-path() {
+  ### python3.6 on Ubuntu; python3.7 in debian:buster-slim in the container
+  shopt -s failglob
+  echo ~/.local/lib/python3.?/site-packages
+}
+
+check-types() {
+  #mypy_ test/py3_parse.py
+
+  local pip3_lib_path
+  pip3_lib_path=$(pip3-lib-path)
+
+  PYTHONPATH=$pip3_lib_path ../oil_DEPS/python3 ~/.local/bin/mypy test/py3_parse.py
+}
+
+soil-run() {
+  check-types
+}
+
 "$@"

types/run.sh

@@ -161,10 +161,9 @@ soil-run() {
   typecheck-more-oil
 }
 
-# Alias for convenience
-check-osh-parse() {
-  types/osh-parse.sh check-some
-}
+#
+# PyAnnotate
+#
 
 collect-types() {
   export PYTHONPATH=".:$PYANN_REPO"