Merge pull request #16560 from mppf/use-python3-python-python2

Use python3/python/python2 instead of just python
[developed by @mppf, reviewed and merged by @lydia-duncan]

This PR adjusts core parts of Chapel to work with whatever `python` it can find - looking for `python3`, then `python`, then `python2`. This should enable the compiler and runtime to build and operate in a variety of systems.

Features that need maximum portability are adapted to use a new `sh` script `util/config/find-python.sh` which returns the version of `python` to run. It chooses (in this order) `python3` `python` `python2`. In each case the thing calling the python script in normal usage computes the name of the interpreter to run.


* Makefiles set `CHPL_MAKE_PYTHON` from `util/config/find-python.sh` and then run Python support scripts like `$(CHPL_MAKE_PYTHON) some-script.py` (which translates to e.g. `python3 some-script.py`)
* updated checkChplInstall similarly (called from `make check`)
* `printchplenv` already included an `sh` wrapper so updated it to run `util/config/find-python.sh`
* added an `sh` wrapper for `compileline` that uses `util/config/find-python.sh`
* setchplenv scripts now call `util/config/find-python.sh` to invoke helpers
* `mason test` also uses find-python.sh

This PR then adjusts many other uses of `python` to now require `python3`. Things in this category are:
 * `make docs`, `chpldoc` and related support
 * `start_test` and related support
 * python scripts in `test/` e.g. `.prediff` files
 * tests that run python scripts by naming the interpreter

- [x] (with #16561) quickstart make / make check on bento-centos-6 with Python 2.6 and no Python 3
- [x] (with #16561) quickstart make / make check / make docs on Ubuntu with python3 but no `python`
- [x] (with #16561) standard config make / make check / make docs / start_test hello on Ubuntu with python3 but no `python`
- [x] (with #16561) full local testing - with `source /data/cf/chapel/setup_python35-byhost.bash ` in my .bashrc

Future work -- revisit PR #3571 if necessary

* Useful trick for finding issues here is `make SHELL='sh -x'`

Makefile

@@ -48,8 +48,9 @@
 MAKEFLAGS = --no-print-directory
 
 export CHPL_MAKE_HOME=$(shell pwd)
+export CHPL_MAKE_PYTHON := $(shell $(CHPL_MAKE_HOME)/util/config/find-python.sh)
 
-NEEDS_LLVM_RUNTIME=${CHPL_MAKE_HOME}/util/chplenv/chpl_llvm.py \
+NEEDS_LLVM_RUNTIME=$(CHPL_MAKE_PYTHON) ${CHPL_MAKE_HOME}/util/chplenv/chpl_llvm.py \
                     --needs-llvm-runtime
 
 default: all

Makefile.devel

@@ -16,7 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PYTHON_VERSION_DIR = py$(shell $(CHPL_MAKE_HOME)/util/chplenv/chpl_python_version.py)
+export CHPL_MAKE_PYTHON := $(shell $(CHPL_MAKE_HOME)/util/config/find-python.sh)
+
+PYTHON_VERSION_DIR = py$(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/chpl_python_version.py)
 
 develall:
 	@$(MAKE) always-build-man

compiler/Makefile

@@ -113,7 +113,8 @@ ifndef CHPL_MAKE_THIRD_PARTY
 CHPL_MAKE_THIRD_PARTY = $(CHPL_MAKE_HOME)/third-party
 endif
 
-FIXPATH_CMD := $(CHPL_MAKE_HOME)/util/config/replace-paths.py \
+FIXPATH_CMD := $(CHPL_MAKE_PYTHON) \
+                  $(CHPL_MAKE_HOME)/util/config/replace-paths.py \
                   --fixpath '$$CHPL_THIRD_PARTY' $(CHPL_MAKE_THIRD_PARTY) \
                   --fixpath '$$CHPL_HOME' $(CHPL_MAKE_HOME)
 
@@ -146,11 +147,11 @@ $(CHPL_MAKE_HOME)/configured-prefix:
 
 $(CONFIGURED_PREFIX_FILE): FORCE $(COMPILER_BUILD) $(CHPL_MAKE_HOME)/configured-prefix
 	@echo '"'`cat $(CHPL_MAKE_HOME)/configured-prefix`'"' \ > $(CONFIGURED_PREFIX_FILE).incoming
-	@$(CHPL_MAKE_HOME)/util/config/update-if-different $(CONFIGURED_PREFIX_FILE) $(CONFIGURED_PREFIX_FILE).incoming
+	@$(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/config/update-if-different $(CONFIGURED_PREFIX_FILE) $(CONFIGURED_PREFIX_FILE).incoming
 
 $(CLANG_SETTINGS_FILE): FORCE $(COMPILER_BUILD)
 	@echo '{"'$(CLANG_CC)'","'$(CLANG_CXX)'","'`$(CHPL_MAKE_HOME)/util/config/gather-clang-sysroot-arguments $(CLANG_CC)`'"}' | $(FIXPATH_CMD) > $(CLANG_SETTINGS_FILE).incoming
-	@$(CHPL_MAKE_HOME)/util/config/update-if-different $(CLANG_SETTINGS_FILE) $(CLANG_SETTINGS_FILE).incoming
+	@$(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/config/update-if-different $(CLANG_SETTINGS_FILE) $(CLANG_SETTINGS_FILE).incoming
 
 $(CHPL_CONFIG_CHECK): | $(CHPL_BIN_DIR)
 	rm -rf $(CHPL_CONFIG_CHECK_PREFIX)

compiler/util/files.cpp

@@ -566,7 +566,7 @@ std::string runPrintChplEnv(std::map<std::string, const char*> varMap) {
 std::string getVenvDir() {
   // Runs `util/chplenv/chpl_home_utils.py --venv` and removes the newline
 
-  std::string command = "CHPL_HOME=" + std::string(CHPL_HOME) + " python ";
+  std::string command = "CHPL_HOME=" + std::string(CHPL_HOME) + " python3 ";
   command += std::string(CHPL_HOME) + "/util/chplenv/chpl_home_utils.py --venv 2> /dev/null";
 
   std::string venvDir = runCommand(command);

doc/rst/developer/bestPractices/TestSystem.rst

@@ -318,7 +318,7 @@ For instance:
 
   .. code-block:: python
 
-     #!/usr/bin/env python
+     #!/usr/bin/env python3
 
      import os
      print(os.getenv('CHPL_TEST_PERF') == 'on' and

doc/util/chpl2rst.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 """chpl2rst converts a chapel program to an rst file, where all comments are

doc/util/extract-rst-tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 """

make/Makefile.base

@@ -99,6 +99,9 @@ $(eval $(CHPL_MAKE_SETTINGS))
 # Try this to debug issues with CHPL_MAKE_* variables
 # $(info $(CHPL_MAKE_CHPLENV_CACHE))
 
+# Compute python command name
+export CHPL_MAKE_PYTHON := $(shell $(CHPL_MAKE_HOME)/util/config/find-python.sh)
+
 # Now do some minor situational clean-ups.
 
 # The config variable for segment is CHPL_GASNET_SEGMENT

man/Makefile

@@ -2,16 +2,20 @@ ifndef CHPL_MAKE_HOME
 export CHPL_MAKE_HOME=$(shell pwd)/..
 endif
 
+ifndef CHPL_MAKE_PYTHON
+export CHPL_MAKE_PYTHON := $(shell $(CHPL_MAKE_HOME)/util/config/find-python.sh)
+endif
+
 PROGRAM = chpl
 CHPLDOC = chpldoc
 
 MANPAGE = man1/$(PROGRAM).1
 CHPLDOC_MANPAGE = man1/$(CHPLDOC).1
 TARGETS = $(MANPAGE) $(PROGRAM).pdf $(CHPLDOC).pdf
 
-VENV_DIR = $(shell python $(CHPL_MAKE_HOME)/util/chplenv/chpl_home_utils.py --venv)
+VENV_DIR = $(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/chpl_home_utils.py --venv)
 
-RST2MAN = $(shell which rst2man.py || echo $(VENV_DIR)/bin/rst2man.py)
+RST2MAN = $(shell which rst2man.py || echo $(CHPL_MAKE_PYTHON) $(VENV_DIR)/bin/rst2man.py)
 
 STARS = \*\*\*\*\*
 

modules/Makefile

@@ -59,11 +59,11 @@ MAKE_SYS_BASIC_TYPES=$(CHPL_MAKE_HOME)/util/config/make_sys_basic_types.py
 
 $(SYS_CTYPES_MODULE): $(MAKE_SYS_BASIC_TYPES)
 	mkdir -p $(@D)
-	cd $(@D) && $(MAKE_SYS_BASIC_TYPES) $(@F)
+	cd $(@D) && $(CHPL_MAKE_PYTHON) $(MAKE_SYS_BASIC_TYPES) $(@F)
 
 $(SYS_CTYPES_MODULE_DOC): $(MAKE_SYS_BASIC_TYPES)
 	mkdir -p $(@D)
-	cd $(@D) && $(MAKE_SYS_BASIC_TYPES) --doc $(@F)
+	cd $(@D) && $(CHPL_MAKE_PYTHON) $(MAKE_SYS_BASIC_TYPES) --doc $(@F)
 
 MODULES_TO_DOCUMENT = \
 	standard/Assert.chpl \

runtime/Makefile

@@ -65,7 +65,7 @@ $(CHPL_ENV_HEADER): $(CHPL_MAKE_HOME)/util/printchplenv $(CHPL_MAKE_HOME)/util/c
           awk '{ print "#define " $$1 "_" toupper($$2) }' >> $(CHPL_ENV_HEADER)
 	@echo "#endif /* _CHPL_ENV_GEN_H_ */" >> $(CHPL_ENV_HEADER)
 
-THIRD_PARTY_PKGS = $(shell $(CHPL_MAKE_HOME)/util/chplenv/third-party-pkgs)
+THIRD_PARTY_PKGS = $(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/third-party-pkgs)
 # NOTE: If any of these builds fail, we won't stop building
 third-party-pkgs:
 ifneq (, $(THIRD_PARTY_PKGS))

runtime/Makefile.config

@@ -41,7 +41,8 @@ ifndef CHPL_MAKE_THIRD_PARTY
 CHPL_MAKE_THIRD_PARTY = $(CHPL_MAKE_HOME)/third-party
 endif
 
-FIXPATH_CMD := $(CHPL_MAKE_HOME)/util/config/replace-paths.py \
+FIXPATH_CMD := $(CHPL_MAKE_PYTHON) \
+                  $(CHPL_MAKE_HOME)/util/config/replace-paths.py \
                   --fixpath '$$CHPL_RUNTIME_LIB' $(CHPL_MAKE_RUNTIME_LIB) \
                   --fixpath '$$CHPL_RUNTIME_INCL' $(CHPL_MAKE_RUNTIME_INCL) \
                   --fixpath '$$CHPL_THIRD_PARTY' $(CHPL_MAKE_THIRD_PARTY) \

test/analysis/alias/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 

test/compflags/bradc/mungeUserIdents/testmunge-export.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # This test ensures that the generated code contains instances of foo,

test/compflags/bradc/mungeUserIdents/testmunge-extern.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # This test ensures that the generated code contains instances of certain

test/compflags/bradc/mungeUserIdents/testmunge.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, os, shutil
 

test/compflags/ferguson/default-binary-name/sub_test

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
 import subprocess
 import sys

test/compflags/link/sungeun/static_dynamic.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, os, subprocess, string
 

test/compflags/sungeun/configs/basic/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, string
 

test/compflags/sungeun/configs/type_variables/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # WARNING: This script is pretty touchy wrt to its input
 #

test/distributions/robust/arithmetic/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Figure out which of TESTNAME.*.good is applicable; copy it to TESTNAME.good.
 #

test/distributions/robust/arithmetic/basics/test_scan1.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Figure out which of TESTNAME.*.good is applicable; copy it to TESTNAME.good.
 #

test/domains/sungeun/assoc/index_not_in_domain_1.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/domains/sungeun/assoc/index_not_in_domain_2.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/domains/sungeun/sparse/index_not_in_domain_1.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/domains/sungeun/sparse/index_not_in_domain_2.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/execflags/bradc/errors/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, os, string
 

test/execflags/bradc/gdbddash/gdbSetConfig.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os, os.path
 

test/execflags/dinan/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, os, string
 

test/execflags/ferguson/help2.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/execflags/shannon/configs/configVarDashSVarEquals.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, os, string
 

test/execflags/shannon/configs/help/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/execflags/shannon/help.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys, re
 

test/execflags/sungeun/about.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This script fixes up the test output to remove the compiler version
 # number and then generates a .good file based on the output of

test/execflags/thomasvandoren/exceedMemInts.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Only run this test on 32-bit platforms."""
 

test/functions/ferguson/main/sub_test

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
 import subprocess
 import sys

test/interop/C/multilocale.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  Multilocale interoperability requires ZMQ

test/interop/python/errorMessages/abiIncompatible.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 

test/interop/python/multilocale.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  Multilocale interoperability requires ZMQ

test/io/ferguson/ctests/qio_bits_test.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """For valgrind, see qio_test.valgrind.test.c. Skip test when atomics are
 implemented with locks and tasking layer is not fifo.

test/io/ferguson/ctests/qio_test.compopts

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 

test/io/ferguson/ctests/qio_test.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Skip test when atomics are implemented with locks and tasking layer is not
 fifo.

test/io/ferguson/ctests/skip_non_fifo_atomic_locks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Skip test when atomics are implemented with locks and tasking layer is not fifo.
 

test/io/ferguson/stdin-is-directory/stdin-is-directory.prediff

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This prediff exists to change an absolute
 # paths like /home/user/chapel/test/io/ferguson/test-dir

test/io/thomasvandoren/lustre/SKIPIF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Skip test if lustre is not set in CHPL_AUX_FILESYS."""
 

test/io/vass/time-write.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 

test/library/draft/Vector/PREDIFF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import sys
 import os.path
 
@@ -13,4 +13,4 @@ with open(sys.argv[2], 'r') as f:
       output += os.path.basename(line[:pos]) + ':xxx' + line[pos2:]
 
 open(sys.argv[2], 'w').write(output)
-
+

test/library/packages/Crypto/saru.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import subprocess
 from distutils.version import LooseVersion

test/library/packages/Crypto/saru/COMPOPTS

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import subprocess
 

test/library/packages/FFTW/SKIPIF

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 #
 # skip this test when FFTW_DIR is not set or we're using pgi (JIRA 195).

test/library/packages/LAPACK.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
 Require BLAS and LAPACK modules

test/library/packages/LinearAlgebra/performance/csrmatmatMult-perf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 """

test/library/packages/MPI/multilocale.skipif

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 
 import os