From c59c2d1f6de662eb6e266f8084f6881741f0d5ad Mon Sep 17 00:00:00 2001 From: gabrieldemarmiesse Date: Tue, 31 Mar 2020 11:51:20 +0000 Subject: [PATCH 1/2] Simplify the build by using python all the time. --- .github/workflows/make_wheel_macOS.sh | 4 ++-- build_deps/build_pip_pkg.sh | 11 ++--------- build_deps/toolchains/gpu/cuda_configure.bzl | 2 +- tools/docker/build_wheel.Dockerfile | 9 +++++---- tools/docker/gpu_tests.Dockerfile | 9 ++++++--- tools/releases/release_linux.sh | 9 ++++----- tools/releases/tf_auditwheel_patch.sh | 2 +- tools/testing/addons_cpu.sh | 13 +++---------- tools/testing/addons_gpu.sh | 8 +++----- 9 files changed, 27 insertions(+), 40 deletions(-) diff --git a/.github/workflows/make_wheel_macOS.sh b/.github/workflows/make_wheel_macOS.sh index e787830e1e..68bb54ff5f 100644 --- a/.github/workflows/make_wheel_macOS.sh +++ b/.github/workflows/make_wheel_macOS.sh @@ -2,8 +2,8 @@ set -e -x export TF_NEED_CUDA=0 -python3 --version -python3 -m pip install delocate wheel setuptools tensorflow==$TF_VERSION +python --version +python -m pip install delocate wheel setuptools tensorflow==$TF_VERSION bash tools/install_deps/bazel_macos.sh $BAZEL_VERSION bash tools/testing/addons_cpu.sh diff --git a/build_deps/build_pip_pkg.sh b/build_deps/build_pip_pkg.sh index 1cca21a98b..6ba70c92ff 100755 --- a/build_deps/build_pip_pkg.sh +++ b/build_deps/build_pip_pkg.sh @@ -48,13 +48,6 @@ function main() { exit 1 fi - # Check if python3 is available. On Windows VM it is not. - if [ -x "$(command -v python3)" ]; then - _PYTHON_BINARY=python3 - else - _PYTHON_BINARY=python - fi - mkdir -p ${DEST} DEST=$(abspath "${DEST}") echo "=== destination directory: ${DEST}" @@ -89,9 +82,9 @@ function main() { if [[ -z ${NIGHTLY_FLAG} ]]; then # Windows has issues with locking library files for deletion so do not fail here - ${_PYTHON_BINARY} ${BUILD_CMD} || true + python ${BUILD_CMD} || true else - ${_PYTHON_BINARY} ${BUILD_CMD} ${NIGHTLY_FLAG} || true + python ${BUILD_CMD} ${NIGHTLY_FLAG} || true fi cp dist/*.whl "${DEST}" diff --git a/build_deps/toolchains/gpu/cuda_configure.bzl b/build_deps/toolchains/gpu/cuda_configure.bzl index c7625d0421..70fd2215f5 100644 --- a/build_deps/toolchains/gpu/cuda_configure.bzl +++ b/build_deps/toolchains/gpu/cuda_configure.bzl @@ -60,7 +60,7 @@ def _get_python_bin(repository_ctx): python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) if python_bin != None: return python_bin - python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python3" + python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python" python_bin_path = repository_ctx.which(python_bin_name) if python_bin_path != None: return str(python_bin_path) diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile index 0f244d502d..2a87021e69 100644 --- a/tools/docker/build_wheel.Dockerfile +++ b/tools/docker/build_wheel.Dockerfile @@ -5,22 +5,23 @@ FROM tensorflow/tensorflow:2.1.0-custom-op-gpu-ubuntu16 as make_wheel RUN apt-get update && apt-get install patchelf ARG PY_VERSION -RUN python$PY_VERSION -m pip install --upgrade pip setuptools auditwheel==2.0.0 +RUN ln -sf $(which python$PY_VERSION) /usr/bin/python +RUN python -m pip install --upgrade pip setuptools auditwheel==2.0.0 COPY tools/install_deps/ /install_deps ARG TF_VERSION -RUN python$PY_VERSION -m pip install \ +RUN python -m pip install \ tensorflow==$TF_VERSION \ -r /install_deps/pytest.txt COPY requirements.txt . -RUN python$PY_VERSION -m pip install -r requirements.txt +RUN python -m pip install -r requirements.txt COPY ./ /addons WORKDIR /addons ARG NIGHTLY_FLAG RUN --mount=type=cache,id=cache_bazel,target=/root/.cache/bazel \ - bash tools/releases/release_linux.sh $PY_VERSION $NIGHTLY_FLAG + bash tools/releases/release_linux.sh $NIGHTLY_FLAG RUN bash tools/releases/tf_auditwheel_patch.sh RUN auditwheel repair --plat manylinux2010_x86_64 artifacts/*.whl diff --git a/tools/docker/gpu_tests.Dockerfile b/tools/docker/gpu_tests.Dockerfile index 261c43fb2a..1b0d7614cd 100644 --- a/tools/docker/gpu_tests.Dockerfile +++ b/tools/docker/gpu_tests.Dockerfile @@ -1,12 +1,15 @@ FROM tensorflow/tensorflow:2.1.0-custom-op-gpu-ubuntu16 -RUN python3 -m pip install --upgrade pip setuptools auditwheel==2.0.0 +ARG PY_VERSION +RUN ln -sf $(which python$PY_VERSION) /usr/bin/python + +RUN python -m pip install --upgrade pip setuptools auditwheel==2.0.0 COPY tools/install_deps/tensorflow.txt ./ -RUN python3 -m pip install -r tensorflow.txt +RUN python -m pip install -r tensorflow.txt COPY requirements.txt ./ -RUN python3 -m pip install -r requirements.txt +RUN python -m pip install -r requirements.txt COPY ./ /addons diff --git a/tools/releases/release_linux.sh b/tools/releases/release_linux.sh index 476bf8ca01..df8f1f3fcc 100644 --- a/tools/releases/release_linux.sh +++ b/tools/releases/release_linux.sh @@ -35,13 +35,12 @@ echo "#! /usr/bin/python2.7" >> /usr/bin/lsb_release2 cat /usr/bin/lsb_release >> /usr/bin/lsb_release2 mv /usr/bin/lsb_release2 /usr/bin/lsb_release -ln -sf $(which python$1) /usr/bin/python3 -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade setuptools +python -m pip install --upgrade pip +python -m pip install --upgrade setuptools #Link TF dependency -python3 --version -python3 ./configure.py +python --version +python ./configure.py bash tools/testing/addons_cpu.sh bazel clean --expunge diff --git a/tools/releases/tf_auditwheel_patch.sh b/tools/releases/tf_auditwheel_patch.sh index 4b64758e33..2b0e0e4322 100644 --- a/tools/releases/tf_auditwheel_patch.sh +++ b/tools/releases/tf_auditwheel_patch.sh @@ -16,7 +16,7 @@ set -e -SITE_PKG_LOCATION=$(python3 -c "import site; print(site.getsitepackages()[0])") +SITE_PKG_LOCATION=$(python -c "import site; print(site.getsitepackages()[0])") TF_SHARED_LIBRARY_NAME=$(grep -r TF_SHARED_LIBRARY_NAME .bazelrc | awk -F= '{print$2}') POLICY_JSON="${SITE_PKG_LOCATION}/auditwheel/policy/policy.json" sed -i "s/libresolv.so.2\"/libresolv.so.2\", $TF_SHARED_LIBRARY_NAME/g" $POLICY_JSON diff --git a/tools/testing/addons_cpu.sh b/tools/testing/addons_cpu.sh index 9f39cf1b35..4abf90dfe7 100644 --- a/tools/testing/addons_cpu.sh +++ b/tools/testing/addons_cpu.sh @@ -21,15 +21,8 @@ set -x -e export CC_OPT_FLAGS='-mavx' export TF_NEED_CUDA=0 -# Check if python3 is available. On Windows VM it is not. -if [ -x "$(command -v python3)" ]; then - PYTHON_BINARY=python3 -else - PYTHON_BINARY=python -fi - -$PYTHON_BINARY -m pip install -r tools/install_deps/pytest.txt -e ./ -$PYTHON_BINARY ./configure.py +python -m pip install -r tools/install_deps/pytest.txt -e ./ +python ./configure.py cat ./.bazelrc bash tools/install_so_files.sh -$PYTHON_BINARY -m pytest -v --durations=25 -n auto ./tensorflow_addons +python -m pytest -v --durations=25 -n auto ./tensorflow_addons diff --git a/tools/testing/addons_gpu.sh b/tools/testing/addons_gpu.sh index 96ef4952ad..61715d3a26 100644 --- a/tools/testing/addons_gpu.sh +++ b/tools/testing/addons_gpu.sh @@ -29,10 +29,8 @@ export CUDA_TOOLKIT_PATH="/usr/local/cuda" export TF_CUDNN_VERSION="7" export CUDNN_INSTALL_PATH="/usr/lib/x86_64-linux-gnu" -# we use only "python3" because we can't run gpu tests on Windows -# so no need to make it work with "python" like in the cpu tests. -python3 -m pip install -r tools/install_deps/pytest.txt -e ./ -python3 ./configure.py +python -m pip install -r tools/install_deps/pytest.txt -e ./ +python ./configure.py cat ./.bazelrc bash tools/install_so_files.sh -python3 -m pytest -v --durations=25 ./tensorflow_addons +python -m pytest -v --durations=25 ./tensorflow_addons From 1074573ee6496748bf47bc05dd7fedc25dfbdc47 Mon Sep 17 00:00:00 2001 From: gabrieldemarmiesse Date: Tue, 31 Mar 2020 12:02:49 +0000 Subject: [PATCH 2/2] Forgot to set variable. --- tools/run_gpu_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_gpu_tests.sh b/tools/run_gpu_tests.sh index bea1e96861..5383996467 100644 --- a/tools/run_gpu_tests.sh +++ b/tools/run_gpu_tests.sh @@ -3,5 +3,5 @@ set -x -e export DOCKER_BUILDKIT=1 -docker build -f tools/docker/gpu_tests.Dockerfile -t tfa_gpu_tests ./ +docker build -f tools/docker/gpu_tests.Dockerfile --build-arg PY_VERSION=3.5 -t tfa_gpu_tests ./ docker run --rm -t --runtime=nvidia tfa_gpu_tests