diff --git a/.github/workflows/make_wheel_macOS.sh b/.github/workflows/make_wheel_macOS.sh
index fc6ec52826..f3731cbbb4 100644
--- a/.github/workflows/make_wheel_macOS.sh
+++ b/.github/workflows/make_wheel_macOS.sh
@@ -2,8 +2,8 @@ set -e -x
 
 export TF_NEED_CUDA=0
 
-python3 --version
-python3 -m pip install delocate wheel setuptools tensorflow==$TF_VERSION
+python --version
+python -m pip install delocate wheel setuptools tensorflow==$TF_VERSION
 
 bash tools/install_deps/bazel_macos.sh $BAZEL_VERSION
 bash tools/testing/build_and_run_tests.sh
diff --git a/build_deps/build_pip_pkg.sh b/build_deps/build_pip_pkg.sh
index 1cca21a98b..6ba70c92ff 100755
--- a/build_deps/build_pip_pkg.sh
+++ b/build_deps/build_pip_pkg.sh
@@ -48,13 +48,6 @@ function main() {
     exit 1
   fi
 
-  # Check if python3 is available. On Windows VM it is not.
-  if [ -x "$(command -v python3)" ]; then
-      _PYTHON_BINARY=python3
-    else
-      _PYTHON_BINARY=python
-  fi
-
   mkdir -p ${DEST}
   DEST=$(abspath "${DEST}")
   echo "=== destination directory: ${DEST}"
@@ -89,9 +82,9 @@ function main() {
 
   if [[ -z ${NIGHTLY_FLAG} ]]; then
     # Windows has issues with locking library files for deletion so do not fail here
-    ${_PYTHON_BINARY} ${BUILD_CMD} || true
+    python ${BUILD_CMD} || true
   else
-    ${_PYTHON_BINARY} ${BUILD_CMD} ${NIGHTLY_FLAG} || true
+    python ${BUILD_CMD} ${NIGHTLY_FLAG} || true
   fi
 
   cp dist/*.whl "${DEST}"
diff --git a/build_deps/toolchains/gpu/cuda_configure.bzl b/build_deps/toolchains/gpu/cuda_configure.bzl
index c7625d0421..70fd2215f5 100644
--- a/build_deps/toolchains/gpu/cuda_configure.bzl
+++ b/build_deps/toolchains/gpu/cuda_configure.bzl
@@ -60,7 +60,7 @@ def _get_python_bin(repository_ctx):
     python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
     if python_bin != None:
         return python_bin
-    python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python3"
+    python_bin_name = "python.exe" if _is_windows(repository_ctx) else "python"
     python_bin_path = repository_ctx.which(python_bin_name)
     if python_bin_path != None:
         return str(python_bin_path)
diff --git a/tools/docker/build_wheel.Dockerfile b/tools/docker/build_wheel.Dockerfile
index 9a70817178..6d14d4aee0 100644
--- a/tools/docker/build_wheel.Dockerfile
+++ b/tools/docker/build_wheel.Dockerfile
@@ -6,23 +6,24 @@ ENV TF_NEED_CUDA="1"
 RUN apt-get update && apt-get install patchelf
 
 ARG PY_VERSION
-RUN python$PY_VERSION -m pip install --upgrade pip setuptools auditwheel==2.0.0
+RUN ln -sf $(which python$PY_VERSION) /usr/bin/python
+RUN python -m pip install --upgrade pip setuptools auditwheel==2.0.0
 
 COPY tools/install_deps/ /install_deps
 ARG TF_VERSION
-RUN python$PY_VERSION -m pip install \
+RUN python -m pip install \
         tensorflow==$TF_VERSION \
         -r /install_deps/pytest.txt
 
 COPY requirements.txt .
-RUN python$PY_VERSION -m pip install -r requirements.txt
+RUN python -m pip install -r requirements.txt
 
 COPY ./ /addons
 WORKDIR /addons
 ARG NIGHTLY_FLAG
 ARG NIGHTLY_TIME
 RUN --mount=type=cache,id=cache_bazel,target=/root/.cache/bazel \
-    bash tools/releases/release_linux.sh $PY_VERSION $NIGHTLY_FLAG
+    bash tools/releases/release_linux.sh $NIGHTLY_FLAG
 
 RUN bash tools/releases/tf_auditwheel_patch.sh
 RUN auditwheel repair --plat manylinux2010_x86_64 artifacts/*.whl
diff --git a/tools/docker/gpu_tests.Dockerfile b/tools/docker/gpu_tests.Dockerfile
index 6f1f52045f..c9ea8de71b 100644
--- a/tools/docker/gpu_tests.Dockerfile
+++ b/tools/docker/gpu_tests.Dockerfile
@@ -1,13 +1,16 @@
 FROM tensorflow/tensorflow:2.1.0-custom-op-gpu-ubuntu16
 ENV TF_NEED_CUDA="1"
 
-RUN python3 -m pip install --upgrade pip setuptools auditwheel==2.0.0
+ARG PY_VERSION
+RUN ln -sf $(which python$PY_VERSION) /usr/bin/python
+
+RUN python -m pip install --upgrade pip setuptools auditwheel==2.0.0
 
 COPY tools/install_deps/tensorflow.txt ./
-RUN python3 -m pip install -r tensorflow.txt
+RUN python -m pip install -r tensorflow.txt
 
 COPY requirements.txt ./
-RUN python3 -m pip install -r requirements.txt
+RUN python -m pip install -r requirements.txt
 
 
 COPY ./ /addons
diff --git a/tools/releases/release_linux.sh b/tools/releases/release_linux.sh
index 0b9b93d2e4..1386785c12 100644
--- a/tools/releases/release_linux.sh
+++ b/tools/releases/release_linux.sh
@@ -31,12 +31,11 @@ echo "#! /usr/bin/python2.7" >> /usr/bin/lsb_release2
 cat /usr/bin/lsb_release >> /usr/bin/lsb_release2
 mv /usr/bin/lsb_release2 /usr/bin/lsb_release
 
-ln -sf $(which python$1) /usr/bin/python3
-python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade setuptools
+python -m pip install --upgrade pip
+python -m pip install --upgrade setuptools
 
 #Link TF dependency
-python3 --version
+python --version
 bash tools/testing/build_and_run_tests.sh
 bazel clean --expunge
 
diff --git a/tools/releases/tf_auditwheel_patch.sh b/tools/releases/tf_auditwheel_patch.sh
index 4b64758e33..2b0e0e4322 100644
--- a/tools/releases/tf_auditwheel_patch.sh
+++ b/tools/releases/tf_auditwheel_patch.sh
@@ -16,7 +16,7 @@
 
 set -e
 
-SITE_PKG_LOCATION=$(python3 -c "import site; print(site.getsitepackages()[0])")
+SITE_PKG_LOCATION=$(python -c "import site; print(site.getsitepackages()[0])")
 TF_SHARED_LIBRARY_NAME=$(grep -r TF_SHARED_LIBRARY_NAME .bazelrc | awk -F= '{print$2}')
 POLICY_JSON="${SITE_PKG_LOCATION}/auditwheel/policy/policy.json"
 sed -i "s/libresolv.so.2\"/libresolv.so.2\", $TF_SHARED_LIBRARY_NAME/g" $POLICY_JSON
diff --git a/tools/run_gpu_tests.sh b/tools/run_gpu_tests.sh
index bea1e96861..5383996467 100644
--- a/tools/run_gpu_tests.sh
+++ b/tools/run_gpu_tests.sh
@@ -3,5 +3,5 @@
 set -x -e
 
 export DOCKER_BUILDKIT=1
-docker build -f tools/docker/gpu_tests.Dockerfile -t tfa_gpu_tests ./
+docker build -f tools/docker/gpu_tests.Dockerfile --build-arg PY_VERSION=3.5 -t tfa_gpu_tests ./
 docker run --rm -t --runtime=nvidia tfa_gpu_tests
diff --git a/tools/testing/addons_gpu.sh b/tools/testing/addons_gpu.sh
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tools/testing/build_and_run_tests.sh b/tools/testing/build_and_run_tests.sh
index 648611ebb6..a6550f55b1 100644
--- a/tools/testing/build_and_run_tests.sh
+++ b/tools/testing/build_and_run_tests.sh
@@ -20,14 +20,7 @@ set -x -e
 
 export CC_OPT_FLAGS='-mavx'
 
-# Check if python3 is available. On Windows VM it is not.
-if [ -x "$(command -v python3)" ]; then
-  PYTHON_BINARY=python3
-else
-  PYTHON_BINARY=python
-fi
-
-$PYTHON_BINARY -m pip install -r tools/install_deps/pytest.txt -e ./
-$PYTHON_BINARY ./configure.py
+python -m pip install -r tools/install_deps/pytest.txt -e ./
+python ./configure.py
 bash tools/install_so_files.sh
-$PYTHON_BINARY -m pytest -v --durations=25 ./tensorflow_addons
+python -m pytest -v --durations=25 ./tensorflow_addons