Temp gha runner v3

.ci/docker/manywheel/Dockerfile_ppc64le

@@ -0,0 +1,130 @@
+FROM quay.io/pypa/manylinux_2_28_ppc64le as base
+
+# Language variables
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8
+
+# there is a bugfix in gcc >= 14 for precompiled headers and s390x vectorization interaction.
+# with earlier gcc versions test/inductor/test_cpu_cpp_wrapper.py will fail.
+ARG DEVTOOLSET_VERSION=14
+# Installed needed OS packages. This is to support all
+# the binary builds (torch, vision, audio, text, data)
+RUN yum -y install epel-release
+RUN yum -y update
+RUN yum install -y \
+  sudo \
+  autoconf \
+  automake \
+  bison \
+  bzip2 \
+  curl \
+  diffutils \
+  file \
+  git \
+  make \
+  patch \
+  perl \
+  unzip \
+  util-linux \
+  wget \
+  which \
+  xz \
+  yasm \
+  less \
+  zstd \
+  libgomp \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
+  gcc-toolset-${DEVTOOLSET_VERSION}-binutils \
+  gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
+  cmake \
+  rust \
+  cargo \
+  llvm-devel \
+  libzstd-devel \
+  python3.12-devel \
+  python3.12-test \
+  python3.12-setuptools \
+  python3.12-pip \
+  python3-virtualenv \
+  python3.12-pyyaml \
+  python3.12-numpy \
+  python3.12-wheel \
+  python3.12-cryptography \
+  blas-devel \
+  openblas-devel \
+  lapack-devel \
+  atlas-devel \
+  libjpeg-devel \
+  libxslt-devel \
+  libxml2-devel \
+  openssl-devel \
+  valgrind \
+  ninja-build
+
+ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
+ENV CC=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin/gcc
+ENV CXX=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin/g++
+ENV LD=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin/ld
+ENV CFLAGS=""
+ENV CXXFLAGS=""
+# git236+ would refuse to run git commands in repos owned by other users
+# Which causes version check to fail, as pytorch repo is bind-mounted into the image
+# Override this behaviour by treating every folder as safe
+# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
+RUN git config --global --add safe.directory "*"
+
+# installed python doesn't have development parts. Rebuild it from scratch
+RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
+
+# EPEL for cmake
+FROM base as patchelf
+RUN git clone --depth 1 --branch temp-gha-runner-v3 https://github.com/sandeepgupta12/pytorch.git /tmp/pytorch && \
+    mkdir -p /build_scripts && \
+    cp /tmp/pytorch/.ci/docker/common/install_patchelf.sh /build_scripts/install_patchelf.sh 
+# Install patchelf
+#ADD ./common/install_patchelf.sh install_patchelf.sh
+#RUN bash ./install_patchelf.sh && rm install_patchelf.sh
+RUN bash /build_scripts/install_patchelf.sh && rm -r /build_scripts
+RUN cp $(which patchelf) /patchelf
+
+FROM patchelf as python
+# build python
+# Clone only required scripts from the PyTorch repo
+RUN mkdir -p /build_scripts && \
+    cp -r /tmp/pytorch/.ci/docker/manywheel/build_scripts/* /build_scripts/ && \
+    cp /tmp/pytorch/.ci/docker/common/install_cpython.sh /build_scripts/install_cpython.sh && \
+    rm -rf /tmp/pytorch
+#COPY manywheel/build_scripts /build_scripts
+#ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
+ENV SSL_CERT_FILE=
+#RUN bash build_scripts/build.sh && rm -r build_scripts
+RUN bash /build_scripts/build.sh && rm -r /build_scripts
+
+FROM base as final
+COPY --from=python             /opt/python                           /opt/python
+COPY --from=python             /opt/_internal                        /opt/_internal
+COPY --from=python             /opt/python/cp39-cp39/bin/auditwheel  /usr/local/bin/auditwheel
+COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
+
+RUN alternatives --set python /usr/bin/python3.12
+RUN alternatives --set python3 /usr/bin/python3.12
+
+RUN pip-3.12 install typing_extensions
+
+ENTRYPOINT []
+CMD ["/bin/bash"]
+
+# install test dependencies:
+# - grpcio requires system openssl, bundled crypto fails to build
+RUN dnf install -y \
+  hdf5-devel \
+  python3-h5py \
+  git
+
+
+#RUN env GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=True pip3 install grpcio
+# cmake-3.28.0 from pip for onnxruntime
+RUN python3 -mpip install cmake==3.28.0

.ci/docker/manywheel/build.sh

@@ -49,6 +49,12 @@ case ${image} in
         DOCKER_GPU_BUILD_ARG=""
         MANY_LINUX_VERSION="s390x"
         ;;
+    manylinuxppc64le-builder:cpu-ppc64le)
+        TARGET=final
+        GPU_IMAGE=redhat/ubi9
+        DOCKER_GPU_BUILD_ARG=""
+        MANY_LINUX_VERSION="ppc64le"
+        ;;
     manylinux2_28-builder:cuda11*)
         TARGET=cuda_final
         GPU_IMAGE=amd64/almalinux:8
@@ -106,7 +112,7 @@ if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then
     DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION}
 fi
 # Only activate this if in CI
-if [ "$(uname -m)" != "s390x" ] && [ -v CI ]; then
+if [ "$(uname -m)" != "s390x" ] && "$(uname -m)" != "ppc64le" ] && [ -v CI ]; then
     # TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
     # is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
     sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service

.ci/docker/manywheel/build_scripts/build.sh

@@ -20,7 +20,7 @@ AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
 # the final image after compiling Python
 PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel libpcap-devel xz-devel libffi-devel"
 
-if [ "$(uname -m)" != "s390x" ] ; then
+if [ "$(uname -m)" != "s390x" ] && [ "$(uname -m)" != "ppc64le" ] ; then
     PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} db4-devel"
 else
     PYTHON_COMPILE_DEPS="${PYTHON_COMPILE_DEPS} libdb-devel"
@@ -39,7 +39,38 @@ yum -y install bzip2 make git patch unzip bison yasm diffutils \
     ${PYTHON_COMPILE_DEPS}
 
 # Install newest autoconf
-build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
+# If the architecture is not ppc64le, use the existing build_autoconf function
+if [ "$(uname -m)" != "ppc64le" ] ; then
+    build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
+# else
+#     curl -sLO http://ftp.gnu.org/gnu/autoconf/$AUTOCONF_ROOT.tar.gz
+
+#     echo "$AUTOCONF_HASH  $AUTOCONF_ROOT.tar.gz" | sha256sum -c -
+
+#     tar -xzf $AUTOCONF_ROOT.tar.gz
+#     cd $AUTOCONF_ROOT
+
+#     mkdir -p build-aux
+
+#     curl -fLo /tmp/config.guess https://git.savannah.gnu.org/cgit/config.git/plain/config.guess
+#     curl -fLo /tmp/config.sub https://git.savannah.gnu.org/cgit/config.git/plain/config.sub
+
+#     chmod +x /tmp/config.guess /tmp/config.sub
+
+#     mv /tmp/config.guess build-aux/config.guess
+#     mv /tmp/config.sub build-aux/config.sub
+
+#     ls -lh build-aux/config.*
+
+#     ./build-aux/config.guess || echo "Failed to detect architecture"
+
+#     ./configure --host=powerpc64le-pc-linux-gnu
+#     make -j$(nproc)
+#     make install
+
+#     cd ..
+#     rm -rf $AUTOCONF_ROOT $AUTOCONF_ROOT.tar.gz
+fi
 autoconf --version
 
 # Compile the latest Python releases.

.ci/docker/manywheel/build_scripts/manylinux1-check.py

@@ -5,7 +5,7 @@ def is_manylinux1_compatible():
     # Only Linux, and only x86-64 / i686
     from distutils.util import get_platform
 
-    if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]:
+    if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x", "linux-ppc64le"]:
         return False
 
     # Check for presence of _manylinux module

.ci/manywheel/build.sh

@@ -11,7 +11,7 @@ case "${GPU_ARCH_TYPE:-BLANK}" in
     rocm)
         bash "${SCRIPTPATH}/build_rocm.sh"
         ;;
-    cpu | cpu-cxx11-abi | cpu-aarch64 | cpu-s390x)
+    cpu | cpu-cxx11-abi | cpu-aarch64 | cpu-s390x | cpu-ppc64le)
         bash "${SCRIPTPATH}/build_cpu.sh"
         ;;
     xpu)

.ci/manywheel/build_common.sh

@@ -363,7 +363,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
     done
 
     # create Manylinux 2_28 tag this needs to happen before regenerate the RECORD
-    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
+    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
         wheel_file=$(echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/WHEEL/g')
         sed -i -e s#linux_x86_64#"${PLATFORM}"# $wheel_file;
     fi
@@ -412,7 +412,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
     fi
 
     # Rename wheel for Manylinux 2_28
-    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
+    if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
         pkg_name=$(echo $(basename $pkg) | sed -e s#linux_x86_64#"${PLATFORM}"#)
         zip -rq $pkg_name $PREIX*
         rm -f $pkg

.ci/manywheel/build_cpu.sh

@@ -65,6 +65,8 @@ elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then
 elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
     if [[ "$ARCH" == "s390x" ]]; then
         LIBGOMP_PATH="/usr/lib/s390x-linux-gnu/libgomp.so.1"
+    elif [[ "$(uname -m)" == "ppc64le" ]]; then
+        LIBGOMP_PATH="/usr/lib64/libgomp.so.1"
     elif [[ "$ARCH" == "aarch64" ]]; then
         LIBGOMP_PATH="/usr/lib/aarch64-linux-gnu/libgomp.so.1"
     else

.ci/pytorch/build.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 
 set -ex -o pipefail
-
+# which python
+# export PATH="/opt/python/cp312-cp312/bin:$PATH"
+# export PYTHON_EXECUTABLE="/opt/python/cp312-cp312/bin/python3.12"
+# which python
 # Required environment variable: $BUILD_ENVIRONMENT
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
@@ -111,6 +114,11 @@ if [[ "$BUILD_ENVIRONMENT" == *riscv64* ]]; then
   export SLEEF_TARGET_EXEC_USE_QEMU=ON
   sudo chown -R jenkins /var/lib/jenkins/workspace /opt
 
+fi
+if [[ "$BUILD_ENVIRONMENT" == *ppc64le* ]]; then
+  export USE_MKLDNN=0
+  export USE_MKLDNN_ACL=0
+
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *libtorch* ]]; then
@@ -243,7 +251,7 @@ fi
 
 # Do not change workspace permissions for ROCm and s390x CI jobs
 # as it can leave workspace with bad permissions for cancelled jobs
-if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *riscv64* && -d /var/lib/jenkins/workspace ]]; then
+if [[ "$BUILD_ENVIRONMENT" != *rocm* && "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *riscv64* && -d /var/lib/jenkins/workspace ]]; then
   # Workaround for dind-rootless userid mapping (https://github.com/pytorch/ci-infra/issues/96)
   WORKSPACE_ORIGINAL_OWNER_ID=$(stat -c '%u' "/var/lib/jenkins/workspace")
   cleanup_workspace() {
@@ -288,7 +296,7 @@ else
     # XLA test build fails when WERROR=1
     # set only when building other architectures
     # or building non-XLA tests.
-    if [[ "$BUILD_ENVIRONMENT" != *rocm*  && "$BUILD_ENVIRONMENT" != *xla* && "$BUILD_ENVIRONMENT" != *riscv64* ]]; then
+    if [[ "$BUILD_ENVIRONMENT" != *rocm*  && "$BUILD_ENVIRONMENT" != *xla* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *riscv64* ]]; then
       # Install numpy-2.0.2 for builds which are backward compatible with 1.X
       python -mpip install numpy==2.0.2
 
@@ -431,6 +439,6 @@ if [[ "$BUILD_ENVIRONMENT" != *libtorch* && "$BUILD_ENVIRONMENT" != *bazel* ]];
   PYTHONPATH=. python tools/stats/export_test_times.py
 fi
 # don't do this for bazel or s390x or riscv64 as they don't use sccache
-if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *riscv64* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
+if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *riscv64* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
   print_sccache_stats
 fi

.ci/pytorch/check_binary.sh

@@ -189,7 +189,7 @@ fi
 if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   echo "Checking that MKL is available"
   build_and_run_example_cpp check-torch-mkl
-elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" ]]; then
+elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
   if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then
     if [[ "$(uname -m)" == "aarch64" ]]; then
       echo "Checking that MKLDNN is available on aarch64"
@@ -213,7 +213,7 @@ if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
   echo "Checking that XNNPACK is available"
   build_and_run_example_cpp check-torch-xnnpack
 else
-  if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x"  ]]; then
+  if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le"  ]]; then
     echo "Checking that XNNPACK is available"
     pushd /tmp
     python -c 'import torch.backends.xnnpack; exit(0 if torch.backends.xnnpack.enabled else 1)'
@@ -243,7 +243,7 @@ fi
 
 # Test that CUDA builds are setup correctly
 # Skip CUDA hardware checks for aarch64 as they run on CPU-only runners
-if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" && "$(uname -m)" != "aarch64" ]]; then
+if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" && "$(uname -m)" != "aarch64" ]]; then
   if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
     build_and_run_example_cpp check-torch-cuda
   else

.circleci/scripts/binary_populate_env.sh

@@ -39,9 +39,9 @@ fi
 
 USE_GOLD_LINKER="OFF"
 # GOLD linker can not be used if CUPTI is statically linked into PyTorch, see https://github.com/pytorch/pytorch/issues/57744
-if [[ ${DESIRED_CUDA} == "cpu" ]]; then
-  USE_GOLD_LINKER="ON"
-fi
+# if [[ ${DESIRED_CUDA} == "cpu" ]]; then
+#   USE_GOLD_LINKER="ON"
+# fi
 
 
 # Default to nightly, since that's where this normally uploads to
@@ -112,7 +112,7 @@ if [[ "$PACKAGE_TYPE" =~ .*wheel.* && -n "$PYTORCH_BUILD_VERSION" && "$PYTORCH_B
 fi
 
 USE_GLOO_WITH_OPENSSL="ON"
-if [[ "$GPU_ARCH_TYPE" =~ .*aarch64.* ]]; then
+if [[ "$GPU_ARCH_TYPE" =~ .*cpu-ppc64le.* ]]; then
   USE_GLOO_WITH_OPENSSL="OFF"
   USE_GOLD_LINKER="OFF"
 fi

.github/actions/test-pytorch-binary/action.yml

@@ -40,9 +40,9 @@ runs:
         docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
 
     - name: Cleanup docker
-      if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
+      if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.BUILD_ENVIRONMENT != 'linux-ppc64le-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
       shell: bash
       run: |
-        # on s390x or xpu stop the container for clean worker stop
+        # on s390x or ppc64le or xpu stop the container for clean worker stop
         # shellcheck disable=SC2046
         docker stop "${{ env.CONTAINER_NAME }}" || true