Skip to content

Commit

Permalink
Add pypi required metadata to all wheels except linux (pytorch#111578)
Browse files Browse the repository at this point in the history 
* Add pypi required metadata to all wheels except linux (pytorch#111042)

Will fix package after publishing pytorch#100974
Poetry install requires all wheels on pypi to have same metadata. Hence including linux dependencies in all non-linux wheels

Pull Request resolved: pytorch#111042
Approved by: https://github.com/malfet

* Regenerate workflows
  • Loading branch information
@atalman @Halmoni100
atalman authored and Halmoni100 committed Nov 25, 2023
1 parent 54925ab commit 7b0fc31
Show file tree
Hide file tree
Showing 10 changed files with 55 additions and 15 deletions.
4 changes: 3 additions & 1 deletion .circleci/scripts/binary_populate_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ else
export PYTORCH_BUILD_VERSION="${BASE_BUILD_VERSION}+$DESIRED_CUDA"
fi

if [[ -n "${PYTORCH_EXTRA_INSTALL_REQUIREMENTS:-}" ]]; then
# The build with with-pypi-cudnn suffix is only applicabe to
# pypi small wheel Linux x86 build
if [[ -n "${PYTORCH_EXTRA_INSTALL_REQUIREMENTS:-}" ]] && [[ "$(uname)" == 'Linux' && "$(uname -m)" == "x86_64" ]]; then
export PYTORCH_BUILD_VERSION="${PYTORCH_BUILD_VERSION}-with-pypi-cudnn"
fi

Expand Down
31 changes: 19 additions & 12 deletions .github/scripts/generate_binary_build_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,21 @@

CPU_AARCH64_ARCH = ["cpu-aarch64"]

PYTORCH_EXTRA_INSTALL_REQUIREMENTS = (
"nvidia-cuda-nvrtc-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | " # noqa: B950
"nvidia-cuda-runtime-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cuda-cupti-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cudnn-cu12==8.9.2.26; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cublas-cu12==12.1.3.1; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cufft-cu12==11.0.2.54; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-curand-cu12==10.3.2.106; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cusolver-cu12==11.4.5.107; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cusparse-cu12==12.1.0.106; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-nccl-cu12==2.18.1; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-nvtx-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"triton==2.1.0; platform_system == 'Linux' and platform_machine == 'x86_64'"
)


def arch_type(arch_version: str) -> str:
if arch_version in CUDA_ARCHES:
Expand Down Expand Up @@ -238,18 +253,7 @@ def generate_wheels_matrix(
"devtoolset": "",
"container_image": WHEEL_CONTAINER_IMAGES[arch_version],
"package_type": package_type,
"pytorch_extra_install_requirements": "nvidia-cuda-nvrtc-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | " # noqa: B950
"nvidia-cuda-runtime-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cuda-cupti-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cudnn-cu12==8.9.2.26; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cublas-cu12==12.1.3.1; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cufft-cu12==11.0.2.54; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-curand-cu12==10.3.2.106; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cusolver-cu12==11.4.5.107; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-cusparse-cu12==12.1.0.106; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-nccl-cu12==2.18.1; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"nvidia-nvtx-cu12==12.1.105; platform_system == 'Linux' and platform_machine == 'x86_64' | "
"triton==2.1.0; platform_system == 'Linux' and platform_machine == 'x86_64'",
"pytorch_extra_install_requirements": PYTORCH_EXTRA_INSTALL_REQUIREMENTS,
"build_name": f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}-with-pypi-cudnn".replace( # noqa: B950
".", "_"
),
Expand All @@ -274,6 +278,9 @@ def generate_wheels_matrix(
"build_name": f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}".replace(
".", "_"
),
"pytorch_extra_install_requirements": PYTORCH_EXTRA_INSTALL_REQUIREMENTS
if os != "linux"
else "",
}
)
return ret
2 changes: 1 addition & 1 deletion .github/templates/linux_binary_build_workflow.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ jobs:
{%- endif %}
build_name: !{{ config["build_name"] }}
build_environment: !{{ build_environment }}
{%- if config.pytorch_extra_install_requirements is defined %}
{%- if config.pytorch_extra_install_requirements is defined and config.pytorch_extra_install_requirements|d('')|length > 0 %}
PYTORCH_EXTRA_INSTALL_REQUIREMENTS: !{{ config.pytorch_extra_install_requirements }}
{%- endif %}
secrets:
Expand Down
3 changes: 3 additions & 0 deletions .github/templates/macos_binary_build_workflow.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ jobs:
runs-on: macos-12-xl
timeout-minutes: !{{ common.timeout_minutes }}
!{{ upload.binary_env(config, true) }}
{%- if config.pytorch_extra_install_requirements is defined and config.pytorch_extra_install_requirements|d('')|length > 0 %}
PYTORCH_EXTRA_INSTALL_REQUIREMENTS: !{{ config.pytorch_extra_install_requirements }}
{%- endif %}
# For sccache access (only on non-forked PRs)
AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
Expand Down
3 changes: 3 additions & 0 deletions .github/templates/windows_binary_build_workflow.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ jobs:
runs-on: windows.4xlarge.nonephemeral
timeout-minutes: !{{ common.timeout_minutes }}
!{{ upload.binary_env(config, True) }}
{%- if config.pytorch_extra_install_requirements is defined and config.pytorch_extra_install_requirements|d('')|length > 0 %}
PYTORCH_EXTRA_INSTALL_REQUIREMENTS: !{{ config.pytorch_extra_install_requirements }}
{%- endif %}
steps:
!{{ common.setup_ec2_windows() }}
!{{ set_runner_specific_vars() }}
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/generated-linux-aarch64-binary-manywheel-nightly.yml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions .github/workflows/generated-macos-arm64-binary-wheel-nightly.yml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 7b0fc31

Please sign in to comment.