diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml index d3e57fd5c..14b7b6395 100644 --- a/.github/workflows/validate-aarch64-linux-binaries.yml +++ b/.github/workflows/validate-aarch64-linux-binaries.yml @@ -17,6 +17,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string workflow_dispatch: inputs: channel: @@ -38,6 +48,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string jobs: generate-aarch64-linux-matrix: @@ -47,7 +67,6 @@ jobs: os: linux-aarch64 channel: ${{ inputs.channel }} with-cuda: disable - linux-aarch64: needs: generate-aarch64-linux-matrix strategy: @@ -72,6 +91,8 @@ jobs: export ENV_NAME="conda-env-${{ github.run_id }}" export TARGET_OS="linux-aarch64" export TORCH_ONLY=${{ inputs.torchonly }} + export RELEASE_VERSION=${{ inputs.version }} + printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json eval "$(conda shell.bash hook)" # Standart case: Validate binaries diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml index 9c877f4ed..f6f73b012 100644 --- a/.github/workflows/validate-binaries.yml +++ b/.github/workflows/validate-binaries.yml @@ -60,44 +60,69 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate' + default: "" + required: false + type: string + jobs: + generate-release-matrix: + uses: pytorch/test-infra/.github/workflows/generate_release_matrix.yml@main + with: + version: ${{ inputs.version }} + win: if: inputs.os == 'windows' || inputs.os == 'all' + needs: generate-release-matrix uses: ./.github/workflows/validate-windows-binaries.yml with: channel: ${{ inputs.channel }} ref: ${{ inputs.ref || github.ref }} torchonly: ${{ inputs.torchonly }} + version: ${{ inputs.version }} + release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} linux: if: inputs.os == 'linux' || inputs.os == 'all' + needs: generate-release-matrix uses: ./.github/workflows/validate-linux-binaries.yml with: channel: ${{ inputs.channel }} ref: ${{ inputs.ref || github.ref }} torchonly: ${{ inputs.torchonly }} + version: ${{ inputs.version }} + release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} linux-aarch64: if: inputs.os == 'linux-aarch64' + needs: generate-release-matrix uses: ./.github/workflows/validate-aarch64-linux-binaries.yml with: channel: ${{ inputs.channel }} ref: ${{ inputs.ref || github.ref }} torchonly: ${{ inputs.torchonly }} - + version: ${{ inputs.version }} + release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} mac: if: inputs.os == 'macos' || inputs.os == 'all' + needs: generate-release-matrix uses: ./.github/workflows/validate-macos-binaries.yml with: channel: ${{ inputs.channel }} ref: ${{ inputs.ref || github.ref }} torchonly: ${{ inputs.torchonly }} + version: ${{ inputs.version }} + release-matrix: ${{ needs.generate-rlease-matrix.outputs.matrix }} mac-arm64: if: inputs.os == 'macos' || inputs.os == 'all' + needs: generate-release-matrix uses: ./.github/workflows/validate-macos-arm64-binaries.yml with: channel: ${{ inputs.channel }} ref: ${{ inputs.ref || github.ref }} torchonly: ${{ inputs.torchonly }} + version: ${{ inputs.version }} + release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml index 3c5aac2eb..dcbfc93a0 100644 --- a/.github/workflows/validate-linux-binaries.yml +++ b/.github/workflows/validate-linux-binaries.yml @@ -17,6 +17,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string workflow_dispatch: inputs: channel: @@ -38,6 +48,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string jobs: generate-linux-matrix: @@ -64,8 +84,11 @@ jobs: set -ex export ENV_NAME="conda-env-${{ github.run_id }}" export TORCH_ONLY=${{ inputs.torchonly }} + export RELEASE_VERSION=${{ inputs.version }} export TARGET_OS="linux" eval "$(conda shell.bash hook)" + printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json + cat release_matrix.json # Special case PyPi installation package. And Install of PyPi package via poetry if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" && ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml index f23dec3f6..541183b9a 100644 --- a/.github/workflows/validate-macos-arm64-binaries.yml +++ b/.github/workflows/validate-macos-arm64-binaries.yml @@ -17,6 +17,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string workflow_dispatch: inputs: channel: @@ -38,6 +48,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string jobs: generate-macos-arm64-matrix: @@ -64,4 +84,6 @@ jobs: export ENV_NAME="conda-env-${{ github.run_id }}" export TARGET_OS="macos-arm64" export TORCH_ONLY=${{ inputs.torchonly }} + export RELEASE_VERSION=${{ inputs.version }} + printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml index 0926dbe93..9610b36f7 100644 --- a/.github/workflows/validate-macos-binaries.yml +++ b/.github/workflows/validate-macos-binaries.yml @@ -17,6 +17,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string workflow_dispatch: inputs: channel: @@ -38,6 +48,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string jobs: generate-macos-matrix: @@ -64,4 +84,6 @@ jobs: export ENV_NAME="conda-env-${{ github.run_id }}" export TARGET_OS="macos" export TORCH_ONLY=${{ inputs.torchonly }} + export RELEASE_VERSION=${{ inputs.version }} + printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml index 96d2b281e..1c501cfb3 100644 --- a/.github/workflows/validate-windows-binaries.yml +++ b/.github/workflows/validate-windows-binaries.yml @@ -17,6 +17,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string workflow_dispatch: inputs: channel: @@ -38,6 +48,16 @@ on: default: false required: false type: boolean + version: + description: 'Version to validate - optional' + default: "" + required: false + type: string + release-matrix: + description: 'Release matrix - optional' + default: "" + required: false + type: string jobs: generate-windows-matrix: @@ -46,7 +66,6 @@ jobs: package-type: all os: windows channel: ${{ inputs.channel }} - win: needs: generate-windows-matrix strategy: @@ -66,9 +85,10 @@ jobs: export ENV_NAME="conda-env-${{ github.run_id }}" export TARGET_OS="windows" export TORCH_ONLY=${{ inputs.torchonly }} + export RELEASE_VERSION=${{ inputs.version }} + printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh if [[ ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then ./windows/internal/driver_update.bat fi - source ./.github/scripts/validate_binaries.sh diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 3d1b6af64..4cb7bac9b 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -3,18 +3,20 @@ import sys import argparse import torch +import json import importlib import subprocess import torch._dynamo import torch.nn as nn import torch.nn.functional as F +from pathlib import Path gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION") gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE") channel = os.getenv("MATRIX_CHANNEL") -stable_version = os.getenv("MATRIX_STABLE_VERSION") package_type = os.getenv("MATRIX_PACKAGE_TYPE") target_os = os.getenv("TARGET_OS") +BASE_DIR = Path(__file__).parent.parent.parent is_cuda_system = gpu_arch_type == "cuda" NIGHTLY_ALLOWED_DELTA = 3 @@ -52,8 +54,27 @@ def forward(self, x): output = self.fc1(x) return output +def load_json_from_basedir(filename: str): + try: + with open(BASE_DIR / filename) as fptr: + return json.load(fptr) + except FileNotFoundError as exc: + raise ImportError(f"File {filename} not found error: {exc.strerror}") from exc + except json.JSONDecodeError as exc: + raise ImportError(f"Invalid JSON {filename}") from exc + +def read_release_matrix(): + return load_json_from_basedir("release_matrix.json") def check_version(package: str) -> None: + release_version = os.getenv("RELEASE_VERSION") + # if release_version is specified, use it to validate the packages + if(release_version): + release_matrix = read_release_matrix() + stable_version = release_matrix["torch"] + else: + stable_version = os.getenv("MATRIX_STABLE_VERSION") + # only makes sense to check nightly package where dates are known if channel == "nightly": check_nightly_binaries_date(package) @@ -62,6 +83,18 @@ def check_version(package: str) -> None: raise RuntimeError( f"Torch version mismatch, expected {stable_version} for channel {channel}. But its {torch.__version__}" ) + + if release_version and package == "all": + for module in MODULES: + imported_module = importlib.import_module(module["name"]) + module_version = imported_module.__version__ + if not module_version.startswith(release_matrix[module["name"]]): + raise RuntimeError( + f"{module['name']} version mismatch, expected {release_matrix[module['name']]} for channel {channel}. But its {module_version}" + ) + else: + print(f"{module['name']} version actual: {module_version} expected: {release_matrix[module['name']]} for channel {channel}.") + else: print(f"Skip version check for channel {channel} as stable version is None") @@ -255,6 +288,7 @@ def main() -> None: ) options = parser.parse_args() print(f"torch: {torch.__version__}") + check_version(options.package) smoke_test_conv2d() smoke_test_linalg()