From bd50384781d970a8b089d23384110492f6679897 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Wed, 28 Feb 2024 16:39:02 -0500 Subject: [PATCH 1/7] This PR's main topic is adding the CUPTI version to the build process for versions 11.3 and 11.6 we don't have nvidia-cupti version that match --- .github/workflows/whl-build-all.yaml | 123 +++++++++++++++++++++------ analyzer/habitat/__init__.py | 2 +- analyzer/habitat/_version.py | 11 --- analyzer/habitat/utils.py | 1 - analyzer/setup.py | 34 ++++++-- cpp/{cmake => }/.gitignore | 0 6 files changed, 123 insertions(+), 48 deletions(-) delete mode 100644 analyzer/habitat/_version.py rename cpp/{cmake => }/.gitignore (100%) diff --git a/.github/workflows/whl-build-all.yaml b/.github/workflows/whl-build-all.yaml index cf2c8c6..b02aff9 100644 --- a/.github/workflows/whl-build-all.yaml +++ b/.github/workflows/whl-build-all.yaml @@ -9,7 +9,7 @@ jobs: runs-on: [self-hosted] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -27,7 +27,7 @@ jobs: runs-on: [self-hosted, cu113] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -42,28 +42,43 @@ jobs: ls -la - name: Fetch repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build Python3.7 wheel run: | ./build_scripts/build_wheel.sh python3.7 + env: + VERSION_TAG: cu113 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 + env: + VERSION_TAG: cu113 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 - + env: + VERSION_TAG: cu113 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 - + env: + VERSION_TAG: cu113 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 - + env: + VERSION_TAG: cu113 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Upload wheels to S3 run: | aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu113/ --recursive --exclude "*" --include "*.whl" @@ -82,7 +97,7 @@ jobs: runs-on: [self-hosted, cu116] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -97,33 +112,42 @@ jobs: ls -la - name: Fetch repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build Python3.7 wheel run: | ./build_scripts/build_wheel.sh python3.7 - + env: + VERSION_TAG: cu116 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 - + env: + VERSION_TAG: cu116 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 - + env: + VERSION_TAG: cu116 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 - + env: + VERSION_TAG: cu116 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 - - - name: Upload wheel artifact - uses: actions/upload-artifact@v3 - with: - name: wheels-cu116 - path: analyzer/dist/*.whl + env: + VERSION_TAG: cu116 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Upload wheels to S3 run: | @@ -143,7 +167,7 @@ jobs: runs-on: [self-hosted, cu117] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -158,27 +182,42 @@ jobs: ls -la - name: Fetch repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build Python3.7 wheel run: | ./build_scripts/build_wheel.sh python3.7 - + env: + VERSION_TAG: cu117 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 + env: + VERSION_TAG: cu117 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 + env: + VERSION_TAG: cu117 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 - + env: + VERSION_TAG: cu117 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 + env: + VERSION_TAG: cu117 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Upload wheels to S3 run: | @@ -198,7 +237,7 @@ jobs: runs-on: [self-hosted, cu118] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -213,27 +252,45 @@ jobs: ls -la - name: Fetch repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build Python3.7 wheel run: | ./build_scripts/build_wheel.sh python3.7 + env: + VERSION_TAG: cu118 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 + env: + VERSION_TAG: cu118 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 + - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 + env: + VERSION_TAG: cu118 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 + env: + VERSION_TAG: cu118 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 + env: + VERSION_TAG: cu118 + EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - name: Upload wheels to S3 run: | @@ -253,7 +310,7 @@ jobs: runs-on: [self-hosted, cu121] steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }} aws-region: ${{ secrets.HABITAT_AWS_REGION }} @@ -268,28 +325,38 @@ jobs: ls -la - name: Fetch repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build Python3.7 wheel run: | ./build_scripts/build_wheel.sh python3.7 + env: + VERSION_TAG: cu121 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 + env: + VERSION_TAG: cu121 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 + env: + VERSION_TAG: cu121 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 - + env: + VERSION_TAG: cu121 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 - + env: + VERSION_TAG: cu121 + - name: Upload wheels to S3 run: | aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu121/ --recursive --exclude "*" --include "*.whl" diff --git a/analyzer/habitat/__init__.py b/analyzer/habitat/__init__.py index 80364c5..2272371 100644 --- a/analyzer/habitat/__init__.py +++ b/analyzer/habitat/__init__.py @@ -10,7 +10,7 @@ traceback.print_exc() print(MISSING_LIBRARY_MESSAGE) -from ._version import __version__ +__version__ = "0.1.5" __description__ = 'Cross-GPU performance predictions for PyTorch neural network training.' diff --git a/analyzer/habitat/_version.py b/analyzer/habitat/_version.py deleted file mode 100644 index 826ee4f..0000000 --- a/analyzer/habitat/_version.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Provides habitat version information. -""" - -# This file is auto-generated! Do not edit! -# Use `python -m incremental.update habitat` to change this file. - -from incremental import Version - -__version__ = Version("habitat", 0, 1, 4) -__all__ = ["__version__"] diff --git a/analyzer/habitat/utils.py b/analyzer/habitat/utils.py index 45f9bb3..c0761f7 100644 --- a/analyzer/habitat/utils.py +++ b/analyzer/habitat/utils.py @@ -1,6 +1,5 @@ import logging import os -import yaml def set_up_logging(): diff --git a/analyzer/setup.py b/analyzer/setup.py index 590d085..b5d1e88 100644 --- a/analyzer/setup.py +++ b/analyzer/setup.py @@ -4,7 +4,7 @@ import re import sysconfig import subprocess - +import sys from setuptools import setup, find_packages from setuptools.command.build import build @@ -14,6 +14,28 @@ ################################################################### +# get Habitat version number from source directory +sys.path.append(os.path.join(os.getcwd())) +from habitat.__init__ import __version__ +sys.path.pop() + +""" +We define two additional environment arguments during build to include dependencies for +different versions of CUDA we're targeting. +1. VERSION_TAG: This follows the "version number". For example, if version="1.0" and VERSION_TAG=cu121, then the + version we pass into setuptools.setup would be "1.0+cu121" +2. EXTRA_REQUIRES: Depends on the CUDA version, we need additional pip libraries to provide CUPTI (among other things). + This is dictated by extra requires. + Packages should be comma separated and can selectively specify version numbers alongside package names. + EXTRA_REQUIRES="nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99" +We also need to handle the "default" scenario where neither is defined. We simply fall back to the default +requirements set by PyTorch. +""" + +VERSION_TAG = os.getenv("VERSION_TAG", default="") +if VERSION_TAG: VERSION_TAG = "+" + VERSION_TAG +EXTRA_REQUIRES = os.getenv("EXTRA_REQUIRES", default="nvidia-cuda-cupti-cu12,nvidia-cuda-runtime-cu12").split(",") + NAME = "deepview-predict" PACKAGES = find_packages() META_PATH = os.path.join("habitat", "__init__.py") @@ -24,7 +46,6 @@ SETUP_REQUIRES = [ "patchelf", - "incremental" ] PACKAGE_DATA = { @@ -47,10 +68,9 @@ "torch>=1.4.0", "pandas>=1.1.2", "tqdm>=4.49.0", - "nvidia-cuda-cupti-cu12", - "nvidia-cuda-runtime-cu12", - "incremental" -] + # "nvidia-cuda-cupti-cu12", + # "nvidia-cuda-runtime-cu12", +] + EXTRA_REQUIRES KEYWORDS = [ "neural networks", @@ -128,6 +148,7 @@ def find_meta(meta): if __name__ == "__main__": setup( name=NAME, + version=__version__ + VERSION_TAG, description=find_meta("description"), license=find_meta("license"), author=find_meta("author"), @@ -135,7 +156,6 @@ def find_meta(meta): maintainer=find_meta("author"), maintainer_email=find_meta("email"), long_description=read(README_PATH), - use_incremental=True, long_description_content_type="text/markdown", cmdclass= { "build": CustomBuildCommand diff --git a/cpp/cmake/.gitignore b/cpp/.gitignore similarity index 100% rename from cpp/cmake/.gitignore rename to cpp/.gitignore From aba6559c8d17c4fef296ed7fc569a9a631a03ed3 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 29 Feb 2024 10:18:41 -0500 Subject: [PATCH 2/7] changed env name from version_tag to version_cuda_tag --- .github/workflows/whl-build-all.yaml | 50 ++++++++++++++-------------- analyzer/setup.py | 8 ++--- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/whl-build-all.yaml b/.github/workflows/whl-build-all.yaml index b02aff9..abb2fc5 100644 --- a/.github/workflows/whl-build-all.yaml +++ b/.github/workflows/whl-build-all.yaml @@ -48,35 +48,35 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_TAG: cu113 + VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_TAG: cu113 + VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_TAG: cu113 + VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_TAG: cu113 + VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_TAG: cu113 + VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Upload wheels to S3 @@ -118,35 +118,35 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_TAG: cu116 + VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_TAG: cu116 + VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_TAG: cu116 + VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_TAG: cu116 + VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_TAG: cu116 + VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Upload wheels to S3 @@ -188,35 +188,35 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_TAG: cu117 + VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_TAG: cu117 + VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_TAG: cu117 + VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_TAG: cu117 + VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_TAG: cu117 + VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Upload wheels to S3 @@ -258,14 +258,14 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_TAG: cu118 + VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_TAG: cu118 + VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 @@ -273,7 +273,7 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_TAG: cu118 + VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 @@ -281,7 +281,7 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_TAG: cu118 + VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 @@ -289,7 +289,7 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_TAG: cu118 + VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - name: Upload wheels to S3 @@ -331,31 +331,31 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_TAG: cu121 + VERSION_CUDA_TAG: cu121 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_TAG: cu121 + VERSION_CUDA_TAG: cu121 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_TAG: cu121 + VERSION_CUDA_TAG: cu121 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_TAG: cu121 + VERSION_CUDA_TAG: cu121 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_TAG: cu121 + VERSION_CUDA_TAG: cu121 - name: Upload wheels to S3 run: | diff --git a/analyzer/setup.py b/analyzer/setup.py index b5d1e88..ef350f2 100644 --- a/analyzer/setup.py +++ b/analyzer/setup.py @@ -22,7 +22,7 @@ """ We define two additional environment arguments during build to include dependencies for different versions of CUDA we're targeting. -1. VERSION_TAG: This follows the "version number". For example, if version="1.0" and VERSION_TAG=cu121, then the +1. VERSION_CUDA_TAG: This follows the "version number". For example, if version="1.0" and VERSION_CUDA_TAG=cu121, then the version we pass into setuptools.setup would be "1.0+cu121" 2. EXTRA_REQUIRES: Depends on the CUDA version, we need additional pip libraries to provide CUPTI (among other things). This is dictated by extra requires. @@ -32,8 +32,8 @@ requirements set by PyTorch. """ -VERSION_TAG = os.getenv("VERSION_TAG", default="") -if VERSION_TAG: VERSION_TAG = "+" + VERSION_TAG +VERSION_CUDA_TAG = os.getenv("VERSION_CUDA_TAG", default="") +if VERSION_CUDA_TAG: VERSION_CUDA_TAG = "+" + VERSION_CUDA_TAG EXTRA_REQUIRES = os.getenv("EXTRA_REQUIRES", default="nvidia-cuda-cupti-cu12,nvidia-cuda-runtime-cu12").split(",") NAME = "deepview-predict" @@ -148,7 +148,7 @@ def find_meta(meta): if __name__ == "__main__": setup( name=NAME, - version=__version__ + VERSION_TAG, + version=__version__ + VERSION_CUDA_TAG, description=find_meta("description"), license=find_meta("license"), author=find_meta("author"), From d362dccaf1aeea7d633e4ed7bb384ca1d78050c1 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 29 Feb 2024 10:26:59 -0500 Subject: [PATCH 3/7] added env variable version_cuda_tag to whl-build-ec2 for the release to pypi --- .github/workflows/whl-build-ec2.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/whl-build-ec2.yaml b/.github/workflows/whl-build-ec2.yaml index 3be01a8..d432579 100644 --- a/.github/workflows/whl-build-ec2.yaml +++ b/.github/workflows/whl-build-ec2.yaml @@ -69,7 +69,7 @@ jobs: v100: false build-release: - runs-on: [self-hosted, cu117] + runs-on: [self-hosted, cu121] needs: - start-runner steps: @@ -81,23 +81,33 @@ jobs: - name: Build Python3.7 distribution run: | ./build_scripts/build_wheel.sh python3.7 - + env: + VERSION_CUDA_TAG: cu121 + - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 - + env: + VERSION_CUDA_TAG: cu121 + - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 + env: + VERSION_CUDA_TAG: cu121 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 + env: + VERSION_CUDA_TAG: cu121 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 - + env: + VERSION_CUDA_TAG: cu121 + - uses: actions/upload-artifact@v3 with: name: ${{ github.event.inputs.tag }} From 47cfe1309273837231984249ea9a028f4b24232e Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 29 Feb 2024 14:17:26 -0500 Subject: [PATCH 4/7] added new env variable for pytorch version --- .github/workflows/whl-build-all.yaml | 46 ++++++++++++++++++++-------- build_scripts/build_wheel.sh | 6 +++- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/.github/workflows/whl-build-all.yaml b/.github/workflows/whl-build-all.yaml index abb2fc5..1416e20 100644 --- a/.github/workflows/whl-build-all.yaml +++ b/.github/workflows/whl-build-all.yaml @@ -50,6 +50,7 @@ jobs: env: VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.8 wheel run: | @@ -57,6 +58,7 @@ jobs: env: VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.9 wheel run: | @@ -64,21 +66,24 @@ jobs: env: VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: VERSION_CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 + - name: Upload wheels to S3 run: | aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu113/ --recursive --exclude "*" --include "*.whl" @@ -120,34 +125,39 @@ jobs: env: VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 + - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 + - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: VERSION_CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Upload wheels to S3 run: | @@ -190,13 +200,15 @@ jobs: env: VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - + TORCH_VERSION: torch==2.0.1 + - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==2.0.1 - name: Build Python3.9 wheel run: | @@ -204,6 +216,7 @@ jobs: env: VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==2.0.1 - name: Build Python3.10 wheel run: | @@ -211,6 +224,7 @@ jobs: env: VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==2.0.1 - name: Build Python3.11 wheel run: | @@ -218,6 +232,7 @@ jobs: env: VERSION_CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 + TORCH_VERSION: torch==2.0.1 - name: Upload wheels to S3 run: | @@ -260,6 +275,7 @@ jobs: env: VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 + TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.8 wheel run: | @@ -267,7 +283,7 @@ jobs: env: VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - + TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.9 wheel run: | @@ -275,7 +291,7 @@ jobs: env: VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - + TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.10 wheel run: | @@ -283,7 +299,7 @@ jobs: env: VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - + TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.11 wheel run: | @@ -291,6 +307,7 @@ jobs: env: VERSION_CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 + TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Upload wheels to S3 run: | @@ -332,31 +349,36 @@ jobs: ./build_scripts/build_wheel.sh python3.7 env: VERSION_CUDA_TAG: cu121 + TORCH_VERSION: torch - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: VERSION_CUDA_TAG: cu121 + TORCH_VERSION: torch - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: VERSION_CUDA_TAG: cu121 + TORCH_VERSION: torch - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: VERSION_CUDA_TAG: cu121 + TORCH_VERSION: torch - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: VERSION_CUDA_TAG: cu121 - + TORCH_VERSION: torch + - name: Upload wheels to S3 run: | aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu121/ --recursive --exclude "*" --include "*.whl" diff --git a/build_scripts/build_wheel.sh b/build_scripts/build_wheel.sh index e806c3d..42bd698 100755 --- a/build_scripts/build_wheel.sh +++ b/build_scripts/build_wheel.sh @@ -1,5 +1,4 @@ PYTHON_VERSION=$1 -CUDA_TAG=$2 VENV_NAME=venv_${PYTHON_VERSION} python3 -m virtualenv ${VENV_NAME} -p ${PYTHON_VERSION} @@ -7,6 +6,9 @@ ln -s /usr/bin/${PYTHON_VERSION}-config ${VENV_NAME}/bin/python3-config source $VENV_NAME/bin/activate +pip install ${TORCH_VERSION} +pip install numpy tqdm pandas + rm -r cpp/build analyzer/habitat/*.so git submodule update --init --recursive git lfs pull @@ -16,6 +18,8 @@ pushd analyzer python3 setup.py bdist_wheel popd +pip list + deactivate rm -r ${VENV_NAME} From 58a4bb6e155cda2e7eadf2af24e054b178d57158 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Thu, 29 Feb 2024 15:00:50 -0500 Subject: [PATCH 5/7] added env var to build-ec2 --- .github/workflows/whl-build-ec2.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/whl-build-ec2.yaml b/.github/workflows/whl-build-ec2.yaml index d432579..91428d2 100644 --- a/.github/workflows/whl-build-ec2.yaml +++ b/.github/workflows/whl-build-ec2.yaml @@ -83,31 +83,36 @@ jobs: ./build_scripts/build_wheel.sh python3.7 env: VERSION_CUDA_TAG: cu121 + TORCH_VERSION: torch - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: VERSION_CUDA_TAG: cu121 - + TORCH_VERSION: torch + - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: VERSION_CUDA_TAG: cu121 - + TORCH_VERSION: torch + - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: VERSION_CUDA_TAG: cu121 - + TORCH_VERSION: torch + - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: VERSION_CUDA_TAG: cu121 - + TORCH_VERSION: torch + - uses: actions/upload-artifact@v3 with: name: ${{ github.event.inputs.tag }} From cb276081cc7b5503c497593ffe46c778d3ba1749 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Mon, 4 Mar 2024 11:33:41 -0500 Subject: [PATCH 6/7] added new bash script to update the version for new releases to replace increment changed setup script to use pip setup instead of python build wheels updated yaml files to reflect changes in setup.py updated version in __init__.py to get the information from importlib commented whl-build-ec2 for initial testing --- .github/workflows/whl-build-all.yaml | 75 ++++++------------ .github/workflows/whl-build-ec2.yaml | 114 +++++++++++++-------------- analyzer/git_update_version.sh | 7 ++ analyzer/habitat/__init__.py | 4 +- analyzer/setup.py | 35 ++++---- build_scripts/build_wheel.sh | 5 +- 6 files changed, 109 insertions(+), 131 deletions(-) create mode 100644 analyzer/git_update_version.sh diff --git a/.github/workflows/whl-build-all.yaml b/.github/workflows/whl-build-all.yaml index 1416e20..59d31fb 100644 --- a/.github/workflows/whl-build-all.yaml +++ b/.github/workflows/whl-build-all.yaml @@ -48,41 +48,36 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu113 + CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu113 + CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu113 + CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu113 + CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu113 + CUDA_TAG: cu113 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 - name: Upload wheels to S3 run: | @@ -123,41 +118,36 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu116 + CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu116 + CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu116 + CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu116 + CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu116 + CUDA_TAG: cu116 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==1.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 - name: Upload wheels to S3 run: | @@ -198,41 +188,36 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu117 + CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==2.0.1 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu117 + CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==2.0.1 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu117 + CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==2.0.1 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu117 + CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==2.0.1 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu117 + CUDA_TAG: cu117 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - TORCH_VERSION: torch==2.0.1 - name: Upload wheels to S3 run: | @@ -273,41 +258,36 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu118 + CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu118 + CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu118 + CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu118 + CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu118 + CUDA_TAG: cu118 EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - TORCH_VERSION: torch==2.1.0 --index-url https://download.pytorch.org/whl/cu118 - name: Upload wheels to S3 run: | @@ -348,36 +328,31 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Upload wheels to S3 run: | diff --git a/.github/workflows/whl-build-ec2.yaml b/.github/workflows/whl-build-ec2.yaml index 91428d2..70d6915 100644 --- a/.github/workflows/whl-build-ec2.yaml +++ b/.github/workflows/whl-build-ec2.yaml @@ -29,9 +29,8 @@ jobs: - name: Update version run: | - pip install incremental pushd analyzer - python3 -m incremental.update habitat --newversion=${{ github.event.inputs.tag }} + bash git_update_version.sh ${{ github.event.inputs.tag }} popd - name: Commit updated version number and tag it @@ -82,36 +81,31 @@ jobs: run: | ./build_scripts/build_wheel.sh python3.7 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.9 wheel run: | ./build_scripts/build_wheel.sh python3.9 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.10 wheel run: | ./build_scripts/build_wheel.sh python3.10 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - name: Build Python3.11 wheel run: | ./build_scripts/build_wheel.sh python3.11 env: - VERSION_CUDA_TAG: cu121 - TORCH_VERSION: torch + CUDA_TAG: cu121 - uses: actions/upload-artifact@v3 with: @@ -167,51 +161,51 @@ jobs: RELEASE_ARTIFACTS=$(find ./dist -name "*${{ github.event.inputs.tag }}*" -type f | paste -s -d ' ' - ) VERSION_TAG="v${{ github.event.inputs.tag }}" gh auth login --with-token <<< "${{ secrets.GH_TOKEN }}" - gh release create "$VERSION_TAG" \ - --title "$VERSION_TAG" \ - --notes "$RELEASE_NOTES" \ - --target "$GITHUB_SHA" \ - $RELEASE_ARTIFACTS - gh pr create --title "Release $VERSION_TAG" --body "$RELEASE_NOTES" - - publish-to-test-pypi: - name: Publish to Test PyPI - needs: build-release - runs-on: ubuntu-latest - environment: Test - concurrency: Test - permissions: - id-token: write - - steps: - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: ${{ github.event.inputs.tag }} - path: dist - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository_url: https://test.pypi.org/legacy/ - - publish-to-pypi: - name: Publish to PyPI - needs: publish-to-test-pypi - runs-on: ubuntu-latest - environment: Production - concurrency: Production - permissions: - id-token: write - - steps: - - name: Download artifact - uses: actions/download-artifact@v3 - with: - name: ${{ github.event.inputs.tag }} - path: dist - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository_url: https://upload.pypi.org/legacy/ \ No newline at end of file + # gh release create "$VERSION_TAG" \ + # --title "$VERSION_TAG" \ + # --notes "$RELEASE_NOTES" \ + # --target "$GITHUB_SHA" \ + # $RELEASE_ARTIFACTS + # gh pr create --title "Release $VERSION_TAG" --body "$RELEASE_NOTES" + + # publish-to-test-pypi: + # name: Publish to Test PyPI + # needs: build-release + # runs-on: ubuntu-latest + # environment: Test + # concurrency: Test + # permissions: + # id-token: write + + # steps: + # - name: Download artifact + # uses: actions/download-artifact@v3 + # with: + # name: ${{ github.event.inputs.tag }} + # path: dist + + # - name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository_url: https://test.pypi.org/legacy/ + + # publish-to-pypi: + # name: Publish to PyPI + # needs: publish-to-test-pypi + # runs-on: ubuntu-latest + # environment: Production + # concurrency: Production + # permissions: + # id-token: write + + # steps: + # - name: Download artifact + # uses: actions/download-artifact@v3 + # with: + # name: ${{ github.event.inputs.tag }} + # path: dist + + # - name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository_url: https://upload.pypi.org/legacy/ \ No newline at end of file diff --git a/analyzer/git_update_version.sh b/analyzer/git_update_version.sh new file mode 100644 index 0000000..be1befa --- /dev/null +++ b/analyzer/git_update_version.sh @@ -0,0 +1,7 @@ +#!bin/bash + +# DO NOT CALL DIRECTLY +# THIS SCRIPT IS USED BY GITHUB ACTIONS TO UPDATE THE VERSION + +NEW_VERSION=$1 +sed -i "s/^VERSION =.*/VERSION = '${NEW_VERSION}'/" ./setup.py \ No newline at end of file diff --git a/analyzer/habitat/__init__.py b/analyzer/habitat/__init__.py index 2272371..e7a3a74 100644 --- a/analyzer/habitat/__init__.py +++ b/analyzer/habitat/__init__.py @@ -10,7 +10,9 @@ traceback.print_exc() print(MISSING_LIBRARY_MESSAGE) -__version__ = "0.1.5" +import importlib.metadata + +__version__ = importlib.metadata.version("deepview-predict") __description__ = 'Cross-GPU performance predictions for PyTorch neural network training.' diff --git a/analyzer/setup.py b/analyzer/setup.py index ef350f2..bc23394 100644 --- a/analyzer/setup.py +++ b/analyzer/setup.py @@ -1,10 +1,9 @@ import codecs import os -import pkg_resources +from importlib.metadata import files import re import sysconfig import subprocess -import sys from setuptools import setup, find_packages from setuptools.command.build import build @@ -14,40 +13,41 @@ ################################################################### -# get Habitat version number from source directory -sys.path.append(os.path.join(os.getcwd())) -from habitat.__init__ import __version__ -sys.path.pop() - """ We define two additional environment arguments during build to include dependencies for different versions of CUDA we're targeting. -1. VERSION_CUDA_TAG: This follows the "version number". For example, if version="1.0" and VERSION_CUDA_TAG=cu121, then the + +1. CUDA_TAG : This follows the "version number". For example, if version="1.0" and CUDA_TAG =cu121, then the version we pass into setuptools.setup would be "1.0+cu121" + 2. EXTRA_REQUIRES: Depends on the CUDA version, we need additional pip libraries to provide CUPTI (among other things). This is dictated by extra requires. + Packages should be comma separated and can selectively specify version numbers alongside package names. + EXTRA_REQUIRES="nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99" + We also need to handle the "default" scenario where neither is defined. We simply fall back to the default requirements set by PyTorch. """ - -VERSION_CUDA_TAG = os.getenv("VERSION_CUDA_TAG", default="") -if VERSION_CUDA_TAG: VERSION_CUDA_TAG = "+" + VERSION_CUDA_TAG -EXTRA_REQUIRES = os.getenv("EXTRA_REQUIRES", default="nvidia-cuda-cupti-cu12,nvidia-cuda-runtime-cu12").split(",") - +VERSION = '0.1.4' NAME = "deepview-predict" PACKAGES = find_packages() META_PATH = os.path.join("habitat", "__init__.py") -README_PATH = "../README.md" +README_PATH = "README.md" PYTHON_REQUIRES = ">=3.7" PYTHON_VERSION = sysconfig.get_python_version().replace('.', '') SETUP_REQUIRES = [ "patchelf", + "incremental" ] +CUDA_TAG = os.getenv("CUDA_TAG", default="") +if CUDA_TAG : CUDA_TAG = "+" + CUDA_TAG +EXTRA_REQUIRES = os.getenv("EXTRA_REQUIRES", default="nvidia-cuda-cupti-cu12,nvidia-cuda-runtime-cu12").split(",") + PACKAGE_DATA = { "habitat": [ "analysis/mlp/devices.csv", @@ -91,7 +91,8 @@ class CustomBuildCommand(build): def run(self): # Need to update the rpath of the habitat_cuda.cpython library # Ensures that it links to the libraries included in the wheel - patchelf_bin_path = pkg_resources.get_distribution("patchelf").location + "/EGG-INFO/scripts/patchelf" + patchelf_bin_loc = [p for p in files("patchelf") if "bin" in str(p)][0] + patchelf_bin_path = str(patchelf_bin_loc.locate()) habitat_dir = os.listdir("habitat") curr_python_ver = "{}".format(PYTHON_VERSION) library_name = "" @@ -148,7 +149,7 @@ def find_meta(meta): if __name__ == "__main__": setup( name=NAME, - version=__version__ + VERSION_CUDA_TAG, + version=VERSION + CUDA_TAG , description=find_meta("description"), license=find_meta("license"), author=find_meta("author"), @@ -172,4 +173,4 @@ def find_meta(meta): "python_tag": "py"+ PYTHON_VERSION } } - ) + ) \ No newline at end of file diff --git a/build_scripts/build_wheel.sh b/build_scripts/build_wheel.sh index 42bd698..e1a867e 100755 --- a/build_scripts/build_wheel.sh +++ b/build_scripts/build_wheel.sh @@ -6,8 +6,7 @@ ln -s /usr/bin/${PYTHON_VERSION}-config ${VENV_NAME}/bin/python3-config source $VENV_NAME/bin/activate -pip install ${TORCH_VERSION} -pip install numpy tqdm pandas +pip install build rm -r cpp/build analyzer/habitat/*.so git submodule update --init --recursive @@ -15,7 +14,7 @@ git lfs pull pushd analyzer ./install-dev.sh --build -python3 setup.py bdist_wheel +python -m build popd pip list From 5dfa4516164991ef8cd0ff0f492179de885cbdd4 Mon Sep 17 00:00:00 2001 From: John Calderon Date: Mon, 4 Mar 2024 13:44:31 -0500 Subject: [PATCH 7/7] removed python3.7 from build because importlib is not found for that version removed start and stop runner because we only need to start-up the ec2 instance updated version in README.md --- .github/workflows/whl-build-all.yaml | 34 ------- .github/workflows/whl-build-ec2.yaml | 143 ++++++++------------------- README.md | 4 +- 3 files changed, 45 insertions(+), 136 deletions(-) diff --git a/.github/workflows/whl-build-all.yaml b/.github/workflows/whl-build-all.yaml index 59d31fb..2edaaee 100644 --- a/.github/workflows/whl-build-all.yaml +++ b/.github/workflows/whl-build-all.yaml @@ -44,13 +44,6 @@ jobs: - name: Fetch repository uses: actions/checkout@v4 - - name: Build Python3.7 wheel - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu113 - EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 @@ -114,13 +107,6 @@ jobs: - name: Fetch repository uses: actions/checkout@v4 - - name: Build Python3.7 wheel - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu116 - EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 @@ -183,13 +169,6 @@ jobs: - name: Fetch repository uses: actions/checkout@v4 - - - name: Build Python3.7 wheel - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu117 - EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.7.101,nvidia-cuda-runtime-cu11==11.7.99 - name: Build Python3.8 wheel run: | @@ -254,13 +233,6 @@ jobs: - name: Fetch repository uses: actions/checkout@v4 - - name: Build Python3.7 wheel - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu118 - EXTRA_REQUIRES: nvidia-cuda-cupti-cu11==11.8.87,nvidia-cuda-runtime-cu11==11.8.89 - - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 @@ -324,12 +296,6 @@ jobs: - name: Fetch repository uses: actions/checkout@v4 - - name: Build Python3.7 wheel - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu121 - - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 diff --git a/.github/workflows/whl-build-ec2.yaml b/.github/workflows/whl-build-ec2.yaml index 70d6915..260a707 100644 --- a/.github/workflows/whl-build-ec2.yaml +++ b/.github/workflows/whl-build-ec2.yaml @@ -42,47 +42,14 @@ jobs: git push -u origin release/${{ github.event.inputs.tag }} git tag ${{ github.event.inputs.tag }} - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - needs: - - create-release-branch - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - aws-region: us-east-1 - aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - - name: Start EC2 runner - id: start-ec2-runner - uses: CentML/GPUClusterSwitchGithubAction@v1.3.10 - with: - mode: start - github-token: ${{ secrets.CENTML_PAT }} - t4: true - a10g: false - v100: false - build-release: runs-on: [self-hosted, cu121] - needs: - - start-runner steps: - name: Fetch repository uses: actions/checkout@v3 with: ref: release/${{ github.event.inputs.tag }} - - name: Build Python3.7 distribution - run: | - ./build_scripts/build_wheel.sh python3.7 - env: - CUDA_TAG: cu121 - - name: Build Python3.8 wheel run: | ./build_scripts/build_wheel.sh python3.8 @@ -111,30 +78,6 @@ jobs: with: name: ${{ github.event.inputs.tag }} path: analyzer/dist/*.whl - - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - build-release # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - aws-region: us-east-1 - aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - - name: Stop EC2 runner - id: stop-ec2-runner - uses: CentML/GPUClusterSwitchGithubAction@v1.3.10 - with: - mode: stop - github-token: ${{ secrets.CENTML_PAT }} - t4: true # required to match the list above - otherwise the runners will not stop - a10g: false # required to match the list above - otherwise the runners will not stop - v100: false publish-to-github: name: Publish to Github @@ -161,51 +104,51 @@ jobs: RELEASE_ARTIFACTS=$(find ./dist -name "*${{ github.event.inputs.tag }}*" -type f | paste -s -d ' ' - ) VERSION_TAG="v${{ github.event.inputs.tag }}" gh auth login --with-token <<< "${{ secrets.GH_TOKEN }}" - # gh release create "$VERSION_TAG" \ - # --title "$VERSION_TAG" \ - # --notes "$RELEASE_NOTES" \ - # --target "$GITHUB_SHA" \ - # $RELEASE_ARTIFACTS - # gh pr create --title "Release $VERSION_TAG" --body "$RELEASE_NOTES" - - # publish-to-test-pypi: - # name: Publish to Test PyPI - # needs: build-release - # runs-on: ubuntu-latest - # environment: Test - # concurrency: Test - # permissions: - # id-token: write + gh release create "$VERSION_TAG" \ + --title "$VERSION_TAG" \ + --notes "$RELEASE_NOTES" \ + --target "$GITHUB_SHA" \ + $RELEASE_ARTIFACTS + gh pr create --title "Release $VERSION_TAG" --body "$RELEASE_NOTES" + + publish-to-test-pypi: + name: Publish to Test PyPI + needs: build-release + runs-on: ubuntu-latest + environment: Test + concurrency: Test + permissions: + id-token: write - # steps: - # - name: Download artifact - # uses: actions/download-artifact@v3 - # with: - # name: ${{ github.event.inputs.tag }} - # path: dist + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ github.event.inputs.tag }} + path: dist - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # repository_url: https://test.pypi.org/legacy/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository_url: https://test.pypi.org/legacy/ - # publish-to-pypi: - # name: Publish to PyPI - # needs: publish-to-test-pypi - # runs-on: ubuntu-latest - # environment: Production - # concurrency: Production - # permissions: - # id-token: write + publish-to-pypi: + name: Publish to PyPI + needs: publish-to-test-pypi + runs-on: ubuntu-latest + environment: Production + concurrency: Production + permissions: + id-token: write - # steps: - # - name: Download artifact - # uses: actions/download-artifact@v3 - # with: - # name: ${{ github.event.inputs.tag }} - # path: dist + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ github.event.inputs.tag }} + path: dist - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # repository_url: https://upload.pypi.org/legacy/ \ No newline at end of file + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository_url: https://upload.pypi.org/legacy/ \ No newline at end of file diff --git a/README.md b/README.md index 722e1c0..89f9ef3 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ DeepView.Predict is a tool that predicts a deep neural network's training iterat

Installation

To run DeepView.Predict, you need: -- [Python 3.6+](https://www.python.org/) -- [Pytorch 1.1.0+](https://pytorch.org/) +- [Python 3.8+](https://www.python.org/) +- [Pytorch 1.13.1+](https://pytorch.org/) - A system equiped with an Nvidia GPU with properly configured CUDA Currently, we have predictors for the following Nvidia GPUs: