Use uv to speed up pip installs & the CI in general (zenml-io#2442)

* Use uv * add to migration testing * typo fix * fixes * api checks * remove -U * test dargli * source * activate * as property * sourced * try fix * remove cache * try export * source * new approach * refactor * refactor2 * setup env * fast ci * environment installation * remove pip upgrade * remove pip completely * try different configuration * uv refactor * Update virtual environment setup and dependencies * line extra * use uv instead of pip * Remove unnecessary environment variable assignment * Add source command to setup environment scripts * remove pip install * remove old venv setup * typo * add pip back (partially) * Update sklearn integration installation in test-migrations scripts * Auto-update of Starter template * Auto-update of NLP template * uv pip list instead of plain pip * Fix source type resolution for defaultdict on Windows * partially disable CI * formatting * formatting * update second test for windows * run the whole ci again * revert test changes * remove old venv commands * debug uv failure * add bash shell for windows * install zenml * fix editable install * add newline * remove double uv for pip debug logging * disable CI temporarily for testing * yamlfix * unfail * more metadata about collections * faster resolution for test * fix escape * attempt fix for windows source resolution * format and windows unit tests * add ubuntu unit tests * revert source_utils changes * try adding cache back in * restore normal CI * add spellcheck back * delete test workflow * use int instead + disable cache again * Auto-update of Starter template * remove file attrib check * increase HTTP timeout for uv (for torch) * update for new migration script * test to ensure that the uv updates work now * no need to run mariadb on fastCI --------- Co-authored-by: GitHub Actions <actions@github.com>
adtygan · Mar 20, 2024 · 41e7c93 · 41e7c93
1 parent a237a53
commit 41e7c93
Show file tree

Hide file tree

Showing 14 changed files with 131 additions and 134 deletions.
diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml
@@ -9,15 +9,12 @@ inputs:
     required: true
   python-version:
     description: Python version
-    type: string
     required: true
   os:
     description: OS
-    type: string
     required: true
   install_integrations:
     description: Install ZenML integrations
-    type: string
     required: false
     default: 'yes'
   runners_cache_access_key_id:
@@ -56,56 +53,57 @@ runs:
       shell: bash
       run: |
         "C:\Program Files\Git\bin\git.exe" config --global user.email "info@zenml.io"
-        "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML GmbH"
-    - name: Get current week
-      shell: bash
-      id: date
-      run: echo "::set-output name=week::$(date +'calendar-week-%W')"
-    - uses: syphar/restore-virtualenv@v1
-      if: ${{ inputs.os != 'arc-runner-set' }}
-      id: cache-virtualenv
-      with:
-        requirement_files: pyproject.toml
-     # The virtualenv cache is invalidated when:
-     # - manually triggered by means of a custom cache version token
-     # - on a weekly basis
-     # - any of the integration requirements change (a hash of the
-     # __init__.py files is included in the cache key)
-        custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{
-          hashFiles('src/zenml/integrations/*/__init__.py') }}
-    - uses: tespkg/actions-cache@v1
-      if: ${{ inputs.os == 'arc-runner-set' }}
-      id: custom-cache-pip
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }}
-      with:
-        endpoint: minio-service.minio.svc.cluster.local  # optional
-        insecure: true  # optional, use http instead of https. default false
-        bucket: caching  # required
-        use-fallback: false  # optional, use github actions cache fallback, default false
-        key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version
-          }}-${{inputs.install_integrations}}
-        path: |
-          ~/.cache/pip
-        restore-keys: |
-          ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}}
-    - name: Custom runners cache miss
-      uses: rjstone/discord-webhook-notify@v1
-      if: steps.custom-cache-pip.outputs.cache-hit != 'true'
-      continue-on-error: true
-      with:
-        severity: warn
-        details: The custom github runnners cache was missed and failed to restore
-          the cache from minio bucket
-        webhookUrl: ${{ inputs.discord_webhook }}
-     # Disabled for now because it doesn't work well with multiple parallel jobs
-     # - uses: syphar/restore-pip-download-cache@v1
-     #   with:
-    #     requirement_files: 'pyproject.toml'
+        "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML
+        GmbH"
+    # - name: Get current week
+    #   shell: bash
+    #   id: date
+    #   run: echo "::set-output name=week::$(date +'calendar-week-%W')"
+    # - uses: syphar/restore-virtualenv@v1
+    #   if: ${{ inputs.os != 'arc-runner-set' }}
+    #   id: cache-virtualenv
+    #   with:
+    #     requirement_files: pyproject.toml
+    #  # The virtualenv cache is invalidated when:
+    #  # - manually triggered by means of a custom cache version token
+    #  # - on a weekly basis
+    #  # - any of the integration requirements change (a hash of the
+    #  # __init__.py files is included in the cache key)
+    #     custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{
+    #       hashFiles('src/zenml/integrations/*/__init__.py') }}
+    # - uses: tespkg/actions-cache@v1
+    #   if: ${{ inputs.os == 'arc-runner-set' }}
+    #   id: custom-cache-pip
+    #   env:
+    #     AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }}
+    #     AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }}
+    #   with:
+    #     endpoint: minio-service.minio.svc.cluster.local  # optional
+    #     insecure: true  # optional, use http instead of https. default false
+    #     bucket: caching  # required
+    #     use-fallback: false  # optional, use github actions cache fallback, default false
+    #     key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version
+    #       }}-${{inputs.install_integrations}}
+    #     path: |
+    #       ~/.cache/pip
+    #     restore-keys: |
+    #       ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}}
+    # - name: Custom runners cache miss
+    #   uses: rjstone/discord-webhook-notify@v1
+    #   if: steps.custom-cache-pip.outputs.cache-hit != 'true'
+    #   continue-on-error: true
+    #   with:
+    #     severity: warn
+    #     details: The custom github runnners cache was missed and failed to restore
+    #       the cache from minio bucket
+    #     webhookUrl: ${{ inputs.discord_webhook }}
+    #  # Disabled for now because it doesn't work well with multiple parallel jobs
+    #  # - uses: syphar/restore-pip-download-cache@v1
+    #  #   with:
+    # #     requirement_files: 'pyproject.toml'
     #  # The pip download cache can be updated on a weekly basis as new packages
     #  # don't appear that often
-     #     custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}}
+    #  #     custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}}
     - name: Install Terraform (Windows)
       if: ${{ inputs.os == 'windows-latest' }}
       shell: bash
@@ -124,5 +122,5 @@ runs:
       shell: bash
       run: |-
         zenml integration list
-        pip list
+        uv pip list
         pip check || true
diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
@@ -24,8 +24,11 @@ jobs:
         uses: actions/setup-python@v5.0.0
         with:
           python-version: '3.10'
-      - name: Install current package as editable
-        run: pip install -U darglint
+      - name: Install darglint using uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          source $HOME/.cargo/env
+          uv pip install --system darglint
       - name: Check docstrings
         run: bash scripts/docstring.sh
   sqlite-db-migration-testing:

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
@@ -1,6 +1,8 @@
 ---
 name: ci-slow
 on:
+  workflow_dispatch:
+  workflow_call:
   push:
     branches: [main]
     paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
@@ -48,7 +50,10 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install current package as editable
-        run: pip install -U darglint
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          source $HOME/.cargo/env
+          uv pip install --system darglint
       - name: Check docstrings
         run: bash scripts/docstring.sh
   sqlite-db-migration-testing:
@@ -80,10 +85,11 @@ jobs:
           python-version: '3.8'
       - name: Install current package as editable
         run: |
-          pip install -U pip
-          pip install -e .
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          source $HOME/.cargo/env
+          uv pip install --system -e .
       - name: Install mlstacks package
-        run: pip install mlstacks
+        run: uv pip install --system mlstacks
       - name: Check for broken dependencies
         run: pip check
       - name: Markdown link check
@@ -98,13 +104,13 @@ jobs:
         continue-on-error: true
       - name: Security check
         run: |
-          pip install bandit
+          uv pip install --system bandit
           bash scripts/check-security.sh
       - name: Check for alembic branch divergence
         env:
           ZENML_DEBUG: 0
         run: |
-          pip install alembic
+          uv pip install --system alembic
           bash scripts/check-alembic-branches.sh
   custom-ubuntu-unit-test:
     if: github.event.pull_request.draft == false
@@ -187,7 +193,7 @@ jobs:
           - docker-server-docker-orchestrator-mysql
           - docker-server-docker-orchestrator-mariadb
         exclude:
-          # docker is time-consuming to run, so we only run it on 3.8
+   # docker is time-consuming to run, so we only run it on 3.8
           - test_environment: docker-server-docker-orchestrator-mysql
             python-version: '3.9'
           - test_environment: docker-server-docker-orchestrator-mysql

diff --git a/.github/workflows/generate-test-duration.yml b/.github/workflows/generate-test-duration.yml
@@ -14,6 +14,7 @@ jobs:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
           # on MAC OS, we need to set this environment variable
           # to fix problems with the fork() calls (see this thread
           # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
@@ -53,5 +54,5 @@ jobs:
       - name: Verify Python Env unaffected
         run: |-
           zenml integration list
-          pip list
+          uv pip list
           pip check || true
diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml
@@ -87,6 +87,7 @@ jobs:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
       # on MAC OS, we need to set this environment variable
       # to fix problems with the fork() calls (see this thread
       # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
@@ -150,8 +151,8 @@ jobs:
       - name: Install docker-compose for non-default environments
         if: inputs.test_environment != 'default'
         run: |
-          pip install pyyaml==5.3.1
-          pip install docker-compose
+          pip install uv
+          uv pip install --system "pyyaml==5.3.1" docker-compose
       - name: Install Linux System Dependencies
         if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
         run: sudo apt install graphviz
@@ -227,5 +228,5 @@ jobs:
       - name: Verify Python Env unaffected
         run: |-
           zenml integration list
-          pip list
+          uv pip list
           pip check || true
diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml
@@ -85,6 +85,7 @@ jobs:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
       # on MAC OS, we need to set this environment variable
       # to fix problems with the fork() calls (see this thread
       # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
@@ -148,8 +149,8 @@ jobs:
       - name: Install docker-compose for non-default environments
         if: inputs.test_environment != 'default'
         run: |
-          pip install pyyaml==5.3.1
-          pip install docker-compose
+          pip install uv
+          uv pip install --system "pyyaml==5.3.1" docker-compose
       - name: Install Linux System Dependencies
         if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
         run: sudo apt install graphviz
@@ -229,5 +230,5 @@ jobs:
       - name: Verify Python Env unaffected
         run: |-
           zenml integration list
-          pip list
+          uv pip list
           pip check || true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -23,9 +23,11 @@ jobs:
         with:
           python-version: '3.8'
       - name: Install current package as editable
-        run: pip install -e .
+        run: |
+          pip install -U uv
+          uv pip install --system -e .
       - name: Install mlstacks package
-        run: pip install mlstacks
+        run: uv pip install --system mlstacks
       - name: Check for broken dependencies
         run: pip check
   mysql-db-migration-testing:

diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -63,6 +63,7 @@ jobs:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
       PYTHONIOENCODING: utf-8
+      UV_HTTP_TIMEOUT: 600
 
       # on MAC OS, we need to set this environment variable
       # to fix problems with the fork() calls (see this thread
@@ -114,5 +115,5 @@ jobs:
       - name: Verify Python Env unaffected
         run: |-
           zenml integration list
-          pip list
+          uv pip list
           pip check || true
diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md
@@ -24,7 +24,7 @@ Along the way we will also show you how to:
 
 You can use Google Colab to see ZenML in action, no signup / installation required!
 
-<a href="https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/quickstart.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
+<a href="https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/run.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
 
 ## :computer: Run Locally
 
@@ -208,4 +208,4 @@ The best way to get a production ZenML instance up and running with all batterie
 Also, make sure to join our <a href="https://zenml.io/slack" target="_blank">
     <img width="15" src="https://cdn3.iconfinder.com/data/icons/logos-and-brands-adobe/512/306_Slack-512.png" alt="Slack"/>
     <b>Slack Community</b> 
-</a> to become part of the ZenML family!
+</a> to become part of the ZenML family!
diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh
@@ -51,8 +51,9 @@ rm docs/mkdocs/index.md || true
 
 ################################################ Install Requirements ##################################################
 if [ -z "$SKIP_INSTALL" ]; then
-  pip3 install -e ".[server,dev]"
-  pip3 install "Jinja2==3.0.3"
+  pip3 install uv
+  uv pip install --system -e ".[server,dev]"
+  uv pip install --system "Jinja2==3.0.3"
 fi
 
 ################################# Initialize DB and delete unnecessary alembic files ###################################

diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh
@@ -24,15 +24,7 @@ parse_args () {
 install_zenml() {
     # install ZenML in editable mode
 
-    touch zenml_requirements.txt
-    echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> zenml_requirements.txt
-
-    cp zenml_requirements.txt zenml_requirements.in
-    uv pip compile zenml_requirements.in -o zenml_requirements-compiled.txt
-
-    pip install -r zenml_requirements-compiled.txt
-    rm zenml_requirements.txt
-    rm zenml_requirements.in
+    uv pip install --system -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]"
 }
 
 install_integrations() {
@@ -63,19 +55,11 @@ install_integrations() {
     echo "pyyaml>=6.0.1" >> integration-requirements.txt
     echo "pyopenssl" >> integration-requirements.txt
     echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt
-    cp integration-requirements.txt integration-requirements.in
-
-    pip install uv
 
-    uv pip compile integration-requirements.in -o integration-requirements-compiled.txt
-
-    pip install -r integration-requirements-compiled.txt
+    uv pip install --system -r integration-requirements.txt
     rm integration-requirements.txt
-    rm integration-requirements.in
-    rm integration-requirements-compiled.txt
 }
 
-
 set -x
 set -e
 
@@ -84,7 +68,7 @@ export ZENML_ANALYTICS_OPT_IN=false
 
 parse_args "$@"
 
-python -m pip install --upgrade pip setuptools wheel uv
+python -m pip install --upgrade setuptools wheel pip uv
 
 install_zenml