From 97def7bb6b4cb080d04e4e02a96ec5dc5d50de86 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 15 Feb 2024 23:59:41 +0100 Subject: [PATCH 01/73] Use uv --- .github/workflows/ci-fast.yml | 2 +- .github/workflows/ci-slow.yml | 10 +++++----- .github/workflows/integration-test-fast.yml | 5 +++-- .github/workflows/integration-test-slow.yml | 5 +++-- .github/workflows/release.yml | 6 ++++-- .github/workflows/update-templates-to-examples.yml | 9 ++++++--- scripts/install-zenml-dev.sh | 6 +++--- 7 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index dc1ee2aef6..eeeadc0394 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -25,7 +25,7 @@ jobs: with: python-version: '3.10' - name: Install current package as editable - run: pip install -U darglint + run: pip install uv uv pip install -U darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 9bb276979b..76d40d0864 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -48,7 +48,7 @@ jobs: with: python-version: '3.10' - name: Install current package as editable - run: pip install -U darglint + run: pip install uv uv pip install -U darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: @@ -76,9 +76,9 @@ jobs: - name: Install current package as editable run: | pip install -U pip - pip install -e . + uv pip install -e . - name: Install mlstacks package - run: pip install mlstacks + run: uv pip install mlstacks - name: Check for broken dependencies run: pip check - name: Markdown link check @@ -93,13 +93,13 @@ jobs: continue-on-error: true - name: Security check run: | - pip install bandit + uv pip install bandit bash scripts/check-security.sh - name: Check for alembic branch divergence env: ZENML_DEBUG: 0 run: | - pip install alembic + uv pip install alembic bash scripts/check-alembic-branches.sh custom-ubuntu-unit-test: if: github.event.pull_request.draft == false diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index d71704462a..d5e323146f 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -150,8 +150,9 @@ jobs: - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | - pip install pyyaml==5.3.1 - pip install docker-compose + pip install uv + uv pip install pyyaml==5.3.1 + uv pip install docker-compose - name: Install Linux System Dependencies if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners') run: sudo apt install graphviz diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 21f83e803f..09b3c6d6d5 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -148,8 +148,9 @@ jobs: - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | - pip install pyyaml==5.3.1 - pip install docker-compose + pip install uv + uv pip install pyyaml==5.3.1 + uv pip install docker-compose - name: Install Linux System Dependencies if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners') run: sudo apt install graphviz diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe21a69b22..2bb86061c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,9 +23,11 @@ jobs: with: python-version: '3.8' - name: Install current package as editable - run: pip install -e . + run: | + pip install uv + uv pip install -e . - name: Install mlstacks package - run: pip install mlstacks + run: uv pip install mlstacks - name: Check for broken dependencies run: pip check mysql-db-migration-testing: diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index fa377b6ff1..a13b34bfcf 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -69,7 +69,8 @@ jobs: rm -rf examples/e2e mkdir -p examples/e2e printf 'info@zenml.io' | zenml init --path examples/e2e --template e2e_batch --template-with-defaults - pip install yamlfix + pip install uv + uv pip install yamlfix bash scripts/format.sh - name: Check for changes id: check_changes @@ -140,7 +141,8 @@ jobs: run: | mkdir -p examples/e2e_nlp printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults - pip install yamlfix + pip install uv + uv install yamlfix bash scripts/format.sh - name: Check for changes id: check_changes @@ -213,7 +215,8 @@ jobs: rm -rf examples/quickstart mkdir -p examples/quickstart printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults - pip install yamlfix + pip install uv + uv install yamlfix bash scripts/format.sh - name: Check for changes id: check_changes diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index ea8c5a6f82..8d22a7edef 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -23,7 +23,7 @@ parse_args () { install_zenml() { # install ZenML in editable mode - pip install -e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks] + uv pip install -e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks] } install_integrations() { @@ -53,7 +53,7 @@ install_integrations() { echo "" >> integration-requirements.txt echo "pyyaml>=6.0.1" >> integration-requirements.txt - pip install -r integration-requirements.txt + uv pip install -r integration-requirements.txt rm integration-requirements.txt # install langchain separately @@ -66,7 +66,7 @@ set -e parse_args "$@" -python -m pip install --upgrade pip +python -m pip install --upgrade pip uv install_zenml From 0be43ebd06a2d5d6ca327595b28a2bd655697539 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:04:38 +0100 Subject: [PATCH 02/73] add to migration testing --- scripts/test-migrations-mariadb.sh | 10 +++++----- scripts/test-migrations-mysql.sh | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index 6d4f351ba7..4c69596f93 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -52,8 +52,8 @@ do source ".venv-$VERSION/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel - pip3 install "zenml[templates,server]==$VERSION" + pip3 install -U pip setuptools wheel uv + uv pip install "zenml[templates,server]==$VERSION" zenml connect --url mysql://127.0.0.1/zenml --username root --password password @@ -72,9 +72,9 @@ set -e python3 -m venv ".venv-current-branch" source ".venv-current-branch/bin/activate" -pip3 install -U pip setuptools wheel -pip3 install -e ".[templates,server]" -pip3 install importlib_metadata +pip3 install -U pip setuptools wheel uv +uv pip install -e ".[templates,server]" +uv pip install importlib_metadata zenml connect --url mysql://127.0.0.1/zenml --username root --password password diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 6f2df4231a..599eb5d546 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -71,10 +71,10 @@ do source ".venv-$VERSION/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel - pip3 install "zenml[templates,server]==$VERSION" + pip install -U pip setuptools wheel uv + uv pip install "zenml[templates,server]==$VERSION" # handles unpinned sqlmodel dependency in older versions - pip3 install "sqlmodel==0.0.8" "bcrypt==4.0.1" + uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" # Get the major and minor version of Python PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') @@ -83,7 +83,7 @@ do if [[ "$PYTHON_VERSION" == "3.9" ]]; then case "$VERSION" in "0.47.0"|"0.50.0"|"0.51.0"|"0.52.0") - pip3 install importlib_metadata + uv pip install importlib_metadata ;; esac fi @@ -111,9 +111,9 @@ set -e python3 -m venv ".venv-current-branch" source ".venv-current-branch/bin/activate" -pip3 install -U pip setuptools wheel -pip3 install -e ".[templates,server]" -pip3 install importlib_metadata +pip3 install -U pip setuptools wheel uv +uv pip install -e ".[templates,server]" +uv pip install importlib_metadata if [ "$1" == "mysql" ]; then zenml connect --url mysql://127.0.0.1/zenml --username root --password password From 340ad1b0bf19b0918934352e40fa12a8eb8855cb Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:09:22 +0100 Subject: [PATCH 03/73] typo fix --- .github/workflows/update-templates-to-examples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index a13b34bfcf..48b0e1d46d 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -142,7 +142,7 @@ jobs: mkdir -p examples/e2e_nlp printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults pip install uv - uv install yamlfix + uv pip install yamlfix bash scripts/format.sh - name: Check for changes id: check_changes @@ -216,7 +216,7 @@ jobs: mkdir -p examples/quickstart printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults pip install uv - uv install yamlfix + uv pip install yamlfix bash scripts/format.sh - name: Check for changes id: check_changes From 11746cebe167056ba173512cef705f49550313f8 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:17:29 +0100 Subject: [PATCH 04/73] fixes --- .github/workflows/ci-fast.yml | 5 ++++- .github/workflows/ci-slow.yml | 8 ++++++-- .github/workflows/integration-test-fast.yml | 1 + .github/workflows/integration-test-slow.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/update-templates-to-examples.yml | 3 +++ scripts/install-zenml-dev.sh | 1 + scripts/test-migrations-mariadb.sh | 2 ++ scripts/test-migrations-mysql.sh | 2 ++ 9 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index eeeadc0394..519afd0178 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -25,7 +25,10 @@ jobs: with: python-version: '3.10' - name: Install current package as editable - run: pip install uv uv pip install -U darglint + run: | + pip install uv + uv venv + uv pip install -U darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 76d40d0864..a9c642d6a3 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -48,7 +48,10 @@ jobs: with: python-version: '3.10' - name: Install current package as editable - run: pip install uv uv pip install -U darglint + run: | + pip install uv + uv venv + uv pip install -U darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: @@ -75,7 +78,8 @@ jobs: python-version: '3.8' - name: Install current package as editable run: | - pip install -U pip + pip install -U pip uv + uv venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index d5e323146f..371444ade5 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -151,6 +151,7 @@ jobs: if: inputs.test_environment != 'default' run: | pip install uv + uv venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 09b3c6d6d5..6eddf499af 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -149,6 +149,7 @@ jobs: if: inputs.test_environment != 'default' run: | pip install uv + uv venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bb86061c7..b4ce5c2c40 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,7 @@ jobs: - name: Install current package as editable run: | pip install uv + uv venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index 48b0e1d46d..21e255a513 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -70,6 +70,7 @@ jobs: mkdir -p examples/e2e printf 'info@zenml.io' | zenml init --path examples/e2e --template e2e_batch --template-with-defaults pip install uv + uv venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -142,6 +143,7 @@ jobs: mkdir -p examples/e2e_nlp printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults pip install uv + uv venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -216,6 +218,7 @@ jobs: mkdir -p examples/quickstart printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults pip install uv + uv venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 8d22a7edef..2d69c25e96 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -67,6 +67,7 @@ set -e parse_args "$@" python -m pip install --upgrade pip uv +uv venv install_zenml diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index 4c69596f93..a8d2dbbfb3 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -53,6 +53,7 @@ do # Install the specific version pip3 install -U pip setuptools wheel uv + uv venv uv pip install "zenml[templates,server]==$VERSION" zenml connect --url mysql://127.0.0.1/zenml --username root --password password @@ -73,6 +74,7 @@ python3 -m venv ".venv-current-branch" source ".venv-current-branch/bin/activate" pip3 install -U pip setuptools wheel uv +uv venv uv pip install -e ".[templates,server]" uv pip install importlib_metadata diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 599eb5d546..50fa150dc6 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -72,6 +72,7 @@ do # Install the specific version pip install -U pip setuptools wheel uv + uv venv uv pip install "zenml[templates,server]==$VERSION" # handles unpinned sqlmodel dependency in older versions uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" @@ -112,6 +113,7 @@ python3 -m venv ".venv-current-branch" source ".venv-current-branch/bin/activate" pip3 install -U pip setuptools wheel uv +uv venv uv pip install -e ".[templates,server]" uv pip install importlib_metadata From c722dbe766f254dd350ba6c9a8ee87f3070d412d Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:20:38 +0100 Subject: [PATCH 05/73] api checks --- scripts/generate-docs.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh index fdb67a6cc5..9acf0a790d 100755 --- a/scripts/generate-docs.sh +++ b/scripts/generate-docs.sh @@ -51,8 +51,10 @@ rm docs/mkdocs/index.md || true ################################################ Install Requirements ################################################## if [ -z "$SKIP_INSTALL" ]; then - pip3 install -e ".[server,dev]" - pip3 install "Jinja2==3.0.3" + pip3 install uv + uv venv + uv pip install -e ".[server,dev]" + uv pip install "Jinja2==3.0.3" fi ################################# Initialize DB and delete unnecessary alembic files ################################### From e25abb47e0988e5acc268e99d9b79931e6943e95 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:21:57 +0100 Subject: [PATCH 06/73] remove -U --- .github/workflows/ci-fast.yml | 2 +- .github/workflows/ci-slow.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 519afd0178..7afa3a02c8 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -28,7 +28,7 @@ jobs: run: | pip install uv uv venv - uv pip install -U darglint + uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index a9c642d6a3..713be41f2c 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -51,7 +51,7 @@ jobs: run: | pip install uv uv venv - uv pip install -U darglint + uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: From 2c74e4f887224b89aa939954b08fcba06f55ccb9 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:41:37 +0100 Subject: [PATCH 07/73] test dargli --- .github/workflows/ci-fast.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 7afa3a02c8..f7e590eee5 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -28,6 +28,7 @@ jobs: run: | pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh From ba7fde83bebfec5eda1462e55e5cfee38a1bb350 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:43:30 +0100 Subject: [PATCH 08/73] source --- scripts/generate-docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh index 9acf0a790d..4b591d02e6 100755 --- a/scripts/generate-docs.sh +++ b/scripts/generate-docs.sh @@ -53,6 +53,7 @@ rm docs/mkdocs/index.md || true if [ -z "$SKIP_INSTALL" ]; then pip3 install uv uv venv + VIRTUAL_ENV=./.venv uv pip install -e ".[server,dev]" uv pip install "Jinja2==3.0.3" fi From 781718a3d73b11bcac63e7dff143d93f069322e8 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:48:08 +0100 Subject: [PATCH 09/73] activate --- .github/workflows/ci-slow.yml | 2 ++ .github/workflows/integration-test-fast.yml | 1 + .github/workflows/integration-test-slow.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/update-templates-to-examples.yml | 3 +++ scripts/install-zenml-dev.sh | 1 + scripts/test-migrations-mariadb.sh | 2 ++ scripts/test-migrations-mysql.sh | 2 ++ 8 files changed, 13 insertions(+) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 713be41f2c..684d751663 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -51,6 +51,7 @@ jobs: run: | pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh @@ -80,6 +81,7 @@ jobs: run: | pip install -U pip uv uv venv + VIRTUAL_ENV=./.venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 371444ade5..2c302286d0 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -152,6 +152,7 @@ jobs: run: | pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 6eddf499af..56956cda2b 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -150,6 +150,7 @@ jobs: run: | pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4ce5c2c40..5321b8053e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,7 @@ jobs: run: | pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index 21e255a513..f010e5ee45 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -71,6 +71,7 @@ jobs: printf 'info@zenml.io' | zenml init --path examples/e2e --template e2e_batch --template-with-defaults pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -144,6 +145,7 @@ jobs: printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -219,6 +221,7 @@ jobs: printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults pip install uv uv venv + VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 2d69c25e96..f353e73c84 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -68,6 +68,7 @@ parse_args "$@" python -m pip install --upgrade pip uv uv venv +VIRTUAL_ENV=./.venv install_zenml diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index a8d2dbbfb3..adcc1ebbc6 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -54,6 +54,7 @@ do # Install the specific version pip3 install -U pip setuptools wheel uv uv venv + VIRTUAL_ENV=./.venv uv pip install "zenml[templates,server]==$VERSION" zenml connect --url mysql://127.0.0.1/zenml --username root --password password @@ -75,6 +76,7 @@ source ".venv-current-branch/bin/activate" pip3 install -U pip setuptools wheel uv uv venv +VIRTUAL_ENV=./.venv uv pip install -e ".[templates,server]" uv pip install importlib_metadata diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 50fa150dc6..0dedcabbbb 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -73,6 +73,7 @@ do # Install the specific version pip install -U pip setuptools wheel uv uv venv + VIRTUAL_ENV=./.venv uv pip install "zenml[templates,server]==$VERSION" # handles unpinned sqlmodel dependency in older versions uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" @@ -114,6 +115,7 @@ source ".venv-current-branch/bin/activate" pip3 install -U pip setuptools wheel uv uv venv +VIRTUAL_ENV=./.venv uv pip install -e ".[templates,server]" uv pip install importlib_metadata From d7d8a7225aa5b30f35215306cd357f07d65c76aa Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:51:17 +0100 Subject: [PATCH 10/73] as property --- .github/workflows/ci-fast.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index f7e590eee5..882dfa93fa 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -32,6 +32,8 @@ jobs: uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh + env: + VIRTUAL_ENV: ./.venv sqlite-db-migration-testing: runs-on: ubuntu-dind-runners # if team member commented, not a draft, on a PR, using /fulltest From 26d0e5aca2db5f3f057404f46fc50ae8c54dd0ab Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 00:54:56 +0100 Subject: [PATCH 11/73] sourced --- .github/workflows/ci-fast.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 882dfa93fa..d755b72eb0 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -28,6 +28,7 @@ jobs: run: | pip install uv uv venv + source .venv/bin/activate VIRTUAL_ENV=./.venv uv pip install darglint - name: Check docstrings From 25fbfc2f17432d78c3bbfc7060c5784e0adca128 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 01:17:37 +0100 Subject: [PATCH 12/73] try fix --- .github/workflows/ci-fast.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index d755b72eb0..816fb3b9f6 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -31,6 +31,11 @@ jobs: source .venv/bin/activate VIRTUAL_ENV=./.venv uv pip install darglint + # Persist the virtual environment for subsequent steps + - name: Configure environment variables + run: | + echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV + echo "PATH=${VIRTUAL_ENV}/bin:$PATH" >> $GITHUB_ENV - name: Check docstrings run: bash scripts/docstring.sh env: From 271b31185b2ac2bdccdc8723d9826ecf715a53bc Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 01:20:45 +0100 Subject: [PATCH 13/73] remove cache --- .github/actions/setup_environment/action.yml | 58 -------------------- 1 file changed, 58 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 7c11084286..c6b16adf64 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -57,55 +57,6 @@ runs: run: | "C:\Program Files\Git\bin\git.exe" config --global user.email "info@zenml.io" "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML GmbH" - - name: Get current week - shell: bash - id: date - run: echo "::set-output name=week::$(date +'calendar-week-%W')" - - uses: syphar/restore-virtualenv@v1 - if: ${{ inputs.os != 'ubuntu-dind-runners' }} - id: cache-virtualenv - with: - requirement_files: pyproject.toml - # The virtualenv cache is invalidated when: - # - manually triggered by means of a custom cache version token - # - on a weekly basis - # - any of the integration requirements change (a hash of the - # __init__.py files is included in the cache key) - custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{ - hashFiles('src/zenml/integrations/*/__init__.py') }} - - uses: tespkg/actions-cache@v1 - if: ${{ inputs.os == 'ubuntu-dind-runners' }} - id: custom-cache-pip - env: - AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }} - with: - endpoint: minio-service.minio.svc.cluster.local # optional - insecure: true # optional, use http instead of https. default false - bucket: caching # required - use-fallback: false # optional, use github actions cache fallback, default false - key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version - }}-${{inputs.install_integrations}} - path: | - ~/.cache/pip - restore-keys: | - ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}} - - name: Custom runners cache miss - uses: rjstone/discord-webhook-notify@v1 - if: steps.custom-cache-pip.outputs.cache-hit != 'true' - continue-on-error: true - with: - severity: warn - details: The custom github runnners cache was missed and failed to restore - the cache from minio bucket - webhookUrl: ${{ inputs.discord_webhook }} - # Disabled for now because it doesn't work well with multiple parallel jobs - # - uses: syphar/restore-pip-download-cache@v1 - # with: - # requirement_files: 'pyproject.toml' - # # The pip download cache can be updated on a weekly basis as new packages - # # don't appear that often - # custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}} - name: Install Terraform (Windows) if: ${{ inputs.os == 'windows-latest' }} shell: bash @@ -117,19 +68,10 @@ runs: brew tap hashicorp/tap brew install hashicorp/tap/terraform - name: Install ZenML and dependencies - if: steps.cache-virtualenv.outputs.cache-hit != 'true' && steps.custom-cache-pip.outputs.cache-hit - != 'true' shell: bash run: | scripts/install-zenml-dev.sh --integrations ${{ inputs.install_integrations }} - # if using a cached virtualenv, just refresh the ZenML installation - - name: Refresh ZenML installation - if: steps.cache-virtualenv.outputs.cache-hit == 'true' || steps.custom-cache-pip.outputs.cache-hit - == 'true' - shell: bash - run: | - scripts/install-zenml-dev.sh --integrations ${{ inputs.install_integrations }} - name: Check Python environment shell: bash run: |- From ae2b11b2df2f7756f454c2df548b35b0721043c8 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 01:26:36 +0100 Subject: [PATCH 14/73] try export --- .github/workflows/ci-fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 816fb3b9f6..511faeb7a9 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -29,7 +29,7 @@ jobs: pip install uv uv venv source .venv/bin/activate - VIRTUAL_ENV=./.venv + export VIRTUAL_ENV=./.venv uv pip install darglint # Persist the virtual environment for subsequent steps - name: Configure environment variables From 6cc351d8c2d973c80e282425c84668c0bedc7d32 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 07:17:10 +0100 Subject: [PATCH 15/73] source --- .github/actions/setup_environment/action.yml | 1 - .github/workflows/ci-fast.yml | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index c6b16adf64..3f5772a61c 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -71,7 +71,6 @@ runs: shell: bash run: | scripts/install-zenml-dev.sh --integrations ${{ inputs.install_integrations }} - - name: Check Python environment shell: bash run: |- diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 511faeb7a9..42f9c715da 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -29,15 +29,11 @@ jobs: pip install uv uv venv source .venv/bin/activate - export VIRTUAL_ENV=./.venv uv pip install darglint - # Persist the virtual environment for subsequent steps - - name: Configure environment variables - run: | - echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV - echo "PATH=${VIRTUAL_ENV}/bin:$PATH" >> $GITHUB_ENV - name: Check docstrings - run: bash scripts/docstring.sh + run: | + source .venv/bin/activate + bash scripts/docstring.sh env: VIRTUAL_ENV: ./.venv sqlite-db-migration-testing: From 8aaf5fd8a6cb7457d0f2f6ed5afd139521424a49 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 07:29:07 +0100 Subject: [PATCH 16/73] new approach --- .github/workflows/ci-fast.yml | 8 ++++---- .github/workflows/ci-slow.yml | 6 ++---- .github/workflows/integration-test-fast.yml | 3 +-- .github/workflows/integration-test-slow.yml | 3 +-- .github/workflows/release.yml | 3 +-- .github/workflows/update-templates-to-examples.yml | 9 +++------ scripts/generate-docs.sh | 2 -- scripts/install-zenml-dev.sh | 3 +-- scripts/test-migrations-mariadb.sh | 5 +---- 9 files changed, 14 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 42f9c715da..5c5d881700 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -26,13 +26,11 @@ jobs: python-version: '3.10' - name: Install current package as editable run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - source .venv/bin/activate uv pip install darglint - name: Check docstrings run: | - source .venv/bin/activate bash scripts/docstring.sh env: VIRTUAL_ENV: ./.venv @@ -71,7 +69,9 @@ jobs: with: python-version: '3.11' - name: Test API docs buildable - run: bash scripts/generate-docs.sh -v DUMMY -c + run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV + bash scripts/generate-docs.sh -v DUMMY -c update-templates-to-examples: # this doesn't work on forked repositories (i.e. outside contributors) # so we disable template updates for those PRs / branches diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 684d751663..2181446b9f 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -49,9 +49,8 @@ jobs: python-version: '3.10' - name: Install current package as editable run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install darglint - name: Check docstrings run: bash scripts/docstring.sh @@ -79,9 +78,8 @@ jobs: python-version: '3.8' - name: Install current package as editable run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install -U pip uv - uv venv - VIRTUAL_ENV=./.venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 2c302286d0..65af077382 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -150,9 +150,8 @@ jobs: - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 56956cda2b..3a93341553 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -148,9 +148,8 @@ jobs: - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install pyyaml==5.3.1 uv pip install docker-compose - name: Install Linux System Dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5321b8053e..f38e0d43f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,9 +24,8 @@ jobs: python-version: '3.8' - name: Install current package as editable run: | + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install -e . - name: Install mlstacks package run: uv pip install mlstacks diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index f010e5ee45..d6f6eb63c6 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -69,9 +69,8 @@ jobs: rm -rf examples/e2e mkdir -p examples/e2e printf 'info@zenml.io' | zenml init --path examples/e2e --template e2e_batch --template-with-defaults + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -143,9 +142,8 @@ jobs: run: | mkdir -p examples/e2e_nlp printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes @@ -219,9 +217,8 @@ jobs: rm -rf examples/quickstart mkdir -p examples/quickstart printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults + echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install yamlfix bash scripts/format.sh - name: Check for changes diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh index 4b591d02e6..8f328f61e7 100755 --- a/scripts/generate-docs.sh +++ b/scripts/generate-docs.sh @@ -52,8 +52,6 @@ rm docs/mkdocs/index.md || true ################################################ Install Requirements ################################################## if [ -z "$SKIP_INSTALL" ]; then pip3 install uv - uv venv - VIRTUAL_ENV=./.venv uv pip install -e ".[server,dev]" uv pip install "Jinja2==3.0.3" fi diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index f353e73c84..ad0595b1f1 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -66,9 +66,8 @@ set -e parse_args "$@" +echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV python -m pip install --upgrade pip uv -uv venv -VIRTUAL_ENV=./.venv install_zenml diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index adcc1ebbc6..1b99adce77 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -53,8 +53,6 @@ do # Install the specific version pip3 install -U pip setuptools wheel uv - uv venv - VIRTUAL_ENV=./.venv uv pip install "zenml[templates,server]==$VERSION" zenml connect --url mysql://127.0.0.1/zenml --username root --password password @@ -74,9 +72,8 @@ set -e python3 -m venv ".venv-current-branch" source ".venv-current-branch/bin/activate" +echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip3 install -U pip setuptools wheel uv -uv venv -VIRTUAL_ENV=./.venv uv pip install -e ".[templates,server]" uv pip install importlib_metadata From 4d40b69cdf38338ad4f0f29f57964e0015b6c23c Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 07:35:38 +0100 Subject: [PATCH 17/73] refactor --- .github/workflows/ci-fast.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 5c5d881700..1ee2db60c6 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -24,9 +24,9 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.10' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install darglint - name: Check docstrings @@ -68,10 +68,9 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.11' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Test API docs buildable - run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - bash scripts/generate-docs.sh -v DUMMY -c + run: bash scripts/generate-docs.sh -v DUMMY -c update-templates-to-examples: # this doesn't work on forked repositories (i.e. outside contributors) # so we disable template updates for those PRs / branches From 7f63c66bd93f9cb67871560016aa66678d15a9de Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 07:42:02 +0100 Subject: [PATCH 18/73] refactor2 --- .github/workflows/ci-slow.yml | 4 ++-- .github/workflows/integration-test-fast.yml | 2 +- .github/workflows/integration-test-slow.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/update-templates-to-examples.yml | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 2181446b9f..c647a0d6e5 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -47,9 +47,9 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.10' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install darglint - name: Check docstrings @@ -76,9 +76,9 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.8' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install -U pip uv uv pip install -e . - name: Install mlstacks package diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 65af077382..526f24fbad 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -147,10 +147,10 @@ jobs: runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }} runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }} discord_webhook: ${{ secrets.DISCORD_WEBHOOK }} + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install pyyaml==5.3.1 uv pip install docker-compose diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 3a93341553..8837c6a87b 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -145,10 +145,10 @@ jobs: runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }} runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }} discord_webhook: ${{ secrets.DISCORD_WEBHOOK }} + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install pyyaml==5.3.1 uv pip install docker-compose diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f38e0d43f8..367603f054 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,9 +22,9 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.8' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install -e . - name: Install mlstacks package diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index d6f6eb63c6..3f8178f765 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -64,12 +64,12 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh E2E template run: | rm -rf examples/e2e mkdir -p examples/e2e printf 'info@zenml.io' | zenml init --path examples/e2e --template e2e_batch --template-with-defaults - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install yamlfix bash scripts/format.sh @@ -138,11 +138,11 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh NLP template run: | mkdir -p examples/e2e_nlp printf 'info@zenml.io' | zenml init --path examples/e2e_nlp --template nlp --template-with-defaults - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install yamlfix bash scripts/format.sh @@ -212,12 +212,12 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh Starter template run: | rm -rf examples/quickstart mkdir -p examples/quickstart printf 'info@zenml.io' | zenml init --path examples/quickstart --template starter --template-with-defaults - echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV pip install uv uv pip install yamlfix bash scripts/format.sh From 9a89404949c1f7b11a1b690744fe7973496b5a63 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 07:52:12 +0100 Subject: [PATCH 19/73] setup env --- .github/actions/setup_environment/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 3f5772a61c..800378c57c 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -36,6 +36,9 @@ runs: uses: actions/setup-python@v5.0.0 with: python-version: ${{ inputs.python-version }} + - name: Set up ENV Var for uvicorn + shell: bash + run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Delete error-causing bash shell: bash if: ${{ inputs.os == 'windows-latest' }} From 7e4c835b400fc1647d9deadef1e80f808d5088c6 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 09:18:08 +0100 Subject: [PATCH 20/73] fast ci --- .github/workflows/ci-fast.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 1ee2db60c6..e2d22394ff 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -24,13 +24,15 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.10' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | pip install uv + uv venv + source .venv/bin/activate uv pip install darglint - name: Check docstrings run: | + source .venv/bin/activate bash scripts/docstring.sh env: VIRTUAL_ENV: ./.venv @@ -68,9 +70,15 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.11' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV + - name: venv loading + run: | + pip install uv + uv venv + source .venv/bin/activate - name: Test API docs buildable - run: bash scripts/generate-docs.sh -v DUMMY -c + run: | + source .venv/bin/activate + bash scripts/generate-docs.sh -v DUMMY -c update-templates-to-examples: # this doesn't work on forked repositories (i.e. outside contributors) # so we disable template updates for those PRs / branches From b55f563700773af46d3f559a317d54285edcf6f4 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 09:22:39 +0100 Subject: [PATCH 21/73] environment installation --- .github/actions/setup_environment/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 800378c57c..1efee2ad71 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -36,9 +36,6 @@ runs: uses: actions/setup-python@v5.0.0 with: python-version: ${{ inputs.python-version }} - - name: Set up ENV Var for uvicorn - shell: bash - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Delete error-causing bash shell: bash if: ${{ inputs.os == 'windows-latest' }} @@ -73,10 +70,14 @@ runs: - name: Install ZenML and dependencies shell: bash run: | + pip install uv + uv venv + source .venv/bin/activate scripts/install-zenml-dev.sh --integrations ${{ inputs.install_integrations }} - name: Check Python environment shell: bash run: |- + source .venv/bin/activate zenml integration list pip list pip check || true From 4ff6df1061c814db2c9487fb3b1d72bd3f0021a4 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 09:32:19 +0100 Subject: [PATCH 22/73] remove pip upgrade --- scripts/install-zenml-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index ad0595b1f1..54c85a0c7f 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -67,7 +67,7 @@ set -e parse_args "$@" echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV -python -m pip install --upgrade pip uv +python -m pip install --upgrade uv install_zenml From afb726f085a5c4e10480fea225d7d2bac21ecb59 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 16 Feb 2024 09:35:36 +0100 Subject: [PATCH 23/73] remove pip completely --- scripts/install-zenml-dev.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 54c85a0c7f..e602966b55 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -66,9 +66,6 @@ set -e parse_args "$@" -echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV -python -m pip install --upgrade uv - install_zenml # install integrations, if requested From 5f5a4d67f539351def540d63ee6955e2e6d8a028 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Mon, 19 Feb 2024 14:26:04 +0100 Subject: [PATCH 24/73] try different configuration --- .github/actions/setup_environment/action.yml | 24 ++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 1efee2ad71..9ed4a75601 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -36,6 +36,26 @@ runs: uses: actions/setup-python@v5.0.0 with: python-version: ${{ inputs.python-version }} + - name: Create and activate a virtual environment (Windows) + shell: bash + if: ${{ runner.os == 'Windows' }} + # uv doesn't (yet) allow us to install packages globally; + # we have to create and activate a virtual environment + run: | + irm https://astral.sh/uv/install.ps1 | iex + uv venv .venv + "VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append + "$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append + - name: Create and activate a virtual environment (Unix) + shell: bash + if: ${{ runner.os != 'Windows' }} + # uv doesn't (yet) allow us to install packages globally; + # we have to create and activate a virtual environment + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + uv venv .venv + echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV + echo "$PWD/.venv/bin" >> $GITHUB_PATH - name: Delete error-causing bash shell: bash if: ${{ inputs.os == 'windows-latest' }} @@ -70,14 +90,10 @@ runs: - name: Install ZenML and dependencies shell: bash run: | - pip install uv - uv venv - source .venv/bin/activate scripts/install-zenml-dev.sh --integrations ${{ inputs.install_integrations }} - name: Check Python environment shell: bash run: |- - source .venv/bin/activate zenml integration list pip list pip check || true From b64937251ab76061a5594c5c4bf79a948381e7aa Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 09:44:51 +0100 Subject: [PATCH 25/73] uv refactor --- scripts/generate-docs.sh | 4 ++-- scripts/install-zenml-dev.sh | 19 ++----------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/scripts/generate-docs.sh b/scripts/generate-docs.sh index 8f328f61e7..2073d29efd 100755 --- a/scripts/generate-docs.sh +++ b/scripts/generate-docs.sh @@ -52,8 +52,8 @@ rm docs/mkdocs/index.md || true ################################################ Install Requirements ################################################## if [ -z "$SKIP_INSTALL" ]; then pip3 install uv - uv pip install -e ".[server,dev]" - uv pip install "Jinja2==3.0.3" + uv pip install --system -e ".[server,dev]" + uv pip install --system "Jinja2==3.0.3" fi ################################# Initialize DB and delete unnecessary alembic files ################################### diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 8f11f110a8..0f0755c0ad 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -24,11 +24,7 @@ parse_args () { install_zenml() { # install ZenML in editable mode - touch zenml_requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> zenml_requirements.txt - - uv pip install -r zenml_requirements.txt - rm zenml_requirements.txt + uv pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" } install_integrations() { @@ -59,17 +55,9 @@ install_integrations() { echo "pyyaml>=6.0.1" >> integration-requirements.txt echo "pyopenssl" >> integration-requirements.txt echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - cp integration-requirements.txt integration-requirements.in - - uv pip compile integration-requirements.in -o integration-requirements-compiled.txt - uv pip install -r integration-requirements-compiled.txt + uv pip install --system -r integration-requirements.txt rm integration-requirements.txt - rm integration-requirements.in - rm integration-requirements-compiled.txt - - # install langchain separately - zenml integration install -y langchain } @@ -88,7 +76,4 @@ install_zenml # install integrations, if requested if [ "$INTEGRATIONS" = yes ]; then install_integrations - - # refresh the ZenML installation after installing integrations - install_zenml fi From cd469f3934e68b3f3a7922e99403b210a3d2106d Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 09:59:31 +0100 Subject: [PATCH 26/73] Update virtual environment setup and dependencies --- scripts/test-migrations-mariadb.sh | 12 +++++++----- scripts/test-migrations-mysql.sh | 15 ++++++++------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index 178f80925d..db4287da16 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -48,18 +48,20 @@ VERSIONS=("0.54.0" "0.54.1" "0.55.0" "0.55.1" "0.55.2" "0.55.3") # Start completely fresh rm -rf ~/.config/zenml +pip install -U uv + for VERSION in "${VERSIONS[@]}" do set -e # Exit immediately if a command exits with a non-zero status # Create a new virtual environment - python3 -m venv ".venv-$VERSION" + uv venv ".venv-$VERSION" source ".venv-$VERSION/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel + uv pip install -U setuptools wheel git checkout release/$VERSION - pip3 install -e ".[templates,server]" + uv pip install -e ".[templates,server]" export ZENML_ANALYTICS_OPT_IN=false export ZENML_DEBUG=true @@ -78,11 +80,11 @@ done # Test the most recent migration with MariaDB echo "===== TESTING CURRENT BRANCH =====" set -e -python3 -m venv ".venv-current-branch" +uv venv ".venv-current-branch" source ".venv-current-branch/bin/activate" echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV -pip3 install -U pip setuptools wheel uv +uv pip install -U setuptools wheel uv pip install -e ".[templates,server]" uv pip install importlib_metadata diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 81ac703886..80080c87e0 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -66,18 +66,20 @@ VERSIONS=("0.40.0" "0.40.3" "0.41.0" "0.43.0" "0.44.1" "0.44.3" "0.45.2" "0.45.3 # Start completely fresh rm -rf ~/.config/zenml +pip install -U uv + for VERSION in "${VERSIONS[@]}" do set -e # Exit immediately if a command exits with a non-zero status # Create a new virtual environment - python3 -m venv ".venv-$VERSION" + uv venv ".venv-$VERSION" source ".venv-$VERSION/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel + uv pip install -U setuptools wheel git checkout release/$VERSION - pip3 install -e ".[templates,server]" + uv pip install -e ".[templates,server]" # handles unpinned sqlmodel dependency in older versions uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" @@ -113,12 +115,11 @@ done # Test the most recent migration with MySQL echo "===== TESTING CURRENT BRANCH =====" set -e -python3 -m venv ".venv-current-branch" +uv venv ".venv-current-branch" source ".venv-current-branch/bin/activate" -pip3 install -U pip setuptools wheel uv -uv venv -VIRTUAL_ENV=./.venv +uv pip install setuptools wheel + uv pip install -e ".[templates,server]" uv pip install importlib_metadata From 09e32101a47426d9869d0dcef60e021ad47b5c7f Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 09:59:54 +0100 Subject: [PATCH 27/73] line extra --- scripts/install-zenml-dev.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 0f0755c0ad..13b3eac654 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -60,7 +60,6 @@ install_integrations() { rm integration-requirements.txt } - set -x set -e From 50580418d5b5cd515303c075e6d43ef9d8af2843 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 10:38:26 +0100 Subject: [PATCH 28/73] use uv instead of pip --- .github/workflows/ci-fast.yml | 21 +++++---------------- .github/workflows/ci-slow.yml | 15 +++++++-------- .github/workflows/integration-test-fast.yml | 6 ++---- .github/workflows/integration-test-slow.yml | 6 ++---- .github/workflows/release.yml | 7 +++---- 5 files changed, 19 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index aefa43be39..1b3cdcf51b 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -24,16 +24,12 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.10' - - name: Install current package as editable + - name: Install darglint using uv run: | - pip install uv - uv venv - source .venv/bin/activate - uv pip install darglint + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install --system darglint - name: Check docstrings - run: | - source .venv/bin/activate - bash scripts/docstring.sh + run: bash scripts/docstring.sh env: VIRTUAL_ENV: ./.venv sqlite-db-migration-testing: @@ -75,15 +71,8 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.11' - - name: venv loading - run: | - pip install uv - uv venv - source .venv/bin/activate - name: Test API docs buildable - run: | - source .venv/bin/activate - bash scripts/generate-docs.sh -v DUMMY -c + run: bash scripts/generate-docs.sh -v DUMMY -c update-templates-to-examples: # this doesn't work on forked repositories (i.e. outside contributors) # so we disable template updates for those PRs / branches diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 29eb97c560..4e9a3c61c9 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -50,8 +50,8 @@ jobs: - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - pip install uv - uv pip install darglint + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install --system darglint - name: Check docstrings run: bash scripts/docstring.sh sqlite-db-migration-testing: @@ -81,13 +81,12 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.8' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - pip install -U pip uv - uv pip install -e . + curl -LsSf https://astral.sh/uv/install.sh | sh + uv pip install --system -e . - name: Install mlstacks package - run: uv pip install mlstacks + run: uv pip install --system mlstacks - name: Check for broken dependencies run: pip check - name: Markdown link check @@ -102,13 +101,13 @@ jobs: continue-on-error: true - name: Security check run: | - uv pip install bandit + uv pip install --system bandit bash scripts/check-security.sh - name: Check for alembic branch divergence env: ZENML_DEBUG: 0 run: | - uv pip install alembic + uv pip install --system alembic bash scripts/check-alembic-branches.sh custom-ubuntu-unit-test: if: github.event.pull_request.draft == false diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 526f24fbad..e99107bc13 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -147,13 +147,11 @@ jobs: runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }} runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }} discord_webhook: ${{ secrets.DISCORD_WEBHOOK }} - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | pip install uv - uv pip install pyyaml==5.3.1 - uv pip install docker-compose + uv pip install --system "pyyaml==5.3.1" docker-compose - name: Install Linux System Dependencies if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners') run: sudo apt install graphviz @@ -229,5 +227,5 @@ jobs: - name: Verify Python Env unaffected run: |- zenml integration list - pip list + uv pip list pip check || true diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 8837c6a87b..1b4df2f17c 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -145,13 +145,11 @@ jobs: runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }} runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }} discord_webhook: ${{ secrets.DISCORD_WEBHOOK }} - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install docker-compose for non-default environments if: inputs.test_environment != 'default' run: | pip install uv - uv pip install pyyaml==5.3.1 - uv pip install docker-compose + uv pip install --system "pyyaml==5.3.1" docker-compose - name: Install Linux System Dependencies if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners') run: sudo apt install graphviz @@ -231,5 +229,5 @@ jobs: - name: Verify Python Env unaffected run: |- zenml integration list - pip list + uv pip list pip check || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0996be9642..8b41c3932f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,13 +22,12 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.8' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | - pip install uv - uv pip install -e . + pip install -U uv + uv pip install --system -e . - name: Install mlstacks package - run: uv pip install mlstacks + run: uv pip install --system mlstacks - name: Check for broken dependencies run: pip check mysql-db-migration-testing: From b7e5a73a2ee40a3c77763c2ae8ae4566936defa7 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 10:59:20 +0100 Subject: [PATCH 29/73] Remove unnecessary environment variable assignment --- .github/workflows/update-templates-to-examples.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index c6f6333b70..dc3c709ecd 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -64,7 +64,6 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh E2E template run: | rm -rf examples/e2e @@ -137,7 +136,6 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh NLP template run: | mkdir -p examples/e2e_nlp @@ -210,7 +208,6 @@ jobs: - uses: actions/checkout@v4.1.1 with: ref: ${{ github.event.pull_request.head.ref }} - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Check-out fresh Starter template run: | rm -rf examples/quickstart From 103ad61f973222ae61485fb104e9229921fc9ebd Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 11:00:25 +0100 Subject: [PATCH 30/73] Add source command to setup environment scripts --- .github/actions/setup_environment/action.yml | 1 + .github/workflows/ci-fast.yml | 1 + .github/workflows/ci-slow.yml | 2 ++ 3 files changed, 4 insertions(+) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 311e9d0ad6..499b259a4c 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -50,6 +50,7 @@ runs: # we have to create and activate a virtual environment run: | curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env uv venv .venv echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV echo "$PWD/.venv/bin" >> $GITHUB_PATH diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 1b3cdcf51b..397082d2dd 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -27,6 +27,7 @@ jobs: - name: Install darglint using uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env uv pip install --system darglint - name: Check docstrings run: bash scripts/docstring.sh diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 4e9a3c61c9..5b843f8ca2 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -51,6 +51,7 @@ jobs: - name: Install current package as editable run: | curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env uv pip install --system darglint - name: Check docstrings run: bash scripts/docstring.sh @@ -84,6 +85,7 @@ jobs: - name: Install current package as editable run: | curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env uv pip install --system -e . - name: Install mlstacks package run: uv pip install --system mlstacks From d69d6b65de8754b285ba3f80acd144bcd60895ee Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 11:16:28 +0100 Subject: [PATCH 31/73] remove pip install --- scripts/install-zenml-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 13b3eac654..dc3e30a544 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -68,7 +68,7 @@ export ZENML_ANALYTICS_OPT_IN=false parse_args "$@" -python -m pip install --upgrade pip setuptools wheel uv +python -m pip install --upgrade setuptools wheel uv install_zenml From c33d0d272e55417cbe19db01d6996c6fad2dab99 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 11:20:43 +0100 Subject: [PATCH 32/73] remove old venv setup --- .github/actions/setup_environment/action.yml | 21 -------------------- 1 file changed, 21 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 499b259a4c..8beb2d13e2 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -33,27 +33,6 @@ runs: uses: actions/setup-python@v5.0.0 with: python-version: ${{ inputs.python-version }} - - name: Create and activate a virtual environment (Windows) - shell: bash - if: ${{ runner.os == 'Windows' }} - # uv doesn't (yet) allow us to install packages globally; - # we have to create and activate a virtual environment - run: | - irm https://astral.sh/uv/install.ps1 | iex - uv venv .venv - "VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append - "$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append - - name: Create and activate a virtual environment (Unix) - shell: bash - if: ${{ runner.os != 'Windows' }} - # uv doesn't (yet) allow us to install packages globally; - # we have to create and activate a virtual environment - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.cargo/env - uv venv .venv - echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV - echo "$PWD/.venv/bin" >> $GITHUB_PATH - name: Delete error-causing bash shell: bash if: ${{ inputs.os == 'windows-latest' }} From ffb636e62867573660e21198e9ce4212664f4891 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 11:35:55 +0100 Subject: [PATCH 33/73] typo --- scripts/install-zenml-dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index dc3e30a544..35ec223a01 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -24,7 +24,7 @@ parse_args () { install_zenml() { # install ZenML in editable mode - uv pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" + uv pip install --system -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" } install_integrations() { From bbe93acec91f3ce8b2716dd0102d134fd9ecc05c Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 12:04:07 +0100 Subject: [PATCH 34/73] add pip back (partially) --- scripts/install-zenml-dev.sh | 2 +- scripts/test-migrations-mariadb.sh | 4 ++-- scripts/test-migrations-mysql.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/install-zenml-dev.sh b/scripts/install-zenml-dev.sh index 35ec223a01..820e639aa0 100755 --- a/scripts/install-zenml-dev.sh +++ b/scripts/install-zenml-dev.sh @@ -68,7 +68,7 @@ export ZENML_ANALYTICS_OPT_IN=false parse_args "$@" -python -m pip install --upgrade setuptools wheel uv +python -m pip install --upgrade setuptools wheel pip uv install_zenml diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index db4287da16..4def7799b8 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -58,7 +58,7 @@ do source ".venv-$VERSION/bin/activate" # Install the specific version - uv pip install -U setuptools wheel + uv pip install -U setuptools wheel pip git checkout release/$VERSION uv pip install -e ".[templates,server]" @@ -84,7 +84,7 @@ uv venv ".venv-current-branch" source ".venv-current-branch/bin/activate" echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV -uv pip install -U setuptools wheel +uv pip install -U setuptools wheel pip uv pip install -e ".[templates,server]" uv pip install importlib_metadata diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 80080c87e0..f80cad7d07 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -76,7 +76,7 @@ do source ".venv-$VERSION/bin/activate" # Install the specific version - uv pip install -U setuptools wheel + uv pip install -U setuptools wheel pip git checkout release/$VERSION uv pip install -e ".[templates,server]" @@ -118,7 +118,7 @@ set -e uv venv ".venv-current-branch" source ".venv-current-branch/bin/activate" -uv pip install setuptools wheel +uv pip install setuptools wheel pip uv pip install -e ".[templates,server]" uv pip install importlib_metadata From f6fc38bca5913e0b8de5ca047c2d37ebf433c126 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 29 Feb 2024 13:29:15 +0100 Subject: [PATCH 35/73] Update sklearn integration installation in test-migrations scripts --- scripts/test-migrations-mariadb.sh | 4 +++- scripts/test-migrations-mysql.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index 4def7799b8..05d098bc4f 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -17,7 +17,9 @@ function run_tests_for_version() { export ZENML_DEBUG=true echo "===== Installing sklearn integration =====" - zenml integration install sklearn -y + zenml integration export-requirements sklearn --output-file sklearn-requirements.txt + uv pip install -r sklearn-requirements.txt + rm sklearn-requirements.txt echo "===== Running starter template pipeline =====" python3 run.py diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index f80cad7d07..c142d05d4e 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -35,7 +35,9 @@ function run_tests_for_version() { export ZENML_DEBUG=true echo "===== Installing sklearn integration =====" - zenml integration install sklearn -y + zenml integration export-requirements sklearn --output-file sklearn-requirements.txt + uv pip install -r sklearn-requirements.txt + rm sklearn-requirements.txt echo "===== Running starter template pipeline =====" python3 run.py From b81b6defe3208a12855316df874fd8971773c52b Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 29 Feb 2024 18:20:55 +0000 Subject: [PATCH 36/73] Auto-update of Starter template --- examples/quickstart/run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/quickstart/run.py b/examples/quickstart/run.py index 32965e2921..d7b1a7f11b 100644 --- a/examples/quickstart/run.py +++ b/examples/quickstart/run.py @@ -188,12 +188,12 @@ def main( test_dataset_name, test_dataset_version_name ) # Use versioned artifacts - run_args_train[ - "train_dataset_id" - ] = train_dataset_artifact_version.id - run_args_train[ - "test_dataset_id" - ] = test_dataset_artifact_version.id + run_args_train["train_dataset_id"] = ( + train_dataset_artifact_version.id + ) + run_args_train["test_dataset_id"] = ( + test_dataset_artifact_version.id + ) # Run the SGD pipeline pipeline_args = {} From 9801dd414eb6f14b08c95dd48d264486f0154285 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 29 Feb 2024 18:30:36 +0000 Subject: [PATCH 37/73] Auto-update of NLP template --- examples/e2e_nlp/run.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/e2e_nlp/run.py b/examples/e2e_nlp/run.py index f59b73d1f6..fd09dfc82d 100644 --- a/examples/e2e_nlp/run.py +++ b/examples/e2e_nlp/run.py @@ -208,9 +208,9 @@ def main( pipeline_args["model"] = model - pipeline_args[ - "run_name" - ] = f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_training_pipeline.with_options(**pipeline_args)( **run_args_train ) @@ -221,9 +221,9 @@ def main( run_args_promoting = {} model = Model(name=zenml_model_name, version=ModelStages.LATEST) pipeline_args["model"] = model - pipeline_args[ - "run_name" - ] = f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_promote_pipeline.with_options(**pipeline_args)( **run_args_promoting ) @@ -243,9 +243,9 @@ def main( "interpretation": deployment_app_interpretation, "example": deployment_app_example, } - pipeline_args[ - "run_name" - ] = f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_deploy_pipeline.with_options(**pipeline_args)( **run_args_deploying ) From 1f79ba66da2d528a42edf2ccdbe1f9fc6830f4d8 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 14:48:52 +0100 Subject: [PATCH 38/73] uv pip list instead of plain pip --- .github/actions/setup_environment/action.yml | 2 +- .github/workflows/generate-test-duration.yml | 2 +- .github/workflows/unit-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 8beb2d13e2..103a6a4c1c 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -72,5 +72,5 @@ runs: shell: bash run: |- zenml integration list - pip list + uv pip list pip check || true diff --git a/.github/workflows/generate-test-duration.yml b/.github/workflows/generate-test-duration.yml index 9985f50649..5a437a5a1f 100644 --- a/.github/workflows/generate-test-duration.yml +++ b/.github/workflows/generate-test-duration.yml @@ -53,5 +53,5 @@ jobs: - name: Verify Python Env unaffected run: |- zenml integration list - pip list + uv pip list pip check || true diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index b6d758026f..f96e5be9ed 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -114,5 +114,5 @@ jobs: - name: Verify Python Env unaffected run: |- zenml integration list - pip list + uv pip list pip check || true From a62b37c8d1b91c667b66577be063a0c14ed66f9e Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 15:17:53 +0100 Subject: [PATCH 39/73] Fix source type resolution for defaultdict on Windows --- tests/unit/utils/test_source_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_source_utils.py b/tests/unit/utils/test_source_utils.py index 009f050355..d3a2d30faf 100644 --- a/tests/unit/utils/test_source_utils.py +++ b/tests/unit/utils/test_source_utils.py @@ -107,10 +107,15 @@ def test_basic_source_resolving(mocker): assert source_utils.resolve(int) == Source( module=int.__module__, attribute=int.__name__, type=SourceType.BUILTIN ) + if sys.platform == "win32": + # On Windows, defaultdict might be classified as SourceType.UNKNOWN + expected_type = SourceType.UNKNOWN + else: + expected_type = SourceType.BUILTIN assert source_utils.resolve(defaultdict) == Source( module=defaultdict.__module__, attribute=defaultdict.__name__, - type=SourceType.BUILTIN, + type=expected_type, ) assert source_utils.resolve(source_utils) == Source( module=source_utils.__name__, From 86cf5c89bac95e68ff2a19dfb0764d3231888a00 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 15:18:21 +0100 Subject: [PATCH 40/73] partially disable CI --- .github/workflows/ci-fast.yml | 12 +- .github/workflows/ci-slow.yml | 324 +++++++++++++++++----------------- 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 90e71ff02b..45ee5492f8 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -3,12 +3,12 @@ name: ci-fast on: workflow_dispatch: workflow_call: - push: - branches: [main] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - pull_request: - types: [opened, synchronize, ready_for_review] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # push: + # branches: [main] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # pull_request: + # types: [opened, synchronize, ready_for_review] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 402c724cf1..3e434a666c 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -37,93 +37,93 @@ jobs: run: | echo "Please add the 'run-slow-ci' label to this PR before merging." exit 1 - docstring-check: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.1 - - name: Set up Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.10' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - - name: Install current package as editable - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.cargo/env - uv pip install --system darglint - - name: Check docstrings - run: bash scripts/docstring.sh - sqlite-db-migration-testing: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - env: - ZENML_ANALYTICS_OPT_IN: false - ZENML_DEBUG: true - runs-on: arc-runner-set - steps: - - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.9' - - name: Test migrations across versions - run: bash scripts/test-migrations-mysql.sh sqlite - small-checks: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.1 - - name: Set up Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.8' - - name: Install current package as editable - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.cargo/env - uv pip install --system -e . - - name: Install mlstacks package - run: uv pip install --system mlstacks - - name: Check for broken dependencies - run: pip check - - name: Markdown link check - uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'no' - folder-path: ./examples, ./docs/book, ./src - file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, - CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md - config-file: .github/workflows/markdown_check_config.json - continue-on-error: true - - name: Security check - run: | - uv pip install --system bandit - bash scripts/check-security.sh - - name: Check for alembic branch divergence - env: - ZENML_DEBUG: 0 - run: | - uv pip install --system alembic - bash scripts/check-alembic-branches.sh - custom-ubuntu-unit-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11'] - fail-fast: false - uses: ./.github/workflows/unit-test.yml - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - secrets: inherit + # docstring-check: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4.1.1 + # - name: Set up Python + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.10' + # - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV + # - name: Install current package as editable + # run: | + # curl -LsSf https://astral.sh/uv/install.sh | sh + # source $HOME/.cargo/env + # uv pip install --system darglint + # - name: Check docstrings + # run: bash scripts/docstring.sh + # sqlite-db-migration-testing: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # env: + # ZENML_ANALYTICS_OPT_IN: false + # ZENML_DEBUG: true + # runs-on: arc-runner-set + # steps: + # - uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 0 + # - name: Set up Python 3.9 + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.9' + # - name: Test migrations across versions + # run: bash scripts/test-migrations-mysql.sh sqlite + # small-checks: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4.1.1 + # - name: Set up Python + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.8' + # - name: Install current package as editable + # run: | + # curl -LsSf https://astral.sh/uv/install.sh | sh + # source $HOME/.cargo/env + # uv pip install --system -e . + # - name: Install mlstacks package + # run: uv pip install --system mlstacks + # - name: Check for broken dependencies + # run: pip check + # - name: Markdown link check + # uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + # with: + # use-quiet-mode: 'yes' + # use-verbose-mode: 'no' + # folder-path: ./examples, ./docs/book, ./src + # file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, + # CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md + # config-file: .github/workflows/markdown_check_config.json + # continue-on-error: true + # - name: Security check + # run: | + # uv pip install --system bandit + # bash scripts/check-security.sh + # - name: Check for alembic branch divergence + # env: + # ZENML_DEBUG: 0 + # run: | + # uv pip install --system alembic + # bash scripts/check-alembic-branches.sh + # custom-ubuntu-unit-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [arc-runner-set] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # fail-fast: false + # uses: ./.github/workflows/unit-test.yml + # with: + # python-version: ${{ matrix.python-version }} + # os: ${{ matrix.os }} + # secrets: inherit windows-unit-test: if: github.event.pull_request.draft == false needs: run-slow-ci-label-is-set @@ -137,78 +137,78 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} secrets: inherit - macos-unit-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - fail-fast: false - uses: ./.github/workflows/unit-test.yml - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - secrets: inherit - windows-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: [default] - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit - macos-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: [default] - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit - custom-ubuntu-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: - - default - - docker-server-docker-orchestrator-mysql - - docker-server-docker-orchestrator-mariadb - exclude: - # docker is time-consuming to run, so we only run it on 3.8 - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.9' - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.10' - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.11' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.9' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.10' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.11' - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit + # macos-unit-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [macos-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # fail-fast: false + # uses: ./.github/workflows/unit-test.yml + # with: + # python-version: ${{ matrix.python-version }} + # os: ${{ matrix.os }} + # secrets: inherit + # windows-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [windows-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: [default] + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit + # macos-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [macos-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: [default] + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit + # custom-ubuntu-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [arc-runner-set] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: + # - default + # - docker-server-docker-orchestrator-mysql + # - docker-server-docker-orchestrator-mariadb + # exclude: + # # docker is time-consuming to run, so we only run it on 3.8 + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.9' + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.10' + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.11' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.9' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.10' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.11' + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit From 94b219cc95e945c9f064d50d96efe168eca491d1 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 16:06:27 +0100 Subject: [PATCH 41/73] formatting --- examples/e2e_nlp/run.py | 18 +++++++++--------- examples/quickstart/run.py | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/e2e_nlp/run.py b/examples/e2e_nlp/run.py index fd09dfc82d..f59b73d1f6 100644 --- a/examples/e2e_nlp/run.py +++ b/examples/e2e_nlp/run.py @@ -208,9 +208,9 @@ def main( pipeline_args["model"] = model - pipeline_args["run_name"] = ( - f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - ) + pipeline_args[ + "run_name" + ] = f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" nlp_use_case_training_pipeline.with_options(**pipeline_args)( **run_args_train ) @@ -221,9 +221,9 @@ def main( run_args_promoting = {} model = Model(name=zenml_model_name, version=ModelStages.LATEST) pipeline_args["model"] = model - pipeline_args["run_name"] = ( - f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - ) + pipeline_args[ + "run_name" + ] = f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" nlp_use_case_promote_pipeline.with_options(**pipeline_args)( **run_args_promoting ) @@ -243,9 +243,9 @@ def main( "interpretation": deployment_app_interpretation, "example": deployment_app_example, } - pipeline_args["run_name"] = ( - f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - ) + pipeline_args[ + "run_name" + ] = f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" nlp_use_case_deploy_pipeline.with_options(**pipeline_args)( **run_args_deploying ) diff --git a/examples/quickstart/run.py b/examples/quickstart/run.py index d7b1a7f11b..32965e2921 100644 --- a/examples/quickstart/run.py +++ b/examples/quickstart/run.py @@ -188,12 +188,12 @@ def main( test_dataset_name, test_dataset_version_name ) # Use versioned artifacts - run_args_train["train_dataset_id"] = ( - train_dataset_artifact_version.id - ) - run_args_train["test_dataset_id"] = ( - test_dataset_artifact_version.id - ) + run_args_train[ + "train_dataset_id" + ] = train_dataset_artifact_version.id + run_args_train[ + "test_dataset_id" + ] = test_dataset_artifact_version.id # Run the SGD pipeline pipeline_args = {} From 23d82d1727f97ab7d27b312407f40249b1423a4f Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 17:18:55 +0100 Subject: [PATCH 42/73] formatting --- .github/workflows/ci-slow.yml | 2 +- examples/e2e_nlp/run.py | 18 +++++++++--------- examples/quickstart/run.py | 12 ++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 3e434a666c..4ed0a49c7f 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -192,7 +192,7 @@ jobs: # - docker-server-docker-orchestrator-mysql # - docker-server-docker-orchestrator-mariadb # exclude: - # # docker is time-consuming to run, so we only run it on 3.8 + # # docker is time-consuming to run, so we only run it on 3.8 # - test_environment: docker-server-docker-orchestrator-mysql # python-version: '3.9' # - test_environment: docker-server-docker-orchestrator-mysql diff --git a/examples/e2e_nlp/run.py b/examples/e2e_nlp/run.py index f59b73d1f6..fd09dfc82d 100644 --- a/examples/e2e_nlp/run.py +++ b/examples/e2e_nlp/run.py @@ -208,9 +208,9 @@ def main( pipeline_args["model"] = model - pipeline_args[ - "run_name" - ] = f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_training_pipeline.with_options(**pipeline_args)( **run_args_train ) @@ -221,9 +221,9 @@ def main( run_args_promoting = {} model = Model(name=zenml_model_name, version=ModelStages.LATEST) pipeline_args["model"] = model - pipeline_args[ - "run_name" - ] = f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_promote_pipeline.with_options(**pipeline_args)( **run_args_promoting ) @@ -243,9 +243,9 @@ def main( "interpretation": deployment_app_interpretation, "example": deployment_app_example, } - pipeline_args[ - "run_name" - ] = f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + pipeline_args["run_name"] = ( + f"nlp_use_case_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ) nlp_use_case_deploy_pipeline.with_options(**pipeline_args)( **run_args_deploying ) diff --git a/examples/quickstart/run.py b/examples/quickstart/run.py index 32965e2921..d7b1a7f11b 100644 --- a/examples/quickstart/run.py +++ b/examples/quickstart/run.py @@ -188,12 +188,12 @@ def main( test_dataset_name, test_dataset_version_name ) # Use versioned artifacts - run_args_train[ - "train_dataset_id" - ] = train_dataset_artifact_version.id - run_args_train[ - "test_dataset_id" - ] = test_dataset_artifact_version.id + run_args_train["train_dataset_id"] = ( + train_dataset_artifact_version.id + ) + run_args_train["test_dataset_id"] = ( + test_dataset_artifact_version.id + ) # Run the SGD pipeline pipeline_args = {} From 1b470c803e04dc4348224da1c9002873e2f7b20f Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 18:01:12 +0100 Subject: [PATCH 43/73] update second test for windows --- tests/unit/utils/test_source_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit/utils/test_source_utils.py b/tests/unit/utils/test_source_utils.py index d3a2d30faf..6721368022 100644 --- a/tests/unit/utils/test_source_utils.py +++ b/tests/unit/utils/test_source_utils.py @@ -214,9 +214,12 @@ def test_module_type_detection(mocker): assert source_utils.get_source_type(builtin_module) == SourceType.BUILTIN standard_lib_module = sys.modules[defaultdict.__module__] - assert ( - source_utils.get_source_type(standard_lib_module) == SourceType.BUILTIN - ) + if sys.platform == "win32": + # On Windows, the module containing defaultdict might be classified as SourceType.DISTRIBUTION_PACKAGE + expected_type = SourceType.DISTRIBUTION_PACKAGE + else: + expected_type = SourceType.BUILTIN + assert source_utils.get_source_type(standard_lib_module) == expected_type assert source_utils.is_standard_lib_file(standard_lib_module.__file__) internal_module = sys.modules[source_utils.__name__] From d2c79fc54761673a1e2d8ec2287ded696be05370 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 1 Mar 2024 18:23:00 +0100 Subject: [PATCH 44/73] run the whole ci again --- .github/workflows/ci-fast.yml | 12 +- .github/workflows/ci-slow.yml | 324 +++++++++++++++++----------------- 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 45ee5492f8..90e71ff02b 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -3,12 +3,12 @@ name: ci-fast on: workflow_dispatch: workflow_call: - # push: - # branches: [main] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - # pull_request: - # types: [opened, synchronize, ready_for_review] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + push: + branches: [main] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + pull_request: + types: [opened, synchronize, ready_for_review] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 4ed0a49c7f..7c8ad81ae7 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -37,93 +37,93 @@ jobs: run: | echo "Please add the 'run-slow-ci' label to this PR before merging." exit 1 - # docstring-check: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4.1.1 - # - name: Set up Python - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.10' - # - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - # - name: Install current package as editable - # run: | - # curl -LsSf https://astral.sh/uv/install.sh | sh - # source $HOME/.cargo/env - # uv pip install --system darglint - # - name: Check docstrings - # run: bash scripts/docstring.sh - # sqlite-db-migration-testing: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # env: - # ZENML_ANALYTICS_OPT_IN: false - # ZENML_DEBUG: true - # runs-on: arc-runner-set - # steps: - # - uses: actions/checkout@v4.1.1 - # with: - # fetch-depth: 0 - # - name: Set up Python 3.9 - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.9' - # - name: Test migrations across versions - # run: bash scripts/test-migrations-mysql.sh sqlite - # small-checks: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4.1.1 - # - name: Set up Python - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.8' - # - name: Install current package as editable - # run: | - # curl -LsSf https://astral.sh/uv/install.sh | sh - # source $HOME/.cargo/env - # uv pip install --system -e . - # - name: Install mlstacks package - # run: uv pip install --system mlstacks - # - name: Check for broken dependencies - # run: pip check - # - name: Markdown link check - # uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 - # with: - # use-quiet-mode: 'yes' - # use-verbose-mode: 'no' - # folder-path: ./examples, ./docs/book, ./src - # file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, - # CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md - # config-file: .github/workflows/markdown_check_config.json - # continue-on-error: true - # - name: Security check - # run: | - # uv pip install --system bandit - # bash scripts/check-security.sh - # - name: Check for alembic branch divergence - # env: - # ZENML_DEBUG: 0 - # run: | - # uv pip install --system alembic - # bash scripts/check-alembic-branches.sh - # custom-ubuntu-unit-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [arc-runner-set] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # fail-fast: false - # uses: ./.github/workflows/unit-test.yml - # with: - # python-version: ${{ matrix.python-version }} - # os: ${{ matrix.os }} - # secrets: inherit + docstring-check: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.10' + - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV + - name: Install current package as editable + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env + uv pip install --system darglint + - name: Check docstrings + run: bash scripts/docstring.sh + sqlite-db-migration-testing: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + env: + ZENML_ANALYTICS_OPT_IN: false + ZENML_DEBUG: true + runs-on: arc-runner-set + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - name: Set up Python 3.9 + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.9' + - name: Test migrations across versions + run: bash scripts/test-migrations-mysql.sh sqlite + small-checks: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.8' + - name: Install current package as editable + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env + uv pip install --system -e . + - name: Install mlstacks package + run: uv pip install --system mlstacks + - name: Check for broken dependencies + run: pip check + - name: Markdown link check + uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'no' + folder-path: ./examples, ./docs/book, ./src + file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, + CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md + config-file: .github/workflows/markdown_check_config.json + continue-on-error: true + - name: Security check + run: | + uv pip install --system bandit + bash scripts/check-security.sh + - name: Check for alembic branch divergence + env: + ZENML_DEBUG: 0 + run: | + uv pip install --system alembic + bash scripts/check-alembic-branches.sh + custom-ubuntu-unit-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [arc-runner-set] + python-version: ['3.8', '3.9', '3.10', '3.11'] + fail-fast: false + uses: ./.github/workflows/unit-test.yml + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + secrets: inherit windows-unit-test: if: github.event.pull_request.draft == false needs: run-slow-ci-label-is-set @@ -137,78 +137,78 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} secrets: inherit - # macos-unit-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [macos-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # fail-fast: false - # uses: ./.github/workflows/unit-test.yml - # with: - # python-version: ${{ matrix.python-version }} - # os: ${{ matrix.os }} - # secrets: inherit - # windows-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [windows-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: [default] - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit - # macos-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [macos-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: [default] - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit - # custom-ubuntu-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [arc-runner-set] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: - # - default - # - docker-server-docker-orchestrator-mysql - # - docker-server-docker-orchestrator-mariadb - # exclude: - # # docker is time-consuming to run, so we only run it on 3.8 - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.9' - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.10' - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.11' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.9' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.10' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.11' - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit + macos-unit-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [macos-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + fail-fast: false + uses: ./.github/workflows/unit-test.yml + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + secrets: inherit + windows-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: [default] + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit + macos-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [macos-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: [default] + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit + custom-ubuntu-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [arc-runner-set] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: + - default + - docker-server-docker-orchestrator-mysql + - docker-server-docker-orchestrator-mariadb + exclude: + # docker is time-consuming to run, so we only run it on 3.8 + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.9' + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.11' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.9' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.11' + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit From 1a9ea7bd2313a55f6f3704b03fc00d87058e4ee7 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 12:21:26 +0100 Subject: [PATCH 45/73] revert test changes --- tests/unit/utils/test_source_utils.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unit/utils/test_source_utils.py b/tests/unit/utils/test_source_utils.py index 6721368022..009f050355 100644 --- a/tests/unit/utils/test_source_utils.py +++ b/tests/unit/utils/test_source_utils.py @@ -107,15 +107,10 @@ def test_basic_source_resolving(mocker): assert source_utils.resolve(int) == Source( module=int.__module__, attribute=int.__name__, type=SourceType.BUILTIN ) - if sys.platform == "win32": - # On Windows, defaultdict might be classified as SourceType.UNKNOWN - expected_type = SourceType.UNKNOWN - else: - expected_type = SourceType.BUILTIN assert source_utils.resolve(defaultdict) == Source( module=defaultdict.__module__, attribute=defaultdict.__name__, - type=expected_type, + type=SourceType.BUILTIN, ) assert source_utils.resolve(source_utils) == Source( module=source_utils.__name__, @@ -214,12 +209,9 @@ def test_module_type_detection(mocker): assert source_utils.get_source_type(builtin_module) == SourceType.BUILTIN standard_lib_module = sys.modules[defaultdict.__module__] - if sys.platform == "win32": - # On Windows, the module containing defaultdict might be classified as SourceType.DISTRIBUTION_PACKAGE - expected_type = SourceType.DISTRIBUTION_PACKAGE - else: - expected_type = SourceType.BUILTIN - assert source_utils.get_source_type(standard_lib_module) == expected_type + assert ( + source_utils.get_source_type(standard_lib_module) == SourceType.BUILTIN + ) assert source_utils.is_standard_lib_file(standard_lib_module.__file__) internal_module = sys.modules[source_utils.__name__] From 1931fbe702c506715aae6408bd8dc522c9d66096 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 12:25:53 +0100 Subject: [PATCH 46/73] remove old venv commands --- .github/workflows/ci-fast.yml | 2 -- .github/workflows/ci-slow.yml | 1 - scripts/test-migrations-mariadb.sh | 1 - 3 files changed, 4 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 90e71ff02b..a5e6cbcfa7 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -31,8 +31,6 @@ jobs: uv pip install --system darglint - name: Check docstrings run: bash scripts/docstring.sh - env: - VIRTUAL_ENV: ./.venv sqlite-db-migration-testing: runs-on: arc-runner-set env: diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 7c8ad81ae7..4e50b99fca 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -47,7 +47,6 @@ jobs: uses: actions/setup-python@v5.0.0 with: python-version: '3.10' - - run: echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV - name: Install current package as editable run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index c23b16d2f5..5faf6dcc07 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -85,7 +85,6 @@ set -e uv venv ".venv-current-branch" source ".venv-current-branch/bin/activate" -echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV uv pip install -U setuptools wheel pip uv pip install -e ".[templates,server]" uv pip install importlib_metadata From 56c50b99d8935560e09934c0ff46e5193ced43dd Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 14:59:40 +0100 Subject: [PATCH 47/73] debug uv failure --- .github/workflows/uv-test.yml | 122 ++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/workflows/uv-test.yml diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml new file mode 100644 index 0000000000..360b357e2d --- /dev/null +++ b/.github/workflows/uv-test.yml @@ -0,0 +1,122 @@ +name: Verify defaultdict resolution +on: + workflow_dispatch: + workflow_call: + pull_request: + types: [opened, synchronize, ready_for_review] + paths-ignore: [docker/**, '*', '!pyproject.toml'] +jobs: + pip-installation: + runs-on: windows-latest + strategy: + matrix: + # python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: gather requirements + run: | + # figure out the python version + python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") + + ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" + # if python version is 3.11, exclude all integrations depending on kfp + # because they are not yet compatible with python 3.11 + if [ "$python_version" = "3.11" ]; then + ignore_integrations="$ignore_integrations kubeflow tekton gcp" + fi + + # turn the ignore integrations into a list of --ignore-integration args + ignore_integrations_args="" + for integration in $ignore_integrations; do + ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" + done + + # install basic ZenML integrations + zenml integration export-requirements \ + --output-file integration-requirements.txt \ + $ignore_integrations_args + + echo "pyyaml>=6.0.1" >> integration-requirements.txt + echo "pyopenssl" >> integration-requirements.txt + echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt + + - name: Install dependencies with pip + run: | + pip install -r integration-requirements.txt + + - name: Log collections version and defaultdict type with pip + run: | + python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + + - name: Install dependencies with uv + run: | + pip install uv + uv venv uvtest + source uvtest/bin/activate + uv pip install --system -r integration-requirements.txt + + - name: Log collections version and defaultdict type with uv + run: | + source uvtest/bin/activate + python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + uv-installation: + runs-on: windows-latest + strategy: + matrix: + # python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: gather requirements + run: | + # figure out the python version + python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") + + ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" + # if python version is 3.11, exclude all integrations depending on kfp + # because they are not yet compatible with python 3.11 + if [ "$python_version" = "3.11" ]; then + ignore_integrations="$ignore_integrations kubeflow tekton gcp" + fi + + # turn the ignore integrations into a list of --ignore-integration args + ignore_integrations_args="" + for integration in $ignore_integrations; do + ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" + done + + # export basic ZenML integration requirements + zenml integration export-requirements \ + --output-file integration-requirements.txt \ + $ignore_integrations_args + + echo "pyyaml>=6.0.1" >> integration-requirements.txt + echo "pyopenssl" >> integration-requirements.txt + echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt + + - name: Install dependencies with uv + run: | + pip install uv + uv pip install --system -r integration-requirements.txt + + - name: Log collections version and defaultdict type with uv + run: | + python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" From 7e284863927f7bc9f18ec3bda7be6dedbaf77b77 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:03:42 +0100 Subject: [PATCH 48/73] add bash shell for windows --- .github/workflows/uv-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index 360b357e2d..dc8bb9021d 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -22,6 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: gather requirements + shell: bash run: | # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -49,15 +50,18 @@ jobs: echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - name: Install dependencies with pip + shell: bash run: | pip install -r integration-requirements.txt - name: Log collections version and defaultdict type with pip + shell: bash run: | python -c "import collections; print(f'collections version: {collections.__version__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - name: Install dependencies with uv + shell: bash run: | pip install uv uv venv uvtest @@ -65,6 +69,7 @@ jobs: uv pip install --system -r integration-requirements.txt - name: Log collections version and defaultdict type with uv + shell: bash run: | source uvtest/bin/activate python -c "import collections; print(f'collections version: {collections.__version__}')" @@ -85,6 +90,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: gather requirements + shell: bash run: | # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -112,11 +118,13 @@ jobs: echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - name: Install dependencies with uv + shell: bash run: | pip install uv uv pip install --system -r integration-requirements.txt - name: Log collections version and defaultdict type with uv + shell: bash run: | python -c "import collections; print(f'collections version: {collections.__version__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" From e5b29084061b78d3ea85df7c26bed37afd330b5d Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:06:05 +0100 Subject: [PATCH 49/73] install zenml --- .github/workflows/uv-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index dc8bb9021d..61fab76d8e 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -24,6 +24,8 @@ jobs: - name: gather requirements shell: bash run: | + pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" + # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -92,6 +94,9 @@ jobs: - name: gather requirements shell: bash run: | + pip install uv + uv pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" + # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -120,7 +125,6 @@ jobs: - name: Install dependencies with uv shell: bash run: | - pip install uv uv pip install --system -r integration-requirements.txt - name: Log collections version and defaultdict type with uv From 35ba2bb1bfe789374013fc7f6467ac1b8fe55934 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:08:44 +0100 Subject: [PATCH 50/73] fix editable install --- .github/workflows/uv-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index 61fab76d8e..4d15d4ea76 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -24,7 +24,7 @@ jobs: - name: gather requirements shell: bash run: | - pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" + pip install -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -95,7 +95,7 @@ jobs: shell: bash run: | pip install uv - uv pip install --system "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" + uv pip install --system -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") From fc342d11f575f75aabc6caac33d61d73361f65a5 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:13:45 +0100 Subject: [PATCH 51/73] add newline --- .github/workflows/uv-test.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index 4d15d4ea76..058585bdd9 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -47,6 +47,7 @@ jobs: --output-file integration-requirements.txt \ $ignore_integrations_args + echo "" >> integration-requirements.txt echo "pyyaml>=6.0.1" >> integration-requirements.txt echo "pyopenssl" >> integration-requirements.txt echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt @@ -70,12 +71,21 @@ jobs: source uvtest/bin/activate uv pip install --system -r integration-requirements.txt - - name: Log collections version and defaultdict type with uv + - name: Log collections version and defaultdict type with pip shell: bash run: | - source uvtest/bin/activate python -c "import collections; print(f'collections version: {collections.__version__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + + python -c "import sys; print(f'Python executable: {sys.executable}')" + python -c "import sys; print(f'Module search paths: {sys.path}')" + + + echo "PYTHONPATH: $PYTHONPATH" + echo "PATH: $PATH" + + pip list + uv-installation: runs-on: windows-latest strategy: @@ -118,6 +128,7 @@ jobs: --output-file integration-requirements.txt \ $ignore_integrations_args + echo "" >> integration-requirements.txt echo "pyyaml>=6.0.1" >> integration-requirements.txt echo "pyopenssl" >> integration-requirements.txt echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt @@ -132,3 +143,14 @@ jobs: run: | python -c "import collections; print(f'collections version: {collections.__version__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + + python -c "import sys; print(f'Python executable: {sys.executable}')" + python -c "import sys; print(f'Module search paths: {sys.path}')" + + + echo "PYTHONPATH: $PYTHONPATH" + echo "PATH: $PATH" + + uv --version + + pip list From eeb530161b78c6628f3b50be9b72121897d985cd Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:20:37 +0100 Subject: [PATCH 52/73] remove double uv for pip debug logging --- .github/workflows/uv-test.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index 058585bdd9..cd8b5ce59c 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -57,20 +57,6 @@ jobs: run: | pip install -r integration-requirements.txt - - name: Log collections version and defaultdict type with pip - shell: bash - run: | - python -c "import collections; print(f'collections version: {collections.__version__}')" - python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - - - name: Install dependencies with uv - shell: bash - run: | - pip install uv - uv venv uvtest - source uvtest/bin/activate - uv pip install --system -r integration-requirements.txt - - name: Log collections version and defaultdict type with pip shell: bash run: | From 6025ce787fdf0b7410afc243c591355dc705a19b Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:21:37 +0100 Subject: [PATCH 53/73] disable CI temporarily for testing --- .github/workflows/ci-fast.yml | 12 ++++++------ .github/workflows/ci-slow.yml | 12 ++++++------ .github/workflows/spellcheck.yml | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index a5e6cbcfa7..b41757b585 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -3,12 +3,12 @@ name: ci-fast on: workflow_dispatch: workflow_call: - push: - branches: [main] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - pull_request: - types: [opened, synchronize, ready_for_review] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # push: + # branches: [main] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # pull_request: + # types: [opened, synchronize, ready_for_review] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 4e50b99fca..3a11c456fa 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -1,12 +1,12 @@ --- name: ci-slow on: - push: - branches: [main] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - pull_request: - types: [opened, synchronize, ready_for_review] - paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # push: + # branches: [main] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + # pull_request: + # types: [opened, synchronize, ready_for_review] + # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 55f481d1ce..fcbcefed82 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -3,12 +3,12 @@ name: Spellcheck on: workflow_dispatch: workflow_call: - push: - branches: [main] - paths-ignore: [docker/**, '*', '!pyproject.toml'] - pull_request: - types: [opened, synchronize, ready_for_review] - paths-ignore: [docker/**, '*', '!pyproject.toml'] + # push: + # branches: [main] + # paths-ignore: [docker/**, '*', '!pyproject.toml'] + # pull_request: + # types: [opened, synchronize, ready_for_review] + # paths-ignore: [docker/**, '*', '!pyproject.toml'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} From 9eb0c8c799204a46a5267eaa6c67307c1f69c383 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:21:55 +0100 Subject: [PATCH 54/73] yamlfix --- .github/workflows/uv-test.yml | 217 +++++++++++++++------------------- 1 file changed, 97 insertions(+), 120 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index cd8b5ce59c..a9c831470b 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -1,3 +1,4 @@ +--- name: Verify defaultdict resolution on: workflow_dispatch: @@ -12,131 +13,107 @@ jobs: matrix: # python-version: ['3.8', '3.9', '3.10', '3.11'] python-version: ['3.10'] - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.0.0 - with: - python-version: ${{ matrix.python-version }} - - - name: gather requirements - shell: bash - run: | - pip install -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' - - # figure out the python version - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") - - ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" - # if python version is 3.11, exclude all integrations depending on kfp - # because they are not yet compatible with python 3.11 - if [ "$python_version" = "3.11" ]; then - ignore_integrations="$ignore_integrations kubeflow tekton gcp" - fi - - # turn the ignore integrations into a list of --ignore-integration args - ignore_integrations_args="" - for integration in $ignore_integrations; do - ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" - done - - # install basic ZenML integrations - zenml integration export-requirements \ - --output-file integration-requirements.txt \ - $ignore_integrations_args - - echo "" >> integration-requirements.txt - echo "pyyaml>=6.0.1" >> integration-requirements.txt - echo "pyopenssl" >> integration-requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - - - name: Install dependencies with pip - shell: bash - run: | - pip install -r integration-requirements.txt - - - name: Log collections version and defaultdict type with pip - shell: bash - run: | - python -c "import collections; print(f'collections version: {collections.__version__}')" - python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - - python -c "import sys; print(f'Python executable: {sys.executable}')" - python -c "import sys; print(f'Module search paths: {sys.path}')" - - - echo "PYTHONPATH: $PYTHONPATH" - echo "PATH: $PATH" - - pip list - + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: gather requirements + shell: bash + run: | + pip install -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' + + # figure out the python version + python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") + ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" + # if python version is 3.11, exclude all integrations depending on kfp + # because they are not yet compatible with python 3.11 + if [ "$python_version" = "3.11" ]; then + ignore_integrations="$ignore_integrations kubeflow tekton gcp" + fi + + # turn the ignore integrations into a list of --ignore-integration args + ignore_integrations_args="" + for integration in $ignore_integrations; do + ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" + done + + # install basic ZenML integrations + zenml integration export-requirements \ + --output-file integration-requirements.txt \ + $ignore_integrations_args + echo "" >> integration-requirements.txt + echo "pyyaml>=6.0.1" >> integration-requirements.txt + echo "pyopenssl" >> integration-requirements.txt + echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt + - name: Install dependencies with pip + shell: bash + run: | + pip install -r integration-requirements.txt + - name: Log collections version and defaultdict type with pip + shell: bash + run: | + python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + python -c "import sys; print(f'Python executable: {sys.executable}')" + python -c "import sys; print(f'Module search paths: {sys.path}')" + echo "PYTHONPATH: $PYTHONPATH" + echo "PATH: $PATH" + pip list uv-installation: runs-on: windows-latest strategy: matrix: # python-version: ['3.8', '3.9', '3.10', '3.11'] python-version: ['3.10'] - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.0.0 - with: - python-version: ${{ matrix.python-version }} - - - name: gather requirements - shell: bash - run: | - pip install uv - uv pip install --system -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' - - # figure out the python version - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") - - ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" - # if python version is 3.11, exclude all integrations depending on kfp - # because they are not yet compatible with python 3.11 - if [ "$python_version" = "3.11" ]; then - ignore_integrations="$ignore_integrations kubeflow tekton gcp" - fi - - # turn the ignore integrations into a list of --ignore-integration args - ignore_integrations_args="" - for integration in $ignore_integrations; do - ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" - done - - # export basic ZenML integration requirements - zenml integration export-requirements \ - --output-file integration-requirements.txt \ - $ignore_integrations_args - - echo "" >> integration-requirements.txt - echo "pyyaml>=6.0.1" >> integration-requirements.txt - echo "pyopenssl" >> integration-requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - - - name: Install dependencies with uv - shell: bash - run: | - uv pip install --system -r integration-requirements.txt - - - name: Log collections version and defaultdict type with uv - shell: bash - run: | - python -c "import collections; print(f'collections version: {collections.__version__}')" - python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - - python -c "import sys; print(f'Python executable: {sys.executable}')" - python -c "import sys; print(f'Module search paths: {sys.path}')" - - - echo "PYTHONPATH: $PYTHONPATH" - echo "PATH: $PATH" - - uv --version - - pip list + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: gather requirements + shell: bash + run: | + pip install uv + uv pip install --system -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' + + # figure out the python version + python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") + ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" + # if python version is 3.11, exclude all integrations depending on kfp + # because they are not yet compatible with python 3.11 + if [ "$python_version" = "3.11" ]; then + ignore_integrations="$ignore_integrations kubeflow tekton gcp" + fi + + # turn the ignore integrations into a list of --ignore-integration args + ignore_integrations_args="" + for integration in $ignore_integrations; do + ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" + done + + # export basic ZenML integration requirements + zenml integration export-requirements \ + --output-file integration-requirements.txt \ + $ignore_integrations_args + echo "" >> integration-requirements.txt + echo "pyyaml>=6.0.1" >> integration-requirements.txt + echo "pyopenssl" >> integration-requirements.txt + echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt + - name: Install dependencies with uv + shell: bash + run: | + uv pip install --system -r integration-requirements.txt + - name: Log collections version and defaultdict type with uv + shell: bash + run: |- + python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + python -c "import sys; print(f'Python executable: {sys.executable}')" + python -c "import sys; print(f'Module search paths: {sys.path}')" + echo "PYTHONPATH: $PYTHONPATH" + echo "PATH: $PATH" + uv --version + pip list From cb9739fc2c5503388170719e0c78f384916fa330 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:22:39 +0100 Subject: [PATCH 55/73] unfail --- .github/workflows/ci-slow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 3a11c456fa..f95cd9fbbc 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -1,6 +1,8 @@ --- name: ci-slow on: + workflow_dispatch: + workflow_call: # push: # branches: [main] # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] From 45a66f1a516d5c57406b880019280e2ecd156ccd Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 15:34:40 +0100 Subject: [PATCH 56/73] more metadata about collections --- .github/workflows/uv-test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index a9c831470b..d22b978fa7 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -54,7 +54,9 @@ jobs: - name: Log collections version and defaultdict type with pip shell: bash run: | - python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "import sys; print(f'Python version: {sys.version}')" + python -c "import collections; print(f'collections module file path: {collections.__file__}')" + python -c "import collections; print(f'collections module name: {collections.__name__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" python -c "import sys; print(f'Python executable: {sys.executable}')" python -c "import sys; print(f'Module search paths: {sys.path}')" @@ -109,7 +111,9 @@ jobs: - name: Log collections version and defaultdict type with uv shell: bash run: |- - python -c "import collections; print(f'collections version: {collections.__version__}')" + python -c "import sys; print(f'Python version: {sys.version}')" + python -c "import collections; print(f'collections module file path: {collections.__file__}')" + python -c "import collections; print(f'collections module name: {collections.__name__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" python -c "import sys; print(f'Python executable: {sys.executable}')" python -c "import sys; print(f'Module search paths: {sys.path}')" From 27e3f2ffa477c8fe13cf6e95fc5d4d741cb7b338 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 16:07:57 +0100 Subject: [PATCH 57/73] faster resolution for test --- .github/workflows/uv-test.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index d22b978fa7..a879cf8abb 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -22,7 +22,13 @@ jobs: - name: gather requirements shell: bash run: | - pip install -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' + pip install uv + touch basic-requirements.txt + echo "-e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]'" >> basic-requirements.txt + cp basic-requirements.txt basic-requirements.in + + uv pip compile basic-requirements.in -o basic-requirements-compiled.txt + pip install -r basic-requirements-compiled.txt # figure out the python version python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") @@ -47,10 +53,12 @@ jobs: echo "pyyaml>=6.0.1" >> integration-requirements.txt echo "pyopenssl" >> integration-requirements.txt echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt + cp integration-requirements.txt integration-requirements.in + uv pip compile integration-requirements.in -o integration-requirements-compiled.txt - name: Install dependencies with pip shell: bash run: | - pip install -r integration-requirements.txt + pip install -r integration-requirements-compiled.txt - name: Log collections version and defaultdict type with pip shell: bash run: | @@ -58,6 +66,7 @@ jobs: python -c "import collections; print(f'collections module file path: {collections.__file__}')" python -c "import collections; print(f'collections module name: {collections.__name__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + python -c "import sys; from collections import defaultdict; from zenml.config.source import Source, SourceType; from zenml.utils import source_utils; print(f'source_utils.resolve(defaultdict): {source_utils.resolve(defaultdict)}'); print(f'Source object: {Source(module=defaultdict.__module__, attribute=defaultdict.__name__, type=SourceType.BUILTIN)}')" python -c "import sys; print(f'Python executable: {sys.executable}')" python -c "import sys; print(f'Module search paths: {sys.path}')" echo "PYTHONPATH: $PYTHONPATH" @@ -115,6 +124,7 @@ jobs: python -c "import collections; print(f'collections module file path: {collections.__file__}')" python -c "import collections; print(f'collections module name: {collections.__name__}')" python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" + python -c "import sys; from collections import defaultdict; from zenml.config.source import Source, SourceType; from zenml.utils import source_utils; print(f'source_utils.resolve(defaultdict): {source_utils.resolve(defaultdict)}'); print(f'Source object: {Source(module=defaultdict.__module__, attribute=defaultdict.__name__, type=SourceType.BUILTIN)}')" python -c "import sys; print(f'Python executable: {sys.executable}')" python -c "import sys; print(f'Module search paths: {sys.path}')" echo "PYTHONPATH: $PYTHONPATH" From 046fa1ebc3c730bb5ffbd864067d262905cc8829 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 16:16:52 +0100 Subject: [PATCH 58/73] fix escape --- .github/workflows/uv-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml index a879cf8abb..59c0b41f4b 100644 --- a/.github/workflows/uv-test.yml +++ b/.github/workflows/uv-test.yml @@ -24,9 +24,8 @@ jobs: run: | pip install uv touch basic-requirements.txt - echo "-e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]'" >> basic-requirements.txt + echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> basic-requirements.txt cp basic-requirements.txt basic-requirements.in - uv pip compile basic-requirements.in -o basic-requirements-compiled.txt pip install -r basic-requirements-compiled.txt From 854cf486747bcb89a9c74aef714febf58302a990 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 17:02:30 +0100 Subject: [PATCH 59/73] attempt fix for windows source resolution --- src/zenml/utils/source_utils.py | 65 ++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/src/zenml/utils/source_utils.py b/src/zenml/utils/source_utils.py index ea1fcfe6d0..881a52d590 100644 --- a/src/zenml/utils/source_utils.py +++ b/src/zenml/utils/source_utils.py @@ -15,6 +15,7 @@ import contextlib import importlib +import importlib.util import inspect import os import site @@ -154,9 +155,7 @@ def resolve( if source_type == SourceType.USER: from zenml.utils import code_repository_utils - local_repo_context = ( - code_repository_utils.find_active_code_repository() - ) + local_repo_context = code_repository_utils.find_active_code_repository() if local_repo_context and not local_repo_context.has_local_changes: module_name = _resolve_module(module) @@ -189,9 +188,7 @@ def resolve( # Fallback to an unknown source if we can't find the package source_type = SourceType.UNKNOWN - return Source( - module=module_name, attribute=attribute_name, type=source_type - ) + return Source(module=module_name, attribute=attribute_name, type=source_type) def get_source_root() -> str: @@ -286,12 +283,50 @@ def is_standard_lib_file(file_path: str) -> bool: file_path: The file path to check. Returns: - True if the file belongs to the Python standard library, False - otherwise. + True if the file belongs to the Python standard library, False otherwise. """ - stdlib_root = get_python_lib(standard_lib=True) - logger.debug("Standard library root: %s", stdlib_root) - return Path(stdlib_root).resolve() in Path(file_path).resolve().parents + # Normalize file path for reliable comparison + normalized_file_path = Path(file_path).resolve() + + # Gather possible locations of the standard library + stdlib_paths = set() + # Default library path + stdlib_paths.add(Path(get_python_lib(standard_lib=True)).resolve()) + # Other common standard library locations, considering virtual environments + stdlib_paths.update(Path(p).resolve() for p in sys.path if "site-packages" not in p) + + # Check if file path is within any known standard library path + if any( + normalized_file_path.is_relative_to(stdlib_path) for stdlib_path in stdlib_paths + ): + return True + + # As a fallback, check if the module is a built-in module + module_name = Path(file_path).stem + try: + spec = importlib.util.find_spec(module_name) + if spec and ( + spec.origin == "built-in" + or ( + spec.loader is not None + and "built-in" in spec.loader.__class__.__name__.lower() + ) + ): + return True + except (ImportError, AttributeError): + pass + + # Fallback to inspect (for dynamically loaded modules, etc.) + try: + source_file = inspect.getsourcefile(normalized_file_path) + if source_file and Path(source_file).resolve().is_file(): + return any( + Path(source_file).resolve().is_relative_to(p) for p in stdlib_paths + ) + except Exception: + pass + + return False def is_distribution_package_file(file_path: str, module_name: str) -> bool: @@ -339,9 +374,7 @@ def get_source_type(module: ModuleType) -> SourceType: if is_internal_module(module_name=module.__name__): return SourceType.INTERNAL - if is_distribution_package_file( - file_path=file_path, module_name=module.__name__ - ): + if is_distribution_package_file(file_path=file_path, module_name=module.__name__): return SourceType.DISTRIBUTION_PACKAGE if is_standard_lib_file(file_path=file_path): @@ -478,9 +511,7 @@ def _resolve_module(module: ModuleType) -> str: return module_source -def _load_module( - module_name: str, import_root: Optional[str] = None -) -> ModuleType: +def _load_module(module_name: str, import_root: Optional[str] = None) -> ModuleType: """Load a module. Args: From 76b0d2d01a32c6ca271112fd3309e79d2995c8d6 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 17:04:12 +0100 Subject: [PATCH 60/73] format and windows unit tests --- .github/workflows/ci-slow.yml | 334 ++++++++++++++++---------------- src/zenml/utils/source_utils.py | 26 ++- 2 files changed, 186 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index f95cd9fbbc..a4cf45de87 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -3,12 +3,12 @@ name: ci-slow on: workflow_dispatch: workflow_call: - # push: - # branches: [main] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - # pull_request: - # types: [opened, synchronize, ready_for_review] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + push: + branches: [main] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + pull_request: + types: [opened, synchronize, ready_for_review] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} @@ -39,92 +39,92 @@ jobs: run: | echo "Please add the 'run-slow-ci' label to this PR before merging." exit 1 - docstring-check: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.1 - - name: Set up Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.10' - - name: Install current package as editable - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.cargo/env - uv pip install --system darglint - - name: Check docstrings - run: bash scripts/docstring.sh - sqlite-db-migration-testing: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - env: - ZENML_ANALYTICS_OPT_IN: false - ZENML_DEBUG: true - runs-on: arc-runner-set - steps: - - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.9' - - name: Test migrations across versions - run: bash scripts/test-migrations-mysql.sh sqlite - small-checks: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.1.1 - - name: Set up Python - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.8' - - name: Install current package as editable - run: | - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.cargo/env - uv pip install --system -e . - - name: Install mlstacks package - run: uv pip install --system mlstacks - - name: Check for broken dependencies - run: pip check - - name: Markdown link check - uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'no' - folder-path: ./examples, ./docs/book, ./src - file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, - CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md - config-file: .github/workflows/markdown_check_config.json - continue-on-error: true - - name: Security check - run: | - uv pip install --system bandit - bash scripts/check-security.sh - - name: Check for alembic branch divergence - env: - ZENML_DEBUG: 0 - run: | - uv pip install --system alembic - bash scripts/check-alembic-branches.sh - custom-ubuntu-unit-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11'] - fail-fast: false - uses: ./.github/workflows/unit-test.yml - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - secrets: inherit + # docstring-check: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4.1.1 + # - name: Set up Python + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.10' + # - name: Install current package as editable + # run: | + # curl -LsSf https://astral.sh/uv/install.sh | sh + # source $HOME/.cargo/env + # uv pip install --system darglint + # - name: Check docstrings + # run: bash scripts/docstring.sh + # sqlite-db-migration-testing: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # env: + # ZENML_ANALYTICS_OPT_IN: false + # ZENML_DEBUG: true + # runs-on: arc-runner-set + # steps: + # - uses: actions/checkout@v4.1.1 + # with: + # fetch-depth: 0 + # - name: Set up Python 3.9 + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.9' + # - name: Test migrations across versions + # run: bash scripts/test-migrations-mysql.sh sqlite + # small-checks: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4.1.1 + # - name: Set up Python + # uses: actions/setup-python@v5.0.0 + # with: + # python-version: '3.8' + # - name: Install current package as editable + # run: | + # curl -LsSf https://astral.sh/uv/install.sh | sh + # source $HOME/.cargo/env + # uv pip install --system -e . + # - name: Install mlstacks package + # run: uv pip install --system mlstacks + # - name: Check for broken dependencies + # run: pip check + # - name: Markdown link check + # uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + # with: + # use-quiet-mode: 'yes' + # use-verbose-mode: 'no' + # folder-path: ./examples, ./docs/book, ./src + # file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, + # CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md + # config-file: .github/workflows/markdown_check_config.json + # continue-on-error: true + # - name: Security check + # run: | + # uv pip install --system bandit + # bash scripts/check-security.sh + # - name: Check for alembic branch divergence + # env: + # ZENML_DEBUG: 0 + # run: | + # uv pip install --system alembic + # bash scripts/check-alembic-branches.sh + # custom-ubuntu-unit-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [arc-runner-set] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # fail-fast: false + # uses: ./.github/workflows/unit-test.yml + # with: + # python-version: ${{ matrix.python-version }} + # os: ${{ matrix.os }} + # secrets: inherit windows-unit-test: if: github.event.pull_request.draft == false needs: run-slow-ci-label-is-set @@ -138,78 +138,78 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} secrets: inherit - macos-unit-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - fail-fast: false - uses: ./.github/workflows/unit-test.yml - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - secrets: inherit - windows-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: [default] - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit - macos-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [macos-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: [default] - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit - custom-ubuntu-integration-test: - if: github.event.pull_request.draft == false - needs: run-slow-ci-label-is-set - strategy: - matrix: - os: [arc-runner-set] - python-version: ['3.8', '3.9', '3.10', '3.11'] - test_environment: - - default - - docker-server-docker-orchestrator-mysql - - docker-server-docker-orchestrator-mariadb - exclude: - # docker is time-consuming to run, so we only run it on 3.8 - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.9' - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.10' - - test_environment: docker-server-docker-orchestrator-mysql - python-version: '3.11' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.9' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.10' - - test_environment: docker-server-docker-orchestrator-mariadb - python-version: '3.11' - fail-fast: false - uses: ./.github/workflows/integration-test-slow.yml - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - test_environment: ${{ matrix.test_environment }} - secrets: inherit + # macos-unit-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [macos-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # fail-fast: false + # uses: ./.github/workflows/unit-test.yml + # with: + # python-version: ${{ matrix.python-version }} + # os: ${{ matrix.os }} + # secrets: inherit + # windows-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [windows-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: [default] + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit + # macos-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [macos-latest] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: [default] + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit + # custom-ubuntu-integration-test: + # if: github.event.pull_request.draft == false + # needs: run-slow-ci-label-is-set + # strategy: + # matrix: + # os: [arc-runner-set] + # python-version: ['3.8', '3.9', '3.10', '3.11'] + # test_environment: + # - default + # - docker-server-docker-orchestrator-mysql + # - docker-server-docker-orchestrator-mariadb + # exclude: + # # docker is time-consuming to run, so we only run it on 3.8 + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.9' + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.10' + # - test_environment: docker-server-docker-orchestrator-mysql + # python-version: '3.11' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.9' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.10' + # - test_environment: docker-server-docker-orchestrator-mariadb + # python-version: '3.11' + # fail-fast: false + # uses: ./.github/workflows/integration-test-slow.yml + # with: + # os: ${{ matrix.os }} + # python-version: ${{ matrix.python-version }} + # test_environment: ${{ matrix.test_environment }} + # secrets: inherit diff --git a/src/zenml/utils/source_utils.py b/src/zenml/utils/source_utils.py index 881a52d590..8b76ca7d25 100644 --- a/src/zenml/utils/source_utils.py +++ b/src/zenml/utils/source_utils.py @@ -155,7 +155,9 @@ def resolve( if source_type == SourceType.USER: from zenml.utils import code_repository_utils - local_repo_context = code_repository_utils.find_active_code_repository() + local_repo_context = ( + code_repository_utils.find_active_code_repository() + ) if local_repo_context and not local_repo_context.has_local_changes: module_name = _resolve_module(module) @@ -188,7 +190,9 @@ def resolve( # Fallback to an unknown source if we can't find the package source_type = SourceType.UNKNOWN - return Source(module=module_name, attribute=attribute_name, type=source_type) + return Source( + module=module_name, attribute=attribute_name, type=source_type + ) def get_source_root() -> str: @@ -293,11 +297,14 @@ def is_standard_lib_file(file_path: str) -> bool: # Default library path stdlib_paths.add(Path(get_python_lib(standard_lib=True)).resolve()) # Other common standard library locations, considering virtual environments - stdlib_paths.update(Path(p).resolve() for p in sys.path if "site-packages" not in p) + stdlib_paths.update( + Path(p).resolve() for p in sys.path if "site-packages" not in p + ) # Check if file path is within any known standard library path if any( - normalized_file_path.is_relative_to(stdlib_path) for stdlib_path in stdlib_paths + normalized_file_path.is_relative_to(stdlib_path) + for stdlib_path in stdlib_paths ): return True @@ -321,7 +328,8 @@ def is_standard_lib_file(file_path: str) -> bool: source_file = inspect.getsourcefile(normalized_file_path) if source_file and Path(source_file).resolve().is_file(): return any( - Path(source_file).resolve().is_relative_to(p) for p in stdlib_paths + Path(source_file).resolve().is_relative_to(p) + for p in stdlib_paths ) except Exception: pass @@ -374,7 +382,9 @@ def get_source_type(module: ModuleType) -> SourceType: if is_internal_module(module_name=module.__name__): return SourceType.INTERNAL - if is_distribution_package_file(file_path=file_path, module_name=module.__name__): + if is_distribution_package_file( + file_path=file_path, module_name=module.__name__ + ): return SourceType.DISTRIBUTION_PACKAGE if is_standard_lib_file(file_path=file_path): @@ -511,7 +521,9 @@ def _resolve_module(module: ModuleType) -> str: return module_source -def _load_module(module_name: str, import_root: Optional[str] = None) -> ModuleType: +def _load_module( + module_name: str, import_root: Optional[str] = None +) -> ModuleType: """Load a module. Args: From 4aa73a5bfa786a5a9d9ce17a55714be4b8610254 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 17:04:40 +0100 Subject: [PATCH 61/73] add ubuntu unit tests --- .github/workflows/ci-slow.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index a4cf45de87..7d63d8b1e4 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -112,19 +112,19 @@ jobs: # run: | # uv pip install --system alembic # bash scripts/check-alembic-branches.sh - # custom-ubuntu-unit-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [arc-runner-set] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # fail-fast: false - # uses: ./.github/workflows/unit-test.yml - # with: - # python-version: ${{ matrix.python-version }} - # os: ${{ matrix.os }} - # secrets: inherit + custom-ubuntu-unit-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [arc-runner-set] + python-version: ['3.8', '3.9', '3.10', '3.11'] + fail-fast: false + uses: ./.github/workflows/unit-test.yml + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + secrets: inherit windows-unit-test: if: github.event.pull_request.draft == false needs: run-slow-ci-label-is-set From 095707d85c0f379a8f0ed5e61e41c2b7be3ba2f3 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 18:07:57 +0100 Subject: [PATCH 62/73] revert source_utils changes --- src/zenml/utils/source_utils.py | 53 ++++----------------------------- 1 file changed, 5 insertions(+), 48 deletions(-) diff --git a/src/zenml/utils/source_utils.py b/src/zenml/utils/source_utils.py index 8b76ca7d25..ea1fcfe6d0 100644 --- a/src/zenml/utils/source_utils.py +++ b/src/zenml/utils/source_utils.py @@ -15,7 +15,6 @@ import contextlib import importlib -import importlib.util import inspect import os import site @@ -287,54 +286,12 @@ def is_standard_lib_file(file_path: str) -> bool: file_path: The file path to check. Returns: - True if the file belongs to the Python standard library, False otherwise. + True if the file belongs to the Python standard library, False + otherwise. """ - # Normalize file path for reliable comparison - normalized_file_path = Path(file_path).resolve() - - # Gather possible locations of the standard library - stdlib_paths = set() - # Default library path - stdlib_paths.add(Path(get_python_lib(standard_lib=True)).resolve()) - # Other common standard library locations, considering virtual environments - stdlib_paths.update( - Path(p).resolve() for p in sys.path if "site-packages" not in p - ) - - # Check if file path is within any known standard library path - if any( - normalized_file_path.is_relative_to(stdlib_path) - for stdlib_path in stdlib_paths - ): - return True - - # As a fallback, check if the module is a built-in module - module_name = Path(file_path).stem - try: - spec = importlib.util.find_spec(module_name) - if spec and ( - spec.origin == "built-in" - or ( - spec.loader is not None - and "built-in" in spec.loader.__class__.__name__.lower() - ) - ): - return True - except (ImportError, AttributeError): - pass - - # Fallback to inspect (for dynamically loaded modules, etc.) - try: - source_file = inspect.getsourcefile(normalized_file_path) - if source_file and Path(source_file).resolve().is_file(): - return any( - Path(source_file).resolve().is_relative_to(p) - for p in stdlib_paths - ) - except Exception: - pass - - return False + stdlib_root = get_python_lib(standard_lib=True) + logger.debug("Standard library root: %s", stdlib_root) + return Path(stdlib_root).resolve() in Path(file_path).resolve().parents def is_distribution_package_file(file_path: str, module_name: str) -> bool: From 1b80e4c4f24ebbdebe6314c19c39d72b9a879421 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 18:27:23 +0100 Subject: [PATCH 63/73] try adding cache back in --- .github/actions/setup_environment/action.yml | 52 +++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 103a6a4c1c..1043f25ac7 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -53,8 +53,58 @@ runs: shell: bash run: | "C:\Program Files\Git\bin\git.exe" config --global user.email "info@zenml.io" - "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML GmbH" + "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML + GmbH" + - name: Get current week + shell: bash + id: date + run: echo "::set-output name=week::$(date +'calendar-week-%W')" + - uses: syphar/restore-virtualenv@v1 + if: ${{ inputs.os != 'arc-runner-set' }} + id: cache-virtualenv + with: + requirement_files: pyproject.toml + # The virtualenv cache is invalidated when: + # - manually triggered by means of a custom cache version token + # - on a weekly basis + # - any of the integration requirements change (a hash of the + # __init__.py files is included in the cache key) + custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{ + hashFiles('src/zenml/integrations/*/__init__.py') }} + - uses: tespkg/actions-cache@v1 + if: ${{ inputs.os == 'arc-runner-set' }} + id: custom-cache-pip + env: + AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }} + with: + endpoint: minio-service.minio.svc.cluster.local # optional + insecure: true # optional, use http instead of https. default false + bucket: caching # required + use-fallback: false # optional, use github actions cache fallback, default false + key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version + }}-${{inputs.install_integrations}} + path: | + ~/.cache/pip + restore-keys: | + ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}} + - name: Custom runners cache miss + uses: rjstone/discord-webhook-notify@v1 + if: steps.custom-cache-pip.outputs.cache-hit != 'true' + continue-on-error: true + with: + severity: warn + details: The custom github runnners cache was missed and failed to restore + the cache from minio bucket + webhookUrl: ${{ inputs.discord_webhook }} + # Disabled for now because it doesn't work well with multiple parallel jobs + # - uses: syphar/restore-pip-download-cache@v1 + # with: - name: Install Terraform (Windows) + # requirement_files: 'pyproject.toml' + # # The pip download cache can be updated on a weekly basis as new packages + # # don't appear that often + # custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}} if: ${{ inputs.os == 'windows-latest' }} shell: bash run: choco install terraform -y From 6b35e94457fc26ad145797d120d958390e50a29a Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 20:00:58 +0100 Subject: [PATCH 64/73] restore normal CI --- .github/actions/setup_environment/action.yml | 6 +- .github/workflows/ci-fast.yml | 12 +- .github/workflows/ci-slow.yml | 296 +++++++++---------- 3 files changed, 157 insertions(+), 157 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 1043f25ac7..1cf4b25eb7 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -100,11 +100,11 @@ runs: # Disabled for now because it doesn't work well with multiple parallel jobs # - uses: syphar/restore-pip-download-cache@v1 # with: - - name: Install Terraform (Windows) # requirement_files: 'pyproject.toml' - # # The pip download cache can be updated on a weekly basis as new packages - # # don't appear that often + # The pip download cache can be updated on a weekly basis as new packages + # don't appear that often # custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}} + - name: Install Terraform (Windows) if: ${{ inputs.os == 'windows-latest' }} shell: bash run: choco install terraform -y diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index b41757b585..a5e6cbcfa7 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -3,12 +3,12 @@ name: ci-fast on: workflow_dispatch: workflow_call: - # push: - # branches: [main] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] - # pull_request: - # types: [opened, synchronize, ready_for_review] - # paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + push: + branches: [main] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] + pull_request: + types: [opened, synchronize, ready_for_review] + paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml index 7d63d8b1e4..470a4f2104 100644 --- a/.github/workflows/ci-slow.yml +++ b/.github/workflows/ci-slow.yml @@ -39,79 +39,79 @@ jobs: run: | echo "Please add the 'run-slow-ci' label to this PR before merging." exit 1 - # docstring-check: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4.1.1 - # - name: Set up Python - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.10' - # - name: Install current package as editable - # run: | - # curl -LsSf https://astral.sh/uv/install.sh | sh - # source $HOME/.cargo/env - # uv pip install --system darglint - # - name: Check docstrings - # run: bash scripts/docstring.sh - # sqlite-db-migration-testing: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # env: - # ZENML_ANALYTICS_OPT_IN: false - # ZENML_DEBUG: true - # runs-on: arc-runner-set - # steps: - # - uses: actions/checkout@v4.1.1 - # with: - # fetch-depth: 0 - # - name: Set up Python 3.9 - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.9' - # - name: Test migrations across versions - # run: bash scripts/test-migrations-mysql.sh sqlite - # small-checks: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4.1.1 - # - name: Set up Python - # uses: actions/setup-python@v5.0.0 - # with: - # python-version: '3.8' - # - name: Install current package as editable - # run: | - # curl -LsSf https://astral.sh/uv/install.sh | sh - # source $HOME/.cargo/env - # uv pip install --system -e . - # - name: Install mlstacks package - # run: uv pip install --system mlstacks - # - name: Check for broken dependencies - # run: pip check - # - name: Markdown link check - # uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 - # with: - # use-quiet-mode: 'yes' - # use-verbose-mode: 'no' - # folder-path: ./examples, ./docs/book, ./src - # file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, - # CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md - # config-file: .github/workflows/markdown_check_config.json - # continue-on-error: true - # - name: Security check - # run: | - # uv pip install --system bandit - # bash scripts/check-security.sh - # - name: Check for alembic branch divergence - # env: - # ZENML_DEBUG: 0 - # run: | - # uv pip install --system alembic - # bash scripts/check-alembic-branches.sh + docstring-check: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.10' + - name: Install current package as editable + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env + uv pip install --system darglint + - name: Check docstrings + run: bash scripts/docstring.sh + sqlite-db-migration-testing: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + env: + ZENML_ANALYTICS_OPT_IN: false + ZENML_DEBUG: true + runs-on: arc-runner-set + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - name: Set up Python 3.9 + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.9' + - name: Test migrations across versions + run: bash scripts/test-migrations-mysql.sh sqlite + small-checks: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.8' + - name: Install current package as editable + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env + uv pip install --system -e . + - name: Install mlstacks package + run: uv pip install --system mlstacks + - name: Check for broken dependencies + run: pip check + - name: Markdown link check + uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'no' + folder-path: ./examples, ./docs/book, ./src + file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, + CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md + config-file: .github/workflows/markdown_check_config.json + continue-on-error: true + - name: Security check + run: | + uv pip install --system bandit + bash scripts/check-security.sh + - name: Check for alembic branch divergence + env: + ZENML_DEBUG: 0 + run: | + uv pip install --system alembic + bash scripts/check-alembic-branches.sh custom-ubuntu-unit-test: if: github.event.pull_request.draft == false needs: run-slow-ci-label-is-set @@ -138,78 +138,78 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} secrets: inherit - # macos-unit-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [macos-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # fail-fast: false - # uses: ./.github/workflows/unit-test.yml - # with: - # python-version: ${{ matrix.python-version }} - # os: ${{ matrix.os }} - # secrets: inherit - # windows-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [windows-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: [default] - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit - # macos-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [macos-latest] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: [default] - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit - # custom-ubuntu-integration-test: - # if: github.event.pull_request.draft == false - # needs: run-slow-ci-label-is-set - # strategy: - # matrix: - # os: [arc-runner-set] - # python-version: ['3.8', '3.9', '3.10', '3.11'] - # test_environment: - # - default - # - docker-server-docker-orchestrator-mysql - # - docker-server-docker-orchestrator-mariadb - # exclude: - # # docker is time-consuming to run, so we only run it on 3.8 - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.9' - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.10' - # - test_environment: docker-server-docker-orchestrator-mysql - # python-version: '3.11' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.9' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.10' - # - test_environment: docker-server-docker-orchestrator-mariadb - # python-version: '3.11' - # fail-fast: false - # uses: ./.github/workflows/integration-test-slow.yml - # with: - # os: ${{ matrix.os }} - # python-version: ${{ matrix.python-version }} - # test_environment: ${{ matrix.test_environment }} - # secrets: inherit + macos-unit-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [macos-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + fail-fast: false + uses: ./.github/workflows/unit-test.yml + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + secrets: inherit + windows-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: [default] + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit + macos-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [macos-latest] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: [default] + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit + custom-ubuntu-integration-test: + if: github.event.pull_request.draft == false + needs: run-slow-ci-label-is-set + strategy: + matrix: + os: [arc-runner-set] + python-version: ['3.8', '3.9', '3.10', '3.11'] + test_environment: + - default + - docker-server-docker-orchestrator-mysql + - docker-server-docker-orchestrator-mariadb + exclude: + # docker is time-consuming to run, so we only run it on 3.8 + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.9' + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mysql + python-version: '3.11' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.9' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.10' + - test_environment: docker-server-docker-orchestrator-mariadb + python-version: '3.11' + fail-fast: false + uses: ./.github/workflows/integration-test-slow.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + test_environment: ${{ matrix.test_environment }} + secrets: inherit From af09d9f6e968b5033ebe8ddd4610a6f20692d809 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 20:02:39 +0100 Subject: [PATCH 65/73] add spellcheck back --- .github/workflows/spellcheck.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index fcbcefed82..55f481d1ce 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -3,12 +3,12 @@ name: Spellcheck on: workflow_dispatch: workflow_call: - # push: - # branches: [main] - # paths-ignore: [docker/**, '*', '!pyproject.toml'] - # pull_request: - # types: [opened, synchronize, ready_for_review] - # paths-ignore: [docker/**, '*', '!pyproject.toml'] + push: + branches: [main] + paths-ignore: [docker/**, '*', '!pyproject.toml'] + pull_request: + types: [opened, synchronize, ready_for_review] + paths-ignore: [docker/**, '*', '!pyproject.toml'] concurrency: # New commit on branch cancels running workflows of the same branch group: ${{ github.workflow }}-${{ github.ref }} From 9e542e1fa3684b1183cee9d78191a60d16f23af9 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Thu, 7 Mar 2024 20:03:43 +0100 Subject: [PATCH 66/73] delete test workflow --- .github/workflows/uv-test.yml | 132 ---------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 .github/workflows/uv-test.yml diff --git a/.github/workflows/uv-test.yml b/.github/workflows/uv-test.yml deleted file mode 100644 index 59c0b41f4b..0000000000 --- a/.github/workflows/uv-test.yml +++ /dev/null @@ -1,132 +0,0 @@ ---- -name: Verify defaultdict resolution -on: - workflow_dispatch: - workflow_call: - pull_request: - types: [opened, synchronize, ready_for_review] - paths-ignore: [docker/**, '*', '!pyproject.toml'] -jobs: - pip-installation: - runs-on: windows-latest - strategy: - matrix: - # python-version: ['3.8', '3.9', '3.10', '3.11'] - python-version: ['3.10'] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.0.0 - with: - python-version: ${{ matrix.python-version }} - - name: gather requirements - shell: bash - run: | - pip install uv - touch basic-requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> basic-requirements.txt - cp basic-requirements.txt basic-requirements.in - uv pip compile basic-requirements.in -o basic-requirements-compiled.txt - pip install -r basic-requirements-compiled.txt - - # figure out the python version - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") - ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" - # if python version is 3.11, exclude all integrations depending on kfp - # because they are not yet compatible with python 3.11 - if [ "$python_version" = "3.11" ]; then - ignore_integrations="$ignore_integrations kubeflow tekton gcp" - fi - - # turn the ignore integrations into a list of --ignore-integration args - ignore_integrations_args="" - for integration in $ignore_integrations; do - ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" - done - - # install basic ZenML integrations - zenml integration export-requirements \ - --output-file integration-requirements.txt \ - $ignore_integrations_args - echo "" >> integration-requirements.txt - echo "pyyaml>=6.0.1" >> integration-requirements.txt - echo "pyopenssl" >> integration-requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - cp integration-requirements.txt integration-requirements.in - uv pip compile integration-requirements.in -o integration-requirements-compiled.txt - - name: Install dependencies with pip - shell: bash - run: | - pip install -r integration-requirements-compiled.txt - - name: Log collections version and defaultdict type with pip - shell: bash - run: | - python -c "import sys; print(f'Python version: {sys.version}')" - python -c "import collections; print(f'collections module file path: {collections.__file__}')" - python -c "import collections; print(f'collections module name: {collections.__name__}')" - python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - python -c "import sys; from collections import defaultdict; from zenml.config.source import Source, SourceType; from zenml.utils import source_utils; print(f'source_utils.resolve(defaultdict): {source_utils.resolve(defaultdict)}'); print(f'Source object: {Source(module=defaultdict.__module__, attribute=defaultdict.__name__, type=SourceType.BUILTIN)}')" - python -c "import sys; print(f'Python executable: {sys.executable}')" - python -c "import sys; print(f'Module search paths: {sys.path}')" - echo "PYTHONPATH: $PYTHONPATH" - echo "PATH: $PATH" - pip list - uv-installation: - runs-on: windows-latest - strategy: - matrix: - # python-version: ['3.8', '3.9', '3.10', '3.11'] - python-version: ['3.10'] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.0.0 - with: - python-version: ${{ matrix.python-version }} - - name: gather requirements - shell: bash - run: | - pip install uv - uv pip install --system -e '.[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]' - - # figure out the python version - python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") - ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure" - # if python version is 3.11, exclude all integrations depending on kfp - # because they are not yet compatible with python 3.11 - if [ "$python_version" = "3.11" ]; then - ignore_integrations="$ignore_integrations kubeflow tekton gcp" - fi - - # turn the ignore integrations into a list of --ignore-integration args - ignore_integrations_args="" - for integration in $ignore_integrations; do - ignore_integrations_args="$ignore_integrations_args --ignore-integration $integration" - done - - # export basic ZenML integration requirements - zenml integration export-requirements \ - --output-file integration-requirements.txt \ - $ignore_integrations_args - echo "" >> integration-requirements.txt - echo "pyyaml>=6.0.1" >> integration-requirements.txt - echo "pyopenssl" >> integration-requirements.txt - echo "-e .[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]" >> integration-requirements.txt - - name: Install dependencies with uv - shell: bash - run: | - uv pip install --system -r integration-requirements.txt - - name: Log collections version and defaultdict type with uv - shell: bash - run: |- - python -c "import sys; print(f'Python version: {sys.version}')" - python -c "import collections; print(f'collections module file path: {collections.__file__}')" - python -c "import collections; print(f'collections module name: {collections.__name__}')" - python -c "from collections import defaultdict; print(f'defaultdict type: {type(defaultdict)}')" - python -c "import sys; from collections import defaultdict; from zenml.config.source import Source, SourceType; from zenml.utils import source_utils; print(f'source_utils.resolve(defaultdict): {source_utils.resolve(defaultdict)}'); print(f'Source object: {Source(module=defaultdict.__module__, attribute=defaultdict.__name__, type=SourceType.BUILTIN)}')" - python -c "import sys; print(f'Python executable: {sys.executable}')" - python -c "import sys; print(f'Module search paths: {sys.path}')" - echo "PYTHONPATH: $PYTHONPATH" - echo "PATH: $PATH" - uv --version - pip list From 53e1fdfb8660f8777fdd661554531fe80b39e604 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 09:47:56 +0100 Subject: [PATCH 67/73] use int instead + disable cache again --- .github/actions/setup_environment/action.yml | 98 ++++++++++---------- tests/unit/utils/test_source_utils.py | 9 +- 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml index 1cf4b25eb7..5ea5d22292 100644 --- a/.github/actions/setup_environment/action.yml +++ b/.github/actions/setup_environment/action.yml @@ -55,55 +55,55 @@ runs: "C:\Program Files\Git\bin\git.exe" config --global user.email "info@zenml.io" "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML GmbH" - - name: Get current week - shell: bash - id: date - run: echo "::set-output name=week::$(date +'calendar-week-%W')" - - uses: syphar/restore-virtualenv@v1 - if: ${{ inputs.os != 'arc-runner-set' }} - id: cache-virtualenv - with: - requirement_files: pyproject.toml - # The virtualenv cache is invalidated when: - # - manually triggered by means of a custom cache version token - # - on a weekly basis - # - any of the integration requirements change (a hash of the - # __init__.py files is included in the cache key) - custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{ - hashFiles('src/zenml/integrations/*/__init__.py') }} - - uses: tespkg/actions-cache@v1 - if: ${{ inputs.os == 'arc-runner-set' }} - id: custom-cache-pip - env: - AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }} - with: - endpoint: minio-service.minio.svc.cluster.local # optional - insecure: true # optional, use http instead of https. default false - bucket: caching # required - use-fallback: false # optional, use github actions cache fallback, default false - key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version - }}-${{inputs.install_integrations}} - path: | - ~/.cache/pip - restore-keys: | - ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}} - - name: Custom runners cache miss - uses: rjstone/discord-webhook-notify@v1 - if: steps.custom-cache-pip.outputs.cache-hit != 'true' - continue-on-error: true - with: - severity: warn - details: The custom github runnners cache was missed and failed to restore - the cache from minio bucket - webhookUrl: ${{ inputs.discord_webhook }} - # Disabled for now because it doesn't work well with multiple parallel jobs - # - uses: syphar/restore-pip-download-cache@v1 - # with: - # requirement_files: 'pyproject.toml' - # The pip download cache can be updated on a weekly basis as new packages - # don't appear that often - # custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}} + # - name: Get current week + # shell: bash + # id: date + # run: echo "::set-output name=week::$(date +'calendar-week-%W')" + # - uses: syphar/restore-virtualenv@v1 + # if: ${{ inputs.os != 'arc-runner-set' }} + # id: cache-virtualenv + # with: + # requirement_files: pyproject.toml + # # The virtualenv cache is invalidated when: + # # - manually triggered by means of a custom cache version token + # # - on a weekly basis + # # - any of the integration requirements change (a hash of the + # # __init__.py files is included in the cache key) + # custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{ + # hashFiles('src/zenml/integrations/*/__init__.py') }} + # - uses: tespkg/actions-cache@v1 + # if: ${{ inputs.os == 'arc-runner-set' }} + # id: custom-cache-pip + # env: + # AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }} + # AWS_SECRET_ACCESS_KEY: ${{ inputs.runners_cache_secret_access_key }} + # with: + # endpoint: minio-service.minio.svc.cluster.local # optional + # insecure: true # optional, use http instead of https. default false + # bucket: caching # required + # use-fallback: false # optional, use github actions cache fallback, default false + # key: ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version + # }}-${{inputs.install_integrations}} + # path: | + # ~/.cache/pip + # restore-keys: | + # ${{ inputs.os }}-${{ inputs.cache_version }}-${{ inputs.python-version }}-${{inputs.install_integrations}} + # - name: Custom runners cache miss + # uses: rjstone/discord-webhook-notify@v1 + # if: steps.custom-cache-pip.outputs.cache-hit != 'true' + # continue-on-error: true + # with: + # severity: warn + # details: The custom github runnners cache was missed and failed to restore + # the cache from minio bucket + # webhookUrl: ${{ inputs.discord_webhook }} + # # Disabled for now because it doesn't work well with multiple parallel jobs + # # - uses: syphar/restore-pip-download-cache@v1 + # # with: + # # requirement_files: 'pyproject.toml' + # # The pip download cache can be updated on a weekly basis as new packages + # # don't appear that often + # # custom_cache_key_element: ${{ inputs.cache_version }}-${{ inputs.python-version }}-${{steps.date.outputs.week}} - name: Install Terraform (Windows) if: ${{ inputs.os == 'windows-latest' }} shell: bash diff --git a/tests/unit/utils/test_source_utils.py b/tests/unit/utils/test_source_utils.py index 009f050355..67d2a5767f 100644 --- a/tests/unit/utils/test_source_utils.py +++ b/tests/unit/utils/test_source_utils.py @@ -14,7 +14,6 @@ import pathlib import sys -from collections import defaultdict from contextlib import ExitStack as does_not_raise from uuid import uuid4 @@ -107,9 +106,9 @@ def test_basic_source_resolving(mocker): assert source_utils.resolve(int) == Source( module=int.__module__, attribute=int.__name__, type=SourceType.BUILTIN ) - assert source_utils.resolve(defaultdict) == Source( - module=defaultdict.__module__, - attribute=defaultdict.__name__, + assert source_utils.resolve(int) == Source( + module=int.__module__, + attribute=int.__name__, type=SourceType.BUILTIN, ) assert source_utils.resolve(source_utils) == Source( @@ -208,7 +207,7 @@ def test_module_type_detection(mocker): builtin_module = sys.modules[int.__module__] assert source_utils.get_source_type(builtin_module) == SourceType.BUILTIN - standard_lib_module = sys.modules[defaultdict.__module__] + standard_lib_module = sys.modules[int.__module__] assert ( source_utils.get_source_type(standard_lib_module) == SourceType.BUILTIN ) From 00c39ff78b23e06edd7943977d102d08451da881 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 8 Mar 2024 08:51:05 +0000 Subject: [PATCH 68/73] Auto-update of Starter template --- examples/quickstart/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index 23c8f24e66..be1d2a1c6b 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -24,7 +24,7 @@ Along the way we will also show you how to: You can use Google Colab to see ZenML in action, no signup / installation required! -Open In Colab +Open In Colab ## :computer: Run Locally @@ -208,4 +208,4 @@ The best way to get a production ZenML instance up and running with all batterie Also, make sure to join our Slack Slack Community - to become part of the ZenML family! + to become part of the ZenML family! \ No newline at end of file From b5e45ac608bb73b3b68f499abefbe03f4aa20270 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 10:20:48 +0100 Subject: [PATCH 69/73] remove file attrib check --- tests/unit/utils/test_source_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/utils/test_source_utils.py b/tests/unit/utils/test_source_utils.py index 67d2a5767f..087d4244b7 100644 --- a/tests/unit/utils/test_source_utils.py +++ b/tests/unit/utils/test_source_utils.py @@ -211,7 +211,6 @@ def test_module_type_detection(mocker): assert ( source_utils.get_source_type(standard_lib_module) == SourceType.BUILTIN ) - assert source_utils.is_standard_lib_file(standard_lib_module.__file__) internal_module = sys.modules[source_utils.__name__] assert source_utils.get_source_type(internal_module) == SourceType.INTERNAL From 996dd804c63ffdda88fc6663596b3daf525f9268 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 10:44:11 +0100 Subject: [PATCH 70/73] increase HTTP timeout for uv (for torch) --- .github/workflows/generate-test-duration.yml | 1 + .github/workflows/integration-test-fast.yml | 1 + .github/workflows/integration-test-slow.yml | 1 + .github/workflows/unit-test.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/generate-test-duration.yml b/.github/workflows/generate-test-duration.yml index f5ea95ea02..2710de2ffb 100644 --- a/.github/workflows/generate-test-duration.yml +++ b/.github/workflows/generate-test-duration.yml @@ -14,6 +14,7 @@ jobs: ZENML_DEBUG: 1 ZENML_ANALYTICS_OPT_IN: false PYTHONIOENCODING: utf-8 + UV_HTTP_TIMEOUT: 600 # on MAC OS, we need to set this environment variable # to fix problems with the fork() calls (see this thread # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html) diff --git a/.github/workflows/integration-test-fast.yml b/.github/workflows/integration-test-fast.yml index 7178fdd1a5..c01a0b84ed 100644 --- a/.github/workflows/integration-test-fast.yml +++ b/.github/workflows/integration-test-fast.yml @@ -87,6 +87,7 @@ jobs: ZENML_DEBUG: 1 ZENML_ANALYTICS_OPT_IN: false PYTHONIOENCODING: utf-8 + UV_HTTP_TIMEOUT: 600 # on MAC OS, we need to set this environment variable # to fix problems with the fork() calls (see this thread # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html) diff --git a/.github/workflows/integration-test-slow.yml b/.github/workflows/integration-test-slow.yml index 7898ae76e7..7abc06925b 100644 --- a/.github/workflows/integration-test-slow.yml +++ b/.github/workflows/integration-test-slow.yml @@ -85,6 +85,7 @@ jobs: ZENML_DEBUG: 1 ZENML_ANALYTICS_OPT_IN: false PYTHONIOENCODING: utf-8 + UV_HTTP_TIMEOUT: 600 # on MAC OS, we need to set this environment variable # to fix problems with the fork() calls (see this thread # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index d91f0ad6b7..24c85f2a36 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -63,6 +63,7 @@ jobs: ZENML_DEBUG: 1 ZENML_ANALYTICS_OPT_IN: false PYTHONIOENCODING: utf-8 + UV_HTTP_TIMEOUT: 600 # on MAC OS, we need to set this environment variable # to fix problems with the fork() calls (see this thread From 22ee1392064b3afcc27e5b2f38029b52495db306 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 13:19:03 +0100 Subject: [PATCH 71/73] update for new migration script --- scripts/test-migrations-mariadb.sh | 10 +++------- scripts/test-migrations-mysql.sh | 12 ++++-------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/scripts/test-migrations-mariadb.sh b/scripts/test-migrations-mariadb.sh index 353e93f71f..12167e2894 100755 --- a/scripts/test-migrations-mariadb.sh +++ b/scripts/test-migrations-mariadb.sh @@ -132,10 +132,6 @@ sleep $DB_STARTUP_DELAY # Test sequential migrations across multiple versions echo "===== TESTING SEQUENTIAL MIGRATIONS =====" set -e -python3 -m venv ".venv-sequential-migrations" -source ".venv-sequential-migrations/bin/activate" - -pip3 install -U pip setuptools wheel # Randomly select versions for sequential migrations MIGRATION_VERSIONS=() @@ -157,14 +153,14 @@ echo "=============================" for i in "${!MIGRATION_VERSIONS[@]}"; do set -e # Exit immediately if a command exits with a non-zero status # Create a new virtual environment - python3 -m venv ".venv-${MIGRATION_VERSIONS[$i]}" + uv venv ".venv-${MIGRATION_VERSIONS[$i]}" source ".venv-${MIGRATION_VERSIONS[$i]}/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel + uv pip install -U pip setuptools wheel git checkout release/${MIGRATION_VERSIONS[$i]} - pip3 install -e ".[templates,server]" + uv pip install -e ".[templates,server]" export ZENML_ANALYTICS_OPT_IN=false export ZENML_DEBUG=true diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 7056f527af..4a52ecfa92 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -204,10 +204,6 @@ fi # Test sequential migrations across multiple versions echo "===== TESTING SEQUENTIAL MIGRATIONS =====" set -e -python3 -m venv ".venv-sequential-migrations" -source ".venv-sequential-migrations/bin/activate" - -pip3 install -U pip setuptools wheel # Randomly select versions for sequential migrations MIGRATION_VERSIONS=() @@ -229,16 +225,16 @@ echo "=============================" for i in "${!MIGRATION_VERSIONS[@]}"; do set -e # Exit immediately if a command exits with a non-zero status # Create a new virtual environment - python3 -m venv ".venv-$VERSION" + uv venv ".venv-$VERSION" source ".venv-$VERSION/bin/activate" # Install the specific version - pip3 install -U pip setuptools wheel + uv pip install -U pip setuptools wheel git checkout release/${MIGRATION_VERSIONS[$i]} - pip3 install -e ".[templates,server]" + uv pip install -e ".[templates,server]" # Handles unpinned sqlmodel dependency in older versions - pip3 install "sqlmodel==0.0.8" "bcrypt==4.0.1" importlib_metadata + uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" importlib_metadata # Get the major and minor version of Python PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') From 040af97283bf3c9bd2acb8234171d003d2046e63 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 13:20:07 +0100 Subject: [PATCH 72/73] test to ensure that the uv updates work now --- .github/workflows/ci-fast.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index a5e6cbcfa7..5e188235c6 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -49,6 +49,24 @@ jobs: python-version: '3.9' - name: Test migrations across versions run: bash scripts/test-migrations-mysql.sh sqlite + mariadb-db-migration-testing: + runs-on: arc-runner-set + env: + ZENML_ANALYTICS_OPT_IN: false + ZENML_DEBUG: true + # if team member commented, not a draft, on a PR, using /fulltest + if: github.event.pull_request.draft == false || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - name: Set up Python 3.9 + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.9' + - name: Test migrations across versions + run: bash scripts/test-migrations-mariadb.sh spellcheck: if: github.event.pull_request.draft == false runs-on: ubuntu-latest From 87bb12676088a88bea501c588bad5cdaf5054104 Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 8 Mar 2024 13:50:12 +0100 Subject: [PATCH 73/73] no need to run mariadb on fastCI --- .github/workflows/ci-fast.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml index 5e188235c6..a5e6cbcfa7 100644 --- a/.github/workflows/ci-fast.yml +++ b/.github/workflows/ci-fast.yml @@ -49,24 +49,6 @@ jobs: python-version: '3.9' - name: Test migrations across versions run: bash scripts/test-migrations-mysql.sh sqlite - mariadb-db-migration-testing: - runs-on: arc-runner-set - env: - ZENML_ANALYTICS_OPT_IN: false - ZENML_DEBUG: true - # if team member commented, not a draft, on a PR, using /fulltest - if: github.event.pull_request.draft == false || github.event_name == 'workflow_dispatch' - steps: - - name: Checkout code - uses: actions/checkout@v4.1.1 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v5.0.0 - with: - python-version: '3.9' - - name: Test migrations across versions - run: bash scripts/test-migrations-mariadb.sh spellcheck: if: github.event.pull_request.draft == false runs-on: ubuntu-latest