Merge branch 'zenml-io:main' into huggingface-model-deployer

.github/actions/setup_environment/action.yml

@@ -62,7 +62,7 @@ runs:
       id: date
       run: echo "::set-output name=week::$(date +'calendar-week-%W')"
     - uses: syphar/restore-virtualenv@v1
-      if: ${{ inputs.os != 'ubuntu-dind-runners' }}
+      if: ${{ inputs.os != 'arc-runner-set' }}
       id: cache-virtualenv
       with:
         requirement_files: pyproject.toml
@@ -74,7 +74,7 @@ runs:
         custom_cache_key_element: ${{ inputs.cache_version }}-${{steps.date.outputs.week}}-${{inputs.install_integrations}}-${{
           hashFiles('src/zenml/integrations/*/__init__.py') }}
     - uses: tespkg/actions-cache@v1
-      if: ${{ inputs.os == 'ubuntu-dind-runners' }}
+      if: ${{ inputs.os == 'arc-runner-set' }}
       id: custom-cache-pip
       env:
         AWS_ACCESS_KEY_ID: ${{ inputs.runners_cache_access_key_id }}

.github/workflows/ci-fast.yml

@@ -16,7 +16,7 @@ concurrency:
 jobs:
   docstring-check:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
@@ -29,7 +29,7 @@ jobs:
       - name: Check docstrings
         run: bash scripts/docstring.sh
   sqlite-db-migration-testing:
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -83,19 +83,19 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [ubuntu-dind-runners]
+        os: [arc-runner-set]
         python-version: ['3.10']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-ubuntu-runners-integration-test:
+  custom-arc-runner-set-integration-test:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [ubuntu-dind-runners]
+        os: [arc-runner-set]
         python-version: ['3.10']
         test_environment: [default, docker-server-docker-orchestrator-mysql]
       fail-fast: false

.github/workflows/ci-slow.yml

@@ -57,7 +57,7 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     steps:
       - uses: actions/checkout@v4.1.1
         with:
@@ -111,7 +111,7 @@ jobs:
     needs: run-slow-ci-label-is-set
     strategy:
       matrix:
-        os: [ubuntu-dind-runners]
+        os: [arc-runner-set]
         python-version: ['3.8', '3.9', '3.10', '3.11']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
@@ -180,7 +180,7 @@ jobs:
     needs: run-slow-ci-label-is-set
     strategy:
       matrix:
-        os: [ubuntu-dind-runners]
+        os: [arc-runner-set]
         python-version: ['3.8', '3.9', '3.10', '3.11']
         test_environment:
           - default

.github/workflows/generate-test-duration.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   generate-test-duration-file:
     name: Generate test duration file
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     strategy:
       fail-fast: false
     env:
@@ -30,7 +30,7 @@ jobs:
         with:
           cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
           python-version: '3.10'
-          os: ubuntu-dind-runners
+          os: arc-runner-set
           runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
           runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
           discord_webhook: ${{ secrets.DISCORD_WEBHOOK }}

.github/workflows/integration-test-fast.yml

@@ -34,7 +34,7 @@ on:
           - ubuntu-latest
           - macos-latest
           - windows-latest
-          - ubuntu-dind-runners
+          - arc-runner-set
         required: false
         default: ubuntu-latest
       python-version:
@@ -153,7 +153,7 @@ jobs:
           pip install pyyaml==5.3.1
           pip install docker-compose
       - name: Install Linux System Dependencies
-        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners')
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
         run: sudo apt install graphviz
       - name: Install MacOS System Dependencies
         if: runner.os=='macOS'
@@ -180,7 +180,7 @@ jobs:
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
           sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
-        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners')
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
       - name: Install kubectl on MacOS
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"

.github/workflows/integration-test-slow.yml

@@ -34,7 +34,7 @@ on:
           - ubuntu-latest
           - macos-latest
           - windows-latest
-          - ubuntu-dind-runners
+          - arc-runner-set
         required: false
         default: ubuntu-latest
       python-version:
@@ -151,7 +151,7 @@ jobs:
           pip install pyyaml==5.3.1
           pip install docker-compose
       - name: Install Linux System Dependencies
-        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners')
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
         run: sudo apt install graphviz
       - name: Install MacOS System Dependencies
         if: runner.os=='macOS'
@@ -178,7 +178,7 @@ jobs:
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
           sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
-        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'ubuntu-dind-runners')
+        if: (inputs.os == 'ubuntu-latest' || inputs.os == 'arc-runner-set')
       - name: Install kubectl on MacOS
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"

.github/workflows/release.yml

@@ -9,12 +9,12 @@ jobs:
   setup-and-test:
     uses: ./.github/workflows/unit-test.yml
     with:
-      os: ubuntu-dind-runners
+      os: arc-runner-set
       python-version: '3.8'
     secrets: inherit
   mlstacks-compatibility-check:
     needs: setup-and-test
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
@@ -29,7 +29,7 @@ jobs:
       - name: Check for broken dependencies
         run: pip check
   mysql-db-migration-testing:
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -45,7 +45,7 @@ jobs:
       - name: Test migrations across versions
         run: bash scripts/test-migrations-mysql.sh mysql
   sqlite-db-migration-testing:
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -61,7 +61,7 @@ jobs:
       - name: Test migrations across versions
         run: bash scripts/test-migrations-mysql.sh sqlite
   mariadb-db-migration-testing:
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
@@ -87,7 +87,7 @@ jobs:
     uses: ./.github/workflows/publish_to_pypi.yml
     secrets: inherit
   wait-for-package-release:
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     needs: publish-python-package
     steps:
       - name: Sleep for 4 minutes

.github/workflows/spellcheck.yml

@@ -16,7 +16,7 @@ concurrency:
 jobs:
   spellcheck:
     if: github.event.pull_request.draft == false && github.repository == 'zenml-io/zenml'
-    runs-on: ubuntu-dind-runners
+    runs-on: arc-runner-set
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1

.github/workflows/unit-test.yml

@@ -35,7 +35,7 @@ on:
           - ubuntu-latest
           - macos-latest
           - windows-latest
-          - ubuntu-dind-runners
+          - arc-runner-set
         required: false
         default: ubuntu-latest
       python-version:
@@ -106,7 +106,7 @@ jobs:
 
       # - name: Upload coverage
       #  # only do it for python 3.8, we don't need to do it for every version
-      #   if: ${{ inputs.os == 'ubuntu-dind-runners' && inputs.python-version == '3.8' }}
+      #   if: ${{ inputs.os == 'arc-runner-set' && inputs.python-version == '3.8' }}
       #   uses: codecov/codecov-action@v2
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}

.github/workflows/update-templates-to-examples.yml

@@ -20,7 +20,7 @@ on:
           - ubuntu-latest
           - macos-latest
           - windows-latest
-          - ubuntu-dind-runners
+          - arc-runner-set
         required: false
         default: ubuntu-latest
       python-version:

README.md

@@ -92,7 +92,7 @@
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.55.3 is out. Check out the release notes
+    🎉 Version 0.55.5 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     <br />

RELEASE_NOTES.md

@@ -1,4 +1,88 @@
 <!-- markdown-link-check-disable -->
+# 0.55.5
+
+This patch contains a number of bug fixes and security improvements.
+
+We improved the isolation of artifact stores so that various artifacts 
+cannot be stored or accessed outside of the configured artifact store scope.
+Such unsafe operations are no longer allowed. This may have an
+impact on existing codebases if you have used unsafe file operations in the past.
+To illustrate such a side effect, let's consider a remote S3 
+artifact store is configured for the path `s3://some_bucket/some_sub_folder`. 
+and in the code you use 
+`artifact_store.open("s3://some_bucket/some_other_folder/dummy.txt","w")` 
+-> this operation is considered unsafe as it accesses the data outside the scope 
+of the artifact store. If you really need this to achieve your goals, 
+consider switching to `s3fs` or similar libraries for such cases.
+
+Also with this release, the server global configuration is no longer stored on the 
+server file system to prevent exposure of sensitive information.
+
+User entities are now uniquely constrained to prevent the creation of duplicate 
+users under certain race conditions.
+
+## What's Changed
+* Change runnerset name to ubuntu-runners by @safoinme in https://github.com/zenml-io/zenml/pull/2489
+* Allow latest `ruff` versions by @strickvl in https://github.com/zenml-io/zenml/pull/2487
+* Uniquely constrained users table by @avishniakov in https://github.com/zenml-io/zenml/pull/2483
+* Add option to add base URL for zenml server (with support for cloud) by @wjayesh in https://github.com/zenml-io/zenml/pull/2464
+* Improve Artifact Store isolation by @avishniakov in https://github.com/zenml-io/zenml/pull/2490
+* Don't write the global config to file on server by @stefannica in https://github.com/zenml-io/zenml/pull/2491
+* Add versions for DB migration testing by @strickvl in https://github.com/zenml-io/zenml/pull/2486
+
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.55.4...0.55.5
+
+# 0.55.4
+
+This release brings a host of enhancements and fixes across the board, including
+significant improvements to our services logging and status, the integration of
+model saving to the registry via CLI methods, and more robust handling of
+parallel pipelines and database entities. We've also made strides in optimizing
+MLflow interactions, enhancing our documentation, and ensuring our CI processes
+are more robust.
+
+Additionally, we've tackled several bug fixes and performance improvements,
+making our platform even more reliable and user-friendly.
+
+We'd like to give a special thanks to @christianversloot and @francoisserra for
+their contributions.
+
+## What's Changed
+* Bump mlflow to 2.10.2 by @christianversloot in https://github.com/zenml-io/zenml/pull/2444
+* Improve services logging and status by @safoinme in https://github.com/zenml-io/zenml/pull/2436
+* Add `save models to registry` setting of a model to CLI methods by @avishniakov in https://github.com/zenml-io/zenml/pull/2447
+* Parallel pipelines can create entities in DB by @avishniakov in https://github.com/zenml-io/zenml/pull/2446
+* Fix MlFlow TF autlogging excessive warnings by @avishniakov in https://github.com/zenml-io/zenml/pull/2449
+* Fix and improve integration deps checker by @stefannica in https://github.com/zenml-io/zenml/pull/2455
+* Add migration test version + use self-hosted runners for release by @strickvl in https://github.com/zenml-io/zenml/pull/2450
+* Enable running pipeline via REST by @schustmi in https://github.com/zenml-io/zenml/pull/2389
+* Faster mlflow `list_model_versions` by @avishniakov in https://github.com/zenml-io/zenml/pull/2460
+* Avoid exposure of tracking uri to metadata by @avishniakov in https://github.com/zenml-io/zenml/pull/2458
+* Some important docs updates by @htahir1 in https://github.com/zenml-io/zenml/pull/2463
+* Fix CI by @strickvl in https://github.com/zenml-io/zenml/pull/2467
+* Fix local Airflow install + docs instructions by @strickvl in https://github.com/zenml-io/zenml/pull/2459
+* Update `.coderabbit.yaml` by @strickvl in https://github.com/zenml-io/zenml/pull/2470
+* Prevent templates update from formatting the whole codebase by @avishniakov in https://github.com/zenml-io/zenml/pull/2469
+* Telemetry guarding for CI & editable installs by @strickvl in https://github.com/zenml-io/zenml/pull/2468
+* Add Vertex Step Operator network parameter by @francoisserra in https://github.com/zenml-io/zenml/pull/2398
+* Allow integration export to overwrite a pre-existing file by @strickvl in https://github.com/zenml-io/zenml/pull/2466
+* Fix `log_model_metadata` with explicit name and version by @avishniakov in https://github.com/zenml-io/zenml/pull/2465
+* Triggers, actions, event sources - base abstractions and github and pipeline run implementations by @AlexejPenner in https://github.com/zenml-io/zenml/pull/2312
+* Mount zenml config path as empty dir by @stefannica in https://github.com/zenml-io/zenml/pull/2472
+* Fix broken docs links by @strickvl in https://github.com/zenml-io/zenml/pull/2473
+* Use `uv pip compile` for environment setup in CI by @strickvl in https://github.com/zenml-io/zenml/pull/2474
+* MLflow fix for tests on Mac Python 3.9 and 3.10 by @strickvl in https://github.com/zenml-io/zenml/pull/2462
+* Improve custom data types docs by @avishniakov in https://github.com/zenml-io/zenml/pull/2476
+* Reflect env variables on global configuration by @safoinme in https://github.com/zenml-io/zenml/pull/2371
+* Fix zenml deploy secret stores by @safoinme in https://github.com/zenml-io/zenml/pull/2454
+* Don't fail when workload manager source fails to load by @schustmi in https://github.com/zenml-io/zenml/pull/2478
+* Add analytics events for cloud onboarding by @schustmi in https://github.com/zenml-io/zenml/pull/2456
+* Race condition on creating new users allows duplicate usernames by @avishniakov in https://github.com/zenml-io/zenml/pull/2479
+
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.55.3...0.55.4
+
 # 0.55.3
 
 This patch comes with a variety of bug fixes and documentation updates.
@@ -50,7 +134,7 @@ Some of the most important changes include:
 - The new [HyperAI](https://hyperai.ai) integration featuring a new orchestrator and service connector
 - Bumping the mlflow version to 2.10.0
 
-We'd like to give a special thanks to @christianversloot and @francoisserra for their contributions. 
+We'd like to give a special thanks to @christianversloot and @francoisserra for their contributions.
 
 ## What's Changed
 * `0.55.1` in migration testing by @avishniakov in https://github.com/zenml-io/zenml/pull/2368

docs/book/user-guide/advanced-guide/data-management/handle-custom-data-types.md

@@ -55,6 +55,9 @@ class MyMaterializer(BaseMaterializer):
 def my_first_step() -> MyObj:
     return 1
 
+# No need to explicitly specify materializer here:
+# it is coupled with Artifact Version generated by
+# `my_first_step` already.
 def my_second_step(a: MyObj):
     print(a)