Integration test examples on kubeflow pipelines

.github/workflows/ci.yml

@@ -35,29 +35,3 @@ jobs:
     needs: poetry-install
     uses: ./.github/workflows/integration-test.yml
     secrets: inherit
-
-  check_comments:
-    runs-on: ubuntu-latest
-    if: github.event.issue.pull_request
-    outputs:
-      kf_trigger: ${{ steps.check.outputs.triggered }}
-    steps:
-      - uses: khan/pull-request-comment-trigger@master
-        id: check
-        with:
-          trigger: "LTKF!"
-          reaction: rocket
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      - run: "echo Found LTKF! in the comments!"
-        if: steps.check.outputs.triggered == 'true'
-
-  kubeflow-tests:
-    needs: [poetry-install, check_comments]
-    # Run this one automatically if 'LTKF! is in the PR comments
-    if: ${{ needs.check_comments.outputs.kf_trigger == 'true' }}
-    uses: ./.github/workflows/kubeflow.yml
-    secrets: inherit
-
-  spell-check:
-    uses: ./.github/workflows/spellcheck.yml

.github/workflows/infrastructure_tests.yml

@@ -0,0 +1,59 @@
+name: Test Examples on Remote Infrastructure
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [ opened, synchronize ]
+  issue_comment:
+    types: [ created ]
+
+jobs:
+  check_comments:
+    runs-on: ubuntu-latest
+    if: github.event.issue.pull_request
+    outputs:
+      kf_trigger: ${{ steps.check.outputs.triggered }}
+    steps:
+      - uses: khan/pull-request-comment-trigger@master
+        id: check
+        with:
+          trigger: 'LTKF!'
+          reaction: rocket
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      - run: 'echo Found LTKF! in the comments!'
+        if: steps.check.outputs.triggered == 'true'
+
+  kubeflow-tests:
+    # needs: check_comments
+    # Run this one automatically if 'LTKF! is in the PR comments
+    # if: ${{ needs.check_comments.outputs.kf_trigger == 'true' }}
+    uses: ./.github/workflows/kubeflow.yml
+    secrets: inherit
+
+  reply-on-pr:
+    needs: kubeflow-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Message success
+        if: ${{ success() }}
+        uses: actions/github-script@v4
+        with:
+          script: |
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Kubeflow tests succeeded! ✅',
+            });
+      - name: Message failure
+        if: ${{ failure() }}
+        uses: actions/github-script@v4
+        with:
+          script: |
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Kubeflow tests failed! ❌',
+            });

.github/workflows/integration-test.yml

@@ -39,6 +39,9 @@ jobs:
           echo -n "  Removing: .NET (frees ~22GB)"
           sudo rm -rf /usr/share/dotnet
           echo "... done"
+          echo -n "  Removing: Android"
+          sudo rm -rf /usr/local/lib/android
+          echo "... done"
           echo
           echo "Available storage:"
           df -h

.github/workflows/kubeflow.yml

@@ -13,6 +13,9 @@ jobs:
   kubeflow_tests:
     name: kubeflow_tests
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -24,20 +27,29 @@ jobs:
       # Workaround from FuseML (https://github.com/fuseml/fuseml/blob/main/.github/workflows/ci.yml)
       # as the TF images are too large for the GH action runner disk
       - name: Free disk space
+        shell: bash
         run: |
           echo "Available storage:"
           df -h
           echo
           echo -n "  Removing: .NET (frees ~22GB)"
           sudo rm -rf /usr/share/dotnet
           echo "... done"
-          echo -n "  Removing: Android"
-          sudo rm -rf /usr/local/lib/android
-          echo "... done"
           echo
           echo "Available storage:"
           df -h
 
+      - name: Install Linux System Dependencies
+        if: runner.os=='Linux'
+        run: sudo apt install graphviz
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::715803424590:role/limited_kubeflow_sandbox_tests_role
+          role-session-name: Kubeflow_Test_Session
+          aws-region: us-east-1
+
       - name: Setup environment with Poetry
         uses: ./.github/actions/setup_environment
         with:
@@ -50,21 +62,33 @@ jobs:
           sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
           source $VENV
           zenml integration install kubeflow -y
+          pip install notebook==6.4.11
 
-      - name: Install Linux System Dependencies
-        if: runner.os=='Linux'
-        run: sudo apt install graphviz
+      - name: Perform docker login and get Kubecontext
+        run: bash scripts/setup_kubeflow_aws_connection.sh
 
       - name: Build docker image
         # this step can be much faster if we use an image that copies the
         # lockfile that was generated in the previous step
-        run: docker build -t zenml-base-image:latest -f docker/base-dev.Dockerfile .
+        run: docker build -t test-base-image:latest -f docker/base-test.Dockerfile .
+
+      - name: Install Prerequisites
+        run: |
+          source $VENV
+          zenml integration install s3 kubeflow \
+            evidently facets huggingface mlflow tensorflow sklearn xgboost lightgbm slack neural_prophet -f
 
       - name: Setup tmate session
         if: ${{ github.event.inputs.tags }}
         uses: mxschmitt/action-tmate@v3
 
       - name: Run tests
+        env:
+          TEST_MLFLOW_TRACKING_URI: https://ac8e6c63af207436194ab675ee71d85a-1399000870.us-east-1.elb.amazonaws.com
+          TEST_MLFLOW_TRACKING_USERNAME: ${{ secrets.TEST_MLFLOW_TRACKING_USERNAME }}
+          TEST_MLFLOW_TRACKING_PASSWORD: ${{ secrets.TEST_MLFLOW_TRACKING_PASSWORD }}
+          TEST_SLACK_TOKEN: ${{ secrets.TEST_SLACK_TOKEN }}
+          TEST_SLACK_CHANNEL_ID: ${{ secrets.TEST_SLACK_CHANNEL_ID }}
         run: |
           source $VENV
           pytest tests/integration/test_examples.py -s --on-kubeflow --use-virtualenv

docker/base-test.Dockerfile

@@ -0,0 +1,51 @@
+FROM ubuntu:20.04
+
+WORKDIR /zenml
+
+# python
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONHASHSEED=random \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    POETRY_HOME=/root/.local
+
+RUN apt-get update && \
+  apt-get install --no-install-recommends -q -y \
+  build-essential \
+  ca-certificates \
+  libsnappy-dev \
+  protobuf-compiler \
+  libprotobuf-dev \
+  python3 \
+  python3-dev \
+  python-is-python3 \
+  python3-venv \
+  python3-pip \
+  curl \
+  unzip \
+  git && \
+  apt-get autoclean && \
+  apt-get autoremove --purge
+
+RUN curl -sSL https://install.python-poetry.org | python
+
+# copy project requirement files here to ensure they will be cached. If poetry.lock file is present it is also copied.
+COPY pyproject.toml poetry.loc[k] /zenml/
+
+ENV ZENML_DEBUG=true
+ENV ZENML_ANALYTICS_OPT_IN=false
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$POETRY_HOME/bin:$PATH"
+RUN python -m venv $VIRTUAL_ENV
+
+RUN pip install --no-cache-dir --upgrade --pre pip
+
+# install dependencies but don't install zenml yet
+# this improves caching as the dependencies don't have to be reinstalled everytime a src file changes
+RUN poetry install --no-root
+
+COPY . /zenml
+
+# install zenml
+RUN poetry update && poetry install

examples/mlflow_tracking/steps/preprocessor/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.

examples/mlflow_tracking/steps/preprocessor/preprocessor.py

@@ -0,0 +1,29 @@
+#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+import numpy as np
+import tensorflow as tf
+
+from zenml.steps import Output, step
+
+
+@step
+def importer_mnist() -> Output(
+    x_train=np.ndarray, y_train=np.ndarray, x_test=np.ndarray, y_test=np.ndarray
+):
+    """Download the MNIST data store it as an artifact"""
+    (x_train, y_train), (
+        x_test,
+        y_test,
+    ) = tf.keras.datasets.mnist.load_data()
+    return x_train, y_train, x_test, y_test

examples/neural_prophet/pipelines/__init__.py

@@ -11,3 +11,8 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
+from .neural_prophet_pipeline.neural_prophet_pipeline import (
+    neural_prophet_pipeline,
+)
+
+__all__ = ["neural_prophet_pipeline"]

examples/neural_prophet/run.py

@@ -10,12 +10,8 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
-from pipelines.neural_prophet_pipeline.neural_prophet_pipeline import (
-    neural_prophet_pipeline,
-)
-from steps.data_loader.data_loader_step import data_loader
-from steps.predictor.predictor_step import predictor
-from steps.trainer.trainer_step import trainer
+from pipelines import neural_prophet_pipeline
+from steps import data_loader, predictor, trainer
 
 if __name__ == "__main__":
     pipeline = neural_prophet_pipeline(

examples/neural_prophet/steps/__init__.py

@@ -11,3 +11,8 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
+from .data_loader.data_loader_step import data_loader
+from .predictor.predictor_step import predictor
+from .trainer.trainer_step import trainer
+
+__all__ = ["data_loader", "trainer", "predictor"]

examples/scipy/pipelines/scipy_example_pipeline/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.

examples/scipy/pipelines/scipy_example_pipeline/scipy_example_pipeline.py

@@ -0,0 +1,23 @@
+#  Copyright (c) ZenML GmbH 2020. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+from zenml.integrations.constants import SKLEARN
+from zenml.pipelines import pipeline
+
+
+@pipeline(required_integrations=[SKLEARN])
+def scipy_example_pipeline(importer, vectorizer, trainer, predictor):
+    X_train, X_test, y_train, y_test = importer()
+    vec_transformer, X_train_vec, X_test_vec = vectorizer(X_train, X_test)
+    model = trainer(X_train_vec, y_train)
+    predictor(vec_transformer, model, X_test)

examples/scipy/steps/__init__.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
-from .loader.loader_step import importer
+from .importer.importer_step import importer
 from .predictor.predictor_step import predictor
 from .trainer.trainer_step import trainer
 from .vectorizer.vectorizer_step import vectorizer

examples/scipy/steps/importer/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright (c) ZenML GmbH 2022. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at:
+#
+#       https://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing
+#  permissions and limitations under the License.