Merge pull request #32 from zenml-io/feature/rework

Big rework

.github/workflows/pr.yaml

@@ -0,0 +1,99 @@
+name: PR Flow
+on:
+  pull_request:
+    branches: [staging, main]
+concurrency:
+  # New commit on branch cancels running workflows of the same branch
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-staging-workflow:
+    runs-on: ubuntu-dind-runners
+    env:
+      GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
+      ZENML_HOST: ${{ secrets.ZENML_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
+      ZENML_STAGING_STACK: ${{ secrets.ZENML_STAGING_STACK }}
+      ZENML_PRODUCTION_STACK: ${{ secrets.ZENML_PRODUCTION_STACK }}
+      ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
+      ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
+      ZENML_DEBUG: true
+      ZENML_ANALYTICS_OPT_IN: false
+      ZENML_LOGGING_VERBOSITY: INFO
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install requirements
+        if: ${{ github.base_ref == 'staging' }}
+        run: |
+          pip3 install -r requirements.txt
+          zenml integration install s3 sklearn mlflow deepchecks -y
+      
+      - name: Install requirements
+        if: ${{ github.base_ref == 'main' }}
+        run: |
+          pip3 install -r requirements.txt
+          zenml integration install s3 sklearn mlflow deepchecks kubernetes -y
+
+      - name: Install wkhtmltopdf
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y wkhtmltopdf
+
+      - name: Connect to ZenML server
+        run: |
+          zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
+
+      - name: Set stack (Staging)
+        if: ${{ github.base_ref == 'staging' }}
+        run: |
+          zenml stack set ${{ env.ZENML_STAGING_STACK }}
+
+      - name: Set stack (Production)
+        if: ${{ github.base_ref == 'main' }}
+        run: |
+          zenml stack set ${{ env.ZENML_PRODUCTION_STACK }}
+
+      - name: Run pipeline (Staging)
+        if: ${{ github.base_ref == 'staging' }}
+        run: |
+          python run.py \
+            --pipeline train \
+            --dataset staging \
+            --version ${{ env.ZENML_GITHUB_SHA }} \
+            --github-pr-url ${{ env.ZENML_GITHUB_URL_PR }} \
+            -o ${{ secrets.ZENML_ORG_ID }} \
+            -t ${{ secrets.ZENML_TENANT_ID }}
+
+
+      - name: Run pipeline (Production)
+        if: ${{ github.base_ref == 'main' }}
+        run: |
+          python run.py \
+            --pipeline end-to-end \
+            --dataset production \
+            --version ${{ env.ZENML_GITHUB_SHA }} \
+            --github-pr-url ${{ env.ZENML_GITHUB_URL_PR }} \
+            -o ${{ secrets.ZENML_ORG_ID }} \
+            -t ${{ secrets.ZENML_TENANT_ID }}
+
+      - name: Read training report
+        id: report
+        uses: juliangruber/read-file-action@v1
+        with:
+          path: ./model_train_results.md
+
+      - name: PR comment with training report
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: ./model_train_results.md
+          comment_tag: training_report
+          mode: recreate
+          create_if_not_exists: true

.github/workflows/production.yaml

@@ -1,4 +1,4 @@
-name: production
+name: Promotion flow
 on:
   push:
     branches: [ main ]
@@ -7,75 +7,40 @@ jobs:
   run-production-workflow:
     runs-on: ubuntu-latest
     env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      KUBEFLOW_USERNAME: ${{ secrets.KUBEFLOW_PRODUCTION_USERNAME }}
-      KUBEFLOW_PASSWORD: ${{ secrets.KUBEFLOW_PRODUCTION_PASSWORD }}
-      KUBEFLOW_NAMESPACE: ${{ secrets.KUBEFLOW_PRODUCTION_NAMESPACE }}
-      ZENML_USERNAME: ${{ secrets.ZENML_USERNAME }}
-      ZENML_PASSWORD: ${{ secrets.ZENML_PASSWORD }}
       ZENML_HOST: ${{ secrets.ZENML_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
       ZENML_DEBUG: true
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_LOGGING_VERBOSITY: INFO
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          role-to-assume: arn:aws:iam::715803424590:role/OrganizationAccountRestrictedAccessRole
-          aws-region: eu-central-1
-
-      - id: install-kubectl
-        uses: azure/setup-kubectl@v3
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        run: |
-          aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.us-east-1.amazonaws.com
-
-      - name: Login to Amazon EKS
-        id: login-eks
+      - name: Get latest SHA before merge
+        id: last-commit
         run: |
-          aws eks --region eu-central-1 update-kubeconfig --name kubeflowmultitenant --alias zenml-eks
+          echo "last_commit_in_pr=$(echo "$GITHUB_CONTEXT" | jq '.event.commits[].id' | tail -n 2 | head -n 1 | sed 's/"//g')" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
 
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
-
-      - uses: syphar/restore-virtualenv@v1
-        id: cache-virtualenv
-        with:
-          requirement_files: 'requirements-aws.txt'
-
-      - uses: syphar/restore-pip-download-cache@v1
-        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
-        with:
-          requirement_files: 'requirements-aws.txt'
 
       - name: Install requirements
-        if: steps.cache-virtualenv.outputs.cache-hit != 'true'
         run: |
-          pip install -r requirements-aws.txt
-
-      - name: Install wkhtmltopdf
-        run: sudo apt-get install wkhtmltopdf
+          make install-local
 
       - name: Connect to ZenML server
         run: |
-          zenml connect --no-verify-ssl --url $ZENML_HOST --username $ZENML_USERNAME --password $ZENML_PASSWORD
+          zenml connect --url $ZENML_HOST --api-key $ZENML_API_KEY
 
       - name: Set stack
         run: |
-          zenml stack set aws_gitflow_stack
+          zenml stack set default
         
       - name: Run pipeline
         run: |
-          python run.py \
-            --pipeline end-to-end \
-            --requirements requirements-aws.txt \
-            --model iris-classifier \
-            --dataset production
+          python promote.py \
+            --version ${{ steps.last-commit.outputs.last_commit_in_pr }}
 

.gitignore

@@ -131,4 +131,8 @@ dmypy.json
 .zen
 mlruns
 
-poetry.lock
+poetry.lock
+model_train_results.md
+
+.DS_Store
+.vscode/

Makefile

@@ -0,0 +1,6 @@
+install-base:
+	pip3 install -r requirements.txt
+	zenml integration install s3 sklearn mlflow deepchecks -y
+
+install-aws: install-base
+	zenml integration install kubernetes -y