Fix 21

.github/workflows/deploy-demo.yaml

@@ -18,15 +18,27 @@ permissions:
   id-token: write
 
 jobs:
-  reuse-build-and-deploy-node-app:
-    uses: SWRdata/workflows-swr-data-lab/.github/workflows/build_and_deploy_node_app.yml@main
-    with:
-      working_directory: 'demo'
-      full_repository: ${{ github.repository }}
-      branch: ${{ github.head_ref || github.ref_name }}
-      install_command: 'npm install -g npm && npm ci'
-      build_command: 'npm run build'
-      build_path: 'build'
-      node_version: 25
-    secrets:
-      SWR_NPM_TOKEN: ${{ secrets.SWR_NPM_TOKEN }}
+  build-deploy:
+    runs-on: ubuntu-latest
+    env:
+      ASSETS_PATH: 'https://static.datenhub.net/apps/${{ github.event.repository.name }}/${{ github.head_ref || github.ref_name }}/'
+    defaults:
+      run:
+        working-directory: ./components
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+      - run: |
+          touch .env
+          echo ASSETS_PATH=$ASSETS_PATH >> .env
+      - run: npm ci
+      - run: npm run sync
+      - run: npm run build
+      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+      - uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a #v3.0.0
+        with:
+          path: './demo/build'
+          destination: 'datenhub-net-static/apps/${{ github.event.repository.name }}/${{ github.head_ref || github.ref_name }}/'
+          parent: false

.github/workflows/deploy-pipeline-.yaml

@@ -0,0 +1,38 @@
+name: deploy pipeline
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - ./dag/**
+      - ./tasks/**
+
+permissions:
+  contents: write
+  id-token: write
+
+env:
+  REGISTRY_NAME: etl-images-airflow-swr-data-lab
+  REGISTRY_REGION: europe-west3
+  PROJECT_ID: swr-datalab-prod
+  BUCKET_NAME: europe-west3-swr-datalab-ai-4eee14a8-bucket
+  GCE_INSTANCE_NAME: datapipelines
+  GCE_ZONE: europe-west3-a
+
+jobs:
+  deploy-pipeline:
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 #v3.0.0
+        id: gcp-auth
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+      - name: get repo name
+        id: get-repo-name
+        run: echo "name=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_OUTPUT
+      - name: deploy
+        run: uv run --project=scripts deploy_pipeline.py --bucket_name="${{ env.BUCKET_NAME }}" --base_dir="tasks" --registry_base=europe-west3-docker.pkg.dev/swr-datalab-prod/etl-images-airflow-swr-data-lab
+        env:
+          GCP_ACCESS_TOKEN: ${{ steps.gcp-auth.outputs.access_token }}

.github/workflows/deploy-pipeline-old.yaml

@@ -0,0 +1,209 @@
+name: deploy pipeline
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "**/dag/**"
+      - "**/tasks/**"
+
+permissions:
+  contents: write
+  id-token: write
+
+env:
+  REGISTRY_NAME: etl-images-airflow-swr-data-lab
+  REGISTRY_REGION: europe-west3
+  PROJECT_ID: swr-datalab-prod
+  BUCKET_NAME: europe-west3-swr-datalab-ai-4eee14a8-bucket
+  GCE_INSTANCE_NAME: datapipelines
+  GCE_ZONE: europe-west3-a
+
+jobs:
+  deploy-data-pipeline:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - name: Authenticate to Google Cloud
+        id: gcp-auth
+        uses: SWRdata/workflows-swr-data-lab/.github/actions/gcp/auth@main
+
+      - name: Get Repo Name
+        id: get-repo-name
+        uses: SWRdata/workflows-swr-data-lab/.github/actions/git/get-repo-name@main
+
+      - name: Ensure the repo name exists
+        uses: SWRdata/workflows-swr-data-lab/.github/actions/git/ensure-repo-name-file@main
+        id: ensure-repo-name-exists
+        with:
+          repo_name: ${{ steps.get-repo-name.outputs.name }}
+
+      - name: Login to GAR
+        uses: SWRdata/workflows-swr-data-lab/.github/actions/gcp/gar-login@main
+        with:
+          access_token: ${{ steps.gcp-auth.outputs.access_token }}
+
+      - name: Clean up CC Bucket and Artifact Registry if repo name changed
+        if: ${{ inputs.enable_cleanup == 'true' && steps.ensure-repo-name-exists.outputs.is_new == 'false' }}
+        run: |
+          REPO_NAME_FILE=".github/REPO_NAME"
+          CURRENT_REPO_NAME="${{ steps.get-repo-name.outputs.name }}"
+
+          if [ "$CURRENT_REPO_NAME" != "$(cat $REPO_NAME_FILE 2>/dev/null)" ]; then
+            echo "[INFO] Repo name has changed! Preparing to clean up CC Bucket and Artifact Registry."
+
+            if [ ! -s "$REPO_NAME_FILE" ]; then
+              echo "[ERROR] .github/REPO_NAME is missing or empty. Aborting cleanup."
+              exit 1
+            fi
+
+            PREV_DAG_NAME=$(cat "$REPO_NAME_FILE" | tr '-' '_')
+
+            if [ -z "$PREV_DAG_NAME" ]; then
+              echo "[ERROR] Resolved GCS folder name is empty. Aborting cleanup."
+              exit 1
+            fi
+
+            GCS_PATH="gs://${{ env.BUCKET_NAME }}/dags/$PREV_DAG_NAME"
+
+            echo "[INFO] Attempting to delete: $GCS_PATH"
+            gcloud storage rm -r "$GCS_PATH" || true
+
+            echo "[INFO] Cleaning up Artifact Registry images starting with: ${PREV_DAG_NAME}_${{ inputs.tasks_path }}*"
+
+            IMAGE_LIST=$(gcloud artifacts docker images list \
+              "${{ env.REGISTRY_REGION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REGISTRY_NAME }}" \
+              --filter="package~${PREV_DAG_NAME}_${{ inputs.tasks_path }}*" \
+              --format="value(package)")
+
+            if [[ -z "$IMAGE_LIST" ]]; then
+                echo "[INFO] No images to delete in Artifact Registry."
+            else
+                for IMAGE in $IMAGE_LIST; do
+                  gcloud artifacts docker images delete "$IMAGE" \
+                    --quiet \
+                    --delete-tags
+                  echo "[INFO] Deleted image: $IMAGE (and all tags)."
+                done
+            fi
+
+            git pull --rebase
+            git config user.name "github-actions"
+            git config user.email "actions@github.com"
+            echo $CURRENT_REPO_NAME > .github/REPO_NAME
+            git add .github/REPO_NAME
+            if git diff --cached --quiet; then
+              echo "[INFO] No changes in REPO_NAME file. Nothing to commit."
+            else
+              git commit -m "chore: update REPO_NAME"
+              git push
+              echo "[INFO] Committed changes in REPO_NAME:"
+            fi
+          else
+            echo "[INFO] Repo name unchanged. Skipping clean up CC Bucket and Artifact Registry."
+          fi
+
+      - name: Build shared-code Docker Image
+        run: |
+          if [[ -f "${{ inputs.tasks_path }}/shared-code/Dockerfile" ]]; then
+            docker build --tag shared-code:latest ./${{ inputs.tasks_path }}/shared-code
+          fi
+
+      - name: Set DAG name
+        id: set-dag-name
+        run: |
+          REPO_NAME=${{ steps.get-repo-name.outputs.name }}
+          CLEAN_NAME=$(echo "$REPO_NAME" | tr '-' '_')
+          echo "name=$CLEAN_NAME" >> $GITHUB_OUTPUT
+
+      - name: Collect Dockerfiles
+        id: collect-dockerfiles
+        run: |
+          DOCKERFILES=$(find . -name Dockerfile | grep -v "tasks/shared-code" || true)
+          echo "dockerfiles=$(echo $DOCKERFILES | tr '\n' ' ')" >> $GITHUB_OUTPUT
+
+      - name: Cleanup Artifact Registry if changes in task meta
+        run: |
+          REMOTE_IMAGES=$(gcloud artifacts docker images list \
+            "${REGISTRY_REGION}-docker.pkg.dev/${PROJECT_ID}/${REGISTRY_NAME}" \
+            --filter="package~${{ steps.set-dag-name.outputs.name }}_${{ inputs.tasks_path }}*" \
+            --format="value(package)")
+
+          declare -A LOCAL_IMAGES_SET=()
+
+          for DOCKERFILE in ${{ steps.collect-dockerfiles.outputs.dockerfiles }}; do
+              IMAGE_NAME=$(echo "$DOCKERFILE" | sed 's/\/Dockerfile//; s/\//_/g; s/\.//g')
+              LOCAL_IMAGE="${REGISTRY_REGION}-docker.pkg.dev/${PROJECT_ID}/${REGISTRY_NAME}/${{ steps.set-dag-name.outputs.name }}${IMAGE_NAME}"
+              LOCAL_IMAGES_SET["$LOCAL_IMAGE"]=1
+          done
+
+          for REMOTE_IMAGE in $REMOTE_IMAGES; do
+              if [[ -z "${LOCAL_IMAGES_SET[$REMOTE_IMAGE]}" ]]; then
+                  gcloud artifacts docker images delete "$REMOTE_IMAGE" --quiet
+                  echo "[INFO] Deleted remote image: $REMOTE_IMAGE"
+              fi
+          done
+
+      - name: Build and push Docker Images
+        env:
+          GAR_READER_SA_KEY: ${{ secrets.GAR_READER_SA_KEY }}
+          DOCKERFILES: ${{ steps.collect-dockerfiles.outputs.dockerfiles }}
+        run: |
+
+          if [[ -z "${{ steps.collect-dockerfiles.outputs.dockerfiles }}" ]]; then
+            echo "[INFO] No Dockerfiles found."
+            exit 0
+          fi
+
+          for DOCKERFILE in ${{ steps.collect-dockerfiles.outputs.dockerfiles }}; do
+            BUILD_CONTEXT_INPUT="${{ inputs.build_context }}"
+            if [[ -n "$BUILD_CONTEXT_INPUT" ]]; then
+              BUILD_CONTEXT_DIR="$BUILD_CONTEXT_INPUT"
+            else
+              BUILD_CONTEXT_DIR=$(dirname "$DOCKERFILE")
+            fi
+
+            IMAGE_NAME=$(echo "$DOCKERFILE" | sed 's/\/Dockerfile//; s/\//_/g; s/\.//g')
+
+            docker build \
+              --file "$DOCKERFILE" \
+              --secret id=google_creds,env=GAR_READER_SA_KEY \
+              --push \
+              --tag "${{ env.REGISTRY_REGION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REGISTRY_NAME }}/${{ steps.set-dag-name.outputs.name }}${IMAGE_NAME}:${{ inputs.image_tag }}" \
+              "$BUILD_CONTEXT_DIR"
+            echo "[INFO] Built image: $IMAGE_NAME"
+          done
+
+      - name: Pull Docker Image into GCE and cleanup
+        if: fromJSON(inputs.deploy_to_gce)
+        uses: google-github-actions/ssh-compute@v2
+        with:
+          instance_name: ${{ env.GCE_INSTANCE_NAME }}
+          zone: ${{ env.GCE_ZONE }}
+          project_id: ${{ env.PROJECT_ID }}
+          ssh_private_key: ${{ secrets.GCE_DATAPIPELINES_SSH_KEY }}
+          command: |
+            TOKEN=$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" | jq -r .access_token)
+            docker login -u oauth2accesstoken --password "$TOKEN" https://${{ env.REGISTRY_REGION }}-docker.pkg.dev
+            if [[ -z "${{ steps.collect-dockerfiles.outputs.dockerfiles }}" ]]; then
+              echo "No Dockerfiles found."
+            else
+              IFS=' '
+              for DOCKERFILE in ${{ steps.collect-dockerfiles.outputs.dockerfiles }}; do
+                IMAGE_NAME=$(echo $DOCKERFILE | sed 's/\/Dockerfile//; s/\//_/g; s/\.//g')
+                docker pull ${{ env.REGISTRY_REGION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REGISTRY_NAME }}/${{ steps.set-dag-name.outputs.name }}$IMAGE_NAME:${{ inputs.image_tag }}
+              done
+            fi
+            docker container prune -f
+            docker image prune -f
+
+      - name: Upload DAG folder
+        uses: SWRdata/workflows-swr-data-lab/.github/actions/gcp/gcs-upload@main
+        with:
+          bucket: ${{ env.BUCKET_NAME }}
+          src_folder: ${{ inputs.dag_path }}
+          dst_folder: dags/${{ steps.set-dag-name.outputs.name }}

.github/workflows/ruff.yaml

@@ -0,0 +1,35 @@
+name: ruff
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "tasks/make_boundaries/**"
+      - "scripts/**"
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "tasks/make_boundaries/**"
+      - "scripts/**"
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  ty:
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # 7.0.0
+        with:
+          fetch-depth: 1
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
+        with:
+          enable-cache: true
+      - run: uv sync
+      - run: uv run --project=tasks/make_boundaries ruff check tasks/make_boundaries/src
+      - run: uv run --project=scripts ruff check scripts
+      - run: uv run --project=common ruff check common

.github/workflows/test.yaml

@@ -0,0 +1,33 @@
+name: tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "scripts/**"
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - "scripts/**"
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  test:
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # 7.0.0
+        with:
+          fetch-depth: 1
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
+        with:
+          enable-cache: true
+      - run: uv sync
+        working-directory: scripts
+      - run: uv run python -m unittest -v
+        working-directory: scripts/src/scripts