Test new CI setup

duneanalytics · May 29, 2024 · e9b3d7d · e9b3d7d
1 parent ab9850a
commit e9b3d7d
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 27 deletions.
diff --git a/.github/workflows/dbt_slim_ci.yml → .github/workflows/dbt_run.yml b/.github/workflows/dbt_slim_ci.yml → .github/workflows/dbt_run.yml
@@ -1,45 +1,31 @@
-# .github/workflows/app.yaml
-name: dbt slim ci
+name: DBT Run Reusable Workflow
 
 on:
-  workflow_dispatch:
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - 'Pipfile'
-      - '.gitignore'
-
-concurrency:
-  group:  ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
+  workflow_call:
+    inputs:
+      project:
+        description: 'DBT run for project'
+        required: true
+        type: string
 
 jobs:
   dbt-test:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
-    strategy:
-      matrix:
-        project: [ 'tokens', 'spellbook', 'daily_spellbook' ]
-      max-parallel: 3
     timeout-minutes: 90
 
     steps:
       - name: Check out repository code
         uses: actions/checkout@v3
 
       - name: Setup variables
-        run: |
+        run: | # Spellbook is a special case because it's not a subdirectory
           echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
           echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
-          if [[ "${{ matrix.project }}" == "tokens" ]]; then
-            echo "PROJECT_DIR=tokens" >> $GITHUB_ENV
-            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/tokens" >> $GITHUB_ENV
-          elif [[ "${{ matrix.project }}" == "spellbook" ]]; then
-            echo "PROJECT_DIR=." >> $GITHUB_ENV
-            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/spellbook" >> $GITHUB_ENV
-          elif [[ "${{ matrix.project }}" == "daily_spellbook" ]]; then
+          echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
+          if [[ "${{ inputs.project }}" == "spellbook" ]]; then
             echo "PROJECT_DIR=." >> $GITHUB_ENV
-            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/daily_spellbook" >> $GITHUB_ENV
+          else [[ "${{ inputs.project }}" == "daily_spellbook" ]]; then
+            echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
           fi
       - name: Add git_sha to schema
         run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
@@ -82,4 +68,4 @@ jobs:
 
       - name: dbt test incremental model(s) if applicable
         if: env.INC_MODEL_COUNT > 0
-        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
+        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental --exclude tag:prod_exclude tag:remove --defer --state . --project-dir ${PROJECT_DIR}"
diff --git a/.github/workflows/tokens.yaml b/.github/workflows/tokens.yaml
@@ -0,0 +1,24 @@
+name: dbt slim ci
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - tokens/**
+      - .github/workflows/tokens.yaml
+      - .github/workflows/dbt_run.yaml
+    paths-ignore:
+      - 'scripts/**'
+      - 'Pipfile'
+      - '.gitignore'
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yaml
+    with:
+      project: 'tokens'
+