Update versions #28
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI test new and modified models | |
| concurrency: ci-${{ github.ref }} | |
| on: | |
| pull_request: | |
| env: | |
| DEBUG: true | |
| DBT_PROFILE_NAME: ${{ vars.DBT_PROFILE_NAME }} | |
| DBT_SNOWFLAKE_ACCOUNT: ${{ secrets.DBT_SNOWFLAKE_ACCOUNT }} | |
| DBT_SNOWFLAKE_USER: ${{ vars.DBT_SNOWFLAKE_USER }} | |
| DBT_SNOWFLAKE_ROLE: ${{ vars.DBT_SNOWFLAKE_ROLE }} | |
| DBT_SNOWFLAKE_PASSWORD: ${{ secrets.DBT_SNOWFLAKE_PASSWORD }} | |
| DBT_SNOWFLAKE_DATABASE: ${{ vars.DBT_SNOWFLAKE_DATABASE }} | |
| DBT_SNOWFLAKE_WAREHOUSE: ${{ vars.DBT_SNOWFLAKE_WAREHOUSE }} | |
| DBT_SNOWFLAKE_THREADS: ${{ vars.DBT_SNOWFLAKE_THREADS }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| BRANCH_NAME: ${{ github.head_ref }} | |
| DBT_PROJECT_FOLDER: ${{ vars.DBT_PROJECT_FOLDER }} | |
| jobs: | |
| initial_compile: | |
| # dependabot does not have access to Github Action secrets | |
| if: github.actor != 'dependabot[bot]' | |
| # For creating the initial manifest.json based on the branch before any commits are made | |
| name: Create initial manifest.json | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Switch to base commit | |
| run: | | |
| git checkout ${{ github.event.pull_request.base.sha }} | |
| - name: Create profiles.yml | |
| run: | | |
| mkdir -p /home/$(whoami)/.dbt/ | |
| echo "${DBT_PROFILE_NAME}:" > /home/$(whoami)/.dbt/profiles.yml | |
| echo " target: development" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " outputs:" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " development:" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " type: snowflake" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " account: ${DBT_SNOWFLAKE_ACCOUNT}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " user: ${DBT_SNOWFLAKE_USER}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " role: ${DBT_SNOWFLAKE_ROLE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " password: ${DBT_SNOWFLAKE_PASSWORD}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " database: ${DBT_SNOWFLAKE_DATABASE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " warehouse: ${DBT_SNOWFLAKE_WAREHOUSE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " schema: pr" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " threads: ${DBT_SNOWFLAKE_THREADS}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " client_session_keep_alive: false" >> /home/$(whoami)/.dbt/profiles.yml | |
| - name: Print profiles.yml | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| echo "----------------------------------------" | |
| echo "Debug mode: Show content of profiles.yml" | |
| echo "Note that secrets will be masked" | |
| echo "----------------------------------------" | |
| cat /home/$(whoami)/.dbt/profiles.yml | |
| - name: Read Python version from .python-version | |
| run: echo "python_version=$(cat .python-version)" >> "$GITHUB_ENV" | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - name: Cache pip dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dbt | |
| run: | | |
| python -m pip install --upgrade pip | |
| grep dbt-snowflake requirements.txt | xargs -n 1 pip install | |
| - name: Run dbt debug | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| echo "----------------------------------------" | |
| echo "Debug mode: Execute dbt debug" | |
| echo "----------------------------------------" | |
| dbt debug | |
| - name: Run dbt deps | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| dbt deps | |
| - name: Run dbt compile | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| dbt compile | |
| - name: Print manifest.json | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| cat target/manifest.json | |
| - name: Upload manifest.json | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: manifest | |
| path: ${{ vars.DBT_PROJECT_FOLDER }}/target/manifest.json | |
| dbt_build: | |
| # dependabot does not have access to Github Action secrets | |
| if: github.actor != 'dependabot[bot]' | |
| name: Test new and updated dbt models | |
| needs: initial_compile | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Create profiles.yml | |
| run: | | |
| BRANCH_NAME=$(echo "${BRANCH_NAME}" | tr '-' '_') | |
| mkdir -p /home/$(whoami)/.dbt/ | |
| echo "${DBT_PROFILE_NAME}:" > /home/$(whoami)/.dbt/profiles.yml | |
| echo " target: development" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " outputs:" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " development:" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " type: snowflake" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " account: ${DBT_SNOWFLAKE_ACCOUNT}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " user: ${DBT_SNOWFLAKE_USER}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " role: ${DBT_SNOWFLAKE_ROLE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " password: ${DBT_SNOWFLAKE_PASSWORD}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " database: ${DBT_SNOWFLAKE_DATABASE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " warehouse: ${DBT_SNOWFLAKE_WAREHOUSE}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " schema: pr_${BRANCH_NAME##*/}_${PR_NUMBER}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " threads: ${DBT_SNOWFLAKE_THREADS}" >> /home/$(whoami)/.dbt/profiles.yml | |
| echo " client_session_keep_alive: false" >> /home/$(whoami)/.dbt/profiles.yml | |
| - name: Print profiles.yml | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| echo "----------------------------------------" | |
| echo "Debug mode: Show content of profiles.yml" | |
| echo "Note that secrets will be masked" | |
| echo "----------------------------------------" | |
| cat /home/$(whoami)/.dbt/profiles.yml | |
| - name: Read Python version from .python-version | |
| run: echo "python_version=$(cat .python-version)" >> "$GITHUB_ENV" | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - name: Cache pip dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dbt | |
| run: | | |
| python -m pip install --upgrade pip | |
| grep dbt-snowflake requirements.txt | xargs -n 1 pip install | |
| - name: Run dbt debug | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| echo "----------------------------------------" | |
| echo "Debug mode: Execute dbt debug" | |
| echo "----------------------------------------" | |
| dbt debug | |
| - name: Run dbt deps | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| dbt deps | |
| - name: Download manifest.json | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: manifest | |
| path: ${{ vars.DBT_PROJECT_FOLDER }}/ | |
| - name: Print initial manifest.json | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| cat manifest.json | |
| - name: Run dbt compile | |
| if: ${{ env.DEBUG == 'true' }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| echo "----------------------------------------" | |
| echo "Debug mode: Execute dbt compile" | |
| echo "----------------------------------------" | |
| dbt compile | |
| - name: Run dbt seed, run, and test | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| echo "" | |
| echo "" | |
| echo "---------- dbt seed ----------" | |
| dbt seed | |
| echo "" | |
| echo "" | |
| echo "---------- dbt run ----------" | |
| dbt run --select +state:modified --state ./ | |
| echo "" | |
| echo "" | |
| echo "---------- dbt test ----------" | |
| dbt test --select state:modified --state ./ | tee dbt_output.log | |
| exit ${PIPESTATUS[0]} | |
| - name: Print sql from failing tests | |
| if: ${{ failure() }} | |
| run: | | |
| cd $DBT_PROJECT_FOLDER | |
| # Define the log file | |
| log_file="dbt_output.log" | |
| # Get the number of failed tests | |
| num_failed_tests=$(grep -o "ERROR=[0-9]*" "$log_file" | awk -F= '{print $2}') | |
| # If there are no failed tests, exit the script | |
| if [ "$num_failed_tests" -eq 0 ]; then | |
| echo "No failed tests found." | |
| exit 0 | |
| fi | |
| # Loop through each failed test and extract the required information | |
| for i in $(seq 1 "$num_failed_tests"); do | |
| header=$(grep -m "$i" "Failure in test" "$log_file" | tail -n 1 | cut -c 15-) | |
| description=$(grep -A 1 -m "$i" "Failure in test" "$log_file" | tail -n 1 | cut -c 15-) | |
| code_path=$(grep -m "$i" "compiled Code at" "$log_file" | tail -n 1 | awk '{print $5}') | |
| # Print the extracted information | |
| echo "$header" | |
| echo "$description" | |
| # Print the content of the code file | |
| if [ -f "$code_path" ]; then | |
| grep -vE '^$' "$code_path" | |
| else | |
| echo "Code file not found: $code_path" | |
| fi | |
| # Add a separator between failed tests, except for the last one | |
| if [ "$i" -ne "$num_failed_tests" ]; then | |
| echo "" | |
| echo "----------------------------------------" | |
| echo "" | |
| fi | |
| done | |
| if [ "$DEBUG" = "true" ]; then | |
| echo "" | |
| echo "----------------------------------------" | |
| echo "Debug mode: Printing the entire dbt_output.log content:" | |
| echo "----------------------------------------" | |
| cat "$log_file" | |
| fi |