Debugging dbt container #52

	name: Ingest pypi data

	permissions:
	id-token: write

	on:
	push:
	branches:
	- fix/ci-pipeline
	pull_request:
	types: [opened, synchronize, reopened]
	workflow_dispatch:
	inputs:
	start_date:
	description: 'Start Date'
	required: true
	default: ''
	end_date:
	description: 'End Date'
	required: true
	default: ''
	schedule:
	- cron: '0 0 * * *'

	jobs:

	transform-pypi-data:
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Pull latest Docker image
	run: docker pull ghcr.io/mehd-io/pypi-duck-flow@sha256:a5fde1539d179cd44d233dbe60454fc266e0866990eadf2f3b413e2270087172

	- name: check dbt pakcages deps
	run: \|
	make list-deps DOCKER=true

	- name: Calculate Dates
	id: dates
	run: \|
	if [ -n "${{ github.event.inputs.start_date }}" ] && [ -n "${{ github.event.inputs.end_date }}" ]; then
	echo "START_DATE=${{ github.event.inputs.start_date }}" >> $GITHUB_ENV
	echo "END_DATE=${{ github.event.inputs.end_date }}" >> $GITHUB_ENV
	else
	START_DATE=$(date -d '2 days ago' +%Y-%m-%d)
	END_DATE=$(date -d '1 day ago' +%Y-%m-%d)
	echo "START_DATE=${START_DATE}" >> $GITHUB_ENV
	echo "END_DATE=${END_DATE}" >> $GITHUB_ENV
	fi

	- name: Transform pypi data using dbt
	env:
	START_DATE: ${{ env.START_DATE }}
	END_DATE: ${{ env.END_DATE }}
	DBT_TARGET: prod
	DOCKER_IMAGE: ghcr.io/${{ github.repository }}:latest
	DBT_FOLDER: transform/pypi_metrics
	motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
	GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.auth.outputs.credentials_file_path }}
	run: \|
	make pypi-transform DOCKER=true DBT_TARGET=prod

Provide feedback