Skip to content

Hourly ETL Pipeline Run #2065

Hourly ETL Pipeline Run

Hourly ETL Pipeline Run #2065

Workflow file for this run

name: ETL Pipeline
run-name: Hourly ETL Pipeline Run
env:
PYTHON_VERSION: "3.10"
AUTOMATION_USERNAME: "ETL Automation"
AUTOMATION_EMAIL: "automation@xetdata.com"
on:
schedule:
- cron: "21 7 * * *"
jobs:
run-etl-pipeline:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: xetdata/checkout@v1
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: ETL Pipeline
run: |
python src/pipeline.py
- name: Peek Data output
run: |
head -n 10 data/air_data.csv
- name: Commit report
run: |
git config --global user.name $AUTOMATION_USERNAME
git config --global user.email $AUTOMATION_EMAIL
git add data/.
git commit -m "Updating Data"
git push