etl.yaml

name: "Extract, transform and load"

on:
  workflow_dispatch:
  schedule:
    - cron: "0 */12 * * *"

jobs:
  etl:
    name: ETL
    runs-on: ubuntu-latest
    steps:
      - id: checkout
        name: Checkout
        uses: actions/checkout@v3

      - id: setup-python
        name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
          cache: 'pipenv'

      - id: install-pipenv
        name: Install pipenv
        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
        shell: bash

      - id: install-python-dependencies
        name: Install Python dependencies
        run: pipenv install --dev --python=`which python`
        shell: bash

      - id: download
        name: Download raw data
        run: |
          pipenv run python src/download.py satnogs
          pipenv run python src/download.py je9pel
        shell: bash

      - id: commit
        name: Commit
        run: |
          git config --global user.name "GitHub Actions"
          git config --global user.email "actions@github.com"
          git config pull.rebase false
          git pull origin $GITHUB_REF
          git add ./data
          git commit -m "Added latest ETL" --author="palewire <palewire@users.noreply.github.com>" && git push || true
        shell: bash