Update local data archive #17063
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
name: Update local data archive | |
on: | |
# Manual button in Github. | |
workflow_dispatch: | |
# Callable from the Wrangle workflow | |
workflow_call: | |
# Run on a schedule | |
schedule: | |
# 5 and 35 past the hour except at 1:35AM | |
- cron: '5 * * * *' | |
- cron: '35 0,2-23 * * *' | |
jobs: | |
scheduled: | |
name: Download data | |
runs-on: ubuntu-latest | |
steps: | |
- name: Check out this repo | |
# https://github.com/actions/checkout | |
uses: actions/checkout@v3 | |
- name: Install Python | |
# https://github.com/actions/setup-python | |
# Will install python version from .python-version | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.10.6' | |
cache: pip | |
- name: Install Python packages | |
# only need minimal requirements for scraping | |
run: |- | |
make install-minimal | |
- name: Fetch latest data and save as JSON | |
run: |- | |
# Download data | |
# Regional forward 48h | |
python run.py download --output_directory data --now | |
# Regional past 24h | |
python run.py download --output_directory data --now --endpoint regional_pt24h | |
# National | |
python run.py download --output_directory data --now --endpoint national_fw48h | |
python run.py download --output_directory data --now --endpoint national_pt24h | |
python run.py download --output_directory data --now --endpoint national_generation_pt24h | |
- name: Commit and push new data | |
run: |- | |
git config user.name "Automated" | |
git config user.email "actions@users.noreply.github.com" | |
git add -A | |
git commit -m "chore: Add latest data" || exit 0 | |
git pull --rebase | |
git push |