-
Notifications
You must be signed in to change notification settings - Fork 0
56 lines (50 loc) · 1.72 KB
/
scrape_data.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
---
name: Update local data archive
on:
# Manual button in Github.
workflow_dispatch:
# Callable from the Wrangle workflow
workflow_call:
# Run on a schedule
schedule:
# 5 and 35 past the hour except at 1:35AM
- cron: '5 * * * *'
- cron: '35 0,2-23 * * *'
jobs:
scheduled:
name: Download data
runs-on: ubuntu-latest
steps:
- name: Check out this repo
# https://github.com/actions/checkout
uses: actions/checkout@v3
- name: Install Python
# https://github.com/actions/setup-python
# Will install python version from .python-version
uses: actions/setup-python@v4
with:
python-version: '3.10.6'
cache: pip
- name: Install Python packages
# only need minimal requirements for scraping
run: |-
make install-minimal
- name: Fetch latest data and save as JSON
run: |-
# Download data
# Regional forward 48h
python run.py download --output_directory data --now
# Regional past 24h
python run.py download --output_directory data --now --endpoint regional_pt24h
# National
python run.py download --output_directory data --now --endpoint national_fw48h
python run.py download --output_directory data --now --endpoint national_pt24h
python run.py download --output_directory data --now --endpoint national_generation_pt24h
- name: Commit and push new data
run: |-
git config user.name "Automated"
git config user.email "actions@users.noreply.github.com"
git add -A
git commit -m "chore: Add latest data" || exit 0
git pull --rebase
git push