-
-
Notifications
You must be signed in to change notification settings - Fork 8
59 lines (56 loc) · 1.63 KB
/
scrape.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
name: Scrape Hacker News
on:
push:
workflow_dispatch:
schedule:
- cron: '17 * * * *'
jobs:
shot-scraper:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- uses: actions/cache@v2
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Cache Playwright browsers
uses: actions/cache@v2
with:
path: ~/.cache/ms-playwright/
key: ${{ runner.os }}-browsers
- name: Install dependencies
run: |
pip install -r requirements.txt
- name: Install Playwright dependencies
run: |
shot-scraper install
- name: Scrape
run: |
shot-scraper javascript \
"https://news.ycombinator.com/from?site=simonwillison.net" \
-i scrape.js -o simonwillison-net.json
sleep 5
shot-scraper javascript \
"https://news.ycombinator.com/from?site=datasette.io" \
-i scrape.js -o datasette-io.json
- name: Submit fresh data to my Datasette Cloud space
env:
DS_TOKEN: ${{ secrets.SIMON_DS_WRITE_TOKEN }}
run: |-
./submit-to-datasette-cloud.sh
- name: Commit and push
run: |-
git config user.name "Automated"
git config user.email "actions@users.noreply.github.com"
git add -A
timestamp=$(date -u)
git commit -m "${timestamp}" || exit 0
git pull --rebase
git push