Skip to content
Permalink
master
Switch branches/tags
View runs Go to file
 
 
Cannot retrieve contributors at this time
name: Deploy spiders
on:
schedule:
# * is a special character in YAML so you have to quote this string
- cron: '0 0 1 * *'
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Redeploy scrapyd image
run: doctl apps create-deployment 1e08d97b-7a8e-4b6c-8635-8aaad64b5e24
- name: Wait for image running
run: sleep 150s
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: 3.7
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install scrapy scrapyd
- name: Update configure files
run: |
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgm/settings.py
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgmapi/settings.py
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgm/settings.py
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgmapi/settings.py
- name: Deploy bgm
run: |
python setup_bgm.py bdist_egg
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgm -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg
- name: Deploy bgmapi
run: |
python setup_bgmapi.py bdist_egg
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgmapi -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg
- name: Create jobs
run: |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgm -d spider=record -d id_max=700000
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgmapi -d spider=subject-api -d id_max=400000