spider.crontab.yaml

name: Deploy spiders

on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    - cron:  '0 0 1 * *'
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - name: Install doctl
      uses: digitalocean/action-doctl@v2
      with:
        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
    - name: Redeploy scrapyd image
      run: doctl apps create-deployment 1e08d97b-7a8e-4b6c-8635-8aaad64b5e24
    - name: Wait for image running
      run: sleep 150s
    - uses: actions/checkout@v2
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: 3.7
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install scrapy scrapyd
    - name: Update configure files
      run: |
        sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgm/settings.py
        sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgmapi/settings.py
        sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgm/settings.py
        sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgmapi/settings.py
    - name: Deploy bgm
      run: |
        python setup_bgm.py bdist_egg
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgm -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg
    - name: Deploy bgmapi
      run: |
        python setup_bgmapi.py bdist_egg
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgmapi -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg
    - name: Create jobs
      run: |
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgm -d spider=record -d id_max=700000
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgmapi -d spider=subject-api -d id_max=400000