spider.crontab.yaml

name: Deploy spiders

on:
  #schedule:
    # * is a special character in YAML so you have to quote this string
  #  - cron:  '0 0 1 * *'
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
    - name: Azure CLI script
      uses: azure/CLI@v1
      with:
        inlineScript: |
          az container start -n cloudynyascrapyd -g cloudymiao
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: '3.10' 
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install scrapy scrapyd
    - name: Update configure files
      run: |
        sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgm/settings.py
        sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgmapi/settings.py
        sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgm/settings.py
        sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgmapi/settings.py
    - name: Deploy bgmapi
      run: |
        python setup_bgmapi.py bdist_egg
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} http://cloudynyascrapyd.westus2.azurecontainer.io:6810/addversion.json -F project=bgmapi -F version=1.0 -F egg=@dist/project-1.0-py3.10.egg
    - name: Create jobs
      run: |
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} http://cloudynyascrapyd.westus2.azurecontainer.io:6810/schedule.json -d project=bgmapi -d spider=subject-api -d id_max=460000 -d token=${{ secrets.BANGUMI_TOKEN }}
        curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} http://cloudynyascrapyd.westus2.azurecontainer.io:6810/schedule.json -d project=bgmapi -d spider=collections-api -d id_max=850000