Permalink
Cannot retrieve contributors at this time
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
47 lines (45 sloc)
2.13 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Deploy spiders | |
on: | |
schedule: | |
# * is a special character in YAML so you have to quote this string | |
- cron: '0 0 1 * *' | |
workflow_dispatch: | |
jobs: | |
build: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Install doctl | |
uses: digitalocean/action-doctl@v2 | |
with: | |
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} | |
- name: Redeploy scrapyd image | |
run: doctl apps create-deployment 1e08d97b-7a8e-4b6c-8635-8aaad64b5e24 | |
- name: Wait for image running | |
run: sleep 150s | |
- uses: actions/checkout@v2 | |
- name: Set up Python 3.7 | |
uses: actions/setup-python@v1 | |
with: | |
python-version: 3.7 | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install scrapy scrapyd | |
- name: Update configure files | |
run: | | |
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgm/settings.py | |
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgmapi/settings.py | |
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgm/settings.py | |
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgmapi/settings.py | |
- name: Deploy bgm | |
run: | | |
python setup_bgm.py bdist_egg | |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgm -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg | |
- name: Deploy bgmapi | |
run: | | |
python setup_bgmapi.py bdist_egg | |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/addversion.json -F project=bgmapi -F version=1.0 -F egg=@dist/project-1.0-py3.7.egg | |
- name: Create jobs | |
run: | | |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgm -d spider=record -d id_max=700000 | |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} https://wattlebird-scrapyd-uedpn.ondigitalocean.app/schedule.json -d project=bgmapi -d spider=subject-api -d id_max=400000 |