Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
58 lines (56 sloc) 2.34 KB
name: Deploy spiders
on:
schedule:
# * is a special character in YAML so you have to quote this string
- cron: '0 0 1 * *'
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: Azure CLI script
uses: azure/CLI@v1
with:
inlineScript: |
az container start -n cloudymiaoscrapyd -g cloudymiao
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: 3.7
- name: Get scrapyd-client
uses: actions/checkout@v2
with:
repository: scrapy/scrapyd-client
ref: 'v1.2.0a1'
path: 'scrapyd-client'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install scrapy scrapyd
cd scrapyd-client
python setup.py install
cd ..
- name: Update configure files
run: |
sed -i 's~^username = username$~username = ike~g' scrapy.cfg
sed -i 's~^password = password$~password = ${{ secrets.SCRAPYD_PASSWORD }}~g' scrapy.cfg
sed -i 's~^url = http:\/\/localhost:6810\/$~url = http:\/\/cloudymiaoscrapyd.westus2.azurecontainer.io:6810\/~g' scrapy.cfg
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgm/settings.py
sed -i 's~^UPLOAD_TO_AZURE_STORAGE = False$~UPLOAD_TO_AZURE_STORAGE = True~g' bgmapi/settings.py
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgm/settings.py
sed -i 's~^AZURE_ACCOUNT_KEY = ""$~AZURE_ACCOUNT_KEY = "${{ secrets.AZURE_BLOB_KEY }}"~g' bgmapi/settings.py
- name: Deploy bgm
run: |
python setup_bgm.py bdist_egg
scrapyd-deploy bgm --egg dist/project-1.0-py3.7.egg
- name: Deploy bgmapi
run: |
python setup_bgmapi.py bdist_egg
scrapyd-deploy bgmapi --egg dist/project-1.0-py3.7.egg
- name: Create jobs
run: |
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} http://cloudymiaoscrapyd.westus2.azurecontainer.io:6810/schedule.json -d project=bgm -d spider=record -d id_max=550000
curl --user ike:${{ secrets.SCRAPYD_PASSWORD }} http://cloudymiaoscrapyd.westus2.azurecontainer.io:6810/schedule.json -d project=bgmapi -d spider=subject-api -d id_max=310000
You can’t perform that action at this time.