Skip to content
neblio-quicksync

neblio-quicksync #4114

Sign in to view logs

neblio-quicksync

neblio-quicksync #4114

Workflow file for this run

.github/workflows/neblio-quicksync.yml at 21bb3e2
name: neblio-quicksync
on:
# push:
workflow_dispatch:
repository_dispatch:
schedule:
- cron: '15 05 * * *'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GHAT: ${{ secrets.GHAT }}
AWS_S3_BUCKET: 'neblio-asset-repo'
AWS_S3_ENDPOINT: 'https://sfo3.digitaloceanspaces.com'
jobs:
quicksync-70516:
runs-on: ubuntu-18.04
strategy:
fail-fast: false
steps:
- uses: actions/checkout@v1
- name: free disk space
run: |
df -h
sudo apt clean || true
df -h
- name: ccache
uses: actions/cache@v2
with:
path: .ccache
key: ${{ runner.OS }}--linux_daemon-${{ github.job }}-${{ github.sha }}-${{ github.run_id }}
restore-keys: |
${{ runner.OS }}--linux_daemon-${{ github.job }}-
- name: Set QuickSync DB_VER
run: |
echo "DB_VER=70516" >> $GITHUB_ENV
- name: Install Dependencies & Setup Environment
run: |
echo "BUILD_DIR=$(echo ${GITHUB_WORKSPACE})" >> $GITHUB_ENV
echo "DEST_DIR=$(echo quicksync/txlmdb-)$(echo ${DB_VER})$(echo /)$(echo ${GITHUB_SHA})" >> $GITHUB_ENV
echo "target_v=linux_daemon" >> $GITHUB_ENV
pip install awscli==1.19.112 --upgrade --user
sudo pip uninstall -y urllib3
sudo pip install urllib3==1.22
sudo pip install requests
- name: Build nebliod for Linux & Sync
run: |
/bin/bash quicksync.sh
- name: Upload Quicksync Files to S3
uses: NeblioTeam/s3-sync-action@v0.5.2
with:
args: --acl public-read
env:
AWS_S3_BUCKET: ${{ env.AWS_S3_BUCKET }}
AWS_S3_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }}
SOURCE_DIR: txlmdb
- name: Verify Uploaded Files on S3
run: |
python -u verify_deploy.py
- name: If Failure Restart Job
if: failure()
run: |
git config --global user.email "info@nebl.io"
git config --global user.name "NeblioTeam"
git checkout master
git commit --allow-empty -m "Daily update of download.json for QuickSync ${DB_VER} Failed. Restarting."
git push https://${GHAT}@github.com/NeblioTeam/neblio-quicksync master
curl POST https://api.github.com/repos/NeblioTeam/neblio-quicksync/dispatches -H 'Accept: application/vnd.github.everest-preview+json' -H "Authorization: token $GHAT" --data '{"event_type": "restart_neblio_quicksync"}'
- name: If Success Update JSON Files & Cleanup
if: success()
run: |
git config --global user.email "info@nebl.io"
git config --global user.name "NeblioTeam"
git checkout master
python -u update_json.py
git add download.json
git commit -m "Daily update of download.json for QuickSync ${DB_VER}"
git push https://${GHAT}@github.com/NeblioTeam/neblio-quicksync master
echo "Removing Old Files"
aws s3 rm s3://neblio-asset-repo/quicksync/txlmdb-${DB_VER}/ --recursive --exclude "$GITHUB_SHA/*" --endpoint=https://sfo3.digitaloceanspaces.com
quicksync-70517:
runs-on: ubuntu-18.04
needs: quicksync-70516
strategy:
fail-fast: false
steps:
- uses: actions/checkout@v1
- name: free disk space
run: |
df -h
sudo apt clean || true
df -h
- name: ccache
uses: actions/cache@v2
with:
path: .ccache
key: ${{ runner.OS }}--linux_daemon-${{ github.job }}-${{ github.sha }}-${{ github.run_id }}
restore-keys: |
${{ runner.OS }}--linux_daemon-${{ github.job }}-
- name: Set QuickSync DB_VER
run: |
echo "DB_VER=70517" >> $GITHUB_ENV
- name: Install Dependencies & Setup Environment
run: |
echo "BUILD_DIR=$(echo ${GITHUB_WORKSPACE})" >> $GITHUB_ENV
echo "DEST_DIR=$(echo quicksync/txlmdb-)$(echo ${DB_VER})$(echo /)$(echo ${GITHUB_SHA})" >> $GITHUB_ENV
echo "target_v=linux_daemon" >> $GITHUB_ENV
pip install awscli==1.19.112 --upgrade --user
sudo pip uninstall -y urllib3
sudo pip install urllib3==1.22
sudo pip install requests
- name: Build nebliod for Linux & Sync
run: |
/bin/bash quicksync.sh
- name: Upload Quicksync Files to S3
uses: NeblioTeam/s3-sync-action@v0.5.2
with:
args: --acl public-read
env:
AWS_S3_BUCKET: ${{ env.AWS_S3_BUCKET }}
AWS_S3_ENDPOINT: ${{ env.AWS_S3_ENDPOINT }}
SOURCE_DIR: txlmdb
- name: Verify Uploaded Files on S3
run: |
python -u verify_deploy.py
- name: If Failure Restart Job
if: failure()
run: |
git config --global user.email "info@nebl.io"
git config --global user.name "NeblioTeam"
git checkout master
git commit --allow-empty -m "Daily update of download.json for QuickSync ${DB_VER} Failed. Restarting."
git push https://${GHAT}@github.com/NeblioTeam/neblio-quicksync master
curl POST https://api.github.com/repos/NeblioTeam/neblio-quicksync/dispatches -H 'Accept: application/vnd.github.everest-preview+json' -H "Authorization: token $GHAT" --data '{"event_type": "restart_neblio_quicksync"}'
- name: If Success Update JSON Files & Cleanup
if: success()
run: |
git config --global user.email "info@nebl.io"
git config --global user.name "NeblioTeam"
git checkout master
python -u update_json.py
git add download.json
git commit -m "Daily update of download.json for QuickSync ${DB_VER}"
git push https://${GHAT}@github.com/NeblioTeam/neblio-quicksync master
echo "Removing Old Files"
aws s3 rm s3://neblio-asset-repo/quicksync/txlmdb-${DB_VER}/ --recursive --exclude "$GITHUB_SHA/*" --endpoint=https://sfo3.digitaloceanspaces.com