Skip to content

Bump werkzeug from 0.16.1 to 3.0.1 #51

Bump werkzeug from 0.16.1 to 3.0.1

Bump werkzeug from 0.16.1 to 3.0.1 #51

Workflow file for this run

name: benchmarks
on:
workflow_dispatch:
pull_request:
types: [labeled]
jobs:
deploy-cloud-runner:
if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }}
runs-on: [ubuntu-latest]
container: docker://dvcorg/cml
steps:
- name: deploy
env:
repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
VPC: ${{ secrets.AWS_CI_VPC }}
run: |
echo "Deploying..."
RUNNER_LABELS="cml,aws"
RUNNER_REPO="https://github.com/${GITHUB_REPOSITORY}"
MACHINE="cml$(date +%s)"
docker-machine create \
--driver amazonec2 \
--amazonec2-instance-type p3.8xlarge \
--amazonec2-vpc-id $VPC \
--amazonec2-region us-east-1 \
--amazonec2-zone c \
--amazonec2-ssh-user ubuntu \
--amazonec2-ami ami-06a25ee8966373068 \
--amazonec2-root-size 150 \
$MACHINE
eval "$(docker-machine env --shell sh $MACHINE)"
(
docker-machine ssh $MACHINE "sudo mkdir -p \
/docker_machine && \
sudo chmod 777 /docker_machine" && \
docker-machine scp -r -q ~/.docker/machine/ \
$MACHINE:/docker_machine && \
docker run --name runner -d \
--gpus all \
-v /docker_machine/machine:/root/.docker/machine \
--net host \
--ipc host \
-e DOCKER_MACHINE=$MACHINE \
-e repo_token=$repo_token \
-e RUNNER_LABELS=$RUNNER_LABELS \
-e RUNNER_REPO=$RUNNER_REPO \
-e RUNNER_IDLE_TIMEOUT=120 \
dvcorg/cml-py3:latest && \
sleep 20 && echo "Deployed $MACHINE"
) || (echo "Shut down machine" && docker-machine rm -y -f $MACHINE && exit 1)
run-benchmark:
if: ${{ (github.event.action == 'labeled' && github.event.label.name == 'benchmark') || github.event.action == 'workflow_dispatch' }}
needs: deploy-cloud-runner
runs-on: [self-hosted,cml]
steps:
- uses: actions/checkout@v2
- name: cml_run
env:
repo_token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CI_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
run: |
apt-get update -y
apt-get install python3-dev -y
pip install -r requirements.txt
pip install .
cd test/benchmarks && python question_answering_accuracy.py
echo -en "## Benchmarks: QA Accuracy\n" >> accuracy_report.md
cat results_accuracy.md >> accuracy_report.md
cml-send-comment accuracy_report.md
python question_answering_components.py
echo -en "## Benchmarks: QA per component\n" >> components_report.md
cat results_per_component.md >> components_report.md
cml-send-comment components_report.md