From 42b36510140913a06fd5ef462caa191542941e19 Mon Sep 17 00:00:00 2001 From: 0x2b3bfa0 <0x2b3bfa0+git@googlemail.com> Date: Fri, 30 Apr 2021 19:03:36 +0200 Subject: [PATCH 1/3] Enforce strict argument parsing Previously, yargs was silently ignoring unknown arguments and producing esoteric bugs on edge cases like --option ---ARGUMENT--- --- bin/cml-pr.js | 1 + bin/cml-publish.js | 1 + bin/cml-runner.js | 1 + bin/cml-send-comment.js | 1 + bin/cml-send-github-check.js | 1 + bin/cml-tensorboard-dev.js | 1 + 6 files changed, 6 insertions(+) diff --git a/bin/cml-pr.js b/bin/cml-pr.js index dcde81bbe..6f713b495 100644 --- a/bin/cml-pr.js +++ b/bin/cml-pr.js @@ -16,6 +16,7 @@ const run = async (opts) => { const opts = decamelize( yargs + .strict() .usage('Usage: $0 ') .describe('md', 'Output in markdown format [](url).') .boolean('md') diff --git a/bin/cml-publish.js b/bin/cml-publish.js index 59f1c738b..f294a02f2 100644 --- a/bin/cml-publish.js +++ b/bin/cml-publish.js @@ -37,6 +37,7 @@ const run = async (opts) => { pipe_args.load('binary'); const data = pipe_args.piped_arg(); const argv = yargs + .strict() .usage(`Usage: $0 `) .describe('md', 'Output in markdown format [title || name](url).') .boolean('md') diff --git a/bin/cml-runner.js b/bin/cml-runner.js index 1a86ffce4..9f59b108d 100755 --- a/bin/cml-runner.js +++ b/bin/cml-runner.js @@ -315,6 +315,7 @@ const run = async (opts) => { const opts = decamelize( yargs + .strict() .usage(`Usage: $0`) .default('labels', RUNNER_LABELS) .describe( diff --git a/bin/cml-send-comment.js b/bin/cml-send-comment.js index 179ce6842..819a75449 100644 --- a/bin/cml-send-comment.js +++ b/bin/cml-send-comment.js @@ -25,6 +25,7 @@ const run = async (opts) => { }; const argv = yargs + .strict() .usage('Usage: $0 ') .default('commit-sha') .describe( diff --git a/bin/cml-send-github-check.js b/bin/cml-send-github-check.js index f35b0ecf7..367be2fcd 100644 --- a/bin/cml-send-github-check.js +++ b/bin/cml-send-github-check.js @@ -18,6 +18,7 @@ const run = async (opts) => { }; const argv = yargs + .strict() .usage('Usage: $0 ') .default('head-sha') .describe( diff --git a/bin/cml-tensorboard-dev.js b/bin/cml-tensorboard-dev.js index c24b6b95c..d374858a6 100644 --- a/bin/cml-tensorboard-dev.js +++ b/bin/cml-tensorboard-dev.js @@ -92,6 +92,7 @@ const run = async (opts) => { }; const argv = yargs + .strict() .usage(`Usage: $0`) .default('credentials') .describe( From f24bc78f2efedc3c848c2defc435ef623bbd8815 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 30 Apr 2021 12:04:32 +0100 Subject: [PATCH 2/3] lint YAML --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2664457a7..f079b1f47 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "*.js": [ "eslint --fix" ], - "*.md": [ + "*.{md,yaml,yml}": [ "prettier --write" ] }, From 7ee050edd105f3bae93d4f1e1e4f00e2c38451f5 Mon Sep 17 00:00:00 2001 From: "Olivaw[bot]" <64868532+iterative-olivaw@users.noreply.github.com> Date: Fri, 30 Apr 2021 12:08:54 +0100 Subject: [PATCH 3/3] lint YAML --- .github/workflows/checkbot.yml | 34 +++--- .github/workflows/dependancron.yml | 60 +++++------ .github/workflows/gitlab.yml | 68 ++++++------ .github/workflows/publish.yml | 160 +++++++++++++++-------------- .gitlab-ci.yml | 16 ++- bitbucket-pipelines.yml | 4 +- 6 files changed, 170 insertions(+), 172 deletions(-) diff --git a/.github/workflows/checkbot.yml b/.github/workflows/checkbot.yml index 7112a4a7c..4a93047ce 100644 --- a/.github/workflows/checkbot.yml +++ b/.github/workflows/checkbot.yml @@ -45,7 +45,7 @@ jobs: with: python-version: '3.x' - - name: "CML test" + - name: 'CML test' env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -79,41 +79,41 @@ jobs: test_machine_aws: needs: check-container - runs-on: [self-hosted,cml-runner-aws-gpu] + runs-on: [self-hosted, cml-runner-aws-gpu] steps: - - name: "tests" - run: | - nvidia-smi + - name: 'tests' + run: | + nvidia-smi test_container_aws: needs: check-container - runs-on: [self-hosted,cml-runner-aws-gpu] + runs-on: [self-hosted, cml-runner-aws-gpu] container: image: docker://dvcorg/cml-test options: --gpus all steps: - - name: "tests" - run: | - nvidia-smi + - name: 'tests' + run: | + nvidia-smi test_machine_az: needs: check-container - runs-on: [self-hosted,cml-runner-az-gpu] + runs-on: [self-hosted, cml-runner-az-gpu] steps: - - name: "tests" - run: | - nvidia-smi + - name: 'tests' + run: | + nvidia-smi test_container_az: needs: check-container - runs-on: [self-hosted,cml-runner-az-gpu] + runs-on: [self-hosted, cml-runner-az-gpu] container: image: docker://dvcorg/cml-test options: --gpus all steps: - - name: "tests" - run: | - nvidia-smi + - name: 'tests' + run: | + nvidia-smi chatbot: if: contains(github.event.comment.body, '/cml-') diff --git a/.github/workflows/dependancron.yml b/.github/workflows/dependancron.yml index bbbbb4378..67264af81 100644 --- a/.github/workflows/dependancron.yml +++ b/.github/workflows/dependancron.yml @@ -1,41 +1,41 @@ -name: "Update dependencies" +name: 'Update dependencies' on: schedule: - # runs every six hours + # runs every six hours - cron: 0 */6 * * * - + env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: dependancron: runs-on: [ubuntu-latest] - + steps: - - uses: actions/checkout@v2 - - - name: "Check dependencies" - id: check - run: | - sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list - sudo apt update && sudo apt -y install dvc - CML_VER=$(docker run dvcorg/cml:latest dvc --version) - DVC_VER=$(dvc --version) - UPDATE=false - MESSAGE="Updating DVC from $CML_VER to $DVC_VER" - if [ $CML_VER != $DVC_VER ]; then - UPDATE=true - echo "$DVC_VER" > .dependancron - fi + - uses: actions/checkout@v2 + + - name: 'Check dependencies' + id: check + run: | + sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list + sudo apt update && sudo apt -y install dvc + CML_VER=$(docker run dvcorg/cml:latest dvc --version) + DVC_VER=$(dvc --version) + UPDATE=false + MESSAGE="Updating DVC from $CML_VER to $DVC_VER" + if [ $CML_VER != $DVC_VER ]; then + UPDATE=true + echo "$DVC_VER" > .dependancron + fi - echo "::set-output name=update::${UPDATE}" - echo "::set-output name=comment::${MESSAGE}" + echo "::set-output name=update::${UPDATE}" + echo "::set-output name=comment::${MESSAGE}" - - name: Create Pull Request - if: steps.check.outputs.update == 'true' - uses: peter-evans/create-pull-request@v3 - with: - title: 'chore: dependancron update dependencies' - body: ${{ steps.check.outputs.comment }} - branch: 'dependancron-update-dependencies' + - name: Create Pull Request + if: steps.check.outputs.update == 'true' + uses: peter-evans/create-pull-request@v3 + with: + title: 'chore: dependancron update dependencies' + body: ${{ steps.check.outputs.comment }} + branch: 'dependancron-update-dependencies' diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 6c6438c4f..905ef82e1 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -7,7 +7,7 @@ jobs: test: runs-on: ubuntu-latest services: - gitlab: + gitlab: image: docker://gitlab/gitlab-ce ports: - 8000:8000 @@ -26,54 +26,44 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - + - name: Configure credentials run: >- - docker exec ${{ job.services.gitlab.id }} bin/gitlab-rails runner " - ; user = User.find_by_username('root') - ; user.password = '${{ github.token }}' - ; user.password_confirmation = '${{ github.token }}' - ; user.save! - ; token = user.personal_access_tokens.create(scopes: [:api], name: 'Token') - ; token.set_token('${{ github.token }}') - ; token.save! - " - + docker exec ${{ job.services.gitlab.id }} bin/gitlab-rails runner " ; + user = User.find_by_username('root') ; user.password = '${{ + github.token }}' ; user.password_confirmation = '${{ github.token }}' + ; user.save! ; token = user.personal_access_tokens.create(scopes: + [:api], name: 'Token') ; token.set_token('${{ github.token }}') ; + token.save! " + - name: Create test project run: >- - curl "http://localhost:8000/gitlab/api/v4/projects" - --header "PRIVATE-TOKEN: ${{ github.token }}" - --request POST - --get - --data "name=test" - + curl "http://localhost:8000/gitlab/api/v4/projects" --header + "PRIVATE-TOKEN: ${{ github.token }}" --request POST --get --data + "name=test" + - name: Create test commit run: >- - curl "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/files/README.md" - --header "PRIVATE-TOKEN: ${{ github.token }}" - --request POST - --get - --data "author_email=test@test" - --data "author_name=Test" - --data "branch=main" - --data "commit_message=Create%20README.md" - --data "content=Test" - + curl + "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/files/README.md" + --header "PRIVATE-TOKEN: ${{ github.token }}" --request POST --get + --data "author_email=test@test" --data "author_name=Test" --data + "branch=main" --data "commit_message=Create%20README.md" --data + "content=Test" + - name: Get last commit id: commit run: >- - curl "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/commits/main" - --header "PRIVATE-TOKEN: ${{ github.token }}" - --request GET | jq -r .id | xargs -0 printf "::set-output name=hash::%s" - + curl + "http://localhost:8000/gitlab/api/v4/projects/root%2Ftest/repository/commits/main" + --header "PRIVATE-TOKEN: ${{ github.token }}" --request GET | jq -r + .id | xargs -0 printf "::set-output name=hash::%s" + - name: Install dependencies run: npm ci - + - name: Run cml-send-comment run: >- - node bin/cml-send-comment.js - --token=${{ github.token }} - --repo=http://localhost:8000/gitlab/root/test - --commit-sha=${{ steps.commit.outputs.hash }} - --driver=gitlab - <(echo message) + node bin/cml-send-comment.js --token=${{ github.token }} + --repo=http://localhost:8000/gitlab/root/test --commit-sha=${{ + steps.commit.outputs.hash }} --driver=gitlab <(echo message) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 02406a188..8bdfc606d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,17 +1,17 @@ -name: "Publish CML dockers" +name: 'Publish CML dockers' on: [push, pull_request] env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} TEST_GITHUB_REPO: https://github.com/iterative/cml_qa_tests_dummy TEST_GITHUB_SHA: 62edc8b3f46a60b3fe1e5c08fd3e0046d350ee29 TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }} TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy TEST_GITLAB_SHA: c4c13286e78dc252dd2611f31a755f10d343fbd4 TEST_BBCLOUD_TOKEN: ${{ secrets.TEST_BBCLOUD_TOKEN }} - TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy + TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy TEST_BBCLOUD_SHA: 9bb9131ce0af294fe1c6eedca1f2bce3983e80bd jobs: @@ -19,83 +19,93 @@ jobs: runs-on: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: "npm ci" - run: npm ci + - name: 'npm ci' + run: npm ci - - name: "lint" - run: npm run lint + - name: 'lint' + run: npm run lint - - name: "tests" - run: | - sudo update-alternatives --install /usr/bin/python python $(which python3) 10 - sudo apt-get update && sudo apt-get install -y python3-pip - sudo pip install --upgrade pip - sudo pip install --upgrade setuptools - sudo pip install tensorboard + - name: 'tests' + run: | + sudo update-alternatives --install /usr/bin/python python $(which python3) 10 + sudo apt-get update && sudo apt-get install -y python3-pip + sudo pip install --upgrade pip + sudo pip install --upgrade setuptools + sudo pip install tensorboard - npm run test + npm run test - - name: Publish CML docker image - # only publish if push to master (dvcorg/cml:latest) - # or create a tag in the repo (dvcorg/cml:tag) - if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: dvcorg/cml - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile - context: ./ - cache: true - tag_names: true + - name: Publish CML docker image + # only publish if push to master (dvcorg/cml:latest) + # or create a tag in the repo (dvcorg/cml:tag) + if: + github.event_name == 'push' && (contains(github.ref, 'tags') || + github.ref == 'refs/heads/master') + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: dvcorg/cml + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + dockerfile: ./docker/Dockerfile + context: ./ + cache: true + tag_names: true - - name: Publish cml-py3 docker image - if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - env: - DOCKER_FROM: cml - with: - name: dvcorg/cml-py3 - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile-py3 - context: ./ - cache: true - tag_names: true - buildargs: DOCKER_FROM + - name: Publish cml-py3 docker image + if: + github.event_name == 'push' && (contains(github.ref, 'tags') || + github.ref == 'refs/heads/master') + uses: elgohr/Publish-Docker-Github-Action@master + env: + DOCKER_FROM: cml + with: + name: dvcorg/cml-py3 + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + dockerfile: ./docker/Dockerfile-py3 + context: ./ + cache: true + tag_names: true + buildargs: DOCKER_FROM - - name: Publish cml-dev docker image - if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: dvcorg/cml-dev - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile-dev - context: ./ - cache: true - tag_names: true + - name: Publish cml-dev docker image + if: + github.event_name == 'push' && (contains(github.ref, 'tags') || + github.ref == 'refs/heads/master') + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: dvcorg/cml-dev + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + dockerfile: ./docker/Dockerfile-dev + context: ./ + cache: true + tag_names: true - # cloud-runner aliases will be deprecated - - name: Alias cloud-runner - if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master') - run: | - echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - - docker tag dvcorg/cml dvcorg/cml-gpu && docker push dvcorg/cml-gpu - docker tag dvcorg/cml-py3 dvcorg/cml-gpu-py3 && docker push dvcorg/cml-gpu-py3 - - docker tag dvcorg/cml dvcorg/cml-cloud-runner && docker push dvcorg/cml-cloud-runner - docker tag dvcorg/cml-py3 dvcorg/cml-py3-cloud-runner && docker push dvcorg/cml-py3-cloud-runner - docker tag dvcorg/cml-gpu dvcorg/cml-gpu-cloud-runner && docker push dvcorg/cml-gpu-cloud-runner - docker tag dvcorg/cml-gpu-py3 dvcorg/cml-gpu-py3-cloud-runner && docker push dvcorg/cml-gpu-py3-cloud-runner + # cloud-runner aliases will be deprecated + - name: Alias cloud-runner + if: + github.event_name == 'push' && (contains(github.ref, 'tags') || + github.ref == 'refs/heads/master') + run: | + echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + + docker tag dvcorg/cml dvcorg/cml-gpu && docker push dvcorg/cml-gpu + docker tag dvcorg/cml-py3 dvcorg/cml-gpu-py3 && docker push dvcorg/cml-gpu-py3 - - name: Publish to NPM - if: github.event_name == 'push' && (contains(github.ref, 'tags') || github.ref == 'refs/heads/master') - run: | - npm config set //registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN - npm publish - env: - NPM_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + docker tag dvcorg/cml dvcorg/cml-cloud-runner && docker push dvcorg/cml-cloud-runner + docker tag dvcorg/cml-py3 dvcorg/cml-py3-cloud-runner && docker push dvcorg/cml-py3-cloud-runner + docker tag dvcorg/cml-gpu dvcorg/cml-gpu-cloud-runner && docker push dvcorg/cml-gpu-cloud-runner + docker tag dvcorg/cml-gpu-py3 dvcorg/cml-gpu-py3-cloud-runner && docker push dvcorg/cml-gpu-py3-cloud-runner + + - name: Publish to NPM + if: + github.event_name == 'push' && (contains(github.ref, 'tags') || + github.ref == 'refs/heads/master') + run: | + npm config set //registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN + npm publish + env: + NPM_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34ee39d56..67e88c507 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,12 +7,8 @@ deploy_job: when: always image: dvcorg/cml script: - - cml-runner - --cloud aws - --cloud-region us-west - --cloud-type g2.2xlarge - --cloud-spot - --labels=cml-runner-gpu + - cml-runner --cloud aws --cloud-region us-west --cloud-type g2.2xlarge + --cloud-spot --labels=cml-runner-gpu test_runner: stage: test @@ -20,15 +16,16 @@ test_runner: tags: - cml-runner-gpu script: - - sudo update-alternatives --install /usr/bin/python python $(which python3) 10 - - sudo apt-get update && sudo apt-get install -y python-pip python3-pip + - sudo update-alternatives --install /usr/bin/python python $(which python3) + 10 + - sudo apt-get update && sudo apt-get install -y python-pip python3-pip - sudo pip install --upgrade pip - sudo pip install --upgrade setuptools - sudo pip install tensorboard - npm ci - npm run lint - - npm run test + - npm run test - nvidia-smi @@ -47,4 +44,3 @@ test_container: - npm run test - nvidia-smi - \ No newline at end of file diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index f39fb47a5..4bf120b8c 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -19,6 +19,8 @@ pipelines: script: # Modify the commands below to build your repository. - npm install - echo "# My first CML report" > report.md - - echo "![](https://static.boredpanda.com/blog/wp-content/uploads/2020/07/funny-expressive-dog-corgi-genthecorgi-1-1-5f0ea719ea38a__700.jpg)" >> report.md + - echo + "![](https://static.boredpanda.com/blog/wp-content/uploads/2020/07/funny-expressive-dog-corgi-genthecorgi-1-1-5f0ea719ea38a__700.jpg)" + >> report.md - echo "So much data viz" >> report.md - node bin/cml-send-comment report.md