From d7c9d6e5a1523234458b979366f47fa231b2cc71 Mon Sep 17 00:00:00 2001 From: Liran BG Date: Wed, 19 May 2021 08:11:02 -0400 Subject: [PATCH] Migrate to Github action [1.1.x] (#2203) --- .github/workflows/ci.yaml | 143 +++++++++++++++++++++++++++ .github/workflows/release.yaml | 97 ++++++++++++++++++ .travis.yml | 72 -------------- Makefile | 8 ++ hack/scripts/ci/add-remove-labels.js | 24 +++++ hack/scripts/ci/free-space.sh | 27 +++++ 6 files changed, 299 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/release.yaml delete mode 100644 .travis.yml create mode 100644 hack/scripts/ci/add-remove-labels.js create mode 100644 hack/scripts/ci/free-space.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..d8bfb78d3e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,143 @@ +name: CI + +on: + pull_request: + branches: + - 1.1.x + push: + branches: + - 1.1.x + +env: + GO_VERSION: 1.12.x + GO111MODULE: off + GOPATH: /home/runner/work/nuclio/nuclio/go + NUCLIO_LABEL: ${{ github.run_number }} + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + path: go/src/github.com/nuclio/nuclio + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Lint + run: | + go get github.com/v3io/v3io-go-http + go get github.com/nuclio/logger + go get github.com/nuclio/nuclio-sdk-go \ + && cd $GOPATH/src/github.com/nuclio/nuclio-sdk-go \ + && git checkout v0.0.1 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + go get github.com/nuclio/amqp + go get github.com/v3io/scaler-types \ + && cd $GOPATH/src/github.com/v3io/scaler-types \ + && git checkout v1.0.0 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + make lint + + test_short: + name: Test short + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + path: go/src/github.com/nuclio/nuclio + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Run short test + run: | + go get github.com/v3io/v3io-go-http + go get github.com/nuclio/logger + go get github.com/nuclio/nuclio-sdk-go \ + && cd $GOPATH/src/github.com/nuclio/nuclio-sdk-go \ + && git checkout v0.0.1 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + go get github.com/nuclio/amqp + go get -d golang.org/x/net/... + go get github.com/v3io/scaler-types \ + && cd $GOPATH/src/github.com/v3io/scaler-types \ + && git checkout v1.0.0 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + make test-short + + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + path: go/src/github.com/nuclio/nuclio + + # since github-actions gives us 14G only, and fills it up with some garbage + # we will free up some space for us (~2GB) + - name: Freeing up disk space + run: | + chmod +x "${GITHUB_WORKSPACE}/go/src/github.com/nuclio/nuclio/hack/scripts/ci/free-space.sh" + "${GITHUB_WORKSPACE}/go/src/github.com/nuclio/nuclio/hack/scripts/ci/free-space.sh" + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Build + run: | + go get github.com/v3io/v3io-go-http + go get github.com/nuclio/logger + go get github.com/nuclio/nuclio-sdk-go \ + && cd $GOPATH/src/github.com/nuclio/nuclio-sdk-go \ + && git checkout v0.0.1 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + go get github.com/nuclio/amqp + go get -d golang.org/x/net/... + go get github.com/v3io/scaler-types \ + && cd $GOPATH/src/github.com/v3io/scaler-types \ + && git checkout v1.0.0 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + make docker-images + env: + NUCLIO_DOCKER_REPO: ${{ env.REPO }}/${{ env.REPO_NAME }} + + - name: Build binaries + run: | + cd $GOPATH/src/github.com/nuclio/nuclio + NUCLIO_OS=linux NUCLIO_ARCH=amd64 make tools + NUCLIO_OS=darwin NUCLIO_ARCH=amd64 make tools + NUCLIO_OS=windows NUCLIO_ARCH=amd64 make tools + env: + NUCLIO_NUCTL_CREATE_SYMLINK: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..0c053c9e42 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,97 @@ +name: Release + +on: + release: + types: + - created + +env: + REPO: quay.io + REPO_NAME: nuclio + GO_VERSION: 1.12.x + GO111MODULE: off + GOPATH: /home/runner/work/nuclio/nuclio/go + +jobs: + release: + name: Release + runs-on: ubuntu-latest + + if: github.repository == 'nuclio/nuclio' + steps: + - name: Dump github context + run: echo "$GITHUB_CONTEXT" + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + + - name: Dump runner context + run: echo "$RUNNER_CONTEXT" + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + + - name: Dump github ref + run: echo "$GITHUB_REF" + + - name: Set NUCLIO_LABEL to release tag + run: | + echo "NUCLIO_LABEL=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + path: go/src/github.com/nuclio/nuclio + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + # since github-actions gives us 14G only, and fills it up with some garbage + # we will free up some space for us (~2GB) + - name: Freeing up disk space + run: | + chmod +x "${GITHUB_WORKSPACE}/go/src/github.com/nuclio/nuclio/hack/scripts/ci/free-space.sh" + "${GITHUB_WORKSPACE}/go/src/github.com/nuclio/nuclio/hack/scripts/ci/free-space.sh" + + - uses: azure/docker-login@v1 + with: + login-server: ${{ env.REPO }} + username: ${{ secrets.QUAYIO_DOCKER_USERNAME }} + password: ${{ secrets.QUAYIO_DOCKER_PASSWORD }} + + - name: Build + run: | + go get github.com/v3io/v3io-go-http + go get github.com/nuclio/logger + go get github.com/nuclio/nuclio-sdk-go \ + && cd $GOPATH/src/github.com/nuclio/nuclio-sdk-go \ + && git checkout v0.0.1 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + go get github.com/nuclio/amqp + go get github.com/v3io/scaler-types \ + && cd $GOPATH/src/github.com/v3io/scaler-types \ + && git checkout v1.0.0 \ + && cd $GOPATH/src/github.com/nuclio/nuclio + make docker-images + env: + NUCLIO_DOCKER_REPO: ${{ env.REPO }}/${{ env.REPO_NAME }} + + - name: Push images + run: | + make push-docker-images + env: + NUCLIO_DOCKER_REPO: ${{ env.REPO }}/${{ env.REPO_NAME }} + + - name: Build binaries + run: | + cd $GOPATH/src/github.com/nuclio/nuclio + NUCLIO_OS=linux NUCLIO_ARCH=amd64 make tools + NUCLIO_OS=darwin NUCLIO_ARCH=amd64 make tools + NUCLIO_OS=windows NUCLIO_ARCH=amd64 make tools + env: + NUCLIO_NUCTL_CREATE_SYMLINK: false + + - name: Upload binaries + uses: AButler/upload-release-assets@v2.0 + with: + files: '/home/runner/work/nuclio/nuclio/go/bin/nuctl-*' + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f2f63f2618..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,72 +0,0 @@ -sudo: required - -services: - - docker - -addons: - apt: - packages: - - docker-ce - -language: go -go_import_path: github.com/nuclio/nuclio -go: "1.12" - -env: - global: - - GO111MODULE=off - -script: - - go get github.com/v3io/v3io-go-http - - go get github.com/nuclio/logger - - go get github.com/nuclio/nuclio-sdk-go - && cd $GOPATH/src/github.com/nuclio/nuclio-sdk-go - && git checkout v0.0.1 - && cd $GOPATH/src/github.com/nuclio/nuclio - - go get github.com/nuclio/amqp - - go get github.com/v3io/scaler-types - && cd $GOPATH/src/github.com/v3io/scaler-types - && git checkout v1.0.0 - && cd $GOPATH/src/github.com/nuclio/nuclio - - make lint test-short - - if [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then - make build test; - fi - -after_success: - - bash <(curl -s https://codecov.io/bash) - - if [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_BRANCH" == "development" ]; then - export NUCLIO_LABEL="unstable"; - fi - - if [ "$TRAVIS_TAG" != "" ]; then - export NUCLIO_LABEL="$TRAVIS_TAG"; - fi - - echo $TRAVIS_PULL_REQUEST - - echo $TRAVIS_BRANCH - - echo $NUCLIO_LABEL - - if [ -n "$NUCLIO_LABEL" ]; then - echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin quay.io && - make docker-images push-docker-images && - if [ "$NUCLIO_LABEL" != "unstable" ]; then - docker tag "quay.io/nuclio/dashboard:$NUCLIO_LABEL-amd64" quay.io/nuclio/dashboard:stable-amd64 && - docker push quay.io/nuclio/dashboard:stable-amd64; - fi - fi - - echo "Done." - -before_deploy: - - NUCLIO_OS=linux NUCLIO_ARCH=amd64 make tools - - NUCLIO_OS=darwin NUCLIO_ARCH=amd64 make tools - - NUCLIO_OS=windows NUCLIO_ARCH=amd64 make tools - -deploy: - provider: releases - api_key: - secure: XYb8aOndiae7AoLyZU2e4s4GKKyoJjjW8tA+gG0HHD6AhqbaaSx6IofwNbQqOhM0iGOTWeeG3SaW34MPRcjV3FX0fYO8lW2GAynXT6Td5uhJmPtmk/msrfdLlCVYhQ42DoWjvtWwaejJRZCYHNlhAwH6c/peSO7KqN0QaRABfO4CK6Npvt9JMo+hCfxnUmaVBzkbtfXIrBlxkXxVOGn/geUzf+2qpnMw85WMIN8RIXedNBe0UBkBasPJA5hKadckMKDCaIbiQ8KsOF2HK7IGwg/goTMfshuOZn/f8aCC+D5lyCQD3bkuHJqrj+HYCVHalAeESXJGmSHr/xmV+XQ9RoxdSUaO2TBU4r+WPPM1JsNNAvQe4zxEaVXOu41bRbZnccxwCHhPcKwT+KD7IniJMUdRkTadGPx+yZoMGwxtGK6elDyr3yzzYRfjBIcfecaPVifny51bRxWpGwcaoWP0qew86vqMcCz9dQBZUvKA/+zdcKXZHvqbouh+Iv79z7BVaW7aTmcsIhn7miqH5qIOnfJUsZiBAAY5EX7FpXMtSx9cpna+vKsvtRtSnywLsLDiLAPLWE0muTDPDksDsmTT0AvHQENv0bd5U0qnTlaR2Q31Q2e8WY0y7DZrYgruNGTvVtwrrAv4KvBHjO0GJya4NvnZB37yMFTKJouZU/o8qm4= - file: - - $GOPATH/bin/nuctl-$NUCLIO_LABEL-linux-amd64 - - $GOPATH/bin/nuctl-$NUCLIO_LABEL-darwin-amd64 - - $GOPATH/bin/nuctl-$NUCLIO_LABEL-windows-amd64 - skip_cleanup: true - on: - tags: true diff --git a/Makefile b/Makefile index 2388f7a389..4cdb071249 100644 --- a/Makefile +++ b/Makefile @@ -145,13 +145,16 @@ print-docker-images: # Tools # +NUCLIO_NUCTL_CREATE_SYMLINK := $(if $(NUCLIO_NUCTL_CREATE_SYMLINK),$(NUCLIO_NUCTL_CREATE_SYMLINK),true) NUCTL_BIN_NAME = nuctl-$(NUCLIO_LABEL)-$(NUCLIO_OS)-$(NUCLIO_ARCH) NUCTL_TARGET = $(GOPATH)/bin/nuctl nuctl: ensure-gopath $(GO_BUILD_TOOL) -o /go/bin/$(NUCTL_BIN_NAME) cmd/nuctl/main.go @rm -f $(NUCTL_TARGET) +ifeq ($(NUCLIO_NUCTL_CREATE_SYMLINK), true) @ln -sF $(GOPATH)/bin/$(NUCTL_BIN_NAME) $(NUCTL_TARGET) +endif processor: ensure-gopath docker build --file cmd/processor/Dockerfile --tag $(NUCLIO_DOCKER_REPO)/processor:$(NUCLIO_DOCKER_IMAGE_TAG) . @@ -313,6 +316,11 @@ IMAGES_TO_PUSH += $(NUCLIO_DOCKER_HANDLER_BUILDER_JAVA_ONBUILD_IMAGE_NAME) # # Testing # + +.PHONY: fmt +fmt: + gofmt -s -w . + .PHONY: lint lint: ensure-gopath @echo Installing linters... diff --git a/hack/scripts/ci/add-remove-labels.js b/hack/scripts/ci/add-remove-labels.js new file mode 100644 index 0000000000..514ec4de54 --- /dev/null +++ b/hack/scripts/ci/add-remove-labels.js @@ -0,0 +1,24 @@ +module.exports = async ({github, context, prNumber, labelsToAdd, labelsToRemove}) => { + + console.log(`Adding labels ${labelsToAdd} to PR #${prNumber}`) + await github.issues.addLabels({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + labels: labelsToAdd + }) + + console.log(`Removing labels ${labelsToRemove} from PR #${prNumber}`) + await Promise.all(labelsToRemove.map(labelName => github.issues.removeLabel({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }).catch(error => { + if (error.toString().toLowerCase().includes('label does not exist')) { + console.log(`Ignoring not existing label error: ${error.toString()}`) + return + } + throw error + }))) +} diff --git a/hack/scripts/ci/free-space.sh b/hack/scripts/ci/free-space.sh new file mode 100644 index 0000000000..9e21a7f7e5 --- /dev/null +++ b/hack/scripts/ci/free-space.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env sh + +print_free_space() { + df --human-readable +} + +# before cleanup +print_free_space + +# clean unneeded os packages and misc +sudo apt-get remove --yes '^dotnet-.*' 'php.*' azure-cli google-cloud-sdk google-chrome-stable firefox powershell +sudo apt-get autoremove --yes +sudo apt clean + +# cleanup unneeded share dirs ~30GB +sudo rm --recursive --force \ + /usr/local/lib/android \ + /usr/share/dotnet \ + /usr/share/miniconda \ + /usr/share/dotnet \ + /usr/share/swift + +# clean unneeded docker images +docker system prune --all --force + +# post cleanup +print_free_space