From d0f1ffe21417af220a4be32019061b9004cb1811 Mon Sep 17 00:00:00 2001 From: n0izn0iz Date: Sat, 30 Nov 2019 14:51:22 +0100 Subject: [PATCH] feat(ci): add buildkite config and agent setup scripts --- .buildkite/agent_setup/linux/README.md | 29 +++++ .../linux/berty-build-agent@.service | 21 +++ .buildkite/agent_setup/linux/create-agent.sh | 47 +++++++ .../linux/debian/bazel_requirements.txt | 1 + .../linux/debian/install_docker.sh | 21 +++ .../agent_setup/linux/debian/install_yarn.sh | 10 ++ .../agent_setup/linux/debian/sysdeps.sh | 14 ++ .buildkite/agent_setup/linux/user/.bazelrc | 1 + .buildkite/agent_setup/linux/user/Makefile | 67 ++++++++++ .../agent_setup/linux/user/entrypoint.sh | 3 + .../linux/user/environment.hook.template.sh | 4 + .buildkite/pipeline.yml | 122 ++++++++++++++++++ .github/workflows/main.yaml | 5 - 13 files changed, 340 insertions(+), 5 deletions(-) create mode 100644 .buildkite/agent_setup/linux/README.md create mode 100644 .buildkite/agent_setup/linux/berty-build-agent@.service create mode 100755 .buildkite/agent_setup/linux/create-agent.sh create mode 100644 .buildkite/agent_setup/linux/debian/bazel_requirements.txt create mode 100755 .buildkite/agent_setup/linux/debian/install_docker.sh create mode 100755 .buildkite/agent_setup/linux/debian/install_yarn.sh create mode 100755 .buildkite/agent_setup/linux/debian/sysdeps.sh create mode 100644 .buildkite/agent_setup/linux/user/.bazelrc create mode 100644 .buildkite/agent_setup/linux/user/Makefile create mode 100755 .buildkite/agent_setup/linux/user/entrypoint.sh create mode 100644 .buildkite/agent_setup/linux/user/environment.hook.template.sh create mode 100644 .buildkite/pipeline.yml diff --git a/.buildkite/agent_setup/linux/README.md b/.buildkite/agent_setup/linux/README.md new file mode 100644 index 0000000000..b53bb1c527 --- /dev/null +++ b/.buildkite/agent_setup/linux/README.md @@ -0,0 +1,29 @@ +# Buildkite linux agent setup + +## Debian, root + +### Sysdeps + +Run `./debian/sysdeps.sh` to install/check apt deps +TODO: replace custom install docker with get.docker.com + +### Create new agent + +Run `./create-agent.sh ` + +### Start agent + +Run `systemctl start berty-build-agent@.service` + +### Start agent at boot + +Run `systemctl enable berty-build-agent@.service` + +## Any linux, no root + +### Init agent for your user + +Run `cd user && BUILDKITE_AGENT_TOKEN=xxx make deps` + +### Start agent +Run `cd user && ./entrypoint.sh` diff --git a/.buildkite/agent_setup/linux/berty-build-agent@.service b/.buildkite/agent_setup/linux/berty-build-agent@.service new file mode 100644 index 0000000000..d5e963a30c --- /dev/null +++ b/.buildkite/agent_setup/linux/berty-build-agent@.service @@ -0,0 +1,21 @@ +[Unit] +Description=Buildkite Agent (%i) +Documentation=https://buildkite.com/agent +After=syslog.target +After=network.target + +[Service] +Type=simple +User=%i +Environment=HOME=/home/%i +ExecStart=/home/%i/entrypoint.sh +RestartSec=5 +Restart=on-failure +RestartForceExitStatus=SIGPIPE +TimeoutStartSec=10 +TimeoutStopSec=0 +KillMode=process + +[Install] +WantedBy=multi-user.target +DefaultInstance=1 diff --git a/.buildkite/agent_setup/linux/create-agent.sh b/.buildkite/agent_setup/linux/create-agent.sh new file mode 100755 index 0000000000..c3c35e5ff9 --- /dev/null +++ b/.buildkite/agent_setup/linux/create-agent.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +set -e + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname "$SCRIPT") + +username="$1" + +[ -n "$username" ] || { + echo "Error: missing username argument" 1>&2 + echo "Usage: $0 " 1>&2 + false +} + +agent_token_file="/root/secrets/buildkite-agent-token" +[ -f "$agent_token_file" ] || { + echo "Error: missing buildkite agent token file '$agent_token_file'" 1>&2 + false +} +bootstrap_vars="BUILDKITE_AGENT_TOKEN=$(cat $agent_token_file)" + +# Add codecov token if present +codecov_token_file="/root/secrets/codecov-token" +[ ! -f "$codecov_token_file" ] || { + bootstrap_vars="$bootstrap_vars CODECOV_TOKEN=$(cat $codecov_token_file)" +} + +useradd -m $username -G docker + +user_home=`getent passwd $username | cut -d: -f6` +cp -r $SCRIPTPATH/user/* $user_home/ +chown -R $username:$username $user_home + +su -l $username -c "$bootstrap_vars make deps" + +echo +echo "WARNING: YOU must add this key to the repo host (GitHub)" +cat $user_home/.ssh/id_ed25519.pub + +echo +echo "WARNING: If this agent must be in special queues, don't forget to edit tags in '$user_home/.buildkite-agent/buildkite-agent.cfg'" + +echo +echo "INFO: To start agent run 'systemctl start berty-build-agent@$username.service" diff --git a/.buildkite/agent_setup/linux/debian/bazel_requirements.txt b/.buildkite/agent_setup/linux/debian/bazel_requirements.txt new file mode 100644 index 0000000000..51fa1544d3 --- /dev/null +++ b/.buildkite/agent_setup/linux/debian/bazel_requirements.txt @@ -0,0 +1 @@ +unzip libssl-dev g++ zip diff --git a/.buildkite/agent_setup/linux/debian/install_docker.sh b/.buildkite/agent_setup/linux/debian/install_docker.sh new file mode 100755 index 0000000000..747e3106c3 --- /dev/null +++ b/.buildkite/agent_setup/linux/debian/install_docker.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +# https://docs.docker.com/v17.12/install/linux/docker-ce/debian/#set-up-the-repository + +apt-get update +apt-get -y install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common +curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +add-apt-repository -y \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" + +apt-get update +apt-get -y install docker-ce diff --git a/.buildkite/agent_setup/linux/debian/install_yarn.sh b/.buildkite/agent_setup/linux/debian/install_yarn.sh new file mode 100755 index 0000000000..874911adc0 --- /dev/null +++ b/.buildkite/agent_setup/linux/debian/install_yarn.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +apt remove -y cmdtest +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list + +apt update +apt install -y yarn diff --git a/.buildkite/agent_setup/linux/debian/sysdeps.sh b/.buildkite/agent_setup/linux/debian/sysdeps.sh new file mode 100755 index 0000000000..a42ff61e92 --- /dev/null +++ b/.buildkite/agent_setup/linux/debian/sysdeps.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e + +# Absolute path to this script, e.g. /home/user/bin/foo.sh +SCRIPT=$(readlink -f "$0") +# Absolute path this script is in, thus /home/user/bin +SCRIPTPATH=$(dirname "$SCRIPT") + +$SCRIPTPATH/install_docker.sh +$SCRIPTPATH/install_yarn.sh +apt-get install -y $(cat $SCRIPTPATH/bazel_requirements.txt) +install $SCRIPTPATH/../berty-build-agent@.service /etc/systemd/system +systemctl daemon-reload diff --git a/.buildkite/agent_setup/linux/user/.bazelrc b/.buildkite/agent_setup/linux/user/.bazelrc new file mode 100644 index 0000000000..2d27116d5f --- /dev/null +++ b/.buildkite/agent_setup/linux/user/.bazelrc @@ -0,0 +1 @@ +build --disk_cache=~/.cache/bazel-disk-cache diff --git a/.buildkite/agent_setup/linux/user/Makefile b/.buildkite/agent_setup/linux/user/Makefile new file mode 100644 index 0000000000..832aba1403 --- /dev/null +++ b/.buildkite/agent_setup/linux/user/Makefile @@ -0,0 +1,67 @@ +AGENT_GO_VERSION=1.12.10 +AGENT_BAZEL_VERSION=0.29.1 +AGENT_GOLANGCILINT_VERSION=1.21.0 +AGENT_KEY_TYPE=ed25519 +BUILDKITE_AGENT_TOKEN ?=xxx +CODECOV_TOKEN ?=xxx + +check-program = $(foreach exec,$(1),$(if $(shell PATH="$(PATH)" which $(exec)),,$(error "No $(exec) in PATH"))) +$(call check-program, sed sh mkdir chmod tar mv ssh-keygen bash curl yarn docker) + +~/.ssh/id_%: + ssh-keygen -t $* -f $@ -P "" + +~/.buildkite-agent: + bash -c "`curl -sL https://raw.githubusercontent.com/buildkite/agent/master/install.sh`" + sed -i 's/^name="%hostname-%n"/name="%hostname-$(shell id -un)-%n"/g' $@/buildkite-agent.cfg + @sed -i 's/^token="xxx"/token="$(BUILDKITE_AGENT_TOKEN)"/g' $@/buildkite-agent.cfg + @echo sed -i 's/^token="xxx"/token="REDACTED"/g' $@/buildkite-agent.cfg + +~/.buildkite-agent/hooks/environment: ~/.buildkite-agent + mkdir -p ~/.buildkite-agent/hooks + cp ~/environment.hook.template.sh $@ + chmod og-rwx $@ + @sed -i 's/^export CODECOV_TOKEN="xxx"/export CODECOV_TOKEN="$(CODECOV_TOKEN)"/g' $@ + @echo sed -i 's/^export CODECOV_TOKEN="xxx"/export CODECOV_TOKEN="REDACTED"/g' $@ + +~/bin: + mkdir -p $@ + +~/goroot: export go_archive=go$(AGENT_GO_VERSION).linux-amd64.tar.gz +~/goroot: + rm -fr ~/tmp/go + mkdir -p ~/tmp/go + cd ~/tmp && curl -LO https://dl.google.com/go/$(go_archive) + cd ~/tmp && tar -xf $(go_archive) + mv ~/tmp/go $@ + +~/bin/golangci-lint: + cd ~ && curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v$(AGENT_GOLANGCILINT_VERSION) + +~/bin/codecov: ~/bin + curl -s https://codecov.io/bash > $@ + chmod +x $@ + +~/go/pkg/mod: + mkdir -p $@ + +~/bin/bazel: export bazel_installer=bazel-$(AGENT_BAZEL_VERSION)-installer-linux-x86_64.sh +~/bin/bazel: + curl -LO https://github.com/bazelbuild/bazel/releases/download/$(AGENT_BAZEL_VERSION)/$(bazel_installer) + chmod +x $(bazel_installer) + ./$(bazel_installer) --user + +.PHONY: deps +deps: ~/.ssh/id_$(AGENT_KEY_TYPE) \ + ~/.buildkite-agent/hooks/environment \ + ~/.buildkite-agent \ + ~/goroot \ + ~/bin/golangci-lint \ + ~/bin/codecov \ + ~/bin/bazel \ + ~/go/pkg/mod \ + + +.PHONY: start +start: deps + ~/entrypoint.sh diff --git a/.buildkite/agent_setup/linux/user/entrypoint.sh b/.buildkite/agent_setup/linux/user/entrypoint.sh new file mode 100755 index 0000000000..79cf9dc2f5 --- /dev/null +++ b/.buildkite/agent_setup/linux/user/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +export GOROOT=${HOME}/goroot +PATH=${HOME}/bin:${GOROOT}/bin:${PATH} exec ${HOME}/.buildkite-agent/bin/buildkite-agent start diff --git a/.buildkite/agent_setup/linux/user/environment.hook.template.sh b/.buildkite/agent_setup/linux/user/environment.hook.template.sh new file mode 100644 index 0000000000..45ce5132ec --- /dev/null +++ b/.buildkite/agent_setup/linux/user/environment.hook.template.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail + +export CODECOV_TOKEN="xxx" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000000..0e676b0402 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,122 @@ +steps: + - label: go-docker + agents: + queue: "bigcores" + commands: + - cd go + - docker build . + + - label: go-generate + plugins: + - n0izn0iz/docker#v3.5.4: + image: bertytech/protoc:21 + propagate-uid-gid: true + workdir: /go/src/berty.tech + environment: [ "GO111MODULE=on", "GOPATH=/go"] + volumes: [ + "~/go/pkg/mod:/go/pkg/mod" + ] + commands: + - go version + - find . -name gen.sum -delete + - cd go + - time go mod vendor + - make generate_local + - make tidy + - cd ../docs + - make generate_local + - cd .. + - git status | cat + - git diff -w | cat + - git diff-index -w --quiet HEAD -- + + - label: githooks + plugins: + - n0izn0iz/docker#v3.5.4: + image: bertytech/githooks:v1 + propagate-uid-gid: true + workdir: /go/src/berty.tech + environment: [ "GO111MODULE=on", "GOPATH=/go" ] + commands: + - ./githooks/pre-commit + - git status | cat + - git diff -w | cat + - git diff-index -w --quiet HEAD -- + + - label: js-generate + plugins: + - n0izn0iz/docker#v3.5.4: + image: bertytech/protoc:21 + propagate-uid-gid: true + workdir: /go/src/berty.tech + environment: [ "GO111MODULE=on", "GOPATH=/go", "HOME=/home/buildkite-agent" ] + volumes: [ + "~:/home/buildkite-agent", + "~/go/pkg/mod:/go/pkg/mod", + ] + agents: + queue: "bigcores" + commands: + - cd js + - make gen.clean + - make gen + - cd .. + - git status | cat + - git diff -w | cat + - git diff-index -w --quiet HEAD -- + + - label: js-lint + plugins: + - n0izn0iz/docker#v3.5.4: + image: bertytech/protoc:21 + propagate-uid-gid: true + workdir: /go/src/berty.tech + environment: [ "GO111MODULE=on", "GOPATH=/go", "HOME=/home/buildkite-agent" ] + volumes: [ + "~:/home/buildkite-agent", + "~/go/pkg/mod:/go/pkg/mod", + ] + agents: + queue: "bigcores" + commands: + - cd js + - make lint + + - label: go-build-112 + key: go-build-112 + env: + GO111MODULE: "on" + agents: + queue: "golint" + # this is needed because golangci-lint is not multiuser + # https://github.com/golangci/golangci-lint/blob/fa69ddfc14ff9da080e14607d07d73e34750b426/pkg/commands/executor.go#L209 + # + this job is not run in a container + go's os.TempDir seems to ignore TMPDIR in this case + # another solution is to put this job in a container or other kind of sandbox + commands: + - go version + - cd go + + # TODO: cache $GOPATH/pkg/mod + - go mod download + + # https://circleci.com/orbs/registry/orb/gotest/tools#commands-mod-tidy-check + - go mod tidy -v + - git --no-pager diff go.mod go.sum + - git --no-pager diff --quiet go.mod go.sum + + - make go.install + - make go.unittest GO_TEST_OPTS="-v -test.timeout=120s" + - make tidy + # we can do this rm because we're in special queue + - rm -f /tmp/golangci-lint.lock + - make lint + - cd .. + - codecov -f ./go/coverage.txt + + - label: bazel-go + commands: + - bazel --version + - go version + - cd go + - make bazel.unittest + - make bazel.build diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c2cb259e32..b3a0adc73c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -84,11 +84,6 @@ jobs: run: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.21.0 - run: PATH=$PATH:$(pwd)/bin make lint working-directory: ./go - - if: matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v1.0.3 - with: - token: ${{secrets.CODECOV_TOKEN}} - file: ./go/coverage.txt bazel-go: name: "Go: Bazel"