feat(ci): add buildkite config and agent setup scripts

.buildkite/agent_setup/linux/README.md

@@ -0,0 +1,29 @@
+# Buildkite linux agent setup
+
+## Debian, root
+
+### Sysdeps
+
+Run `./debian/sysdeps.sh` to install/check apt deps
+TODO: replace custom install docker with get.docker.com
+
+### Create new agent
+
+Run `./create-agent.sh <username>`
+
+### Start agent
+
+Run `systemctl start berty-build-agent@<username>.service`
+
+### Start agent at boot
+
+Run `systemctl enable berty-build-agent@<username>.service`
+
+## Any linux, no root
+
+### Init agent for your user
+
+Run `cd user && BUILDKITE_AGENT_TOKEN=xxx make deps`
+
+### Start agent
+Run `cd user && ./entrypoint.sh`

.buildkite/agent_setup/linux/berty-build-agent@.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=Buildkite Agent (%i)
+Documentation=https://buildkite.com/agent
+After=syslog.target
+After=network.target
+
+[Service]
+Type=simple
+User=%i
+Environment=HOME=/home/%i
+ExecStart=/home/%i/entrypoint.sh
+RestartSec=5
+Restart=on-failure
+RestartForceExitStatus=SIGPIPE
+TimeoutStartSec=10
+TimeoutStopSec=0
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
+DefaultInstance=1

.buildkite/agent_setup/linux/create-agent.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+set -e
+
+# Absolute path to this script, e.g. /home/user/bin/foo.sh
+SCRIPT=$(readlink -f "$0")
+# Absolute path this script is in, thus /home/user/bin
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+username="$1"
+
+[ -n "$username" ] || {
+  echo "Error: missing username argument" 1>&2
+  echo "Usage: $0 <username>" 1>&2
+  false
+}
+
+agent_token_file="/root/secrets/buildkite-agent-token"
+[ -f "$agent_token_file" ] || {
+  echo "Error: missing buildkite agent token file '$agent_token_file'" 1>&2
+  false
+}
+bootstrap_vars="BUILDKITE_AGENT_TOKEN=$(cat $agent_token_file)"
+
+# Add codecov token if present
+codecov_token_file="/root/secrets/codecov-token"
+[ ! -f "$codecov_token_file" ] || {
+  bootstrap_vars="$bootstrap_vars CODECOV_TOKEN=$(cat $codecov_token_file)"
+}
+
+useradd -m $username -G docker
+
+user_home=`getent passwd $username | cut -d: -f6`
+cp -r $SCRIPTPATH/user/* $user_home/
+chown -R $username:$username $user_home
+
+su -l $username -c "$bootstrap_vars make deps"
+
+echo
+echo "WARNING: YOU must add this key to the repo host (GitHub)"
+cat $user_home/.ssh/id_ed25519.pub
+
+echo
+echo "WARNING: If this agent must be in special queues, don't forget to edit tags in '$user_home/.buildkite-agent/buildkite-agent.cfg'"
+
+echo
+echo "INFO: To start agent run 'systemctl start berty-build-agent@$username.service"

.buildkite/agent_setup/linux/debian/bazel_requirements.txt

@@ -0,0 +1 @@
+unzip libssl-dev g++ zip

.buildkite/agent_setup/linux/debian/install_docker.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+
+# https://docs.docker.com/v17.12/install/linux/docker-ce/debian/#set-up-the-repository
+
+apt-get update
+apt-get -y install \
+     apt-transport-https \
+     ca-certificates \
+     curl \
+     gnupg2 \
+     software-properties-common
+curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+add-apt-repository -y \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable"
+
+apt-get update
+apt-get -y install docker-ce

.buildkite/agent_setup/linux/debian/install_yarn.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+apt remove -y cmdtest
+curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+apt update
+apt install -y yarn

.buildkite/agent_setup/linux/debian/sysdeps.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+# Absolute path to this script, e.g. /home/user/bin/foo.sh
+SCRIPT=$(readlink -f "$0")
+# Absolute path this script is in, thus /home/user/bin
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+$SCRIPTPATH/install_docker.sh
+$SCRIPTPATH/install_yarn.sh
+apt-get install -y $(cat $SCRIPTPATH/bazel_requirements.txt)
+install $SCRIPTPATH/../berty-build-agent@.service /etc/systemd/system
+systemctl daemon-reload

.buildkite/agent_setup/linux/user/.bazelrc

@@ -0,0 +1 @@
+build --disk_cache=~/.cache/bazel-disk-cache

.buildkite/agent_setup/linux/user/Makefile

@@ -0,0 +1,67 @@
+AGENT_GO_VERSION=1.12.10
+AGENT_BAZEL_VERSION=0.29.1
+AGENT_GOLANGCILINT_VERSION=1.21.0
+AGENT_KEY_TYPE=ed25519
+BUILDKITE_AGENT_TOKEN ?=xxx
+CODECOV_TOKEN ?=xxx
+
+check-program = $(foreach exec,$(1),$(if $(shell PATH="$(PATH)" which $(exec)),,$(error "No $(exec) in PATH")))
+$(call check-program, sed sh mkdir chmod tar mv ssh-keygen bash curl yarn docker)
+
+~/.ssh/id_%:
+	ssh-keygen -t $* -f $@ -P ""
+
+~/.buildkite-agent:
+	bash -c "`curl -sL https://raw.githubusercontent.com/buildkite/agent/master/install.sh`"
+	sed -i 's/^name="%hostname-%n"/name="%hostname-$(shell id -un)-%n"/g' $@/buildkite-agent.cfg
+	@sed -i 's/^token="xxx"/token="$(BUILDKITE_AGENT_TOKEN)"/g' $@/buildkite-agent.cfg
+	@echo sed -i 's/^token="xxx"/token="REDACTED"/g' $@/buildkite-agent.cfg
+
+~/.buildkite-agent/hooks/environment: ~/.buildkite-agent
+	mkdir -p ~/.buildkite-agent/hooks
+	cp ~/environment.hook.template.sh $@
+	chmod og-rwx $@
+	@sed -i 's/^export CODECOV_TOKEN="xxx"/export CODECOV_TOKEN="$(CODECOV_TOKEN)"/g' $@
+	@echo sed -i 's/^export CODECOV_TOKEN="xxx"/export CODECOV_TOKEN="REDACTED"/g' $@
+
+~/bin:
+	mkdir -p $@
+
+~/goroot: export go_archive=go$(AGENT_GO_VERSION).linux-amd64.tar.gz
+~/goroot:
+	rm -fr ~/tmp/go
+	mkdir -p ~/tmp/go
+	cd ~/tmp && curl -LO https://dl.google.com/go/$(go_archive)
+	cd ~/tmp && tar -xf $(go_archive)
+	mv ~/tmp/go $@
+
+~/bin/golangci-lint:
+	cd ~ && curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v$(AGENT_GOLANGCILINT_VERSION)
+
+~/bin/codecov: ~/bin
+	curl -s https://codecov.io/bash > $@
+	chmod +x $@
+
+~/go/pkg/mod:
+	mkdir -p $@
+
+~/bin/bazel: export bazel_installer=bazel-$(AGENT_BAZEL_VERSION)-installer-linux-x86_64.sh
+~/bin/bazel:
+	curl -LO https://github.com/bazelbuild/bazel/releases/download/$(AGENT_BAZEL_VERSION)/$(bazel_installer)
+	chmod +x $(bazel_installer)
+	./$(bazel_installer) --user
+
+.PHONY: deps
+deps: ~/.ssh/id_$(AGENT_KEY_TYPE) \
+      ~/.buildkite-agent/hooks/environment \
+      ~/.buildkite-agent \
+      ~/goroot \
+      ~/bin/golangci-lint \
+      ~/bin/codecov \
+      ~/bin/bazel \
+      ~/go/pkg/mod \
+
+
+.PHONY: start
+start: deps
+	~/entrypoint.sh

.buildkite/agent_setup/linux/user/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+export GOROOT=${HOME}/goroot
+PATH=${HOME}/bin:${GOROOT}/bin:${PATH} exec ${HOME}/.buildkite-agent/bin/buildkite-agent start

.buildkite/agent_setup/linux/user/environment.hook.template.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euo pipefail
+
+export CODECOV_TOKEN="xxx"

.buildkite/pipeline.yml

@@ -0,0 +1,122 @@
+steps:
+  - label: go-docker
+    agents:
+      queue: "bigcores"
+    commands:
+        - cd go
+        - docker build .
+
+  - label: go-generate
+    plugins:
+      - n0izn0iz/docker#v3.5.4:
+          image: bertytech/protoc:21
+          propagate-uid-gid: true
+          workdir: /go/src/berty.tech
+          environment: [ "GO111MODULE=on", "GOPATH=/go"]
+          volumes: [
+              "~/go/pkg/mod:/go/pkg/mod"
+            ]
+    commands:
+        - go version
+        - find . -name gen.sum -delete
+        - cd go
+        - time go mod vendor
+        - make generate_local
+        - make tidy
+        - cd ../docs
+        - make generate_local
+        - cd ..
+        - git status | cat
+        - git diff -w | cat
+        - git diff-index -w --quiet HEAD --
+
+  - label: githooks
+    plugins:
+      - n0izn0iz/docker#v3.5.4:
+          image: bertytech/githooks:v1
+          propagate-uid-gid: true
+          workdir: /go/src/berty.tech
+          environment: [ "GO111MODULE=on", "GOPATH=/go" ]
+    commands:
+      - ./githooks/pre-commit
+      - git status | cat
+      - git diff -w | cat
+      - git diff-index -w --quiet HEAD --
+
+  - label: js-generate
+    plugins:
+      - n0izn0iz/docker#v3.5.4:
+          image: bertytech/protoc:21
+          propagate-uid-gid: true
+          workdir: /go/src/berty.tech
+          environment: [ "GO111MODULE=on", "GOPATH=/go", "HOME=/home/buildkite-agent" ]
+          volumes: [
+              "~:/home/buildkite-agent",
+              "~/go/pkg/mod:/go/pkg/mod",
+            ]
+    agents:
+      queue: "bigcores"
+    commands:
+      - cd js
+      - make gen.clean
+      - make gen
+      - cd ..
+      - git status | cat
+      - git diff -w | cat
+      - git diff-index -w --quiet HEAD --
+
+  - label: js-lint
+    plugins:
+      - n0izn0iz/docker#v3.5.4:
+          image: bertytech/protoc:21
+          propagate-uid-gid: true
+          workdir: /go/src/berty.tech
+          environment: [ "GO111MODULE=on", "GOPATH=/go", "HOME=/home/buildkite-agent" ]
+          volumes: [
+              "~:/home/buildkite-agent",
+              "~/go/pkg/mod:/go/pkg/mod",
+            ]
+    agents:
+      queue: "bigcores"
+    commands:
+      - cd js
+      - make lint
+
+  - label: go-build-112
+    key: go-build-112
+    env:
+      GO111MODULE: "on"
+    agents:
+      queue: "golint"
+      # this is needed because golangci-lint is not multiuser
+      # https://github.com/golangci/golangci-lint/blob/fa69ddfc14ff9da080e14607d07d73e34750b426/pkg/commands/executor.go#L209
+      # + this job is not run in a container + go's os.TempDir seems to ignore TMPDIR in this case
+      # another solution is to put this job in a container or other kind of sandbox
+    commands:
+      - go version
+      - cd go
+
+      # TODO: cache $GOPATH/pkg/mod
+      - go mod download
+
+      # https://circleci.com/orbs/registry/orb/gotest/tools#commands-mod-tidy-check
+      - go mod tidy -v
+      - git --no-pager diff go.mod go.sum
+      - git --no-pager diff --quiet go.mod go.sum
+
+      - make go.install
+      - make go.unittest GO_TEST_OPTS="-v -test.timeout=120s"
+      - make tidy
+      # we can do this rm because we're in special queue
+      - rm -f /tmp/golangci-lint.lock
+      - make lint
+      - cd ..
+      - codecov -f ./go/coverage.txt
+
+  - label: bazel-go
+    commands:
+    - bazel --version
+    - go version
+    - cd go
+    - make bazel.unittest
+    - make bazel.build

.github/workflows/main.yaml

@@ -84,11 +84,6 @@ jobs:
         run: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.21.0
       - run: PATH=$PATH:$(pwd)/bin  make lint
         working-directory: ./go
-      - if: matrix.os == 'ubuntu-latest'
-        uses: codecov/codecov-action@v1.0.3
-        with:
-          token: ${{secrets.CODECOV_TOKEN}}
-          file: ./go/coverage.txt
 
   bazel-go:
     name: "Go: Bazel"