Merge branch 'main' into rgl-fix-capture-actions-logs

tinkerbell · Nov 2, 2021 · c6ce199 · c6ce199
2 parents c362740 + 948f687
commit c6ce199
Show file tree

Hide file tree

Showing 31 changed files with 2,806 additions and 2,722 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -6,7 +6,7 @@ env:
   CGO_ENABLED: 0
 
 jobs:
-  lint:
+  verify:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -15,8 +15,8 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: "1.17.0"
-      - name: make lint
-        run: make lint
+      - name: make verify
+        run: make verify
   test:
     runs-on: ubuntu-latest
     steps:
@@ -43,8 +43,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - ci-checks
-      - lint
       - test
+      - verify
     steps:
       - name: fake
         run: echo ":+1:"

diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,4 @@ terraform.tfstate.backup
 envrc
 .env
 deploy/state
+out/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -177,8 +177,11 @@ It has a base server that implements the API.
 ### protos
 
 The `protos` package contains all the protobuf files used by the gRPC server.
-Handling of protobuf deps and generating the go files are both handled by the [protoc.sh](./protos/protoc.sh) script.
-Also, both `go`, and `protoc` are required by `protoc.sh`.
+Handling of protobuf deps and generating the go files are both handled [buf] via `make pbfiles`.
+The protubuf/grpc files are not generated as part of the build pipeline to keep builds fast.
+CI will ensure generated files are up to date.
+
+[buf]: https://buf.build/
 
 ```
 .
@@ -188,3 +191,7 @@ Also, both `go`, and `protoc` are required by `protoc.sh`.
 │   ├── template
 │   └── workflow
 ```
+
+### environment variables
+
+Tink Server, CLI, and Worker environment variables are documented [here](docs/ENVVARS.md).
diff --git a/Makefile b/Makefile
@@ -1,6 +1,10 @@
+help: ## Print this help
+	@grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*##/·/' | sort | column -ts '·' -c 120
+
 all: cli server worker ## Build all binaries for host OS and CPU
 
 -include rules.mk
+-include lint.mk
 
 crosscompile: $(crossbinaries) ## Build all binaries for Linux and all supported CPU arches
 images: tink-cli-image tink-server-image tink-worker-image ## Build all docker images
@@ -10,53 +14,12 @@ test: ## Run tests
 	go clean -testcache
 	go test ./... -v
 
-verify: lint
+verify: lint check-generated # Verify code style, is lint free, freshness ...
+	gofumpt -s -d .
 
-help: ## Print this help
-	@grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*##/·/' | sort | column -ts '·' -c 120
+generated: pbfiles protomocks ## Generate dynamically created files
+check-generated: check-pbfiles check-protomocks ## Check if generated files are up to date
+
+tools: ${toolsBins} ## Build Go based build tools
 
-# BEGIN: lint-install --dockerfile=warn .
-# http://github.com/tinkerbell/lint-install
-
-GOLINT_VERSION ?= v1.42.0
-HADOLINT_VERSION ?= v2.7.0
-SHELLCHECK_VERSION ?= v0.7.2
-LINT_OS := $(shell uname)
-LINT_ARCH := $(shell uname -m)
-LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
-
-# shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation
-ifeq ($(LINT_OS),Darwin)
-	ifeq ($(LINT_ARCH),arm64)
-		LINT_ARCH=x86_64
-	endif
-endif
-
-LINT_LOWER_OS  = $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
-GOLINT_CONFIG:=$(LINT_ROOT)/.golangci.yml
-
-lint: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-	out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run
-	out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) --no-fail $(shell find . -name "*Dockerfile")
-	out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh")
-
-fix: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-	out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run --fix
-	out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh") -f diff | git apply -p2 -
-
-out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck:
-	mkdir -p out/linters
-	curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_LOWER_OS).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
-	mv out/linters/shellcheck-$(SHELLCHECK_VERSION) out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
-
-out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH):
-	mkdir -p out/linters
-	curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
-	chmod u+x out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
-
-out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH):
-	mkdir -p out/linters
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLINT_VERSION)
-	mv out/linters/golangci-lint out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-
-# END: lint-install --dockerfile=warn .
+.PHONY: all check-generated crosscompile generated help images run test tools verify
diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -0,0 +1,13 @@
+version: v1
+plugins:
+  - name: go
+    out: .
+    opt: paths=source_relative
+  - name: go-grpc
+    out: .
+    opt:
+      - paths=source_relative
+      - require_unimplemented_servers=false
+  - name: grpc-gateway
+    out: .
+    opt: paths=source_relative
diff --git a/buf.lock b/buf.lock
@@ -0,0 +1,10 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    branch: main
+    commit: 389e82394fcc4e18b2463c26674170b3
+    digest: b1-j-IWUPcGJWMkqbee6hR7IJCXlLF9Y5P4ARue8gLwLP0=
+    create_time: 2021-10-01T15:05:49.20888Z
diff --git a/buf.yaml b/buf.yaml
@@ -0,0 +1,9 @@
+version: v1
+lint:
+  use:
+    - DEFAULT
+deps:
+  - buf.build/googleapis/googleapis
+breaking:
+  use:
+    - FILE
diff --git a/ci-checks.sh b/ci-checks.sh
@@ -12,7 +12,7 @@ if ! git ls-files '*.sh' '*.go' '*.md' | xargs codespell -q 3 -I .codespell-whit
 fi
 
 # --check doesn't show what line number fails, so write the result to disk for the diff to catch
-if ! git ls-files '*.yml' '*.json' '*.md' | xargs prettier --list-different --write; then
+if ! git ls-files '*.yaml' '*.yml' '*.json' '*.md' | xargs prettier --list-different --write; then
 	failed=1
 fi
 

diff --git a/docs/ENVVARS.md b/docs/ENVVARS.md
@@ -0,0 +1,38 @@
+# Environment Variables
+
+The follow describes environment variables available to be set when running Tink Server, Tink CLI, or Tink Worker.
+
+| Name                                                                                           | Type   | Service(s)        | Description                                                                                                                                   |
+| ---------------------------------------------------------------------------------------------- | ------ | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
+| `TINK_AUTH_USERNAME=tink`                                                                      | string | server            | username to use for basic auth to http endpoints                                                                                              |
+| `TINK_AUTH_PASSWORD=tink`                                                                      | string | server            | password to use for basic auth to http endpoints                                                                                              |
+| `TINKERBELL_CERT_URL=http://127.0.0.1:42114/cert`                                              | string | cli/worker        | url from which to get a TLS certificate, needed when Tink Server's TLS cert is signed by an unknown certificate authority, ie self-signed     |
+| `CERTS_DIR=/certs`                                                                             | string | server            | a directory which contains the `bundle.pem` and `server-key.pem` files, for use when running Tink with TLS                                    |
+| `TINKERBELL_CERTS_DIR=/certs`                                                                  | string | server            | same as `CERTS_DIR`, deprecated in server                                                                                                     |
+| `TLS_CERT="-----BEGIN RSA PRIVATE KEY-----\n....\n-----END RSA PRIVATE KEY-----\n"`            | string | server            | a TLS certificate for use with Tink server                                                                                                    |
+| `TINKERBELL_TLS_CERT="-----BEGIN RSA PRIVATE KEY-----\n....\n-----END RSA PRIVATE KEY-----\n"` | string | server            | same as `TLS_CERT`, deprecated in server                                                                                                      |
+| `GRPC_AUTHORITY=127.0.0.1:42113`                                                               | string | server            | url of the Tink gRPC server                                                                                                                   |
+| `TINKERBELL_GRPC_AUTHORITY=127.0.0.1:42113`                                                    | string | server/cli/worker | same as `GRPC_AUTHORITY`, deprecated in server                                                                                                |
+| `HTTP_AUTHORITY=127.0.0.1:42114`                                                               | string | server            | url of the Tink HTTP server                                                                                                                   |
+| `TINKERBELL_HTTP_AUTHORITY=127.0.0.1:42114`                                                    | string | server            | same as `HTTP_AUTHORITY`, deprecated in server                                                                                                |
+| `FACILITY=onprem`                                                                              | string | server/cli        | location for which the Tink server serves, deprecated in server                                                                               |
+| `POSTGRES_DATABASE=tinkerbell`                                                                 | string | server            | name of the PostgreSQL database for use in the Tink server                                                                                    |
+| `PGDATABASE=tinkerbell`                                                                        | string | server            | same as `POSTGRES_DATABASE`, deprecated in server                                                                                             |
+| `POSTGRES_USER=tink`                                                                           | string | server            | PostgreSQL username for connecting to the DB                                                                                                  |
+| `PGUSER=tink`                                                                                  | string | server            | same as `POSTGRES_USER`, deprecated in server                                                                                                 |
+| `POSTGRES_PASSWORD=tink`                                                                       | string | server            | PostgreSQL password for connecting to the DB                                                                                                  |
+| `PGPASSWORD=tink`                                                                              | string | server            | same as `POSTGRES_PASSWORD`, deprecated in server                                                                                             |
+| `POSTGRES_SSLMODE=disable`                                                                     | string | server            | sets the PostgreSQL SSL priority [docs](https://www.postgresql.org/docs/10/libpq-connect.html#LIBPQ-CONNECT-SSLMODE)                          |
+| `PGSSLMODE=disable`                                                                            | string | server            | same as `POSTGRES_SSLMODE`, deprecated in server                                                                                              |
+| `MAX_WORKFLOW_DATA_VERSIONS=`                                                                  | int    | server            | maximum number of workflow data versions to be kept in database                                                                               |
+| `EVENTS_TTL=60`                                                                                | string | server            | purges the events in the events table that have passed this TTL in minutes                                                                    |
+| `ONLY_MIGRATION=true`                                                                          | bool   | server            | if set to true, only POSTGRES migrations are executed                                                                                         |
+| `TINK_CLI_VERSION="0.0.0"`                                                                     | string | cli               | if set to `0.0.0`, the old get command is used                                                                                                |
+| `DOCKER_REGISTRY=`                                                                             | string | worker            | the docker registry to use for pulling images                                                                                                 |
+| `REGISTRY_PASSWORD=`                                                                           | string | worker            | the password for the docker registry                                                                                                          |
+| `REGISTRY_USERNAME=`                                                                           | string | worker            | the username for the docker registry                                                                                                          |
+| `ID=`                                                                                          | string | worker            | the id of the workflow to be executed                                                                                                         |
+| `RETRY_INTERVAL=`                                                                              | int    | worker            | the interval in seconds between retries for setting up connections to, querying for workflows from, and sending status reports to Tink Server |
+| `MAX_RETRIES=`                                                                                 | int    | worker            | the maximum number of retries for setting up connections and sending status reports to Tink Server                                            |
+| `MAX_FILE_SIZE=`                                                                               | int    | worker            | the maximum size in bytes for the Tink worker data file                                                                                       |
+| `CAPTURE_ACTION_LOGS=`                                                                         | bool   | worker            | Capture action container output as part of worker logs                                                                                        |
diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/tinkerbell/tink
 go 1.13
 
 require (
+	github.com/bufbuild/buf v1.0.0-rc2
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/equinix-labs/otel-init-go v0.0.1
@@ -16,6 +17,7 @@ require (
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/ktr0731/evans v0.10.0
 	github.com/lib/pq v1.10.1
+	github.com/matryer/moq v0.2.3
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550
 	github.com/pkg/errors v0.9.1
@@ -24,15 +26,16 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
-	github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816
 	github.com/stretchr/testify v1.7.0
 	github.com/testcontainers/testcontainers-go v0.11.1
 	go.mongodb.org/mongo-driver v1.1.2 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.22.0
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
-	google.golang.org/grpc v1.39.0
+	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6
+	google.golang.org/grpc v1.41.0-dev.0.20210907181116-2f3355d2244e
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v2 v2.4.0
+	mvdan.cc/gofumpt v0.1.1
 )
 
 replace github.com/stormcat24/protodep => github.com/ackintosh/protodep v0.0.0-20200728152107-abf8eb579d6c