diff --git a/.crd-docs.yaml b/.crd-docs.yaml new file mode 100644 index 00000000..892566f0 --- /dev/null +++ b/.crd-docs.yaml @@ -0,0 +1,10 @@ +processor: + ignoreTypes: + - "(EtcdCluster)List$" + - "(EtcdCluster)Status$" + ignoreFields: + - "status$" + - "TypeMeta$" + +render: + kubernetesVersion: 1.30.0 diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 97389b05..0b6f370f 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer @@ -49,7 +49,7 @@ jobs: # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} if: github.event_name != 'pull_request' - uses: docker/login-action@v3.1.0 + uses: docker/login-action@v3.2.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/helm-publish.yaml b/.github/workflows/helm-publish.yaml index e9e5d214..98d7cf28 100644 --- a/.github/workflows/helm-publish.yaml +++ b/.github/workflows/helm-publish.yaml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 - name: Install Helm uses: azure/setup-helm@v4.2.0 diff --git a/.github/workflows/hugo.yaml b/.github/workflows/hugo.yaml index 82db238d..6fe53973 100644 --- a/.github/workflows/hugo.yaml +++ b/.github/workflows/hugo.yaml @@ -41,7 +41,7 @@ jobs: - name: Install Dart Sass run: sudo snap install dart-sass - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 with: submodules: recursive fetch-depth: 0 diff --git a/.github/workflows/make-test-e2e.yaml b/.github/workflows/make-test-e2e.yaml index 0f597722..ccac6da4 100644 --- a/.github/workflows/make-test-e2e.yaml +++ b/.github/workflows/make-test-e2e.yaml @@ -15,22 +15,28 @@ jobs: strategy: matrix: k8s: - - version: 1.28.3 + - version: v1.28.0 attribute: penultimate - - version: 1.29.3 + - version: v1.29.0 attribute: previous - - version: 1.30.0 + - version: default attribute: latest runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - uses: actions/setup-go@v5.0.1 with: - go-version: 1.22.2 + go-version: 1.22.4 - uses: docker/setup-buildx-action@v3.3.0 - uses: tale/kubectl-action@v1.4.0 with: kubectl-version: v1.30.0 # Empty kubeconfig file base64-kube-config: "YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCnByZWZlcmVuY2VzOiB7fQo=" - - run: ENVTEST_K8S_VERSION=${{ matrix.k8s.version }} make test-e2e + - run: | + if [ "${{ matrix.k8s.version }}" = "default" ]; then + # For latest version use default from Makefile + make test-e2e + else + K8S_VERSION=${{ matrix.k8s.version }} make test-e2e + fi diff --git a/.github/workflows/make-test.yaml b/.github/workflows/make-test.yaml index 060b2c54..8d58d1bd 100644 --- a/.github/workflows/make-test.yaml +++ b/.github/workflows/make-test.yaml @@ -15,16 +15,22 @@ jobs: strategy: matrix: k8s: - - version: 1.28.3 + - version: v1.28.0 attribute: penultimate - - version: 1.29.3 + - version: v1.29.0 attribute: previous - - version: 1.30.0 + - version: default attribute: latest runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - uses: actions/setup-go@v5.0.1 with: - go-version: 1.22.2 - - run: ENVTEST_K8S_VERSION=${{ matrix.k8s.version }} make test + go-version: 1.22.4 + - run: | + if [ "${{ matrix.k8s.version }}" = "default" ]; then + # For latest version use default from Makefile + make test + else + K8S_VERSION=${{ matrix.k8s.version }} make test + fi diff --git a/.github/workflows/nilaway-lint.yaml b/.github/workflows/nilaway-lint.yaml index 6484dfce..3da57c48 100644 --- a/.github/workflows/nilaway-lint.yaml +++ b/.github/workflows/nilaway-lint.yaml @@ -7,8 +7,8 @@ jobs: nilaway-lint: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - uses: actions/setup-go@v5.0.1 with: - go-version: 1.22.2 + go-version: 1.22.4 - run: make nilaway-lint diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 8139f70c..d5479a01 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -7,8 +7,8 @@ jobs: pre-commit: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - uses: actions/setup-go@v5.0.1 with: - go-version: 1.22.2 + go-version: 1.22.4 - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release-assets.yaml b/.github/workflows/release-assets.yaml index df6d9d83..8d58970d 100644 --- a/.github/workflows/release-assets.yaml +++ b/.github/workflows/release-assets.yaml @@ -14,10 +14,10 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - uses: actions/setup-go@v5.0.1 with: - go-version: 1.22.2 + go-version: 1.22.4 - name: Get tag from current run run: | TAG=${{ github.ref_name }} diff --git a/.gitignore b/.gitignore index b447f47a..ab2a770c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ vendor # editor and IDE paraphernalia .idea +.vscode *.swp *.swo *~ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a612aa6f..e8203552 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer - exclude: "^charts/etcd-operator/(values.schema.json|README.md)$" + exclude: "^charts/etcd-operator/(values.schema.json|README.md)|site/content/en/docs/v([.0-9]+)/reference/api.md$" - id: check-added-large-files - repo: local hooks: @@ -24,6 +24,11 @@ repos: entry: sh -c "make lint-fix" language: system require_serial: true + - id: make-generate-docs + name: make-generate-docs + entry: sh -c "make generate-docs" + language: system + require_serial: true - id: make-mod-tidy name: make-mod-tidy entry: sh -c "make mod-tidy" diff --git a/CODEOWNERS b/CODEOWNERS index c676aa43..d81b6e67 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,10 +1,10 @@ -/.github @hiddenmarten @AlexGluck +/.github @hiddenmarten @AlexGluck @aobort /api @sircthulhu @Kirill-Garbar -/charts @hiddenmarten @AlexGluck +/charts @hiddenmarten @AlexGluck @aobort /cmd @sircthulhu @Kirill-Garbar /config @sircthulhu @Kirill-Garbar /hack @sircthulhu @Kirill-Garbar /internal @sircthulhu @Kirill-Garbar @sergeyshevch -/test @sircthulhu @Kirill-Garbar @hiddenmarten @AlexGluck +/test @sircthulhu @Kirill-Garbar @hiddenmarten @AlexGluck @aobort /site @sergeyshevch @Kirill-Garbar /*.md @kvaps @Kirill-Garbar diff --git a/Dockerfile b/Dockerfile index ce6e3f30..f3175bcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22.2 AS builder +FROM golang:1.22.4 AS builder ARG TARGETOS TARGETARCH WORKDIR /workspace @@ -11,7 +11,7 @@ COPY go.mod go.sum ./ RUN go mod download # Copy the go source -COPY cmd/main.go ./cmd/ +COPY cmd/ ./cmd/ COPY api/ ./api/ COPY internal/ ./internal/ diff --git a/Makefile b/Makefile index 64e324e1..722b17f4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # Image URL to use all building/pushing image targets IMG ?= ghcr.io/aenix-io/etcd-operator:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +# K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. # renovate: datasource=github-tags depName=kubernetes/kubernetes -ENVTEST_K8S_VERSION ?= v1.30.0 -ENVTEST_K8S_VERSION_TRIMMED_V = $(subst v,,$(ENVTEST_K8S_VERSION)) +K8S_VERSION ?= v1.30.0 +K8S_VERSION_TRIMMED_V = $(subst v,,$(K8S_VERSION)) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -55,6 +55,11 @@ manifests: controller-gen yq ## Generate WebhookConfiguration, ClusterRole and C generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." +.PHONY: generate-docs +generate-docs: crd-ref-docs ## Generate CRD reference documentation. + @$(eval VERSION := $(shell $(YQ) '.params.version' site/hugo.yaml)) + $(CRD_REF_DOCS) --config=.crd-docs.yaml --renderer=markdown --templates-dir="site/reference-templates" --output-path="site/content/en/docs/$(VERSION)/reference/api.md" + .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... @@ -69,7 +74,9 @@ mod-tidy: ## Run go mod tidy against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + @echo "Check for kubernetes version $(K8S_VERSION_TRIMMED_V) in $(ENVTEST)" + @$(ENVTEST) list | grep -q $(K8S_VERSION_TRIMMED_V) + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. @@ -155,9 +162,9 @@ KIND_CLUSTER_NAME ?= etcd-operator-kind NAMESPACE ?= etcd-operator-system # renovate: datasource=github-tags depName=prometheus-operator/prometheus-operator -PROMETHEUS_OPERATOR_VERSION ?= v0.73.2 +PROMETHEUS_OPERATOR_VERSION ?= v0.74.0 # renovate: datasource=github-tags depName=jetstack/cert-manager -CERT_MANAGER_VERSION ?= v1.14.5 +CERT_MANAGER_VERSION ?= v1.15.0 ifndef ignore-not-found ignore-not-found = false @@ -191,9 +198,13 @@ kind-load: docker-build kind ## Build and upload docker image to the local Kind $(KIND) load docker-image ${IMG} --name $(KIND_CLUSTER_NAME) .PHONY: kind-create -kind-create: kind ## Create kubernetes cluster using Kind. +kind-create: kind yq ## Create kubernetes cluster using Kind. @if ! $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then \ - $(KIND) create cluster --name $(KIND_CLUSTER_NAME); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ + fi + @if ! $(CONTAINER_TOOL) container inspect $$($(KIND) get nodes) | $(YQ) e '.[0].Config.Image' | grep -q $(K8S_VERSION); then \ + $(KIND) delete cluster --name $(KIND_CLUSTER_NAME); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ fi .PHONY: kind-delete @@ -234,6 +245,7 @@ KIND ?= $(LOCALBIN)/kind HELM ?= $(LOCALBIN)/helm HELM_DOCS ?= $(LOCALBIN)/helm-docs YQ = $(LOCALBIN)/yq +CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs ## Tool Versions # renovate: datasource=github-tags depName=kubernetes-sigs/kustomize @@ -242,17 +254,17 @@ KUSTOMIZE_VERSION ?= v5.3.0 CONTROLLER_TOOLS_VERSION ?= v0.15.0 ENVTEST_VERSION ?= latest # renovate: datasource=github-tags depName=golangci/golangci-lint -GOLANGCI_LINT_VERSION ?= v1.58.0 +GOLANGCI_LINT_VERSION ?= v1.59.0 # renovate: datasource=github-tags depName=kubernetes-sigs/kind -KIND_VERSION ?= v0.22.0 +KIND_VERSION ?= v0.23.0 # renovate: datasource=github-tags depName=helm/helm -HELM_VERSION ?= v3.14.4 +HELM_VERSION ?= v3.15.1 # renovate: datasource=github-tags depName=losisin/helm-values-schema-json -HELM_SCHEMA_VERSION ?= v1.3.0 +HELM_SCHEMA_VERSION ?= v1.4.0 # renovate: datasource=github-tags depName=norwoodj/helm-docs HELM_DOCS_VERSION ?= v1.13.1 # renovate: datasource=github-tags depName=mikefarah/yq -YQ_VERSION ?= v4.43.1 +YQ_VERSION ?= v4.44.1 ## Tool install scripts KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" @@ -274,6 +286,10 @@ controller-gen: $(LOCALBIN) envtest: $(LOCALBIN) @test -x $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION) +.PHONY: crd-ref-docs +crd-ref-docs: $(LOCALBIN) + @test -x $(CRD_REF_DOCS) || GOBIN=$(LOCALBIN) go install github.com/elastic/crd-ref-docs@latest + .PHONY: golangci-lint golangci-lint: $(LOCALBIN) @test -x $(GOLANGCI_LINT) && $(GOLANGCI_LINT) version | grep -q $(GOLANGCI_LINT_VERSION) || \ diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index 205861ff..71cf65a6 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -98,6 +98,18 @@ func (r *EtcdCluster) CalculateQuorumSize() int { return int(*r.Spec.Replicas)/2 + 1 } +func (c *EtcdCluster) IsClientSecurityEnabled() bool { + return c.Spec.Security != nil && c.Spec.Security.TLS.ClientSecret != "" +} + +func (c *EtcdCluster) IsServerSecurityEnabled() bool { + return c.Spec.Security != nil && c.Spec.Security.TLS.ServerSecret != "" +} + +func (c *EtcdCluster) IsServerTrustedCADefined() bool { + return c.Spec.Security != nil && c.Spec.Security.TLS.ServerTrustedCASecret != "" +} + // +kubebuilder:object:root=true // EtcdClusterList contains a list of EtcdCluster @@ -174,24 +186,36 @@ type SecuritySpec struct { // Section for user-managed tls certificates // +optional TLS TLSSpec `json:"tls,omitempty"` + // Section to enable etcd auth + EnableAuth bool `json:"enableAuth,omitempty"` } // TLSSpec defines user-managed certificates names. type TLSSpec struct { - // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerTrustedCASecret string `json:"peerTrustedCASecret,omitempty"` // Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional PeerSecret string `json:"peerSecret,omitempty"` + // Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + // It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used. + // This secret must be created in the namespace with etcdCluster CR. + // +optional + ServerTrustedCASecret string `json:"serverTrustedCASecret,omitempty"` // Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). // It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ServerSecret string `json:"serverSecret,omitempty"` - // Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + // Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ClientTrustedCASecret string `json:"clientTrustedCASecret,omitempty"` // Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + // This secret must be created in the namespace with etcdCluster CR. // +optional ClientSecret string `json:"clientSecret,omitempty"` } diff --git a/api/v1alpha1/etcdcluster_types_test.go b/api/v1alpha1/etcdcluster_types_test.go index 144131c4..e9640a2b 100644 --- a/api/v1alpha1/etcdcluster_types_test.go +++ b/api/v1alpha1/etcdcluster_types_test.go @@ -20,3 +20,105 @@ var _ = Context("CalculateQuorumSize", func() { Expect(etcdCluster.CalculateQuorumSize()).To(Equal(3)) }) }) + +var _ = Describe("Aux Functions", func() { + + Context("When running IsClientSecurityEnabled function", func() { + It("should return true if ClientSecret is set", func() { + cluster := EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{ + ClientSecret: "some-client-secret", + }, + }, + }, + } + Expect(cluster.IsClientSecurityEnabled()).To(BeTrue()) + }) + + It("should return false if ClientSecret is not set", func() { + cluster := EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{}, + }, + }, + } + Expect(cluster.IsClientSecurityEnabled()).To(BeFalse()) + }) + + It("should return false if Security is nil", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{}, + } + Expect(cluster.IsClientSecurityEnabled()).To(BeFalse()) + }) + }) + + Context("When running IsServerSecurityEnabled function", func() { + It("should return true if ServerSecret is set", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{ + ServerSecret: "some-server-secret", + }, + }, + }, + } + Expect(cluster.IsServerSecurityEnabled()).To(BeTrue()) + }) + + It("should return false if ServerSecret is not set", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{}, + }, + }, + } + Expect(cluster.IsServerSecurityEnabled()).To(BeFalse()) + }) + + It("should return false if Security is nil", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{}, + } + Expect(cluster.IsServerSecurityEnabled()).To(BeFalse()) + }) + }) + + Context("When running IsServerTrustedCADefined function", func() { + It("should return true if ServerTrustedCASecret is set", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{ + ServerTrustedCASecret: "some-ca-secret", + }, + }, + }, + } + Expect(cluster.IsServerTrustedCADefined()).To(BeTrue()) + }) + + It("should return false if ServerTrustedCASecret is not set", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Security: &SecuritySpec{ + TLS: TLSSpec{}, + }, + }, + } + Expect(cluster.IsServerTrustedCADefined()).To(BeFalse()) + }) + + It("should return false if Security is nil", func() { + cluster := &EtcdCluster{ + Spec: EtcdClusterSpec{}, + } + Expect(cluster.IsServerTrustedCADefined()).To(BeFalse()) + }) + }) +}) diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 210efb02..18c7971b 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -281,6 +281,15 @@ func (r *EtcdCluster) validateSecurity() field.ErrorList { ) } + if security.EnableAuth && (security.TLS.ClientSecret == "" || security.TLS.ServerSecret == "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security"), + security.TLS, + "if auth is enabled, client secret and server secret must be provided"), + ) + } + if len(allErrors) > 0 { return allErrors } diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index 8181f012..23fe9c79 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -213,6 +213,66 @@ var _ = Describe("EtcdCluster Webhook", func() { } } }) + + It("Shouldn't reject if auth is enabled and security client certs are defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security = &SecuritySpec{ + EnableAuth: true, + TLS: TLSSpec{ + ClientTrustedCASecret: "test-client-trusted-ca-cert", + ClientSecret: "test-client-cert", + ServerSecret: "test-server-cert", + }, + } + err := localCluster.validateSecurity() + Expect(err).To(BeNil()) + }) + + It("Should reject if auth is enabled and one of client and server certs is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security = &SecuritySpec{ + EnableAuth: true, + TLS: TLSSpec{ + ClientSecret: "test-client-cert", + ClientTrustedCASecret: "test-client-trusted-ca-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security"), + localCluster.Spec.Security.TLS, + "if auth is enabled, client secret and server secret must be provided", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + + }) + + It("Should reject if auth is enabled and one of client and server certs is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security = &SecuritySpec{ + EnableAuth: true, + TLS: TLSSpec{ + ServerSecret: "test-server-cert", + }, + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security"), + localCluster.Spec.Security.TLS, + "if auth is enabled, client secret and server secret must be provided", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + + }) + }) Context("Validate PDB", func() { diff --git a/charts/etcd-operator/README.md b/charts/etcd-operator/README.md index 729dab25..33838106 100644 --- a/charts/etcd-operator/README.md +++ b/charts/etcd-operator/README.md @@ -1,63 +1,54 @@ # etcd-operator -![Version: 0.0.0](https://img.shields.io/badge/Version-0.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.0.0](https://img.shields.io/badge/AppVersion-v0.0.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| affinity | object | `{}` | | +| affinity | object | `{}` | ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity | | etcdOperator.args[0] | string | `"--health-probe-bind-address=:8081"` | | | etcdOperator.args[1] | string | `"--metrics-bind-address=127.0.0.1:8080"` | | | etcdOperator.args[2] | string | `"--leader-elect"` | | -| etcdOperator.envVars | object | `{}` | | -| etcdOperator.image.pullPolicy | string | `"IfNotPresent"` | | -| etcdOperator.image.repository | string | `"ghcr.io/aenix-io/etcd-operator"` | | -| etcdOperator.image.tag | string | `""` | | -| etcdOperator.livenessProbe.httpGet.path | string | `"/healthz"` | | -| etcdOperator.livenessProbe.httpGet.port | int | `8081` | | -| etcdOperator.livenessProbe.initialDelaySeconds | int | `15` | | -| etcdOperator.livenessProbe.periodSeconds | int | `20` | | -| etcdOperator.readinessProbe.httpGet.path | string | `"/readyz"` | | -| etcdOperator.readinessProbe.httpGet.port | int | `8081` | | -| etcdOperator.readinessProbe.initialDelaySeconds | int | `5` | | -| etcdOperator.readinessProbe.periodSeconds | int | `10` | | -| etcdOperator.resources.limits.cpu | string | `"500m"` | | -| etcdOperator.resources.limits.memory | string | `"128Mi"` | | -| etcdOperator.resources.requests.cpu | string | `"100m"` | | -| etcdOperator.resources.requests.memory | string | `"64Mi"` | | -| etcdOperator.securityContext.allowPrivilegeEscalation | bool | `false` | | -| etcdOperator.securityContext.capabilities.drop[0] | string | `"ALL"` | | -| etcdOperator.service.port | int | `9443` | | -| etcdOperator.service.type | string | `"ClusterIP"` | | -| fullnameOverride | string | `""` | | +| etcdOperator.envVars | object | `{}` | Empty environment variables section | +| etcdOperator.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| etcdOperator.image.repository | string | `"ghcr.io/aenix-io/etcd-operator"` | Image repository | +| etcdOperator.image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| etcdOperator.livenessProbe.httpGet.path | string | `"/healthz"` | Healthcheck liveness probe path | +| etcdOperator.livenessProbe.httpGet.port | int | `8081` | Healthcheck port | +| etcdOperator.livenessProbe.initialDelaySeconds | int | `15` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes | +| etcdOperator.livenessProbe.periodSeconds | int | `20` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes | +| etcdOperator.readinessProbe.httpGet.path | string | `"/readyz"` | Healthcheck readiness probe path | +| etcdOperator.readinessProbe.httpGet.port | int | `8081` | Healthcheck port | +| etcdOperator.readinessProbe.initialDelaySeconds | int | `5` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes | +| etcdOperator.readinessProbe.periodSeconds | int | `10` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes | +| etcdOperator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}` | ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| etcdOperator.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | +| etcdOperator.service.port | int | `9443` | Service port | +| etcdOperator.service.type | string | `"ClusterIP"` | Service type | +| fullnameOverride | string | `""` | Override a full name of helm release | | imagePullSecrets | list | `[]` | | | kubeRbacProxy.args[0] | string | `"--secure-listen-address=0.0.0.0:8443"` | | | kubeRbacProxy.args[1] | string | `"--upstream=http://127.0.0.1:8080/"` | | | kubeRbacProxy.args[2] | string | `"--logtostderr=true"` | | | kubeRbacProxy.args[3] | string | `"--v=0"` | | -| kubeRbacProxy.image.pullPolicy | string | `"IfNotPresent"` | | -| kubeRbacProxy.image.repository | string | `"gcr.io/kubebuilder/kube-rbac-proxy"` | | -| kubeRbacProxy.image.tag | string | `"v0.16.0"` | | -| kubeRbacProxy.livenessProbe | object | `{}` | | -| kubeRbacProxy.readinessProbe | object | `{}` | | -| kubeRbacProxy.resources.limits.cpu | string | `"500m"` | | -| kubeRbacProxy.resources.limits.memory | string | `"128Mi"` | | -| kubeRbacProxy.resources.requests.cpu | string | `"100m"` | | -| kubeRbacProxy.resources.requests.memory | string | `"64Mi"` | | -| kubeRbacProxy.securityContext.allowPrivilegeEscalation | bool | `false` | | -| kubeRbacProxy.securityContext.capabilities.drop[0] | string | `"ALL"` | | -| kubeRbacProxy.service.port | int | `8443` | | -| kubeRbacProxy.service.type | string | `"ClusterIP"` | | -| kubernetesClusterDomain | string | `"cluster.local"` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| podAnnotations | object | `{}` | | -| podLabels | object | `{}` | | -| podSecurityContext | object | `{}` | | -| replicaCount | int | `1` | | -| securityContext.runAsNonRoot | bool | `true` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| tolerations | list | `[]` | | +| kubeRbacProxy.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| kubeRbacProxy.image.repository | string | `"gcr.io/kubebuilder/kube-rbac-proxy"` | Image repository | +| kubeRbacProxy.image.tag | string | `"v0.16.0"` | Version of image | +| kubeRbacProxy.livenessProbe | object | `{}` | https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ | +| kubeRbacProxy.readinessProbe | object | `{}` | https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ | +| kubeRbacProxy.resources | object | `{"limits":{"cpu":"250m","memory":"128Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}` | ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | +| kubeRbacProxy.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | +| kubeRbacProxy.service.port | int | `8443` | Service port | +| kubeRbacProxy.service.type | string | `"ClusterIP"` | Service type | +| kubernetesClusterDomain | string | `"cluster.local"` | Kubernetes cluster domain prefix | +| nameOverride | string | `""` | Override a name of helm release | +| nodeSelector | object | `{}` | ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ | +| podAnnotations | object | `{}` | ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | +| podLabels | object | `{}` | ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ | +| podSecurityContext | object | `{}` | ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | +| replicaCount | int | `1` | Count of pod replicas | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| tolerations | list | `[]` | ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | diff --git a/charts/etcd-operator/README.md.gotmpl b/charts/etcd-operator/README.md.gotmpl new file mode 100644 index 00000000..e1c25058 --- /dev/null +++ b/charts/etcd-operator/README.md.gotmpl @@ -0,0 +1,19 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.typeBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/charts/etcd-operator/crds/etcd-cluster.yaml b/charts/etcd-operator/crds/etcd-cluster.yaml index 3abbf976..279e4b63 100644 --- a/charts/etcd-operator/crds/etcd-cluster.yaml +++ b/charts/etcd-operator/crds/etcd-cluster.yaml @@ -3,7 +3,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: etcd-operator-system/etcd-operator-serving-cert - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: etcdclusters.etcd.aenix.io spec: conversion: @@ -202,25 +202,43 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: + enableAuth: + description: Section to enable etcd auth + type: boolean tls: description: Section for user-managed tls certificates properties: clientSecret: - description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string clientTrustedCASecret: - description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerSecret: - description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerTrustedCASecret: - description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string serverSecret: description: |- Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. + type: string + serverTrustedCASecret: + description: |- + Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used. + This secret must be created in the namespace with etcdCluster CR. type: string type: object type: object @@ -581,6 +599,7 @@ spec: not set, the implementation will apply its default routing strategy. If set to "PreferClose", implementations should prioritize endpoints that are topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. type: string type: description: |- diff --git a/charts/etcd-operator/templates/workload/deployment.yml b/charts/etcd-operator/templates/workload/deployment.yml index ece5bae8..17848ada 100644 --- a/charts/etcd-operator/templates/workload/deployment.yml +++ b/charts/etcd-operator/templates/workload/deployment.yml @@ -58,6 +58,12 @@ spec: - configMapRef: name: {{ include "etcd-operator.fullname" . }}-env {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert diff --git a/charts/etcd-operator/values.schema.json b/charts/etcd-operator/values.schema.json index 6fd0f134..65836a6b 100644 --- a/charts/etcd-operator/values.schema.json +++ b/charts/etcd-operator/values.schema.json @@ -256,14 +256,6 @@ "replicaCount": { "type": "integer" }, - "securityContext": { - "properties": { - "runAsNonRoot": { - "type": "boolean" - } - }, - "type": "object" - }, "serviceAccount": { "properties": { "annotations": { diff --git a/charts/etcd-operator/values.yaml b/charts/etcd-operator/values.yaml index 9c9ff75a..dcef802b 100644 --- a/charts/etcd-operator/values.yaml +++ b/charts/etcd-operator/values.yaml @@ -1,98 +1,184 @@ etcdOperator: + image: + + # -- Image repository repository: ghcr.io/aenix-io/etcd-operator + + # -- Image pull policy pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. + + # -- Overrides the image tag whose default is the chart appVersion. tag: "" + args: - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 - --leader-elect + service: + + # -- Service type type: ClusterIP + + # -- Service port port: 9443 + + # -- Empty environment variables section envVars: {} + livenessProbe: + httpGet: + + # -- Healthcheck liveness probe path path: /healthz + + # -- Healthcheck port port: 8081 + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes initialDelaySeconds: 15 + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes periodSeconds: 20 + readinessProbe: + httpGet: + + # -- Healthcheck readiness probe path path: /readyz + + # -- Healthcheck port port: 8081 + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes initialDelaySeconds: 5 + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes periodSeconds: 10 + + # -- ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + memory: 64Mi + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: - ALL kubeRbacProxy: + image: + + # -- Image repository repository: gcr.io/kubebuilder/kube-rbac-proxy + + # -- Image pull policy pullPolicy: IfNotPresent + + # -- Version of image tag: v0.16.0 + args: - --secure-listen-address=0.0.0.0:8443 - --upstream=http://127.0.0.1:8080/ - --logtostderr=true - --v=0 + service: + + # -- Service type type: ClusterIP + + # -- Service port port: 8443 + + # -- https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ livenessProbe: {} + + # -- https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ readinessProbe: {} + + # -- ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: + limits: - cpu: 500m + + cpu: 250m + memory: 128Mi + requests: + cpu: 100m + memory: 64Mi + + # -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: - - ALL + - ALL +# -- Kubernetes cluster domain prefix kubernetesClusterDomain: cluster.local +# -- Count of pod replicas replicaCount: 1 imagePullSecrets: [] +# -- Override a name of helm release nameOverride: "" +# -- Override a full name of helm release fullnameOverride: "" serviceAccount: - # Specifies whether a service account should be created + + # -- Specifies whether a service account should be created create: true - # Annotations to add to the service account + + # -- Annotations to add to the service account annotations: {} +# -- ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ podAnnotations: {} +# -- ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ podLabels: {} +# -- ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ podSecurityContext: {} # fsGroup: 2000 -securityContext: - runAsNonRoot: true - +# -- ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ nodeSelector: {} +# -- ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ tolerations: [] +# -- ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} diff --git a/cmd/app/commandline.go b/cmd/app/commandline.go new file mode 100644 index 00000000..9ba7fcb4 --- /dev/null +++ b/cmd/app/commandline.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/aenix-io/etcd-operator/internal/log" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +type Flags struct { + Kubeconfig string + MetricsAddress string + ProbeAddress string + LeaderElection bool + SecureMetrics bool + EnableHTTP2 bool + DisableWebhooks bool + LogLevel string + StacktraceLevel string + Dev bool +} + +func ParseCmdLine() Flags { + pflag.Usage = usage + pflag.ErrHelp = nil + pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + + pflag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") + pflag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + pflag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + pflag.Bool("leader-elect", false, "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + pflag.Bool("metrics-secure", false, "If set the metrics endpoint is served securely.") + pflag.Bool("enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers.") + pflag.Bool("disable-webhooks", false, "If set, the webhooks will be disabled.") + pflag.String("log-level", "info", "Logger verbosity level.Applicable values are debug, info, warn, error.") + pflag.String("stacktrace-level", "error", "Logger level to add stacktrace. "+ + "Applicable values are debug, info, warn, error.") + pflag.Bool("dev", false, "development mode.") + + var help bool + pflag.BoolVarP(&help, "help", "h", false, "Show this help message.") + err := viper.BindPFlags(pflag.CommandLine) + if err != nil { + exitUsage(err) + } + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() + err = pflag.CommandLine.Parse(os.Args[1:]) + if err != nil { + exitUsage(err) + } + if help { + exitUsage(nil) + } + + return Flags{ + Kubeconfig: viper.GetString("kubeconfig"), + MetricsAddress: viper.GetString("metrics-bind-address"), + ProbeAddress: viper.GetString("health-probe-bind-address"), + LeaderElection: viper.GetBool("leader-elect"), + SecureMetrics: viper.GetBool("metrics-secure"), + EnableHTTP2: viper.GetBool("enable-http2"), + DisableWebhooks: viper.GetBool("disable-webhooks"), + LogLevel: viper.GetString("log-level"), + StacktraceLevel: viper.GetString("stacktrace-level"), + Dev: viper.GetBool("dev"), + } +} + +func usage() { + name := filepath.Base(os.Args[0]) + _, _ = fmt.Fprintf(os.Stderr, "Usage: %s [--option]...\n", name) + _, _ = fmt.Fprintf(os.Stderr, "Options:\n") + pflag.PrintDefaults() +} + +func exitUsage(err error) { + code := 0 + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s: %s\n", filepath.Base(os.Args[0]), err) + code = 2 + } + pflag.Usage() + os.Exit(code) +} + +func LogParameters(flags Flags) log.Parameters { + return log.Parameters{ + LogLevel: flags.LogLevel, + StacktraceLevel: flags.StacktraceLevel, + Development: flags.Dev, + } +} diff --git a/cmd/main.go b/cmd/main.go index 6ff10b1b..ddbb5bbc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,62 +17,48 @@ limitations under the License. package main import ( + "context" "crypto/tls" - "flag" "os" + "syscall" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - + "github.com/aenix-io/etcd-operator/cmd/app" + "github.com/aenix-io/etcd-operator/internal/log" + "github.com/aenix-io/etcd-operator/internal/signal" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/controller" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + signals = []os.Signal{os.Interrupt, syscall.SIGTERM} ) func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(etcdaenixiov1alpha1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme } func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + flags := app.ParseCmdLine() + ctx := signal.NotifyContext(log.Setup(context.TODO(), app.LogParameters(flags)), signals...) + ctrl.SetLogger(logr.FromContextOrDiscard(ctx)) // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will @@ -81,12 +67,12 @@ func main() { // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 // - https://github.com/advisories/GHSA-4374-p667-p6c8 disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") + log.Info(ctx, "disabling http/2") c.NextProtos = []string{"http/1.1"} } - tlsOpts := []func(*tls.Config){} - if !enableHTTP2 { + tlsOpts := make([]func(*tls.Config), 0) + if !flags.EnableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } @@ -97,13 +83,13 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, + BindAddress: flags.MetricsAddress, + SecureServing: flags.SecureMetrics, TLSOpts: tlsOpts, }, WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, + HealthProbeBindAddress: flags.ProbeAddress, + LeaderElection: flags.LeaderElection, LeaderElectionID: "1b04a718.etcd.aenix.io", // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the @@ -118,7 +104,7 @@ func main() { // LeaderElectionReleaseOnCancel: true, }) if err != nil { - setupLog.Error(err, "unable to start manager") + log.Error(ctx, err, "unable to setup manager") os.Exit(1) } @@ -126,29 +112,29 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "EtcdCluster") + log.Error(ctx, err, "unable to create controller", "controller", "EtcdCluster") os.Exit(1) } - if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if !flags.DisableWebhooks { if err = (&etcdaenixiov1alpha1.EtcdCluster{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "EtcdCluster") + log.Error(ctx, err, "unable to create webhook", "webhook", "EtcdCluster") os.Exit(1) } } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(ctx, err, "unable to set up health check") os.Exit(1) } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(ctx, err, "unable to set up ready check") os.Exit(1) } - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + log.Info(ctx, "starting manager") + if err = mgr.Start(ctx); err != nil { + log.Error(ctx, err, "problem running manager") os.Exit(1) } } diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index a824d508..b4134dbf 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: etcdclusters.etcd.aenix.io spec: group: etcd.aenix.io @@ -192,25 +192,43 @@ spec: security: description: Security describes security settings of etcd (authentication, certificates, rbac) properties: + enableAuth: + description: Section to enable etcd auth + type: boolean tls: description: Section for user-managed tls certificates properties: clientSecret: - description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string clientTrustedCASecret: - description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerSecret: - description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + description: |- + Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string peerTrustedCASecret: - description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + description: |- + Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret. + This secret must be created in the namespace with etcdCluster CR. type: string serverSecret: description: |- Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). It is expected to have tls.crt and tls.key fields in the secret. + This secret must be created in the namespace with etcdCluster CR. + type: string + serverTrustedCASecret: + description: |- + Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd. + It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used. + This secret must be created in the namespace with etcdCluster CR. type: string type: object type: object @@ -571,6 +589,7 @@ spec: not set, the implementation will apply its default routing strategy. If set to "PreferClose", implementations should prioritize endpoints that are topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. type: string type: description: |- diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 70c3437f..d008961a 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -37,3 +37,4 @@ spec: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - "--leader-elect" + - "--log-level=debug" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b62c4319..d657a5e9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -99,5 +99,11 @@ spec: requests: cpu: 10m memory: 64Mi + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9b8adb43..1d3153cd 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -16,6 +16,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - view + - watch - apiGroups: - "" resources: diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml index 5b40dddc..7471b6aa 100644 --- a/examples/manifests/etcdcluster-with-external-certificates.yaml +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -3,13 +3,15 @@ apiVersion: etcd.aenix.io/v1alpha1 kind: EtcdCluster metadata: name: test - namespace: default + namespace: test-tls-auth-etcd-cluster spec: storage: {} security: + enableAuth: true tls: peerTrustedCASecret: ca-peer-secret peerSecret: peer-secret + serverTrustedCASecret: ca-server-secret serverSecret: server-secret clientTrustedCASecret: ca-client-secret clientSecret: client-secret @@ -18,7 +20,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: selfsigned-issuer - namespace: default + namespace: test-tls-auth-etcd-cluster spec: selfSigned: {} --- @@ -26,7 +28,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: ca-certificate-peer - namespace: default + namespace: test-tls-auth-etcd-cluster spec: isCA: true usages: @@ -52,7 +54,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: ca-certificate-server - namespace: default + namespace: test-tls-auth-etcd-cluster spec: isCA: true usages: @@ -78,7 +80,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: ca-certificate-client - namespace: default + namespace: test-tls-auth-etcd-cluster spec: isCA: true usages: @@ -104,7 +106,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer-peer - namespace: default + namespace: test-tls-auth-etcd-cluster spec: ca: secretName: ca-peer-secret @@ -113,7 +115,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer-server - namespace: default + namespace: test-tls-auth-etcd-cluster spec: ca: secretName: ca-server-secret @@ -122,7 +124,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer-client - namespace: default + namespace: test-tls-auth-etcd-cluster spec: ca: secretName: ca-client-secret @@ -131,7 +133,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: server-certificate - namespace: default + namespace: test-tls-auth-etcd-cluster spec: secretName: server-secret isCA: false @@ -142,19 +144,19 @@ spec: dnsNames: - test-0 - test-0.test-headless - - test-0.test-headless.default.svc - - test-0.test-headless.default.svc.cluster.local + - test-0.test-headless.test-tls-auth-etcd-cluster.svc + - test-0.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - test-1 - test-1.test-headless - - test-1.test-headless.default.svc - - test-1.test-headless.default.svc.cluster.local + - test-1.test-headless.test-tls-auth-etcd-cluster.svc + - test-1.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - test-2 - test-2.test-headless - - test-2.test-headless.default.svc - - test-2.test-headless.default.svc.cluster.local + - test-2.test-headless.test-tls-auth-etcd-cluster.svc + - test-2.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - test - - test.default.svc - - test.default.svc.cluster.local + - test.test-tls-auth-etcd-cluster.svc + - test.test-tls-auth-etcd-cluster.svc.cluster.local - localhost - "127.0.0.1" privateKey: @@ -168,7 +170,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: peer-certificate - namespace: default + namespace: test-tls-auth-etcd-cluster spec: secretName: peer-secret isCA: false @@ -180,16 +182,16 @@ spec: dnsNames: - test-0 - test-0.test-headless - - test-0.test-headless.default.svc - - test-0.test-headless.default.svc.cluster.local + - test-0.test-headless.test-tls-auth-etcd-cluster.svc + - test-0.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - test-1 - test-1.test-headless - - test-1.test-headless.default.svc - - test-1.test-headless.default.svc.cluster.local + - test-1.test-headless.test-tls-auth-etcd-cluster.svc + - test-1.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - test-2 - test-2.test-headless - - test-2.test-headless.default.svc - - test-2.test-headless.default.svc.cluster.local + - test-2.test-headless.test-tls-auth-etcd-cluster.svc + - test-2.test-headless.test-tls-auth-etcd-cluster.svc.cluster.local - localhost - "127.0.0.1" privateKey: @@ -203,7 +205,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: client-certificate - namespace: default + namespace: test-tls-auth-etcd-cluster spec: commonName: root secretName: client-secret diff --git a/go.mod b/go.mod index 29bed8f7..c4454997 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,22 @@ module github.com/aenix-io/etcd-operator -go 1.22.2 +go 1.22.4 require ( + github.com/go-logr/logr v1.4.2 github.com/google/uuid v1.6.0 - github.com/onsi/ginkgo/v2 v2.17.2 + github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 - go.etcd.io/etcd/client/v3 v3.5.13 - k8s.io/api v0.30.0 - k8s.io/apimachinery v0.30.0 - k8s.io/client-go v0.30.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 + go.etcd.io/etcd/client/v3 v3.5.14 + go.uber.org/zap v1.27.0 + go.uber.org/zap/exp v0.2.0 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - sigs.k8s.io/controller-runtime v0.18.1 + sigs.k8s.io/controller-runtime v0.18.4 ) require ( @@ -19,11 +24,10 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -36,43 +40,51 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.etcd.io/etcd/api/v3 v3.5.13 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.14 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.20.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.59.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/grpc v1.62.1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.0 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 63d213e9..ed763278 100644 --- a/go.sum +++ b/go.sum @@ -8,18 +8,21 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -35,11 +38,13 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -50,6 +55,8 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -65,10 +72,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -76,14 +87,17 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= @@ -94,86 +108,117 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= -go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= -go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= -go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= -go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= -go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0= +go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU= +go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ= +go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI= +go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg= +go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= +go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -181,28 +226,30 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= -k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= -k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= -k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= -k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= -k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.18.1 h1:RpWbigmuiylbxOCLy0tGnq1cU1qWPwNIQzoJk+QeJx4= -sigs.k8s.io/controller-runtime v0.18.1/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/internal/controller/etcdcluster_controller.go b/internal/controller/etcdcluster_controller.go index 1f6ce78a..b01576b3 100644 --- a/internal/controller/etcdcluster_controller.go +++ b/internal/controller/etcdcluster_controller.go @@ -18,11 +18,17 @@ package controller import ( "context" + "crypto/tls" + "crypto/x509" goerrors "errors" "fmt" + "slices" + "strconv" + "strings" + "time" + "github.com/aenix-io/etcd-operator/internal/log" policyv1 "k8s.io/api/policy/v1" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -37,6 +43,8 @@ import ( etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/controller/factory" + + clientv3 "go.etcd.io/etcd/client/v3" ) // EtcdClusterReconciler reconciles a EtcdCluster object @@ -51,18 +59,18 @@ type EtcdClusterReconciler struct { // +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;watch;delete;patch // +kubebuilder:rbac:groups="",resources=services,verbs=get;create;delete;update;patch;list;watch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=view;list;watch // +kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;create;delete;update;patch;list;watch // +kubebuilder:rbac:groups="policy",resources=poddisruptionbudgets,verbs=get;create;delete;update;patch;list;watch // Reconcile checks CR and current cluster state and performs actions to transform current state to desired. func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - logger.V(2).Info("reconciling object", "namespaced_name", req.NamespacedName) + log.Debug(ctx, "reconciling object") instance := &etcdaenixiov1alpha1.EtcdCluster{} err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if errors.IsNotFound(err) { - logger.V(2).Info("object not found", "namespaced_name", req.NamespacedName) + log.Debug(ctx, "object not found") return ctrl.Result{}, nil } // Error retrieving object, requeue @@ -130,8 +138,7 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // ensure managed resources - if err := r.ensureClusterObjects(ctx, instance); err != nil { - logger.Error(err, "cannot create Cluster auxiliary objects") + if err = r.ensureClusterObjects(ctx, instance); err != nil { return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot create Cluster auxiliary objects: %w", err)) } @@ -145,10 +152,17 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // check sts condition clusterReady, err := r.isStatefulSetReady(ctx, instance) if err != nil { - logger.Error(err, "failed to check etcd cluster state") + log.Error(ctx, err, "failed to check etcd cluster state") return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot check Cluster readiness: %w", err)) } + if clusterReady && *instance.Spec.Replicas != int32(0) { + err := r.configureAuth(ctx, instance) + if err != nil { + return ctrl.Result{}, err + } + } + // set cluster readiness condition existingCondition := factory.GetCondition(instance, etcdaenixiov1alpha1.EtcdConditionReady) if existingCondition != nil && @@ -179,21 +193,37 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // ensureClusterObjects creates or updates all objects owned by cluster CR func (r *EtcdClusterReconciler) ensureClusterObjects( ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) error { + if err := factory.CreateOrUpdateClusterStateConfigMap(ctx, cluster, r.Client); err != nil { + log.Error(ctx, err, "reconcile cluster state configmap failed") return err } + log.Debug(ctx, "cluster state configmap reconciled") + if err := factory.CreateOrUpdateHeadlessService(ctx, cluster, r.Client); err != nil { + log.Error(ctx, err, "reconcile headless service failed") return err } + log.Debug(ctx, "headless service reconciled") + if err := factory.CreateOrUpdateStatefulSet(ctx, cluster, r.Client); err != nil { + log.Error(ctx, err, "reconcile statefulset failed") return err } + log.Debug(ctx, "statefulset reconciled") + if err := factory.CreateOrUpdateClientService(ctx, cluster, r.Client); err != nil { + log.Error(ctx, err, "reconcile client service failed") return err } + log.Debug(ctx, "client service reconciled") + if err := factory.CreateOrUpdatePdb(ctx, cluster, r.Client); err != nil { + log.Error(ctx, err, "reconcile pdb failed") return err } + log.Debug(ctx, "pdb reconciled") + return nil } @@ -211,16 +241,15 @@ func (r *EtcdClusterReconciler) updateStatusOnErr(ctx context.Context, cluster * // updateStatus updates EtcdCluster status and returns error and requeue in case status could not be updated due to conflict func (r *EtcdClusterReconciler) updateStatus(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (ctrl.Result, error) { - logger := log.FromContext(ctx) err := r.Status().Update(ctx, cluster) if err == nil { return ctrl.Result{}, nil } if errors.IsConflict(err) { - logger.V(2).Info("conflict during cluster status update") + log.Debug(ctx, "conflict during cluster status update") return ctrl.Result{Requeue: true}, nil } - logger.Error(err, "cannot update cluster status") + log.Error(ctx, err, "cannot update cluster status") return ctrl.Result{}, err } @@ -244,3 +273,280 @@ func (r *EtcdClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&policyv1.PodDisruptionBudget{}). Complete(r) } + +func (r *EtcdClusterReconciler) configureAuth(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) error { + + var err error + + cli, err := r.GetEtcdClient(ctx, cluster) + if err != nil { + return err + } + + defer func() { + err = cli.Close() + }() + + err = testMemberList(ctx, cli) + if err != nil { + return err + } + + auth := clientv3.NewAuth(cli) + + if cluster.Spec.Security != nil && cluster.Spec.Security.EnableAuth { + + if err := r.createRoleIfNotExists(ctx, auth, "root"); err != nil { + return err + } + + rootUserResponse, err := r.createUserIfNotExists(ctx, auth, "root") + if err != nil { + return err + } + + if err := r.grantRoleToUser(ctx, auth, "root", "root", rootUserResponse); err != nil { + return err + } + + if err := r.enableAuth(ctx, auth); err != nil { + return err + } + } else { + if err := r.disableAuth(ctx, auth); err != nil { + return err + } + } + + err = testMemberList(ctx, cli) + if err != nil { + return err + } + + return err +} + +// This is auxiliary self-test function, that shows that connection to etcd cluster works. +// As soon as operator has functionality to operate etcd-cluster, this function can be removed. +func testMemberList(ctx context.Context, cli *clientv3.Client) error { + + etcdCluster := clientv3.NewCluster(cli) + + _, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + memberList, err := etcdCluster.MemberList(ctx) + + if err != nil { + log.Error(ctx, err, "failed to get member list", "endpoints", cli.Endpoints()) + return err + } + log.Debug(ctx, "member list got", "member list", memberList) + + return err +} + +func (r *EtcdClusterReconciler) GetEtcdClient(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (*clientv3.Client, error) { + + endpoints := getEndpointsSlice(cluster) + log.Debug(ctx, "endpoints built", "endpoints", endpoints) + + tlsConfig, err := r.getTLSConfig(ctx, cluster) + if err != nil { + log.Error(ctx, err, "failed to build tls config") + return nil, err + } + log.Debug(ctx, "tls config built", "tls config", tlsConfig) + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + TLS: tlsConfig, + }, + ) + if err != nil { + log.Error(ctx, err, "failed to create etcd client", "endpoints", endpoints) + return nil, err + } + log.Debug(ctx, "etcd client created", "endpoints", endpoints) + + return cli, nil + +} + +func (r *EtcdClusterReconciler) getTLSConfig(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) (*tls.Config, error) { + + var err error + + caCertPool := &x509.CertPool{} + + if cluster.IsServerTrustedCADefined() { + + serverCASecret := &corev1.Secret{} + + if err = r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Spec.Security.TLS.ServerTrustedCASecret}, serverCASecret); err != nil { + log.Error(ctx, err, "failed to get server trusted CA secret") + return nil, err + } + log.Debug(ctx, "secret read", "server trusted CA secret") // serverCASecret, + + caCertPool = x509.NewCertPool() + + if !caCertPool.AppendCertsFromPEM(serverCASecret.Data["tls.crt"]) { + log.Error(ctx, err, "failed to parse CA certificate") + return nil, err + } + + } + + cert := tls.Certificate{} + + if cluster.IsClientSecurityEnabled() { + + rootSecret := &corev1.Secret{} + if err = r.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Spec.Security.TLS.ClientSecret}, rootSecret); err != nil { + log.Error(ctx, err, "failed to get root client secret") + return nil, err + } + log.Debug(ctx, "secret read", "root client secret") // rootSecret, + + cert, err = tls.X509KeyPair(rootSecret.Data["tls.crt"], rootSecret.Data["tls.key"]) + if err != nil { + log.Error(ctx, err, "failed to parse key pair", "cert", cert) + return nil, err + } + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: !cluster.IsServerTrustedCADefined(), + RootCAs: caCertPool, + Certificates: []tls.Certificate{ + cert, + }, + } + + return tlsConfig, err +} + +func getEndpointsSlice(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { + + endpoints := []string{} + for podNumber := 0; podNumber < int(*cluster.Spec.Replicas); podNumber++ { + endpoints = append( + endpoints, + strings.Join( + []string{ + factory.GetServerProtocol(cluster) + cluster.Name + "-" + strconv.Itoa(podNumber), + factory.GetHeadlessServiceName(cluster), + cluster.Namespace, + "svc:2379"}, + ".")) + } + return endpoints +} + +func (r *EtcdClusterReconciler) createRoleIfNotExists(ctx context.Context, authClient clientv3.Auth, roleName string) error { + + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := authClient.RoleGet(ctx, roleName) + if err != nil { + if err.Error() != "etcdserver: role name not found" { + log.Error(ctx, err, "failed to get role", "role name", "root") + return err + } + ctx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err = authClient.RoleAdd(ctx, roleName) + if err != nil { + log.Error(ctx, err, "failed to add role", "role name", "root") + return err + } + log.Debug(ctx, "role added", "role name", "root") + return nil + } + log.Debug(ctx, "role exists, nothing to do", "role name", "root") + + return nil +} + +func (r *EtcdClusterReconciler) createUserIfNotExists(ctx context.Context, authClient clientv3.Auth, userName string) (*clientv3.AuthUserGetResponse, error) { + + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + userResponse, err := authClient.UserGet(ctx, userName) + if err != nil { + if err.Error() != "etcdserver: user name not found" { + log.Error(ctx, err, "failed to get user", "user name", "root") + return nil, err + } + + _, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err = authClient.UserAddWithOptions(ctx, "root", "", &clientv3.UserAddOptions{ + NoPassword: true, + }) + if err != nil { + log.Error(ctx, err, "failed to add user", "user name", "root") + return nil, err + } + log.Debug(ctx, "user added", "user name", "root") + return nil, nil + } + log.Debug(ctx, "user exists, nothing to do", "user name", "root") + + return userResponse, err +} + +func (r *EtcdClusterReconciler) grantRoleToUser(ctx context.Context, authClient clientv3.Auth, userName, roleName string, userResponse *clientv3.AuthUserGetResponse) error { + + var err error + + if userResponse == nil || !slices.Contains(userResponse.Roles, roleName) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + _, err := authClient.UserGrantRole(ctx, userName, roleName) + + if err != nil { + log.Error(ctx, err, "failed to grant user to role", "user:role name", "root:root") + return err + } + log.Debug(ctx, "user:role granted", "user:role name", "root:root") + } else { + log.Debug(ctx, "user:role already granted, nothing to do", "user:role name", "root:root") + } + + return err +} + +func (r *EtcdClusterReconciler) enableAuth(ctx context.Context, authClient clientv3.Auth) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := authClient.AuthEnable(ctx) + + if err != nil { + log.Error(ctx, err, "failed to enable auth") + return err + } + log.Debug(ctx, "auth enabled") + + return err +} + +func (r *EtcdClusterReconciler) disableAuth(ctx context.Context, authClient clientv3.Auth) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, err := authClient.AuthDisable(ctx) + if err != nil { + log.Error(ctx, err, "failed to disable auth") + return err + } + log.Debug(ctx, "auth disabled") + + return nil +} diff --git a/internal/controller/etcdcluster_controller_test.go b/internal/controller/etcdcluster_controller_test.go index 8f67311b..d7c2c9a1 100644 --- a/internal/controller/etcdcluster_controller_test.go +++ b/internal/controller/etcdcluster_controller_test.go @@ -37,7 +37,7 @@ var _ = Describe("EtcdCluster Controller", func() { ns *corev1.Namespace ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { reconciler = &EtcdClusterReconciler{ Client: k8sClient, Scheme: k8sClient.Scheme(), @@ -52,6 +52,37 @@ var _ = Describe("EtcdCluster Controller", func() { DeferCleanup(k8sClient.Delete, ns) }) + Context("When running getEtcdEndpoints", func() { + It("Should get etcd empty string slice if etcd has 0 replicas", func() { + cluster := &etcdaenixiov1alpha1.EtcdCluster{ + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Replicas: ptr.To(int32(0)), + }, + } + Expect(getEndpointsSlice(cluster)).To(BeEmpty()) + Expect(getEndpointsSlice(cluster)).To(Equal([]string{})) + + }) + + It("Should get etcd correct string slice if etcd has 3 replicas", func() { + cluster := &etcdaenixiov1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-test", + Namespace: "ns-test", + }, + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Replicas: ptr.To(int32(3)), + }, + } + Expect(getEndpointsSlice(cluster)).To(Equal([]string{ + "http://etcd-test-0.etcd-test-headless.ns-test.svc:2379", + "http://etcd-test-1.etcd-test-headless.ns-test.svc:2379", + "http://etcd-test-2.etcd-test-headless.ns-test.svc:2379", + })) + }) + + }) + Context("When reconciling the EtcdCluster", func() { var ( etcdcluster etcdaenixiov1alpha1.EtcdCluster @@ -63,7 +94,7 @@ var _ = Describe("EtcdCluster Controller", func() { err error ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { etcdcluster = etcdaenixiov1alpha1.EtcdCluster{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-etcdcluster-", @@ -110,7 +141,7 @@ var _ = Describe("EtcdCluster Controller", func() { DeferCleanup(k8sClient.Delete, &statefulSet) }) - It("should reconcile a new EtcdCluster", func(ctx SpecContext) { + It("should reconcile a new EtcdCluster", func() { By("reconciling the EtcdCluster", func() { _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&etcdcluster)}) Expect(err).ToNot(HaveOccurred()) @@ -142,7 +173,8 @@ var _ = Describe("EtcdCluster Controller", func() { }) }) - It("should successfully reconcile the resource twice and mark as ready", func(ctx SpecContext) { + It("should successfully reconcile the resource twice and mark as ready", func() { + Skip("Skipped because it is checked in e2e tests. Waiting for wrapper interface implementation from @ArtemBortnikov") By("reconciling the EtcdCluster", func() { _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&etcdcluster)}) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/controller/factory/builders.go b/internal/controller/factory/builders.go index da1cc719..8f152377 100644 --- a/internal/controller/factory/builders.go +++ b/internal/controller/factory/builders.go @@ -20,46 +20,62 @@ import ( "context" "fmt" + "github.com/aenix-io/etcd-operator/internal/log" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/controller-runtime/pkg/log" ) -func reconcileOwnedResource(ctx context.Context, c client.Client, resource client.Object) error { +// contextWithGVK returns a new context with the GroupVersionKind (GVK) information +// of the given resource added to the context values. It uses the provided scheme +// to determine the GVK. +// If the resource is nil, it returns an error with the message "resource cannot be nil". +// If there is an error while obtaining the GVK, it returns an error with the message +// "failed to get GVK" followed by the detailed error message. +// The context value is updated with the following key-value pairs: +// - "group": GVK's GroupVersion string +// - "kind": GVK's Kind +// - "name": Resource's name +func contextWithGVK(ctx context.Context, resource client.Object, scheme *runtime.Scheme) (context.Context, error) { if resource == nil { - return fmt.Errorf("resource cannot be nil") + return nil, fmt.Errorf("resource cannot be nil") } - gvk, err := apiutil.GVKForObject(resource, c.Scheme()) + gvk, err := apiutil.GVKForObject(resource, scheme) if err != nil { - return fmt.Errorf("failed to get GVK: %w", err) + return nil, fmt.Errorf("failed to get GVK: %w", err) } - logger := log.FromContext(ctx).WithValues("group", gvk.GroupVersion().String(), "kind", gvk.Kind, "name", resource.GetName()) - logger.V(2).Info("reconciling owned resource") + ctx = log.WithValues(ctx, "group", gvk.GroupVersion().String(), "kind", gvk.Kind, "name", resource.GetName()) + return ctx, nil +} + +func reconcileOwnedResource(ctx context.Context, c client.Client, resource client.Object) error { + if resource == nil { + return fmt.Errorf("resource cannot be nil") + } + log.Debug(ctx, "reconciling owned resource") base := resource.DeepCopyObject().(client.Object) - err = c.Get(ctx, client.ObjectKeyFromObject(resource), base) + err := c.Get(ctx, client.ObjectKeyFromObject(resource), base) if err == nil { - logger.V(2).Info("updating owned resource") + log.Debug(ctx, "updating owned resource") resource.SetAnnotations(labels.Merge(base.GetAnnotations(), resource.GetAnnotations())) resource.SetResourceVersion(base.GetResourceVersion()) - logger.V(2).Info("owned resource annotations merged", "annotations", resource.GetAnnotations()) + log.Debug(ctx, "owned resource annotations merged", "annotations", resource.GetAnnotations()) return c.Update(ctx, resource) } if errors.IsNotFound(err) { - logger.V(2).Info("creating new owned resource") + log.Debug(ctx, "creating new owned resource") return c.Create(ctx, resource) } return fmt.Errorf("error getting owned resource: %w", err) } func deleteOwnedResource(ctx context.Context, c client.Client, resource client.Object) error { - gvk, err := apiutil.GVKForObject(resource, c.Scheme()) - if err != nil { - return err + if resource == nil { + return fmt.Errorf("resource cannot be nil") } - logger := log.FromContext(ctx).WithValues("group", gvk.GroupVersion().String(), "kind", gvk.Kind, "name", resource.GetName()) - logger.V(2).Info("deleting owned resource") + log.Debug(ctx, "deleting owned resource") return client.IgnoreNotFound(c.Delete(ctx, resource)) } diff --git a/internal/controller/factory/configMap.go b/internal/controller/factory/configMap.go index 5c7bbc38..66862cfd 100644 --- a/internal/controller/factory/configMap.go +++ b/internal/controller/factory/configMap.go @@ -21,11 +21,11 @@ import ( "fmt" etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" + "github.com/aenix-io/etcd-operator/internal/log" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) func GetClusterStateConfigMapName(cluster *etcdaenixiov1alpha1.EtcdCluster) string { @@ -37,6 +37,7 @@ func CreateOrUpdateClusterStateConfigMap( cluster *etcdaenixiov1alpha1.EtcdCluster, rclient client.Client, ) error { + var err error initialCluster := "" clusterService := fmt.Sprintf("%s.%s.svc:2380", GetHeadlessServiceName(cluster), cluster.Namespace) for i := int32(0); i < *cluster.Spec.Replicas; i++ { @@ -49,7 +50,6 @@ func CreateOrUpdateClusterStateConfigMap( ) } - logger := log.FromContext(ctx) configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, @@ -61,13 +61,17 @@ func CreateOrUpdateClusterStateConfigMap( "ETCD_INITIAL_CLUSTER_TOKEN": cluster.Name + "-" + cluster.Namespace, }, } + ctx, err = contextWithGVK(ctx, configMap, rclient.Scheme()) + if err != nil { + return err + } if isEtcdClusterReady(cluster) { // update cluster state to existing - logger.V(2).Info("updating cluster state", "cluster_name", cluster.Name) + log.Debug(ctx, "updating cluster state") configMap.Data["ETCD_INITIAL_CLUSTER_STATE"] = "existing" } - logger.V(2).Info("configmap spec generated", "cm_name", configMap.Name, "cm_spec", configMap.Data) + log.Debug(ctx, "configmap data generated", "data", configMap.Data) if err := ctrl.SetControllerReference(cluster, configMap, rclient.Scheme()); err != nil { return fmt.Errorf("cannot set controller reference: %w", err) diff --git a/internal/controller/factory/configmap_test.go b/internal/controller/factory/configmap_test.go index 2b83f994..2d744600 100644 --- a/internal/controller/factory/configmap_test.go +++ b/internal/controller/factory/configmap_test.go @@ -34,7 +34,7 @@ import ( var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { var ns *corev1.Namespace - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", @@ -52,7 +52,7 @@ var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { err error ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { etcdcluster = etcdaenixiov1alpha1.EtcdCluster{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-etcdcluster-", @@ -75,7 +75,7 @@ var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { } }) - AfterEach(func(ctx SpecContext) { + AfterEach(func() { err = Get(&configMap)() if err == nil { Expect(k8sClient.Delete(ctx, &configMap)).Should(Succeed()) @@ -84,7 +84,7 @@ var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { } }) - It("should successfully ensure the configmap", func(ctx SpecContext) { + It("should successfully ensure the configmap", func() { var configMapUID types.UID By("processing new etcd cluster", func() { Expect(CreateOrUpdateClusterStateConfigMap(ctx, &etcdcluster, k8sClient)).To(Succeed()) @@ -114,7 +114,7 @@ var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { }) }) - It("should fail to create the configmap with invalid owner reference", func(ctx SpecContext) { + It("should fail to create the configmap with invalid owner reference", func() { Expect(CreateOrUpdateClusterStateConfigMap(ctx, &etcdcluster, clientWithEmptyScheme)).NotTo(Succeed()) }) }) diff --git a/internal/controller/factory/pdb.go b/internal/controller/factory/pdb.go index efadbb93..e3ef2e3c 100644 --- a/internal/controller/factory/pdb.go +++ b/internal/controller/factory/pdb.go @@ -21,13 +21,13 @@ import ( "fmt" etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" + "github.com/aenix-io/etcd-operator/internal/log" v1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) func CreateOrUpdatePdb( @@ -35,15 +35,18 @@ func CreateOrUpdatePdb( cluster *etcdaenixiov1alpha1.EtcdCluster, rclient client.Client, ) error { + var err error + if cluster.Spec.PodDisruptionBudgetTemplate == nil { + ctx = log.WithValues(ctx, "group", "policy/v1", "kind", "PodDisruptionBudget", "name", cluster.Name) return deleteOwnedResource(ctx, rclient, &v1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, Name: cluster.Name, - }}) + }, + }) } - logger := log.FromContext(ctx) pdb := &v1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, @@ -62,10 +65,13 @@ func CreateOrUpdatePdb( if pdb.Spec.MinAvailable == nil && pdb.Spec.MaxUnavailable == nil { pdb.Spec.MinAvailable = ptr.To(intstr.FromInt32(int32(cluster.CalculateQuorumSize()))) } + ctx, err = contextWithGVK(ctx, pdb, rclient.Scheme()) + if err != nil { + return err + } + log.Debug(ctx, "pdb spec generated", "spec", pdb.Spec) - logger.V(2).Info("pdb spec generated", "pdb_name", pdb.Name, "pdb_spec", pdb.Spec) - - if err := ctrl.SetControllerReference(cluster, pdb, rclient.Scheme()); err != nil { + if err = ctrl.SetControllerReference(cluster, pdb, rclient.Scheme()); err != nil { return fmt.Errorf("cannot set controller reference: %w", err) } diff --git a/internal/controller/factory/pdb_test.go b/internal/controller/factory/pdb_test.go index 436a11cf..19d3a09c 100644 --- a/internal/controller/factory/pdb_test.go +++ b/internal/controller/factory/pdb_test.go @@ -34,7 +34,7 @@ import ( var _ = Describe("CreateOrUpdatePdb handlers", func() { var ns *corev1.Namespace - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", @@ -52,7 +52,7 @@ var _ = Describe("CreateOrUpdatePdb handlers", func() { err error ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { etcdcluster = etcdaenixiov1alpha1.EtcdCluster{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-etcdcluster-", @@ -76,7 +76,7 @@ var _ = Describe("CreateOrUpdatePdb handlers", func() { } }) - AfterEach(func(ctx SpecContext) { + AfterEach(func() { err = Get(&podDisruptionBudget)() if err == nil { Expect(k8sClient.Delete(ctx, &podDisruptionBudget)).Should(Succeed()) @@ -85,7 +85,7 @@ var _ = Describe("CreateOrUpdatePdb handlers", func() { } }) - It("should create PDB with pre-filled data", func(ctx SpecContext) { + It("should create PDB with pre-filled data", func() { etcdcluster.Spec.PodDisruptionBudgetTemplate.Spec.MinAvailable = ptr.To(intstr.FromInt32(int32(3))) Expect(CreateOrUpdatePdb(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Get(&podDisruptionBudget)).Should(Succeed()) @@ -94,7 +94,7 @@ var _ = Describe("CreateOrUpdatePdb handlers", func() { Expect(podDisruptionBudget.Spec.MaxUnavailable).To(BeNil()) }) - It("should create PDB with empty data", func(ctx SpecContext) { + It("should create PDB with empty data", func() { Expect(CreateOrUpdatePdb(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Get(&podDisruptionBudget)).Should(Succeed()) Expect(etcdcluster.Spec.PodDisruptionBudgetTemplate.Spec.MinAvailable).To(BeNil()) @@ -102,12 +102,12 @@ var _ = Describe("CreateOrUpdatePdb handlers", func() { Expect(podDisruptionBudget.Spec.MaxUnavailable).To(BeNil()) }) - It("should skip deletion of PDB if not filled and not exist", func(ctx SpecContext) { + It("should skip deletion of PDB if not filled and not exist", func() { etcdcluster.Spec.PodDisruptionBudgetTemplate = nil Expect(CreateOrUpdatePdb(ctx, &etcdcluster, k8sClient)).NotTo(HaveOccurred()) }) - It("should delete created PDB after updating CR", func(ctx SpecContext) { + It("should delete created PDB after updating CR", func() { Expect(CreateOrUpdatePdb(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Get(&podDisruptionBudget)).Should(Succeed()) etcdcluster.Spec.PodDisruptionBudgetTemplate = nil diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index c29bf5c9..68dc9b1c 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -19,21 +19,25 @@ package factory import ( "context" "fmt" + "math" + "strconv" + "github.com/aenix-io/etcd-operator/internal/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/k8sutils" ) const ( - etcdContainerName = "etcd" + etcdContainerName = "etcd" + defaultBackendQuotaBytesFraction = 0.95 ) func CreateOrUpdateStatefulSet( @@ -101,8 +105,11 @@ func CreateOrUpdateStatefulSet( VolumeClaimTemplates: volumeClaimTemplates, }, } - logger := log.FromContext(ctx) - logger.V(2).Info("statefulset spec generated", "sts_name", statefulSet.Name, "sts_spec", statefulSet.Spec) + ctx, err = contextWithGVK(ctx, statefulSet, rclient.Scheme()) + if err != nil { + return err + } + log.Debug(ctx, "statefulset spec generated", "spec", statefulSet.Spec) if err = ctrl.SetControllerReference(cluster, statefulSet, rclient.Scheme()); err != nil { return fmt.Errorf("cannot set controller reference: %w", err) @@ -247,6 +254,25 @@ func generateEtcdCommand() []string { func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { args := []string{} + if value, ok := cluster.Spec.Options["quota-backend-bytes"]; !ok || value == "" { + var size resource.Quantity + if cluster.Spec.Storage.EmptyDir != nil { + if cluster.Spec.Storage.EmptyDir.SizeLimit != nil { + size = *cluster.Spec.Storage.EmptyDir.SizeLimit + } + } else { + size = *cluster.Spec.Storage.VolumeClaimTemplate.Spec.Resources.Requests.Storage() + } + quota := float64(size.Value()) * defaultBackendQuotaBytesFraction + quota = math.Floor(quota) + if quota > 0 { + if cluster.Spec.Options == nil { + cluster.Spec.Options = make(map[string]string, 1) + } + cluster.Spec.Options["quota-backend-bytes"] = strconv.FormatInt(int64(quota), 10) + } + } + for name, value := range cluster.Spec.Options { flag := "--" + name if len(value) == 0 { @@ -270,38 +296,42 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { } serverTlsSettings := []string{} - serverProtocol := "http" if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { serverTlsSettings = []string{ "--cert-file=/etc/etcd/pki/server/cert/tls.crt", "--key-file=/etc/etcd/pki/server/cert/tls.key", } - serverProtocol = "https" } clientTlsSettings := []string{} - if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + if cluster.IsClientSecurityEnabled() { clientTlsSettings = []string{ "--trusted-ca-file=/etc/etcd/pki/client/ca/ca.crt", "--client-cert-auth", } } + autoCompactionSettings := []string{ + "--auto-compaction-retention=5m", + "--snapshot-count=10000", + } + args = append(args, []string{ "--name=$(POD_NAME)", "--listen-metrics-urls=http://0.0.0.0:2381", "--listen-peer-urls=https://0.0.0.0:2380", - fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", serverProtocol), + fmt.Sprintf("--listen-client-urls=%s0.0.0.0:2379", GetServerProtocol(cluster)), fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", GetHeadlessServiceName(cluster)), "--data-dir=/var/run/etcd/default.etcd", - fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", serverProtocol, GetHeadlessServiceName(cluster)), + fmt.Sprintf("--advertise-client-urls=%s$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", GetServerProtocol(cluster), GetHeadlessServiceName(cluster)), }...) args = append(args, peerTlsSettings...) args = append(args, serverTlsSettings...) args = append(args, clientTlsSettings...) + args = append(args, autoCompactionSettings...) return args } @@ -389,3 +419,11 @@ func getLivenessProbe() *corev1.Probe { PeriodSeconds: 5, } } + +func GetServerProtocol(cluster *etcdaenixiov1alpha1.EtcdCluster) string { + serverProtocol := "http://" + if cluster.IsServerSecurityEnabled() { + serverProtocol = "https://" + } + return serverProtocol +} diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index a3aaf3e3..be36bd97 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -37,7 +37,7 @@ import ( var _ = Describe("CreateOrUpdateStatefulSet handler", func() { var ns *corev1.Namespace - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", @@ -55,7 +55,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { err error ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { etcdcluster = etcdaenixiov1alpha1.EtcdCluster{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-etcdcluster-", @@ -78,7 +78,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { } }) - AfterEach(func(ctx SpecContext) { + AfterEach(func() { err = Get(&statefulSet)() if err == nil { Expect(k8sClient.Delete(ctx, &statefulSet)).Should(Succeed()) @@ -87,14 +87,14 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { } }) - It("should successfully ensure the statefulSet with empty spec", func(ctx SpecContext) { + It("should successfully ensure the statefulSet with empty spec", func() { Expect(CreateOrUpdateStatefulSet(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Object(&statefulSet)).Should( HaveField("Spec.Replicas", Equal(etcdcluster.Spec.Replicas)), ) }) - It("should successfully ensure the statefulSet with filled spec", func(ctx SpecContext) { + It("should successfully ensure the statefulSet with filled spec", func() { etcdcluster.Spec.Storage = etcdaenixiov1alpha1.StorageSpec{ VolumeClaimTemplate: etcdaenixiov1alpha1.EmbeddedPersistentVolumeClaim{ EmbeddedObjectMetadata: etcdaenixiov1alpha1.EmbeddedObjectMetadata{ @@ -276,7 +276,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }) }) - It("should successfully override probes", func(ctx SpecContext) { + It("should successfully override probes", func() { etcdcluster.Spec.PodTemplate.Spec = corev1.PodSpec{ Containers: []corev1.Container{ { @@ -353,7 +353,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }) }) - It("should successfully create statefulSet with emptyDir", func(ctx SpecContext) { + It("should successfully create statefulSet with emptyDir", func() { size := resource.MustParse("1Gi") etcdcluster.Spec.Storage = etcdaenixiov1alpha1.StorageSpec{ EmptyDir: &corev1.EmptyDirVolumeSource{ @@ -368,7 +368,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }) }) - It("should fail on creating the statefulset with invalid owner reference", func(ctx SpecContext) { + It("should fail on creating the statefulset with invalid owner reference", func() { Expect(CreateOrUpdateStatefulSet(ctx, &etcdcluster, clientWithEmptyScheme)).NotTo(Succeed()) }) }) @@ -392,6 +392,58 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { "--key2=value2", })) }) + It("should not override user defined quota-backend-bytes", func() { + etcdCluster := &etcdaenixiov1alpha1.EtcdCluster{ + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Options: map[string]string{ + "quota-backend-bytes": "2147483648", // 2Gi + }, + Storage: etcdaenixiov1alpha1.StorageSpec{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: ptr.To(resource.MustParse("2Gi")), + }, + }, + }, + } + args := generateEtcdArgs(etcdCluster) + Expect(args).To(ContainElement("--quota-backend-bytes=2147483648")) + }) + It("should set quota-backend-bytes to 0.95 of EmptyDir size", func() { + etcdCluster := &etcdaenixiov1alpha1.EtcdCluster{ + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Storage: etcdaenixiov1alpha1.StorageSpec{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: ptr.To(resource.MustParse("2Gi")), + }, + }, + }, + } + args := generateEtcdArgs(etcdCluster) + // 2Gi * 0.95 = 2040109465,6 + Expect(args).To(ContainElement("--quota-backend-bytes=2040109465")) + }) + It("should set quota-backend-bytes to 0.95 of PVC size", func() { + etcdCluster := &etcdaenixiov1alpha1.EtcdCluster{ + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Storage: etcdaenixiov1alpha1.StorageSpec{ + VolumeClaimTemplate: etcdaenixiov1alpha1.EmbeddedPersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: ptr.To("local-path"), + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + }, + } + args := generateEtcdArgs(etcdCluster) + // 2Gi * 0.95 = 2040109465,6 + Expect(args).To(ContainElement("--quota-backend-bytes=2040109465")) + }) }) /* TODO: all of the following tests validate merging logic, but all merging logic is now handled externally. diff --git a/internal/controller/factory/suite_test.go b/internal/controller/factory/suite_test.go index 7260b791..b0b47bea 100644 --- a/internal/controller/factory/suite_test.go +++ b/internal/controller/factory/suite_test.go @@ -17,11 +17,13 @@ limitations under the License. package factory import ( + "context" "fmt" "path/filepath" "runtime" "testing" + "github.com/aenix-io/etcd-operator/internal/log" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" kruntime "k8s.io/apimachinery/pkg/runtime" @@ -45,6 +47,13 @@ var cfg *rest.Config var k8sClient, clientWithEmptyScheme client.Client var testEnv *envtest.Environment +// global context for test suites +var ctx = log.Setup(context.TODO(), log.Parameters{ + LogLevel: "debug", + StacktraceLevel: "error", + Development: true, +}) + func TestFactories(t *testing.T) { RegisterFailHandler(Fail) @@ -87,6 +96,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) SetClient(k8sClient) + SetContext(ctx) }) var _ = AfterSuite(func() { diff --git a/internal/controller/factory/svc.go b/internal/controller/factory/svc.go index 9d857dd6..8a2ffb63 100644 --- a/internal/controller/factory/svc.go +++ b/internal/controller/factory/svc.go @@ -20,12 +20,12 @@ import ( "context" "fmt" + "github.com/aenix-io/etcd-operator/internal/log" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" "github.com/aenix-io/etcd-operator/internal/k8sutils" @@ -52,7 +52,6 @@ func CreateOrUpdateHeadlessService( cluster *etcdaenixiov1alpha1.EtcdCluster, rclient client.Client, ) error { - logger := log.FromContext(ctx) var err error metadata := metav1.ObjectMeta{ @@ -81,10 +80,13 @@ func CreateOrUpdateHeadlessService( PublishNotReadyAddresses: true, }, } + ctx, err = contextWithGVK(ctx, svc, rclient.Scheme()) + if err != nil { + return err + } + log.Debug(ctx, "cluster service spec generated", "spec", svc.Spec) - logger.V(2).Info("cluster service spec generated", "svc_name", svc.Name, "svc_spec", svc.Spec) - - if err := ctrl.SetControllerReference(cluster, svc, rclient.Scheme()); err != nil { + if err = ctrl.SetControllerReference(cluster, svc, rclient.Scheme()); err != nil { return fmt.Errorf("cannot set controller reference: %w", err) } @@ -96,7 +98,6 @@ func CreateOrUpdateClientService( cluster *etcdaenixiov1alpha1.EtcdCluster, rclient client.Client, ) error { - logger := log.FromContext(ctx) var err error svc := corev1.Service{ @@ -123,10 +124,13 @@ func CreateOrUpdateClientService( return fmt.Errorf("cannot strategic-merge base svc with serviceTemplate: %w", err) } } + ctx, err = contextWithGVK(ctx, &svc, rclient.Scheme()) + if err != nil { + return err + } + log.Debug(ctx, "client service spec generated", "spec", svc.Spec) - logger.V(2).Info("client service spec generated", "svc_name", svc.Name, "svc_spec", svc.Spec) - - if err := ctrl.SetControllerReference(cluster, &svc, rclient.Scheme()); err != nil { + if err = ctrl.SetControllerReference(cluster, &svc, rclient.Scheme()); err != nil { return fmt.Errorf("cannot set controller reference: %w", err) } diff --git a/internal/controller/factory/svc_test.go b/internal/controller/factory/svc_test.go index 0e27752c..a2aabaa8 100644 --- a/internal/controller/factory/svc_test.go +++ b/internal/controller/factory/svc_test.go @@ -34,7 +34,7 @@ import ( var _ = Describe("CreateOrUpdateService handlers", func() { var ns *corev1.Namespace - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-", @@ -53,7 +53,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { err error ) - BeforeEach(func(ctx SpecContext) { + BeforeEach(func() { etcdcluster = etcdaenixiov1alpha1.EtcdCluster{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-etcdcluster-", @@ -82,7 +82,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { } }) - AfterEach(func(ctx SpecContext) { + AfterEach(func() { err = Get(&headlessService)() if err == nil { Expect(k8sClient.Delete(ctx, &headlessService)).Should(Succeed()) @@ -97,7 +97,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { } }) - It("should successfully ensure headless service", func(ctx SpecContext) { + It("should successfully ensure headless service", func() { Expect(CreateOrUpdateHeadlessService(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Object(&headlessService)).Should(SatisfyAll( HaveField("Spec.Type", Equal(corev1.ServiceTypeClusterIP)), @@ -105,7 +105,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { )) }) - It("should successfully ensure headless service with custom metadata", func(ctx SpecContext) { + It("should successfully ensure headless service with custom metadata", func() { etcdcluster.Spec.HeadlessServiceTemplate = &etcdaenixiov1alpha1.EmbeddedMetadataResource{ EmbeddedObjectMetadata: etcdaenixiov1alpha1.EmbeddedObjectMetadata{ Name: "headless-name", @@ -129,7 +129,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { Expect(k8sClient.Delete(ctx, svc)).Should(Succeed()) }) - It("should successfully ensure client service", func(ctx SpecContext) { + It("should successfully ensure client service", func() { Expect(CreateOrUpdateClientService(ctx, &etcdcluster, k8sClient)).To(Succeed()) Eventually(Object(&clientService)).Should(SatisfyAll( HaveField("Spec.Type", Equal(corev1.ServiceTypeClusterIP)), @@ -137,7 +137,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { )) }) - It("should successfully ensure client service with custom metadata", func(ctx SpecContext) { + It("should successfully ensure client service with custom metadata", func() { etcdcluster.Spec.ServiceTemplate = &etcdaenixiov1alpha1.EmbeddedService{ EmbeddedObjectMetadata: etcdaenixiov1alpha1.EmbeddedObjectMetadata{ Name: "client-name", @@ -161,7 +161,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { Expect(k8sClient.Delete(ctx, svc)).Should(Succeed()) }) - It("should successfully ensure client service with custom spec", func(ctx SpecContext) { + It("should successfully ensure client service with custom spec", func() { etcdcluster.Spec.ServiceTemplate = &etcdaenixiov1alpha1.EmbeddedService{ Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, @@ -191,7 +191,7 @@ var _ = Describe("CreateOrUpdateService handlers", func() { )) }) - It("should fail on creating the client service with invalid owner reference", func(ctx SpecContext) { + It("should fail on creating the client service with invalid owner reference", func() { Expect(CreateOrUpdateHeadlessService(ctx, &etcdcluster, clientWithEmptyScheme)).NotTo(Succeed()) Expect(CreateOrUpdateClientService(ctx, &etcdcluster, clientWithEmptyScheme)).NotTo(Succeed()) }) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index fe147008..dccc5bf0 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -17,12 +17,14 @@ limitations under the License. package controller import ( + "context" "fmt" "path/filepath" "runtime" "testing" "time" + "github.com/aenix-io/etcd-operator/internal/log" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" @@ -45,6 +47,13 @@ var cfg *rest.Config var k8sClient client.Client var testEnv *envtest.Environment +// global context for test suites +var ctx = log.Setup(context.TODO(), log.Parameters{ + LogLevel: "debug", + StacktraceLevel: "error", + Development: true, +}) + func TestControllers(t *testing.T) { SetDefaultEventuallyTimeout(time.Second * 5) RegisterFailHandler(Fail) @@ -85,6 +94,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) SetClient(k8sClient) + SetContext(ctx) }) var _ = AfterSuite(func() { diff --git a/internal/log/logger.go b/internal/log/logger.go new file mode 100644 index 00000000..3bd4415d --- /dev/null +++ b/internal/log/logger.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "context" + "log/slog" + "os" + + "github.com/go-logr/logr" + "go.uber.org/zap" + "go.uber.org/zap/exp/zapslog" + "go.uber.org/zap/zapcore" +) + +func mapLogLevel(level string) zapcore.Level { + switch level { + case "debug": + return zapcore.DebugLevel + case "info": + return zapcore.InfoLevel + case "warn": + return zapcore.WarnLevel + case "error": + return zapcore.ErrorLevel + default: + return zapcore.InfoLevel + } +} + +type Parameters struct { + LogLevel string + StacktraceLevel string + Development bool +} + +// Setup initializes the logger and returns a new context with the logger attached. +// The logger is configured based on the provided Parameters. The encoder and writer +// are selected based on the Development flag. The LogLevel parameter determines the +// log level of the logger. The StacktraceLevel parameter determines the log level at +// which a stack trace is added to log entries. +// The function does not modify the original context. +// +// Example usage: +// +// ctx := Setup(context.Background(), Parameters{ +// LogLevel: "debug", +// StacktraceLevel: "error", +// Development: true, +// }) +func Setup(ctx context.Context, p Parameters) context.Context { + encoderConfig := zapcore.EncoderConfig{ + MessageKey: "message", + LevelKey: "level", + TimeKey: "time", + CallerKey: "caller", + StacktraceKey: "stacktrace", + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.RFC3339TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + EncodeName: zapcore.FullNameEncoder, + } + encoder := zapcore.NewJSONEncoder(encoderConfig) + writer := os.Stderr + if p.Development { + encoder = zapcore.NewConsoleEncoder(encoderConfig) + writer = os.Stdout + } + + core := zapcore.NewCore(encoder, writer, mapLogLevel(p.LogLevel)) + logger := zap.New(core, zap.AddStacktrace(mapLogLevel(p.StacktraceLevel))) + l := slog.New(zapslog.NewHandler(logger.Core(), &zapslog.HandlerOptions{AddSource: p.Development})) + return logr.NewContextWithSlogLogger(ctx, l) +} + +// Info logs an informational message with optional key-value pairs. +func Info(ctx context.Context, msg string, keysAndValues ...interface{}) { + logr.FromContextAsSlogLogger(ctx).With(keysAndValues...).Info(msg) +} + +// Debug logs a debug message with optional key-value pairs. +func Debug(ctx context.Context, msg string, keysAndValues ...interface{}) { + logr.FromContextAsSlogLogger(ctx).With(keysAndValues...).Debug(msg) +} + +// Warn logs a warning message with optional key-value pairs. +func Warn(ctx context.Context, msg string, keysAndValues ...interface{}) { + logr.FromContextAsSlogLogger(ctx).With(keysAndValues...).Warn(msg) +} + +// Error logs an error message with optional key-value pairs. +func Error(ctx context.Context, err error, msg string, keysAndValues ...interface{}) { + logr.FromContextAsSlogLogger(ctx).With(keysAndValues...).Error(msg, slog.Any("error", err)) +} + +// WithValues adds additional key-value pairs to the context's logger. +func WithValues(ctx context.Context, keysAndValues ...interface{}) context.Context { + return logr.NewContextWithSlogLogger(ctx, logr.FromContextAsSlogLogger(ctx).With(keysAndValues...)) +} diff --git a/internal/signal/signal.go b/internal/signal/signal.go new file mode 100644 index 00000000..223c080a --- /dev/null +++ b/internal/signal/signal.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signal + +import ( + "context" + "os" + "os/signal" +) + +// NotifyContext returns a derived context from the parent context with a cancel +// function. It sets up a signal channel to listen for the specified signals and +// cancels the context when any of those signals are received. If a second signal +// is received, it exits the program with a status code of 1. +// +// Example usage: +// +// ctx := signal.NotifyContext(parentContext, os.Interrupt, syscall.SIGTERM) +// go func() { +// <-ctx.Done() +// // Perform cleanup or other tasks before exiting +// }() +func NotifyContext(parent context.Context, signals ...os.Signal) context.Context { + ctx, cancel := context.WithCancel(parent) + c := make(chan os.Signal, 2) + signal.Notify(c, signals...) + go func() { + <-c + cancel() + <-c + os.Exit(1) + }() + return ctx +} diff --git a/site/content/en/docs/v0.2/examples/_index.md b/site/content/en/docs/v0.2/examples/_index.md index 45f82d78..38e6239f 100755 --- a/site/content/en/docs/v0.2/examples/_index.md +++ b/site/content/en/docs/v0.2/examples/_index.md @@ -2,7 +2,9 @@ title: Examples weight: 3 date: 2017-01-05 -description: See your project in action! +description: See etcd-operator in action! --- -TODO +You can find examples for deploying etcd clusters in various configuration in the GitHub repository under [examples/manifests](https://github.com/aenix-io/etcd-operator/tree/main/examples/manifests) directory. + +This directory provides various manifests that can help you understand how to set up etcd clusters using the etcd-operator. diff --git a/site/content/en/docs/v0.2/reference/_index.md b/site/content/en/docs/v0.2/reference/_index.md index 99acc582..faacee21 100644 --- a/site/content/en/docs/v0.2/reference/_index.md +++ b/site/content/en/docs/v0.2/reference/_index.md @@ -2,6 +2,5 @@ title: Reference description: Low level reference docs for your project. weight: 9 +simple_list: true --- - -TODO diff --git a/site/content/en/docs/v0.2/reference/api.md b/site/content/en/docs/v0.2/reference/api.md new file mode 100644 index 00000000..7c972d28 --- /dev/null +++ b/site/content/en/docs/v0.2/reference/api.md @@ -0,0 +1,246 @@ +--- +title: API Reference +description: Autogenerated API Reference for the CRD +weight: 9 +simple_list: true +--- + +## Packages +- [etcd.aenix.io/v1alpha1](#etcdaenixiov1alpha1) + + +## etcd.aenix.io/v1alpha1 + +Package v1alpha1 contains API Schema definitions for the etcd.aenix.io v1alpha1 API group + +### Resource Types +- [EtcdCluster](#etcdcluster) + + + +#### EmbeddedMetadataResource + + + + + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | + + +#### EmbeddedObjectMetadata + + + +EmbeddedObjectMetadata contains a subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta +Only fields which are relevant to embedded resources are included. + + + +_Appears in:_ +- [EmbeddedMetadataResource](#embeddedmetadataresource) +- [EmbeddedPersistentVolumeClaim](#embeddedpersistentvolumeclaim) +- [EmbeddedPodDisruptionBudget](#embeddedpoddisruptionbudget) +- [EmbeddedService](#embeddedservice) +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name must be unique within a namespace. Is required when creating resources, although
some resources may allow a client to request the generation of an appropriate name
automatically. Name is primarily intended for creation idempotence and configuration
definition.
Cannot be updated.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names | | | +| `labels` _object (keys:string, values:string)_ | Labels Map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | | +| `annotations` _object (keys:string, values:string)_ | Annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | | + + +#### EmbeddedPersistentVolumeClaim + + + +EmbeddedPersistentVolumeClaim is an embedded version of k8s.io/api/core/v1.PersistentVolumeClaim. +It contains TypeMeta and a reduced ObjectMeta. + + + +_Appears in:_ +- [StorageSpec](#storagespec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30.0/#persistentvolumeclaimspec-v1-core)_ | Spec defines the desired characteristics of a volume requested by a pod author.
More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims | | | + + +#### EmbeddedPodDisruptionBudget + + + +EmbeddedPodDisruptionBudget describes PDB resource for etcd cluster members + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodDisruptionBudgetSpec](#poddisruptionbudgetspec)_ | Spec defines the desired characteristics of a PDB.
More info: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/#pod-disruption-budgets | | | + + +#### EmbeddedService + + + + + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[ServiceSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30.0/#servicespec-v1-core)_ | Spec defines the behavior of the service. | | | + + +#### EtcdCluster + + + +EtcdCluster is the Schema for the etcdclusters API + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `etcd.aenix.io/v1alpha1` | | | +| `kind` _string_ | `EtcdCluster` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30.0/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EtcdClusterSpec](#etcdclusterspec)_ | | | | + + +#### EtcdClusterSpec + + + +EtcdClusterSpec defines the desired state of EtcdCluster + + + +_Appears in:_ +- [EtcdCluster](#etcdcluster) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `replicas` _integer_ | Replicas is the count of etcd instances in cluster. | 3 | Minimum: 0
| +| `options` _object (keys:string, values:string)_ | Options are the extra arguments to pass to the etcd container. | | | +| `podTemplate` _[PodTemplate](#podtemplate)_ | PodTemplate defines the desired state of PodSpec for etcd members. If not specified, default values will be used. | | | +| `serviceTemplate` _[EmbeddedService](#embeddedservice)_ | Service defines the desired state of Service for etcd members. If not specified, default values will be used. | | | +| `headlessServiceTemplate` _[EmbeddedMetadataResource](#embeddedmetadataresource)_ | HeadlessService defines the desired state of HeadlessService for etcd members. If not specified, default values will be used. | | | +| `podDisruptionBudgetTemplate` _[EmbeddedPodDisruptionBudget](#embeddedpoddisruptionbudget)_ | PodDisruptionBudgetTemplate describes PDB resource to create for etcd cluster members. Nil to disable. | | | +| `storage` _[StorageSpec](#storagespec)_ | | | | +| `security` _[SecuritySpec](#securityspec)_ | Security describes security settings of etcd (authentication, certificates, rbac) | | | + + + + + + +#### PodDisruptionBudgetSpec + + + + + + + +_Appears in:_ +- [EmbeddedPodDisruptionBudget](#embeddedpoddisruptionbudget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `minAvailable` _[IntOrString](#intorstring)_ | MinAvailable describes minimum ready replicas. If both are empty, controller will implicitly
calculate MaxUnavailable based on number of replicas
Mutually exclusive with MaxUnavailable. | | | +| `maxUnavailable` _[IntOrString](#intorstring)_ | MinAvailable describes maximum not ready replicas. If both are empty, controller will implicitly
calculate MaxUnavailable based on number of replicas
Mutually exclusive with MinAvailable | | | + + +#### PodTemplate + + + +PodTemplate allows overrides, such as sidecars, init containers, changes to the security context, etc to the pod template generated by the operator. + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30.0/#podspec-v1-core)_ | Spec follows the structure of a regular Pod spec. Overrides defined here will be strategically merged with the default pod spec, generated by the operator. | | | + + +#### SecuritySpec + + + +SecuritySpec defines security settings for etcd. + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `tls` _[TLSSpec](#tlsspec)_ | Section for user-managed tls certificates | | | +| `enableAuth` _boolean_ | Section to enable etcd auth | | | + + +#### StorageSpec + + + +StorageSpec defines the configured storage for a etcd members. +If neither `emptyDir` nor `volumeClaimTemplate` is specified, then by default an [EmptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) will be used. + + + +_Appears in:_ +- [EtcdClusterSpec](#etcdclusterspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `emptyDir` _[EmptyDirVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30.0/#emptydirvolumesource-v1-core)_ | EmptyDirVolumeSource to be used by the StatefulSets. If specified, used in place of any volumeClaimTemplate. More
info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir | | | +| `volumeClaimTemplate` _[EmbeddedPersistentVolumeClaim](#embeddedpersistentvolumeclaim)_ | A PVC spec to be used by the StatefulSets. | | | + + +#### TLSSpec + + + +TLSSpec defines user-managed certificates names. + + + +_Appears in:_ +- [SecuritySpec](#securityspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `peerTrustedCASecret` _string_ | Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have ca.crt field in the secret.
This secret must be created in the namespace with etcdCluster CR. | | | +| `peerSecret` _string_ | Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret.
This secret must be created in the namespace with etcdCluster CR. | | | +| `serverTrustedCASecret` _string_ | Trusted CA for etcd server certificates for client-server communication. Is necessary to set trust between operator and etcd.
It is expected to have ca.crt field in the secret. If it is not specified, then insecure communication will be used.
This secret must be created in the namespace with etcdCluster CR. | | | +| `serverSecret` _string_ | Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default).
It is expected to have tls.crt and tls.key fields in the secret.
This secret must be created in the namespace with etcdCluster CR. | | | +| `clientTrustedCASecret` _string_ | Trusted CA for client certificates that are provided by client to etcd. It is expected to have ca.crt field in the secret.
This secret must be created in the namespace with etcdCluster CR. | | | +| `clientSecret` _string_ | Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret.
This secret must be created in the namespace with etcdCluster CR. | | | + + diff --git a/site/package-lock.json b/site/package-lock.json index c33a395d..8abd072f 100644 --- a/site/package-lock.json +++ b/site/package-lock.json @@ -11,7 +11,7 @@ "devDependencies": { "autoprefixer": "^10.4.14", "cross-env": "^7.0.3", - "hugo-extended": "0.125.4", + "hugo-extended": "0.127.0", "postcss-cli": "^11.0.0" } }, @@ -1150,11 +1150,12 @@ } }, "node_modules/hugo-extended": { - "version": "0.125.4", - "resolved": "https://registry.npmjs.org/hugo-extended/-/hugo-extended-0.125.4.tgz", - "integrity": "sha512-t2P9kz/CN6UQCX/XConZLb4Xxlx/YU8A11L4ysGYdPWUZB4XxOOe/1jcY23KChIqH3joEeBcz5DnzmgmwXrmcA==", + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/hugo-extended/-/hugo-extended-0.127.0.tgz", + "integrity": "sha512-TDjizLf8cUpkRPkHXeDq3rZJKSAUdguV7KPddOZrclNcnL1QQh/9BgKddCZP0+4/nolPOb6GI+2LhIehfjrUHw==", "dev": true, "hasInstallScript": true, + "license": "MIT", "dependencies": { "careful-downloader": "^3.0.0", "log-symbols": "^5.1.0", @@ -1165,7 +1166,7 @@ "hugo-extended": "lib/cli.js" }, "engines": { - "node": ">=14.14" + "node": ">=16.14" } }, "node_modules/ieee754": { diff --git a/site/package.json b/site/package.json index dc9730d4..dd2efc72 100644 --- a/site/package.json +++ b/site/package.json @@ -36,7 +36,7 @@ "devDependencies": { "autoprefixer": "^10.4.14", "cross-env": "^7.0.3", - "hugo-extended": "0.125.4", + "hugo-extended": "0.127.0", "postcss-cli": "^11.0.0" } } diff --git a/site/reference-templates/gv_detail.tpl b/site/reference-templates/gv_detail.tpl new file mode 100644 index 00000000..30ad0d75 --- /dev/null +++ b/site/reference-templates/gv_detail.tpl @@ -0,0 +1,19 @@ +{{- define "gvDetails" -}} +{{- $gv := . -}} + +## {{ $gv.GroupVersionString }} + +{{ $gv.Doc }} + +{{- if $gv.Kinds }} +### Resource Types +{{- range $gv.SortedKinds }} +- {{ $gv.TypeForKind . | markdownRenderTypeLink }} +{{- end }} +{{ end }} + +{{ range $gv.SortedTypes }} +{{ template "type" . }} +{{ end }} + +{{- end -}} diff --git a/site/reference-templates/gv_list.tpl b/site/reference-templates/gv_list.tpl new file mode 100644 index 00000000..e39be9b0 --- /dev/null +++ b/site/reference-templates/gv_list.tpl @@ -0,0 +1,20 @@ +{{- define "gvList" -}} +{{- $groupVersions := . -}} + +--- +title: API Reference +description: Autogenerated API Reference for the CRD +weight: 9 +simple_list: true +--- + +## Packages +{{- range $groupVersions }} +- {{ markdownRenderGVLink . }} +{{- end }} + +{{ range $groupVersions }} +{{ template "gvDetails" . }} +{{ end }} + +{{- end -}} diff --git a/site/reference-templates/type.tpl b/site/reference-templates/type.tpl new file mode 100644 index 00000000..405a32af --- /dev/null +++ b/site/reference-templates/type.tpl @@ -0,0 +1,40 @@ +{{- define "type" -}} +{{- $type := . -}} +{{- if markdownShouldRenderType $type -}} + +#### {{ $type.Name }} + +{{ if $type.IsAlias }}_Underlying type:_ _{{ markdownRenderTypeLink $type.UnderlyingType }}_{{ end }} + +{{ $type.Doc }} + +{{ if $type.Validation -}} +_Validation:_ +{{- range $type.Validation }} +- {{ . }} +{{- end }} +{{- end }} + +{{ if $type.References -}} +_Appears in:_ +{{- range $type.SortedReferences }} +- {{ markdownRenderTypeLink . }} +{{- end }} +{{- end }} + +{{ if $type.Members -}} +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +{{ if $type.GVK -}} +| `apiVersion` _string_ | `{{ $type.GVK.Group }}/{{ $type.GVK.Version }}` | | | +| `kind` _string_ | `{{ $type.GVK.Kind }}` | | | +{{ end -}} + +{{ range $type.Members -}} +| `{{ .Name }}` _{{ markdownRenderType .Type }}_ | {{ template "type_members" . }} | {{ markdownRenderDefault .Default }} | {{ range .Validation -}} {{ . }}
{{ end }} | +{{ end -}} + +{{ end -}} + +{{- end -}} +{{- end -}} diff --git a/site/reference-templates/type_members.tpl b/site/reference-templates/type_members.tpl new file mode 100644 index 00000000..041758a8 --- /dev/null +++ b/site/reference-templates/type_members.tpl @@ -0,0 +1,8 @@ +{{- define "type_members" -}} +{{- $field := . -}} +{{- if eq $field.Name "metadata" -}} +Refer to Kubernetes API documentation for fields of `metadata`. +{{- else -}} +{{ markdownRenderFieldDoc $field.Doc }} +{{- end -}} +{{- end -}} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 968c8056..5a0976a2 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -17,16 +17,37 @@ limitations under the License. package e2e import ( + "context" "fmt" "testing" + "github.com/aenix-io/etcd-operator/internal/log" + "github.com/go-logr/logr" + logctrl "sigs.k8s.io/controller-runtime/pkg/log" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +var ctx context.Context + +var _ = BeforeSuite(func() { + ctx = log.Setup(context.TODO(), log.Parameters{ + LogLevel: "error", + StacktraceLevel: "error", + Development: true, + }) + + // This line prevents controller-runtime from complaining about log.SetLogger never being called + logctrl.SetLogger(logr.FromContextOrDiscard(ctx)) +}) + // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting etcd-operator suite\n") + _, err := fmt.Fprintf(GinkgoWriter, "Starting etcd-operator suite\n") + if err != nil { + return + } RunSpecs(t, "e2e suite") } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e245fb11..c9967858 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,12 +17,15 @@ limitations under the License. package e2e import ( + "fmt" + "os" "os/exec" - "strconv" "sync" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + clientv3 "go.etcd.io/etcd/client/v3" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aenix-io/etcd-operator/test/utils" ) @@ -31,26 +34,43 @@ var _ = Describe("etcd-operator", Ordered, func() { BeforeAll(func() { var err error - By("prepare kind environment") - cmd := exec.Command("make", "kind-prepare") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - By("upload latest etcd-operator docker image to kind cluster") - cmd = exec.Command("make", "kind-load") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - By("deploy etcd-operator") - cmd = exec.Command("make", "deploy") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }) + By("prepare kind environment", func() { + cmd := exec.Command("make", "kind-prepare") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + By("upload latest etcd-operator docker image to kind cluster", func() { + cmd := exec.Command("make", "kind-load") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + By("deploy etcd-operator", func() { + cmd := exec.Command("make", "deploy") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + By("wait while etcd-operator is ready", func() { + cmd := exec.Command("kubectl", "wait", "--namespace", + "etcd-operator-system", "deployment/etcd-operator-controller-manager", + "--for", "jsonpath={.status.availableReplicas}=1", "--timeout=5m") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - AfterAll(func() { - By("Delete kind environment") - cmd := exec.Command("make", "kind-delete") - _, _ = utils.Run(cmd) }) + if os.Getenv("DO_NOT_CLEANUP_AFTER_E2E") == "true" { + AfterAll(func() { + By("Delete kind environment", func() { + cmd := exec.Command("make", "kind-delete") + _, _ = utils.Run(cmd) + }) + }) + } + Context("Simple", func() { It("should deploy etcd cluster", func() { var err error @@ -58,47 +78,113 @@ var _ = Describe("etcd-operator", Ordered, func() { var wg sync.WaitGroup wg.Add(1) - By("create namespace") - cmd := exec.Command("kubectl", "create", "namespace", namespace) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("create namespace", func() { + cmd := exec.Command("sh", "-c", + fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) // nolint:lll + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - By("apply simple etcd cluster manifest") - dir, _ := utils.GetProjectDir() - cmd = exec.Command("kubectl", "apply", - "--filename", dir+"/examples/manifests/etcdcluster-simple.yaml", - "--namespace", namespace, - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("apply simple etcd cluster manifest", func() { + dir, _ := utils.GetProjectDir() + cmd := exec.Command("kubectl", "apply", + "--filename", dir+"/examples/manifests/etcdcluster-simple.yaml", + "--namespace", namespace, + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - By("wait for statefulset is ready") - cmd = exec.Command("kubectl", "wait", - "statefulset/test", - "--for", "jsonpath={.status.availableReplicas}=3", - "--namespace", namespace, - "--timeout", "5m", - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("wait for statefulset is ready", func() { + cmd := exec.Command("kubectl", "wait", + "statefulset/test", + "--for", "jsonpath={.status.readyReplicas}=3", + "--namespace", namespace, + "--timeout", "5m", + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := client.Close() + Expect(err).NotTo(HaveOccurred()) + }() + + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + }) + + }) + }) + + Context("TLS and enabled auth", func() { + It("should deploy etcd cluster with auth", func() { + var err error + const namespace = "test-tls-auth-etcd-cluster" + var wg sync.WaitGroup + wg.Add(1) + + By("create namespace", func() { + cmd := exec.Command("sh", "-c", fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) //nolint:lll + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + By("apply tls with enabled auth etcd cluster manifest", func() { + dir, _ := utils.GetProjectDir() + cmd := exec.Command("kubectl", "apply", + "--filename", dir+"/examples/manifests/etcdcluster-with-external-certificates.yaml", + "--namespace", namespace, + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) - By("port-forward service to localhost") - port, _ := utils.GetFreePort() - go func() { - defer GinkgoRecover() - defer wg.Done() - cmd = exec.Command("kubectl", "port-forward", - "service/test", strconv.Itoa(port)+":2379", + By("wait for statefulset is ready", func() { + cmd := exec.Command("kubectl", "wait", + "statefulset/test", + "--for", "jsonpath={.status.availableReplicas}=3", "--namespace", namespace, + "--timeout", "5m", ) _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := client.Close() + Expect(err).NotTo(HaveOccurred()) }() - By("check etcd cluster is healthy") - endpoints := []string{"localhost:" + strconv.Itoa(port)} - for i := 0; i < 3; i++ { - Expect(utils.IsEtcdClusterHealthy(endpoints)).To(BeTrue()) - } + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + }) + + auth := clientv3.NewAuth(client) + + By("check root role is created", func() { + _, err = auth.RoleGet(ctx, "root") + Expect(err).NotTo(HaveOccurred()) + }) + + By("check root user is created and has root role", func() { + userResponce, err := auth.UserGet(ctx, "root") + Expect(err).NotTo(HaveOccurred()) + Expect(userResponce.Roles).To(ContainElement("root")) + }) + + By("check auth is enabled", func() { + authStatus, err := auth.AuthStatus(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(authStatus.Enabled).To(BeTrue()) + }) + }) }) + }) diff --git a/test/utils/utils.go b/test/utils/utils.go index cba3878f..02d77295 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -19,17 +19,27 @@ package utils import ( "context" "fmt" + "regexp" + "strconv" clientv3 "go.etcd.io/etcd/client/v3" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" - "log" "net" "os" "os/exec" "strings" "time" + etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" + "github.com/aenix-io/etcd-operator/internal/controller" + "github.com/aenix-io/etcd-operator/internal/log" + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) // Run executes the provided command within this context @@ -38,12 +48,18 @@ func Run(cmd *exec.Cmd) ([]byte, error) { cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, err := fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + if err != nil { + return nil, err + } } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, err := fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + if err != nil { + return nil, err + } output, err := cmd.CombinedOutput() if err != nil { return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) @@ -76,8 +92,8 @@ func GetProjectDir() (string, error) { return wd, nil } -// GetFreePort asks the kernel for a free open port that is ready to use. -func GetFreePort() (port int, err error) { +// getFreePort asks the kernel for a free open port that is ready to use. +func getFreePort(ctx context.Context) (port int, err error) { var a *net.TCPAddr if a, err = net.ResolveTCPAddr("tcp", "localhost:0"); err == nil { var l *net.TCPListener @@ -85,7 +101,7 @@ func GetFreePort() (port int, err error) { defer func(l *net.TCPListener) { err := l.Close() if err != nil { - log.Fatal(err) + log.Error(ctx, err, "failed to get free port") } }(l) return l.Addr().(*net.TCPAddr).Port, nil @@ -94,44 +110,52 @@ func GetFreePort() (port int, err error) { return } -// GetEtcdClient creates client for interacting with etcd. -func GetEtcdClient(endpoints []string) *clientv3.Client { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 5 * time.Second, - }) +func GetK8sClient(ctx context.Context) (controller.EtcdClusterReconciler, error) { + + cfg, err := config.GetConfig() + if err != nil { + return controller.EtcdClusterReconciler{}, err + } + + c, err := client.New(cfg, client.Options{Scheme: runtimeScheme}) if err != nil { - log.Fatal(err) + return controller.EtcdClusterReconciler{}, err } - return cli + + r := controller.EtcdClusterReconciler{ + Client: c, + } + + return r, err +} + +var ( + runtimeScheme = runtime.NewScheme() +) + +func init() { + _ = etcdaenixiov1alpha1.AddToScheme(runtimeScheme) + _ = corev1.AddToScheme(runtimeScheme) } // IsEtcdClusterHealthy checks etcd cluster health. -func IsEtcdClusterHealthy(endpoints []string) bool { +func IsEtcdClusterHealthy(ctx context.Context, client *clientv3.Client) (bool, error) { // Should be changed when etcd is healthy health := false - - // Configure client - client := GetEtcdClient(endpoints) - defer func(client *clientv3.Client) { - err := client.Close() - if err != nil { - log.Fatal(err) - } - }(client) + var err error // Prepare the maintenance client maint := clientv3.NewMaintenance(client) // Context for the call - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() // Perform the status call to check health - for i := range endpoints { - resp, err := maint.Status(ctx, endpoints[i]) + for _, endpoint := range client.Endpoints() { + resp, err := maint.Status(ctx, endpoint) if err != nil { - log.Fatalf("Failed to get endpoint health: %v", err) + return false, err } else { if resp.Errors == nil { fmt.Printf("Endpoint is healthy: %s\n", resp.Version) @@ -139,5 +163,44 @@ func IsEtcdClusterHealthy(endpoints []string) bool { } } } - return health + return health, err +} + +func GetEtcdClient(ctx context.Context, namespacedName types.NamespacedName) (*clientv3.Client, error) { + + r, err := GetK8sClient(ctx) + if err != nil { + return nil, err + } + + cluster := &etcdaenixiov1alpha1.EtcdCluster{} + err = r.Get(ctx, namespacedName, cluster) + if err != nil { + return nil, err + } + + client, err := r.GetEtcdClient(ctx, cluster) + if err != nil { + return nil, err + } + + port, _ := getFreePort(ctx) + go func() { + cmd := exec.Command("kubectl", "port-forward", + "service/test", strconv.Itoa(port)+":2379", + "--namespace", cluster.Namespace, + ) + _, err = Run(cmd) + }() + + localEndpoints := []string{} + + for _, endpoint := range client.Endpoints() { + re := regexp.MustCompile(`:\/\/.*`) + localEndpoints = append(localEndpoints, re.ReplaceAllString(endpoint, "://localhost:"+strconv.Itoa(port))) + } + + client.SetEndpoints(localEndpoints...) + + return client, err }