Merge pull request #293 from 3scale-ops/feat/reproducible-dev-env

feat/local-setup

Makefile

@@ -49,11 +49,10 @@ endif
 IMG ?= $(IMAGE_TAG_BASE):v$(VERSION)
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.24
+ENVTEST_K8S_VERSION = 1.27
 
 # KIND_K8S_VERSION refers to the version of the kind k8s cluster for e2e testing.
-# OCP 4.11 uses k8s 1.24
-KIND_K8S_VERSION = v1.24.0
+KIND_K8S_VERSION = v1.27.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -108,14 +107,23 @@ vet: ## Run go vet against code.
 TEST_PKG = ./api/... ./controllers/... ./pkg/...
 KUBEBUILDER_ASSETS = "$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)"
 
-test: manifests generate fmt vet envtest assets ginkgo ## Run tests.
+test/assets/external-apis/crds.yaml: kustomize
+	mkdir -p $(@D)
+	$(KUSTOMIZE) build config/dependencies/external-secrets-crds > $@
+	echo "---" >> $@ && $(KUSTOMIZE) build config/dependencies/grafana-crds >> $@
+	echo "---" >> $@ && $(KUSTOMIZE) build config/dependencies/marin3r-crds >> $@
+	echo "---" >> $@ && $(KUSTOMIZE) build config/dependencies/prometheus-crds >> $@
+	echo "---" >> $@ && $(KUSTOMIZE) build config/dependencies/tekton-crds >> $@
+
+test: manifests generate fmt vet envtest assets ginkgo test/assets/external-apis/crds.yaml ## Run tests.
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -p -r $(TEST_PKG)  -coverprofile cover.out
 
-test-debug: manifests generate fmt vet envtest assets ginkgo ## Run tests.
+test-debug: manifests generate fmt vet envtest assets ginkgo test/assets/external-apis/crds.yaml ## Run tests.
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -v -r $(TEST_PKG)  -coverprofile cover.out
 
+TEST_E2E_DEPLOY = marin3r-crds prometheus-crds tekton-crds grafana-crds external-secrets-crds minio
 test-e2e: export KUBECONFIG = $(PWD)/kubeconfig
-test-e2e: manifests ginkgo kind-create kind-deploy kind-deploy-backup-assets ## Runs e2e tests
+test-e2e: manifests ginkgo kind-create $(foreach elem,$(TEST_E2E_DEPLOY),install-$(elem)) kind-deploy-controller kind-load-redis-with-ssh ## Runs e2e tests
 	$(GINKGO) -p -r ./test/e2e
 	$(MAKE) kind-delete
 
@@ -250,48 +258,87 @@ catalog-retag-latest:
 ##@ Kind Deployment
 
 kind-create: export KUBECONFIG = $(PWD)/kubeconfig
-kind-create: docker-build kind ## Runs a k8s kind cluster with a local registry in "localhost:5000" and ports 1080 and 1443 exposed to the host
-	$(KIND) create cluster --wait 5m --image kindest/node:$(KIND_K8S_VERSION) || true
+kind-create: kind ## Runs a k8s kind cluster
+	docker inspect kind-saas-operator > /dev/null || docker network create -d bridge --subnet 172.27.27.0/24 kind-saas-operator
+	KIND_EXPERIMENTAL_DOCKER_NETWORK=kind-saas-operator $(KIND) create cluster --wait 5m --image kindest/node:$(KIND_K8S_VERSION)
+
+install-%: export KUBECONFIG = $(PWD)/kubeconfig
+install-%: kustomize yq helm
+	echo
+	KUSTOMIZE_BIN=$(KUSTOMIZE) YQ_BIN=$(YQ) BASE_PATH=config/dependencies hack/apply-kustomize.sh $*
 
 kind-delete: ## Deletes the kind cluster and the registry
 kind-delete: kind
 	$(KIND) delete cluster
 
-kind-deploy: export KUBECONFIG = $(PWD)/kubeconfig
-kind-deploy: manifests kustomize ## Deploy operator to the Kind K8s cluster
-	kubectl apply -f config/test/external-apis/ && \
-		find config/test/external-apis/ -name '*yaml' -type f \
-            | sed -n 's/.*\/\(.*\).yaml/\1/p' \
-            | xargs -n1 kubectl wait --for condition=established --timeout=60s crd
+CONTROLLER_DEPS = prometheus-crds grafana-crds
+kind-deploy-controller: export KUBECONFIG = $(PWD)/kubeconfig
+kind-deploy-controller: manifests kustomize docker-build $(foreach elem,$(CONTROLLER_DEPS),install-$(elem)) ## Deploy operator to the Kind K8s cluster
 	$(KIND) load docker-image $(IMG)
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/test | kubectl apply -f -
+	$(KUSTOMIZE) build config/test --load-restrictor LoadRestrictionsNone | kubectl apply -f -
 
-kind-refresh-operator: export KUBECONFIG = ${PWD}/kubeconfig
-kind-refresh-operator: manifests kind docker-build ## Reloads the operator image into the K8s cluster and deletes the old Pod
+kind-refresh-controller: export KUBECONFIG = ${PWD}/kubeconfig
+kind-refresh-controller: manifests kind docker-build ## Reloads the controller image into the K8s cluster and deletes the old Pod
 	$(KIND) load docker-image $(IMG)
 	kubectl delete pod -l control-plane=controller-manager
 
+kind-deploy-databases: export KUBECONFIG = $(PWD)/kubeconfig
+kind-deploy-databases: kind-deploy-controller
+	$(KUSTOMIZE) build config/local-setup/databases | kubectl apply -f -
+	sleep 10
+	kubectl wait --for condition=ready --timeout=300s pod --all
+
 kind-undeploy: export KUBECONFIG = $(PWD)/kubeconfig
 kind-undeploy: ## Undeploy controller from the Kind K8s cluster
 	$(KUSTOMIZE) build config/test | kubectl delete -f -
 
-kind-deploy-backup-assets: export KUBECONFIG = $(PWD)/kubeconfig
-kind-deploy-backup-assets: kind-load-redis-with-ssh
-	$(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone --enable-helm | kubectl apply -f -
-
 REDIS_WITH_SSH_IMG = redis-with-ssh:6.2.13-alpine
 kind-load-redis-with-ssh:
 	docker build -t $(REDIS_WITH_SSH_IMG) test/assets/redis-with-ssh
 	$(KIND) load docker-image $(REDIS_WITH_SSH_IMG)
 
+kind-deploy-saas: export KUBECONFIG = ${PWD}/kubeconfig
+kind-deploy-saas: kind-load-redis-with-ssh ## Deploys a 3scale SaaS dev environment
+	$(KUSTOMIZE) build config/local-setup | kubectl apply -f -
+	sleep 5
+	kubectl wait --for condition=ready --timeout=300s pod system-console-0
+	kubectl get pods --no-headers -o name | grep -v system | xargs kubectl wait --for condition=ready --timeout=300s
+	kubectl -ti exec system-console-0 -c system-console -- bash -c '\
+		MASTER_DOMAIN=multitenant-admin \
+		MASTER_ACCESS_TOKEN=mtoken \
+		MASTER_PASSWORD=mpass \
+		MASTER_USER=admin \
+		TENANT_NAME=provider \
+		PROVIDER_NAME="3scale SaaS Dev Provider" \
+		USER_LOGIN=admin \
+		USER_PASSWORD=ppass \
+		ADMIN_ACCESS_TOKEN=ptoken \
+		USER_EMAIL="admin@cluster.local" \
+		DISABLE_DATABASE_ENVIRONMENT_CHECK=1 \
+		bundle exec rake db:setup'
+	kubectl get pods --no-headers -o name | grep system | xargs kubectl wait --for condition=ready --timeout=300s
+
+kind-cleanup-saas: export KUBECONFIG = ${PWD}/kubeconfig
+kind-cleanup-saas:
+	-$(KUSTOMIZE) build config/local-setup/databases | kubectl delete -f -
+	-$(KUSTOMIZE) build config/local-setup | kubectl delete -f -
+	-kubectl get pod --no-headers -o name | grep -v saas-operator | xargs kubectl delete --grace-period=0 --force
+	-kubectl get pvc --no-headers -o name | xargs kubectl delete
+
+LOCAL_SETUP_DEPS = metallb cert-manager marin3r prometheus-crds tekton-crds grafana-crds external-secrets-crds minio
+kind-local-setup: export KUBECONFIG = ${PWD}/kubeconfig
+kind-local-setup: $(foreach elem,$(LOCAL_SETUP_DEPS),install-$(elem)) kind-deploy-controller kind-deploy-databases kind-deploy-saas
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to
 LOCALBIN ?= $(shell pwd)/bin
 $(LOCALBIN):
 	mkdir -p $(LOCALBIN)
 
+export PATH := $(LOCALBIN):$(PATH)
+
 ## Tool Binaries
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
@@ -300,6 +347,8 @@ GINKGO ?= $(LOCALBIN)/ginkgo
 CRD_REFDOCS ?= $(LOCALBIN)/crd-ref-docs
 KIND ?= $(LOCALBIN)/kind
 GOBINDATA ?= $(LOCALBIN)/go-bindata
+YQ ?= $(LOCALBIN)/yq
+HELM ?= $(LOCALBIN)/helm
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.1.1
@@ -310,6 +359,8 @@ KIND_VERSION ?= v0.16.0
 ENVTEST_VERSION ?= latest
 GOBINDATA_VERSION ?= latest
 TEKTON_VERSION ?= v0.49.0
+YQ_VERSION ?= v4.40.5
+HELM_VERSION ?= v3.14.0
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
@@ -345,6 +396,17 @@ go-bindata: $(GOBINDATA) ## Download go-bindata locally if necessary.
 $(GOBINDATA):
 	test -s $(GOBINDATA) || GOBIN=$(LOCALBIN) go install github.com/go-bindata/go-bindata/...@$(GOBINDATA_VERSION)
 
+.PHONY: yq
+yq: $(YQ)
+$(YQ):
+	test -s $(YQ) || GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION)
+
+HELM_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3"
+.PHONY: helm
+helm: $(HELM)
+$(HELM):
+	curl -s $(HELM_INSTALL_SCRIPT) | HELM_INSTALL_DIR=$(LOCALBIN) bash -s -- --no-sudo --version $(HELM_VERSION)
+
 ##@ Other
 
 .PHONY: operator-sdk

api/v1alpha1/system_types.go

@@ -430,13 +430,15 @@ type SystemConfig struct {
 	// DSN of system's main database
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	DatabaseDSN SecretReference `json:"databaseDSN"`
-	// EventsSharedSecret
+	// EventsSharedSecret is a password that protects System's event
+	// hooks endpoint.
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	EventsSharedSecret SecretReference `json:"eventsSharedSecret"`
 	// Holds recaptcha configuration options
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	Recaptcha SystemRecaptchaSpec `json:"recaptcha"`
-	// SecretKeyBase
+	// SecretKeyBase: https://api.rubyonrails.org/classes/Rails/Application.html#method-i-secret_key_base
+	// You can generate one random key using 'bundle exec rake secret'
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	SecretKeyBase SecretReference `json:"secretKeyBase"`
 	// AccessCode to protect admin urls
@@ -456,7 +458,8 @@ type SystemConfig struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// +optional
 	Bugsnag *BugsnagSpec `json:"bugsnag,omitempty"`
-	// Database secret
+	// DatabaseSecret is a site key stored off-database for improved more secure password hashing
+	// See https://github.com/3scale/porta/blob/ae498814cef3d856613f60d29330882fa870271d/config/initializers/site_keys.rb#L2-L19
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	DatabaseSecret SecretReference `json:"databaseSecret"`
 	// Memcached servers

config/crd/bases/saas.3scale.net_systems.yaml

@@ -861,7 +861,8 @@ spec:
                         type: string
                     type: object
                   databaseSecret:
-                    description: Database secret
+                    description: DatabaseSecret is a site key stored off-database
+                      for improved more secure password hashing See https://github.com/3scale/porta/blob/ae498814cef3d856613f60d29330882fa870271d/config/initializers/site_keys.rb#L2-L19
                     properties:
                       fromVault:
                         description: FromVault is a reference to a secret key/value
@@ -883,7 +884,8 @@ spec:
                         type: string
                     type: object
                   eventsSharedSecret:
-                    description: EventsSharedSecret
+                    description: EventsSharedSecret is a password that protects System's
+                      event hooks endpoint.
                     properties:
                       fromVault:
                         description: FromVault is a reference to a secret key/value
@@ -1156,7 +1158,8 @@ spec:
                         type: integer
                     type: object
                   secretKeyBase:
-                    description: SecretKeyBase
+                    description: 'SecretKeyBase: https://api.rubyonrails.org/classes/Rails/Application.html#method-i-secret_key_base
+                      You can generate one random key using ''bundle exec rake secret'''
                     properties:
                       fromVault:
                         description: FromVault is a reference to a secret key/value

config/dependencies/cert-manager/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml
+  - resources.yaml

config/dependencies/cert-manager/resources.yaml

@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned-cluster-issuer
+spec:
+  selfSigned: {}

config/dependencies/external-secrets-crds/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - https://github.com/external-secrets/external-secrets/config/crds/bases/?ref=v0.9.11

config/dependencies/grafana-crds/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+ - https://github.com/grafana/grafana-operator/config/crd/?ref=v4.10.1

config/dependencies/marin3r-crds/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - https://github.com/3scale-ops/marin3r/config/crd/?ref=v0.12.3

config/dependencies/marin3r/kustomization.yaml

@@ -0,0 +1,49 @@
+namespace: marin3r
+resources:
+  - https://github.com/3scale-ops/marin3r/config/default/?ref=v0.12.3
+
+# somehow this works in upstream repo but not here
+patches:
+  - target:
+      group: admissionregistration.k8s.io
+      version: v1
+      kind: MutatingWebhookConfiguration
+    patch: |-
+      - op: remove
+        path: /metadata/creationTimestamp
+  # use cluster scoped installation
+  - target:
+      group: rbac.authorization.k8s.io
+      kind: Role
+      name: manager-role
+      version: v1
+    patch: |-
+      - op: replace
+        path: /kind
+        value: ClusterRole
+      - op: remove
+        path: /metadata/namespace
+  - target:
+      group: rbac.authorization.k8s.io
+      kind: RoleBinding
+      name: manager-rolebinding
+      version: v1
+    patch: |-
+      - op: replace
+        path: /kind
+        value: ClusterRoleBinding
+      - op: replace
+        path: /roleRef/kind
+        value: ClusterRole
+      - op: remove
+        path: /metadata/namespace
+  # no HA
+  - target:
+      group: apps
+      kind: Deployment
+      name: controller-webhook
+      version: v1
+    patch: |-
+      - op: replace
+        path: /spec/replicas
+        value: 1

config/dependencies/metallb/kustomization.yaml

@@ -0,0 +1,4 @@
+namespace: metallb
+resources:
+  - https://github.com/metallb/metallb/config/native/?ref=v0.13.12
+  - resources.yaml