test: support testing a cluster-scoped deployment

Signed-off-by: Paulo Pires <pjpires@gmail.com>

.dockerignore

@@ -4,3 +4,4 @@ _output/
 docker
 docs
 example
+tmp/

.gitignore

@@ -12,5 +12,8 @@ vendor
 build/
 
 # CSRs and TLS material
-test/certs/*.csr
-test/certs/*.pem
+hack/e2e/certs/*.csr
+hack/e2e/certs/*.pem
+
+# Temporary directory used to template skaffold manifests
+tmp/

.travis.yml

@@ -6,7 +6,7 @@ go:
 go_import_path: github.com/nats-io/nats-operator
 
 env:
-- KUBERNETES_CONFIG_FILE=$HOME/.kube/config CHANGE_MINIKUBE_NONE_USER=true E2E_NAMESPACE=nats-io
+- KUBERNETES_CONFIG_FILE=$HOME/.kube/config CHANGE_MINIKUBE_NONE_USER=true CLUSTER_SCOPED_E2E_NAMESPACE=nats-io NAMESPACE_SCOPED_E2E_NAMESPACE=nats-operator-e2e
 
 install:
 - go get -u github.com/golang/dep/cmd/dep
@@ -25,6 +25,8 @@ before_script:
 
 script:
 - kubectl cluster-info
-- kubectl create namespace ${E2E_NAMESPACE}
-- cd $GOPATH/src/github.com/nats-io/nats-operator/ && NAMESPACE=${E2E_NAMESPACE} make e2e
+- cd $GOPATH/src/github.com/nats-io/nats-operator/ && make e2e EXPERIMENTAL_CLUSTER_SCOPED=0 NAMESPACE=${NAMESPACE_SCOPED_E2E_NAMESPACE}
+- kubectl delete namespace ${NAMESPACE_SCOPED_E2E_NAMESPACE}
+- cd $GOPATH/src/github.com/nats-io/nats-operator/ && make e2e EXPERIMENTAL_CLUSTER_SCOPED=1 NAMESPACE=${CLUSTER_SCOPED_E2E_NAMESPACE}
+- kubectl delete namespace ${CLUSTER_SCOPED_E2E_NAMESPACE}
 - cd $GOPATH/src/github.com/nats-io/nats-operator/test/reloader/ && go test ./... -v

Makefile

@@ -32,32 +32,23 @@ dep:
 
 # e2e runs the end-to-end test suite.
 .PHONY: e2e
+e2e: EXPERIMENTAL_CLUSTER_SCOPED ?= false
 e2e: KUBECONFIG ?= $(HOME)/.kube/config
 e2e: NAMESPACE ?= default
 e2e:
-	@./test/prepare-secrets.sh $(NAMESPACE)
-	MODE=run NAMESPACE=$(NAMESPACE) PROFILE=local TARGET=operator $(MAKE) run
-	MODE=run NAMESPACE=$(NAMESPACE) PROFILE=local TARGET=e2e $(MAKE) run
-	@go test -tags e2e -v ./test/e2e/main_test.go -kubeconfig $(KUBECONFIG) -namespace $(NAMESPACE) -wait
+	EXPERIMENTAL_CLUSTER_SCOPED=$(EXPERIMENTAL_CLUSTER_SCOPED) MODE=run NAMESPACE=$(NAMESPACE) PROFILE=local TARGET=operator $(MAKE) run
+	EXPERIMENTAL_CLUSTER_SCOPED=$(EXPERIMENTAL_CLUSTER_SCOPED) MODE=run NAMESPACE=$(NAMESPACE) PROFILE=local TARGET=e2e $(MAKE) run
+	@go test -tags e2e -v ./test/e2e/main_test.go -experimental-cluster-scoped=$(EXPERIMENTAL_CLUSTER_SCOPED) -kubeconfig $(KUBECONFIG) -namespace $(NAMESPACE) -wait
 
 # run deploys either nats-operator or nats-operator-e2e to the Kubernetes cluster targeted by the current kubeconfig.
 .PHONY: run
-.SECONDEXPANSION:
+run: EXPERIMENTAL_CLUSTER_SCOPED ?= false
 run: MODE ?= dev
 run: NAMESPACE ?= default
 run: PROFILE ?= local
 run: TARGET ?= operator
-run: build.$$(TARGET)
 run:
-	@if [[ "${TARGET}" == "operator" ]]; then \
-		if [[ "${MODE}" == "delete" ]]; then \
-			kubectl delete -n $(NAMESPACE) -f $(PWD)/deploy/00-prereqs.yaml --ignore-not-found; \
-		else \
-			kubectl apply -n $(NAMESPACE) -f $(PWD)/deploy/00-prereqs.yaml; \
-			$(PWD)/hack/skaffold/patch-cluster-role-binding.sh $(NAMESPACE); \
-		fi \
-	fi
-	@skaffold $(MODE) -f $(PWD)/hack/skaffold/$(TARGET)/skaffold.yml -n $(NAMESPACE) -p $(PROFILE)
+	@EXPERIMENTAL_CLUSTER_SCOPED=$(EXPERIMENTAL_CLUSTER_SCOPED) MODE=$(MODE) NAMESPACE=$(NAMESPACE) PROFILE=$(PROFILE) TARGET=$(TARGET) $(PWD)/hack/skaffold.sh
 
 # gen executes the code generation step.
 .PHONY: gen

docs/development/development.md

@@ -74,28 +74,56 @@ The build toolchain leverages on `skaffold` to build a container image of `nats-
 After performing the deployment, `skaffold` will stream the logs of the `nats-operator` pod, and will keep on monitoring the `build/nats-operator` binary for changes.
 When such changes occur (e.g. as a result of running `make build`), `skaffold` will re-deploy `nats-operator` to the Kubernetes cluster, and the process will repeat itself.
 
-The exact command you must execute to run `nats-operator` depends on whether you are using a local (Minikube or Docker for Desktop) or a Google Kubernetes Engine cluster.
+The exact command you must execute to run `nats-operator` depends on whether you are using a local (Minikube or Docker for Desktop) or a Google Kubernetes Engine cluster, and on whether you want to perform a namespace-scoped or cluster-scoped deployment.
 
 ### Local
 
-To run `nats-operator` against the local Kubernetes cluster targeted by the current context, you must run:
+#### Namespace-scoped
+
+To run a namespace-scoped instance of `nats-operator` against the local Kubernetes cluster targeted by the current context, you must run:
 
 ```console
-$ make run PROFILE=local
+$ make run NAMESPACE=<namespace> PROFILE=local
 ```
 
 To stop execution and cleanup the deployment, hit `Ctrl+C`.
 
+#### Cluster-scoped
+
+To run a cluster-scoped instance of `nats-operator` against the local Kubernetes cluster targeted by the current context, you must run:
+
+```console
+$ make run EXPERIMENTAL_CLUSTER_SCOPED=1 PROFILE=local
+```
+
+To stop execution and cleanup the deployment, hit `Ctrl+C`.
+
+**NOTE:** Cluster-scoped deployments of `nats-operator` _always_ run on the `nats-io` namespace.
+
 ### Google Kubernetes Engine
 
-To run `nats-operator` against the Google Kubernetes Engine cluster targeted by the current context, you must run:
+#### Namespace-scoped
+
+To run a namespace-scoped instance of `nats-operator` against the Google Kubernetes Engine cluster targeted by the current context, you must run:
 
 ```console
-$ make run PROFILE=gke
+$ make run NAMESPACE=<namespace> PROFILE=gke
 ```
 
 To stop execution and cleanup the deployment, hit `Ctrl+C`.
 
+#### Cluster-scoped
+
+To run a cluster-scoped instance of `nats-operator` against the Google Kubernetes Engine cluster targeted by the current context, you must run:
+
+```console
+$ make run EXPERIMENTAL_CLUSTER_SCOPED=1 PROFILE=gke
+```
+
+To stop execution and cleanup the deployment, hit `Ctrl+C`.
+
+**NOTE:** Cluster-scoped deployments of `nats-operator` _always_ run on the `nats-io` namespace.
+
 ## Testing
 
 `nats-operator` includes an end-to-end test suite that is used to validate the implementation.
@@ -135,14 +163,20 @@ This allows for running the test suite from _within_ the Kubernetes cluster (hen
 
 By default, the end-to-end test suite tests an installation of `nats-operator` in the `default` namespace.
 It is however possible to test installation in a different namespace.
-In order to do so, you must first create the desired namespace (in the example below, `nats-io`):
+To run the test suite against a different namespace, you may simply run:
 
 ```console
-$ kubectl create namespace nats-io
+$ make e2e NAMESPACE=<namespace>
 ```
 
-Then, to run the test suite against the resulting namespace, you may simply run:
+`<namespace>` will be automatically created if it doesn't already exist.
+
+### Testing the cluster-scoped mode
+
+To perform a cluster-scoped installation of `nats-operator` and run the end-to-end test suite against it, you may run:
 
 ```console
-$ NAMESPACE=nats-io make e2e
+$ make e2e EXPERIMENTAL_CLUSTER_SCOPED=1
 ```
+
+The required `nats-io` namespace will be automatically created if it doesn't already exist. 

hack/skaffold/e2e/pod.yml → hack/e2e/skaffold/pod.yml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nats-operator-e2e
+  namespace: __NAMESPACE__
 spec:
   containers:
   - name: nats-operator-e2e
@@ -11,6 +12,7 @@ spec:
     - -namespace
     - "$(NAMESPACE)"
     - -test.v
+    - -experimental-cluster-scoped=__EXPERIMENTAL_CLUSTER_SCOPED__
     env:
     - name: NAMESPACE
       valueFrom:

hack/skaffold/e2e/skaffold.yml → hack/e2e/skaffold/skaffold.yml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1alpha5
+apiVersion: skaffold/v1beta2
 kind: Config
 build:
   artifacts:
@@ -7,12 +7,12 @@ build:
   - image: gcr.io/project-id/nats-operator-e2e
     docker:
       # Use a Dockerfile specific for end-to-end testing only.
-      dockerfile: hack/skaffold/e2e/Dockerfile
+      dockerfile: __TMP_DIR__/Dockerfile
 deploy:
   kubectl:
     manifests:
     # Deploy the end-to-end test suite pod.
-    - hack/skaffold/e2e/pod.yml
+    - __TMP_DIR__/pod.yml
 profiles:
 - name: gke
   build:

hack/operator/skaffold/deployment.yml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nats-operator
+  name: nats-operator
+  namespace: __NAMESPACE__
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: nats-operator
+  template:
+    metadata:
+      labels:
+        name: nats-operator
+    spec:
+      containers:
+      - name: nats-operator
+        # "gcr.io/project-id/nats-operator" is used to make it easy to support both local (Minikube, ...) and GKE deployments.
+        # To deploy to GKE, "project-id" must be replaced by the correct project ID, and the Cloud Build API must be enabled.
+        image: gcr.io/project-id/nats-operator
+        # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
+        imagePullPolicy: IfNotPresent
+        args:
+        - /nats-operator
+        - --experimental-cluster-scoped=__EXPERIMENTAL_CLUSTER_SCOPED__
+        ports:
+        - name: readyz
+          containerPort: 8080
+        env:
+        - name: MY_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        resources:
+          requests:
+            memory: 64Mi
+            cpu: 100m
+          limits:
+            memory: 64Mi
+            cpu: 100m
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: readyz
+          initialDelaySeconds: 15
+          timeoutSeconds: 3
+      serviceAccountName: nats-operator

hack/operator/skaffold/prereqs.yml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nats-operator
+  namespace: __NAMESPACE__
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nats-operator-__NAMESPACE__
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nats-operator
+subjects:
+  - kind: ServiceAccount
+    name: nats-operator
+    namespace: __NAMESPACE__
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: nats-operator
+rules:
+  # Allow creating CRDs
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs: ["*"]
+  # Allow all actions on NatsClusters
+  - apiGroups:
+      - nats.io
+    resources:
+      - natsclusters
+      - natsserviceroles
+    verbs: ["*"]
+  # Allow actions on basic Kubernetes objects
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - secrets
+      - pods
+      - pods/exec
+      - pods/log
+      - services
+      - serviceaccounts
+      - serviceaccounts/token
+      - endpoints
+      - events
+    verbs: ["*"]
+  # Allow creating, deleting and listing namespaces
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - create
+      - delete
+      - list
+  # Allow listing and watching deployments (required by the end-to-end test suite).
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    verbs:
+      - list
+      - watch

hack/skaffold/operator/skaffold.yml → hack/operator/skaffold/skaffold.yml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1alpha5
+apiVersion: skaffold/v1beta2
 kind: Config
 build:
   artifacts:
@@ -7,13 +7,15 @@ build:
   - image: gcr.io/project-id/nats-operator
     docker:
       # Use a Dockerfile specific for development only.
-      dockerfile: hack/skaffold/operator/Dockerfile
+      dockerfile: __TMP_DIR__/Dockerfile
 deploy:
   kubectl:
     manifests:
-    # We deploy a single "nats-operator" pod to make debugging easier.
+    # Make sure the prerequisites are in place.
+    - __TMP_DIR__/prereqs.yml
+    # Create a "nats-operator" deployment with a single replica to make debugging easier.
     # NOTE: Any changes made to the podspec in "deploy/deployment.yaml" will have to be replayed on this file.
-    - hack/skaffold/operator/pod.yml
+    - __TMP_DIR__/deployment.yml
 profiles:
 - name: gke
   build: