allow ClusterOutput and ClusterFlow resources defined in arbitrary na…

…mespaces (kube-logging#499)
simonkey007 · Jun 15, 2020 · df9794c · df9794c
1 parent 5816354
commit df9794c
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 14 deletions.
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml
@@ -779,6 +779,7 @@ spec:
             selectors:
               additionalProperties:
                 type: string
+              description: Deprecated
               type: object
           required:
           - outputRefs

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml
@@ -39,6 +39,8 @@ spec:
         spec:
           description: LoggingSpec defines the desired state of Logging
           properties:
+            allowClusterResourcesFromAllNamespaces:
+              type: boolean
             controlNamespace:
               type: string
             defaultFlow:

diff --git a/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml b/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml
@@ -779,6 +779,7 @@ spec:
             selectors:
               additionalProperties:
                 type: string
+              description: Deprecated
               type: object
           required:
           - outputRefs

diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml
@@ -39,6 +39,8 @@ spec:
         spec:
           description: LoggingSpec defines the desired state of Logging
           properties:
+            allowClusterResourcesFromAllNamespaces:
+              type: boolean
             controlNamespace:
               type: string
             defaultFlow:

diff --git a/controllers/logging_controller.go b/controllers/logging_controller.go
@@ -263,8 +263,14 @@ func (r *LoggingReconciler) GetResources(logging *loggingv1beta1.Logging) (*mode
 	loggingResources := model.NewLoggingResources(logging, r.Client, r.Log)
 	var err error
 
+	listOptsForClusterResources := []client.ListOption{}
+
+	if !logging.Spec.AllowClusterResourcesFromAllNamespaces {
+		listOptsForClusterResources = append(listOptsForClusterResources, client.InNamespace(logging.Spec.ControlNamespace))
+	}
+
 	clusterFlows := &loggingv1beta1.ClusterFlowList{}
-	err = r.List(context.TODO(), clusterFlows, client.InNamespace(logging.Spec.ControlNamespace))
+	err = r.List(context.TODO(), clusterFlows, listOptsForClusterResources...)
 	if err != nil {
 		return nil, err
 	}
@@ -288,7 +294,7 @@ func (r *LoggingReconciler) GetResources(logging *loggingv1beta1.Logging) (*mode
 	}
 
 	clusterOutputs := &loggingv1beta1.ClusterOutputList{}
-	err = r.List(context.TODO(), clusterOutputs, client.InNamespace(logging.Spec.ControlNamespace))
+	err = r.List(context.TODO(), clusterOutputs, listOptsForClusterResources...)
 	if err != nil {
 		return nil, err
 	}

diff --git a/controllers/logging_controller_test.go b/controllers/logging_controller_test.go
@@ -264,6 +264,58 @@ func TestSingleFlowWithClusterOutput(t *testing.T) {
 	g.Expect(string(secret.Data[fluentd.AppConfigKey])).Should(gomega.ContainSubstring("a:b"))
 }
 
+func TestSingleClusterFlowWithClusterOutputFromExternalNamespace(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	defer beforeEach(t)()
+
+	logging := &v1beta1.Logging{
+		ObjectMeta: v1.ObjectMeta{
+			Name: "test-" + uuid.New()[:8],
+		},
+		Spec: v1beta1.LoggingSpec{
+			WatchNamespaces:                        []string{testNamespace},
+			FluentdSpec:                            &v1beta1.FluentdSpec{},
+			FlowConfigCheckDisabled:                true,
+			ControlNamespace:                       controlNamespace,
+			AllowClusterResourcesFromAllNamespaces: true,
+		},
+	}
+
+	output := &v1beta1.ClusterOutput{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "test-cluster-output",
+			Namespace: testNamespace,
+		},
+		Spec: v1beta1.ClusterOutputSpec{
+			OutputSpec: v1beta1.OutputSpec{
+				NullOutputConfig: output.NewNullOutputConfig(),
+			},
+		},
+	}
+
+	flow := &v1beta1.ClusterFlow{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "test-flow",
+			Namespace: testNamespace,
+		},
+		Spec: v1beta1.ClusterFlowSpec{
+			Selectors: map[string]string{
+				"a": "b",
+			},
+			OutputRefs: []string{"test-cluster-output"},
+		},
+	}
+
+	defer ensureCreated(t, logging)()
+	defer ensureCreated(t, output)()
+	defer ensureCreated(t, flow)()
+
+	secret := &corev1.Secret{}
+	defer ensureCreatedEventually(t, controlNamespace, logging.QualifiedName(fluentd.AppSecretConfigName), secret)()
+
+	g.Expect(string(secret.Data[fluentd.AppConfigKey])).Should(gomega.ContainSubstring("a:b"))
+}
+
 func TestClusterFlowWithNamespacedOutput(t *testing.T) {
 	defer beforeEach(t)()
 

diff --git a/pkg/sdk/api/v1beta1/clusterflow_types.go b/pkg/sdk/api/v1beta1/clusterflow_types.go
@@ -61,6 +61,7 @@ type ClusterExclude struct {
 
 // FlowSpec is the Kubernetes spec for Flows
 type ClusterFlowSpec struct {
+	// Deprecated
 	Selectors  map[string]string `json:"selectors,omitempty"`
 	Match      []ClusterMatch    `json:"match,omitempty"`
 	Filters    []Filter          `json:"filters,omitempty"`

diff --git a/pkg/sdk/api/v1beta1/logging_types.go b/pkg/sdk/api/v1beta1/logging_types.go
@@ -41,14 +41,15 @@ type _metaLoggingSpec interface{}
 
 // LoggingSpec defines the desired state of Logging
 type LoggingSpec struct {
-	LoggingRef              string           `json:"loggingRef,omitempty"`
-	FlowConfigCheckDisabled bool             `json:"flowConfigCheckDisabled,omitempty"`
-	FlowConfigOverride      string           `json:"flowConfigOverride,omitempty"`
-	FluentbitSpec           *FluentbitSpec   `json:"fluentbit,omitempty"`
-	FluentdSpec             *FluentdSpec     `json:"fluentd,omitempty"`
-	DefaultFlowSpec         *DefaultFlowSpec `json:"defaultFlow,omitempty"`
-	WatchNamespaces         []string         `json:"watchNamespaces,omitempty"`
-	ControlNamespace        string           `json:"controlNamespace"`
+	LoggingRef                             string           `json:"loggingRef,omitempty"`
+	FlowConfigCheckDisabled                bool             `json:"flowConfigCheckDisabled,omitempty"`
+	FlowConfigOverride                     string           `json:"flowConfigOverride,omitempty"`
+	FluentbitSpec                          *FluentbitSpec   `json:"fluentbit,omitempty"`
+	FluentdSpec                            *FluentdSpec     `json:"fluentd,omitempty"`
+	DefaultFlowSpec                        *DefaultFlowSpec `json:"defaultFlow,omitempty"`
+	WatchNamespaces                        []string         `json:"watchNamespaces,omitempty"`
+	ControlNamespace                       string           `json:"controlNamespace"`
+	AllowClusterResourcesFromAllNamespaces bool             `json:"allowClusterResourcesFromAllNamespaces,omitempty"`
 
 	// EnableRecreateWorkloadOnImmutableFieldChange enables the operator to recreate the
 	// fluentbit daemonset and the fluentd statefulset (and possibly other resource in the future)

diff --git a/pkg/sdk/static/gen/crds/generated.go b/pkg/sdk/static/gen/crds/generated.go