fix image

apis/fluentd/v1alpha1/plugins/params/model.go

@@ -72,6 +72,12 @@ func (ps *PluginStore) Hash() string {
 	return utils.HashCode(c.String())
 }
 
+// Returns the @label value string of this plugin store.
+func (ps *PluginStore) GetTag() string {
+	return ps.Store["tag"]
+}
+
+
 // Returns the @label value string of this plugin store.
 func (ps *PluginStore) RouteLabel() string {
 	if ps.Name != "route" {

cmd/fluent-watcher/fluentbit/Dockerfile

@@ -7,7 +7,7 @@ RUN echo $(ls -al /code)
 RUN CGO_ENABLED=0 go build -i -ldflags '-w -s' -o /fluent-bit/fluent-bit /code/cmd/fluent-watcher/fluentbit/main.go
 
 FROM fluent/fluent-bit:1.8.3
-LABEL Description="Fluent Bit docker image" Vendor="KubeSphere" Version="1.0"
+LABEL Description="Fluent Bit docker image" Vendor="Fluent" Version="1.0"
 
 COPY conf/fluent-bit.conf conf/parsers.conf /fluent-bit/etc/
 COPY --from=buildergo /fluent-bit/fluent-bit /fluent-bit/bin/fluent-bit-watcher

cmd/fluent-watcher/fluentd/Dockerfile.complete

@@ -9,7 +9,7 @@ RUN CGO_ENABLED=0 go build -i -ldflags '-w -s' -o /fluentd/fluentd-watcher /code
 
 # Fluentd main image
 FROM alpine:3.13
-LABEL Description="Fluentd docker image" Vendor="Kubesphere Organization" Version="1.14.4"
+LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.14.4"
 
 # Do not split this into multiple RUN!
 # Docker creates a layer for every RUN-Statement

cmd/fluent-watcher/fluentd/base/Dockerfile

@@ -2,7 +2,7 @@
 # DO NOT EDIT THIS FILE DIRECTLY, USE /Dockerfile.template.erb
 
 FROM alpine:3.13
-LABEL Description="Fluentd docker image" Vendor="Kubesphere Organization" Version="1.14.4"
+LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.14.4"
 
 # Do not split this into multiple RUN!
 # Docker creates a layer for every RUN-Statement

controllers/fluentdconfig_controller.go

@@ -313,12 +313,13 @@ func (r *FluentdConfigReconciler) CfgsForFluentd(ctx context.Context, fdSelector
 		errs := make([]string, 0)
 
 		// Combine the cluster filter/output pluginstores in this fluentd config.
-		cfgResouces, cerrs := pgr.PatchAndFilterClusterLevelResources(sl, cfg.GetCfgId(), clusterfilters, clusteroutputs)
-		pgr.WithCfgResources(cfgRouterLabel, cfgResouces)
+		clustercfgResouces, cerrs := pgr.PatchAndFilterClusterLevelResources(sl, cfg.GetCfgId(), clusterfilters, clusteroutputs)
 		errs = append(errs, cerrs...)
 
 		// Combine the namespaced filter/output pluginstores in this fluentd config.
 		cfgResouces, nerrs := pgr.PatchAndFilterNamespacedLevelResources(sl, cfg.GetCfgId(), filters, outputs)
+		cfgResouces.FilterPlugins = append(cfgResouces.FilterPlugins, clustercfgResouces.FilterPlugins...)
+		cfgResouces.OutputPlugins = append(cfgResouces.OutputPlugins, clustercfgResouces.OutputPlugins...)
 		pgr.WithCfgResources(cfgRouterLabel, cfgResouces)
 		errs = append(errs, nerrs...)
 

docs/proposals/fluent_operator_20210907.md

@@ -2,29 +2,50 @@
 
 ## Table of contents
 
-- [Introduction](#Introduction)
-  - [Why adding Fluentd CRDs to Fluent Operator? ](#Why-adding-Fluentd-CRDs-to-Fluent-Operator? )
-  - [How to integrate fluentd ?](#How-to-integrate-fluentd)
-      - [Introduction to the types of CRD](#Introduction-to-the-types-of-CRD-and-workflow)
-      - [Support for namespaces isolation and cloud native selectors](#Support-for-namespaces-isolation-and-cloud-native-selectors)
-      - [Support for strict configuration validation](#Support-for-strict-configuration-validation)
-      - [Select popular Fluentd plugins to add CRD support](#Select-popular-fluentd-plugins-to-add-crd-support)
-      - [Support for custom plugins installation](#Support-for-custom-plugins-installation)
-      - [Enable filesystem pvc for buffer](#Enable-filesystem-pvc-for-buffer)
-      - [Add ClusterDirectives CRD to support directives predefined](#Add-ClusterDirectives-CRD-to-support-directives-predefined)
-- [Fluentd CRDs Design](#Fluentd-CRDs-Design)
-  - [Fluentd CRD](#Fluentd-CRD)
-  - [FluentdClusterConfig CRD Or FluentdConfig CRD](#FluentdClusterConfig-CRD-Or-FluentdConfig-CRD)
-  - [Output CRD Or ClusterOutput CRD](#Output-CRD-Or-ClusterOutput-CRD)
-  - [Configuration Supplement](#Configuration-Supplement)
-      - [Common Directive Configuration Supplement](#Common-Directive-Configuration-Supplement)
-      - [Buffer Directive Configuration Supplement](#Buffer-Directive-Configuration-Supplement)
-      - [Format Directive Configuration Supplement](#Format-Directive-Configuration-Supplement)
-      - [Inject Directive Configuration Supplement](#Inject-Directive-Configuration-Supplement)
-      - [Transport Directive Configuration Supplement](#Transport-Directive-Configuration-Supplement)
-      - [Storage Directive Configuration Supplement](#Storage-Directive-Configuration-Supplement)
-      - [Service Discovery Directive Configuration Supplement](#Service-Discovery-Directive-Configuration-Supplement)
-- [Parts that need to be discussed or added](#Parts-that-need-to-be-discussed-or-added)
+- [Proposal： Add Fluentd to Fluent Operator as an optional log aggregation and forwarding layer](#proposal-add-fluentd-to-fluent-operator-as-an-optional-log-aggregation-and-forwarding-layer)
+  - [Table of contents](#table-of-contents)
+  - [Introduction](#introduction)
+    - [FluentBit CRDs refactoring](#fluentbit-crds-refactoring)
+    - [Why adding Fluentd CRDs to Fluent Operator?](#why-adding-fluentd-crds-to-fluent-operator)
+    - [How to integrate fluentd?](#how-to-integrate-fluentd)
+      - [Introduction to the types of CRD and workflow:](#introduction-to-the-types-of-crd-and-workflow)
+      - [Support for namespaces isolation and cloud native selectors](#support-for-namespaces-isolation-and-cloud-native-selectors)
+      - [Support for strict configuration validation](#support-for-strict-configuration-validation)
+      - [Select popular Fluentd plugins to add CRD support](#select-popular-fluentd-plugins-to-add-crd-support)
+      - [Support for custom plugins installation](#support-for-custom-plugins-installation)
+      - [Enable filesystem pvc for buffer](#enable-filesystem-pvc-for-buffer)
+      - [Add ClusterDirectives CRD to support directives predefined](#add-clusterdirectives-crd-to-support-directives-predefined)
+  - [Fluentd CRDs Design](#fluentd-crds-design)
+    - [Fluentd CRD](#fluentd-crd)
+    - [FluentdClusterConfig CRD Or FluentdConfig CRD](#fluentdclusterconfig-crd-or-fluentdconfig-crd)
+      - [Input Section Of FluentdClusterConfig CRD or FluentdConfig CRD](#input-section-of-fluentdclusterconfig-crd-or-fluentdconfig-crd)
+        - [forward](#forward)
+        - [http](#http)
+      - [Filters Section Of the FluentdClusterConfig Crd or FluentdConfig Crd](#filters-section-of-the-fluentdclusterconfig-crd-or-fluentdconfig-crd)
+        - [record_transformer](#record_transformer)
+        - [grep](#grep)
+        - [parser](#parser)
+        - [stdout](#stdout)
+    - [Output CRD Or ClusterOutput CRD](#output-crd-or-clusteroutput-crd)
+      - [forward](#forward-1)
+      - [http](#http-1)
+      - [stdout](#stdout-1)
+      - [kafka](#kafka)
+      - [elasticsearch](#elasticsearch)
+      - [s3](#s3)
+    - [Configuration Supplement](#configuration-supplement)
+      - [Common Directive Configuration Supplement](#common-directive-configuration-supplement)
+        - [type](#type)
+        - [id](#id)
+        - [log_level](#log_level)
+          - [label](#label)
+      - [Buffer Directive Configuration Supplement](#buffer-directive-configuration-supplement)
+      - [Format Directive Configuration Supplement](#format-directive-configuration-supplement)
+      - [Inject Directive Configuration Supplement](#inject-directive-configuration-supplement)
+      - [Transport Directive Configuration Supplement](#transport-directive-configuration-supplement)
+      - [Storage Directive Configuration Supplement](#storage-directive-configuration-supplement)
+      - [Service Discovery Directive Configuration Supplement](#service-discovery-directive-configuration-supplement)
+  - [Parts that need to be discussed or added](#parts-that-need-to-be-discussed-or-added)
 
 
 ## Introduction
@@ -43,7 +64,7 @@ Now all FluentBit CRDs are namespace level. We should use cluster level CRDs for
 
 The following table from [Fluent Bit documentation](https://docs.fluentbit.io/manual/about/fluentd-and-fluent-bit) describes a comparison in different areas of the two projects:
 
-![image-20210618114950689](https://raw.githubusercontent.com/zhu733756/bedpic/main/images/image-20210618114950689.png)
+![image-20210618114950689](https://raw.githubusercontent.com/fluent/fluentbit-operator/master/docs/images/fluent-operator.svg)
 
 The rich plugins is one of the reasons, which helps to do better handling of logging in some complex scenarios. 
 

manifests/forward/fluentd-cluster-cfg-output-buffer-example.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 3
-  image: zhu733756/fluentd:v1.14.4-amd64
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/forward/fluentd-cluster-cfg-output-es.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 1
-  image: zhu733756/fluentd:config-reloader
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/forward/fluentd-cluster-cfg-output-kafka.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 1
-  image: kubesphere/fluentd:v1.14.4-amd64
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/forward/fluentd-mixed-cfgs-output-es.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 1
-  image: kubesphere/fluentd:v1.14.4-amd64
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/forward/fluentd-namespaced-cfg-multi-tenant-output.yaml

@@ -0,0 +1,77 @@
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: Fluentd
+metadata:
+  name: fluentd-forward
+  namespace: kubesphere-logging-system
+  labels:
+    app.kubernetes.io/name: fluentd
+spec:
+  globalInputs:
+    - forward: 
+        bind: 0.0.0.0
+        port: 24224
+  replicas: 1
+  image: kubesphere/fluentd:v1.14.4
+  fluentdCfgSelector: 
+    matchLabels:
+      config.fluentd.fluent.io/enabled: "true"
+
+---
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: FluentdConfig
+metadata:
+  name: fluentd-config
+  namespace: kubesphere-logging-system
+  labels:
+    config.fluentd.fluent.io/enabled: "true"
+spec:
+  filterSelector:
+    matchLabels:
+      filter.fluentd.fluent.io/enabled: "true"
+  outputSelector:
+    matchLabels:
+      output.fluentd.fluent.io/enabled: "true"
+
+---
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterFilter
+metadata:
+  name: fluentd-filter
+  labels:
+    filter.fluentd.fluent.io/enabled: "true"
+spec: 
+  filters: 
+    - recordTransformer:
+        enableRuby: true
+        records:
+          - key: kubernetes_ns
+            value: ${record["kubernetes"]["namespace_name"]}
+
+---
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterOutput
+metadata:
+  name: fluentd-output-stdout
+  labels:
+    output.fluentd.fluent.io/enabled: "true"
+    type: global
+spec: 
+  outputs: 
+    - stdout: {}
+
+---
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: Output
+metadata:
+  name: fluentd-output-es
+  namespace: kubesphere-logging-system
+  labels:
+    output.fluentd.fluent.io/enabled: "true"
+    type: custom
+spec: 
+  outputs: 
+    - elasticsearch:
+        host: elasticsearch-logging-data.kubesphere-logging-system.svc
+        port: 9200
+        logstashFormat: true
+        logstashPrefix: ks-logstash-log

manifests/forward/fluentd-namespaced-cfg-output-es.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 1
-  image: zhu733756/fluentd:config-reloader
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/quick-start/fluentd-forward.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 24224
   replicas: 1
-  image: zhu733756/fluentd:config-reloader
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/quick-start/fluentd-http.yaml

@@ -11,7 +11,7 @@ spec:
         bind: 0.0.0.0
         port: 9880
   replicas: 1
-  image: zhu733756/fluentd:latest
+  image: kubesphere/fluentd:v1.14.4
   fluentdCfgSelector: 
     matchLabels:
       config.fluentd.fluent.io/enabled: "true"

manifests/setup/fluent-operator-deployment.yaml

@@ -39,7 +39,7 @@ spec:
           mountPath: /var/run/docker.sock
       containers:
       - name: fluent-operator
-        image: zhu733756/fluent-operator:latest-amd64
+        image: kubesphere/fluent-operator:v1.0.0
         resources:
           limits:
             cpu: 200m

manifests/setup/setup.yaml

@@ -11112,7 +11112,7 @@ spec:
         app.kubernetes.io/name: fluent-operator
     spec:
       containers:
-      - image: zhu733756/fluent-operator:latest-amd64
+      - image: kubesphere/fluent-operator:v1.0.0
         name: fluent-operator
         resources:
           limits: