update docs-gen cmd && update README

Signed-off-by: zhu733756 <zhu733756@kubesphere.io>
fluent · Feb 15, 2022 · b807840 · b807840
1 parent 90d2b48
commit b807840
Show file tree

Hide file tree

Showing 78 changed files with 1,678 additions and 395 deletions.
diff --git a/Makefile b/Makefile
@@ -152,3 +152,6 @@ endef
 
 go-deps: # download go dependencies
 	go mod download
+
+docs-update:
+	go run ./cmd/doc-gen/main.go
diff --git a/README.md b/README.md
@@ -10,31 +10,44 @@ Once installed, the Fluent Bit Operator provides the following features:
 
 ## Table of contents
 
-- [Overview](#overview)
-- [Get Started](#get-started)
-  - [Prerequisites](#prerequisites)
-  - [Install](#Install)
-    - [Deploy Fluent Bit Operator with YAML](#deploy-fluent-bit-operator-with-yaml)
-    - [Deploy Fluent Bit Operator with Helm](#deploy-fluent-bit-operator-with-helm)
-  - [Quick Start](#quick-start)
-  - [Configure Custom Watch Namespaces](#configure-custom-watch-namespaces)
-  - [Collect Kubernetes logs](#collect-kubernetes-logs)
-    - [Deploy the Kubernetes logging stack with YAML](#deploy-the-kubernetes-logging-stack-with-yaml)
-    - [Deploy the Kubernetes logging stack with Helm](#deploy-the-kubernetes-logging-stack-with-helm)
-    - [Collect Auditd logs](#collect-auditd-logs)
-- [Monitoring](#monitoring)
-- [API Doc](#api-doc)
-- [Best Practice](#best-practice)
-  - [Plugin Grouping](#plugin-grouping)
-  - [Path Convention](#path-convention)
-- [Custom Parser](#custom-parser)
-- [Roadmap](#roadmap)
-- [Development](#development)
-  - [Requirements](#requirements)
-  - [Running](#running)
-- [Contributing](#contributing)
-  - [Documentation](#documentation)
-  - [Manifests](#manifests)
+- [Fluent Bit Operator](#fluent-bit-operator)
+  - [Table of contents](#table-of-contents)
+  - [Overview](#overview)
+  - [Get Started](#get-started)
+    - [Prerequisites](#prerequisites)
+    - [Install](#install)
+        - [Deploy Fluent Bit Operator with YAML](#deploy-fluent-bit-operator-with-yaml)
+        - [Deploy Fluent Bit Operator with Helm](#deploy-fluent-bit-operator-with-helm)
+    - [Quick Start](#quick-start)
+    - [Configure Custom Watch Namespaces](#configure-custom-watch-namespaces)
+    - [Collect Kubernetes logs](#collect-kubernetes-logs)
+        - [Deploy the Kubernetes logging stack with YAML](#deploy-the-kubernetes-logging-stack-with-yaml)
+        - [Deploy the Kubernetes logging stack with Helm](#deploy-the-kubernetes-logging-stack-with-helm)
+    - [Collect auditd logs](#collect-auditd-logs)
+    - [Fluentd collects logs from fluentbit](#fluentd-collects-logs-from-fluentbit)
+      - [Enable Fluentbit forward plugin](#enable-fluentbit-forward-plugin)
+      - [Fluentd ClusterFluentdConfig example](#fluentd-clusterfluentdconfig-example)
+      - [Fluentd FluentdConfig example](#fluentd-fluentdconfig-example)
+      - [Fluentd mixed FluentdConfig and ClusterFluentdConfig example](#fluentd-mixed-fluentdconfig-and-clusterfluentdconfig-example)
+      - [Fluentd ClusterOutput and Output example in multi-tenant scenario](#fluentd-clusteroutput-and-output-example-in-multi-tenant-scenario)
+      - [Fluentd outputs to kafka/es example](#fluentd-outputs-to-kafkaes-example)
+      - [Fluentd buffer example](#fluentd-buffer-example)
+    - [Fluentd collects logs through http plugin](#fluentd-collects-logs-through-http-plugin)
+  - [Monitoring](#monitoring)
+  - [API Doc](#api-doc)
+    - [Fluent Bit](#fluent-bit)
+    - [Fluentd](#fluentd)
+  - [Best Practice](#best-practice)
+    - [Plugin Grouping](#plugin-grouping)
+    - [Path Convention](#path-convention)
+  - [Custom Parser](#custom-parser)
+  - [Roadmap](#roadmap)
+  - [Development](#development)
+    - [Requirements](#requirements)
+    - [Running](#running)
+  - [Contributing](#contributing)
+    - [Documentation](#documentation)
+    - [Manifests](#manifests)
 
 ## Overview
 
@@ -55,6 +68,10 @@ To enable fluent-bit to pick up and use the latest config whenever the fluent-bi
 
 ![Kubesphere-logging-fluentbit](docs/images/kubesphere-logging-fluentbit.svg)
 
+Since we have finished [the pr #189](https://github.com/fluent/fluentbit-operator/pull/189) to integrate fluent-operator as a forward log layer to collect logs from fluentbit or other apps. The whole workflow changes as below.
+
+![Fluent-operator](docs/images/fluent-operator.svg)
+
 ## Get Started
 
 ### Prerequisites
@@ -212,6 +229,61 @@ $ curl localhost:9200/_cat/indices
 green open ks-logstash-log-2021.04.06 QeI-k_LoQZ2h1z23F3XiHg  5 1 404879 0 298.4mb 149.2mb
 ```
 
+### Fluentd collects logs from fluentbit
+
+#### Enable Fluentbit forward plugin
+
+Since the fluentd acts as a forward log layer to collect logs from fluentbit. Here we should the fluentbit forward plugin.
+
+```shell
+kubectl apply -f manifests/fluentd/fluentbit-output-forward.yaml
+```
+
+#### Fluentd ClusterFluentdConfig example
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-cluster-cfg-output-es.yaml
+```
+
+#### Fluentd FluentdConfig example
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-namespaced-cfg-output-es.yaml
+```
+
+#### Fluentd mixed FluentdConfig and ClusterFluentdConfig example
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-mixed-cfgs-output-es.yaml
+```
+
+#### Fluentd ClusterOutput and Output example in multi-tenant scenario
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-mixed-cfgs-multi-tenant-output.yaml
+```
+
+#### Fluentd outputs to kafka/es example
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-cluster-cfg-output-kafka.yaml
+kubectl apply -f manifests/fluentd/fluentd-cluster-cfg-output-es.yaml
+```
+
+#### Fluentd buffer example
+
+```shell
+kubectl apply -f manifests/fluentd/fluentd-cluster-cfg-output-buffer-example.yaml
+```
+
+### Fluentd collects logs through http plugin
+
+The following example enables the http plugin in fluentd:
+
+```shell
+kubectl apply -f manifests/quick-start/fluentd-http.yaml
+```
+
 ## Monitoring
 
 Fluent Bit comes with a built-in HTTP Server. According to the official [documentation](https://docs.fluentbit.io/manual/administration/monitoring) of fluentbit You can enable this by enabling the HTTP server from the fluent bit configuration file:
@@ -223,7 +295,7 @@ Fluent Bit comes with a built-in HTTP Server. According to the official [documen
     HTTP_PORT    2020
 ```
 
-When you use the kubesphere/fluentbit-operator, You can enable this from `FluentBitConfig` manifest. Example is below:
+When you use the fluent-operator, You can enable this from `FluentBitConfig` manifest. Example is below:
 
 ```yaml
 apiVersion: fluentbit.fluent.io/v1alpha2
@@ -291,18 +363,20 @@ curl <podIP>:2020 | jq .
 
 ## API Doc
 
+### Fluent Bit
+
 The list below shows supported plugins which are based on Fluent Bit v1.7.x+. For more information, please refer to the API docs of each plugin.
 
-- [Input](docs/crd.md#input)
+- [Input](docs/fluentbit.md#input)
   - [dummy](docs/plugins/fluentbit/input/dummy.md)
   - [tail](docs/plugins/fluentbit/input/tail.md)
   - [systemd](docs/plugins/fluentbit/input/systemd.md)
-- [Parser](docs/crd.md#parser)
+- [Parser](docs/fluentbit.md#parser)
   - [json](docs/plugins/fluentbit/parser/json.md)
   - [logfmt](docs/plugins/fluentbit/parser/logfmt.md)
   - [lstv](docs/plugins/fluentbit/parser/lstv.md)
   - [regex](docs/plugins/fluentbit/parser/regex.md)
-- [Filter](docs/crd.md#filter)
+- [Filter](docs/fluentbit.md#filter)
   - [kubernetes](docs/plugins/fluentbit/filter/kubernetes.md)
   - [modify](docs/plugins/fluentbit/filter/modify.md)
   - [nest](docs/plugins/fluentbit/filter/nest.md)
@@ -313,7 +387,7 @@ The list below shows supported plugins which are based on Fluent Bit v1.7.x+. Fo
   - [throttle](docs/plugins/fluentbit/filter/throttle.md)
   - [aws](docs/plugins/fluentbit/filter/aws.md)
   - [multiline](docs/plugins/fluentbit/filter/multiline.md)
-- [Output](docs/crd.md#output)
+- [Output](docs/fluentbit.md#output)
   - [elasticsearch](docs/plugins/fluentbit/output/elasticsearch.md)
   - [file](docs/plugins/fluentbit/output/file.md)
   - [forward](docs/plugins/fluentbit/output/forward.md)
@@ -326,6 +400,39 @@ The list below shows supported plugins which are based on Fluent Bit v1.7.x+. Fo
   - [syslog](docs/plugins/fluentbit/output/syslog.md)
   - [datadog](docs/plugins/fluentbit/output/datadog.md)
 
+### Fluentd
+
+The list below shows supported plugins which are based on Fluentd v1.14.4+. For more information, please refer to the API docs of each plugin.
+
+- Common
+  - [buffer](docs/plugins/fluentd/common/buffer.md)
+  - [format](docs/plugins/fluentd/common/format.md)
+  - [parse](docs/plugins/fluentd/common/parse.md)
+  - [time](docs/plugins/fluentd/common/common.md#time)
+  - [inject](docs/plugins/fluentd/common/common.md#inject)
+  - [security](docs/plugins/fluentd/common/common.md#security)
+  - [user](docs/plugins/fluentd/common/common.md#user)
+  - [transport](docs/plugins/fluentd/common/common.md#transport)
+  - [client](docs/plugins/fluentd/common/common.md#client)
+  - [auth](docs/plugins/fluentd/common/common.md#auth)
+  - [server](docs/plugins/fluentd/common/common.md#server)
+  - [service_discovery](docs/plugins/fluentd/common/common.md#ServiceDiscovery)
+- [Input]((docs/fluentd/input/types.md)
+  - [http](docs/plugins/fluentd/input/http.md)
+  - [forward](docs/plugins/fluentd/input/forward.md)
+- [Filter](docs/fluentd/filter/types.md)
+  - [parser](docs/plugins/fluentd/filter/parser.md)
+  - [grep](docs/plugins/fluentd/filter/grep.md)
+  - [record modifier](docs/plugins/fluentd/filter/record_modifier.md)
+  - [stdout](docs/plugins/fluentd/filter/stdout.md)
+- [Output](docs/plugins/fluentd/output/types.md)
+  - [elasticsearch](docs/plugins/fluentd/output/elasticsearch.md)
+  - [forward](docs/plugins/fluentd/output/forward.md)
+  - [http](docs/plugins/fluentd/output/http.md)
+  - [kafka](docs/plugins/fluentd/output/kafka.md)
+  - stdout
+
+
 ## Best Practice
 
 ### Plugin Grouping
@@ -340,8 +447,8 @@ Path to file in Fluent Bit config should be well regulated. Fluent Bit Operator
 
 |Dir Path|Description|
 |---|---|
-|/fluent-bit/tail|Stores tail related files, eg. file tracking db. Using [fluentbit.spec.positionDB](docs/crd.md#fluentbitspec) will mount a file `pos.db` under this dir by default.|
-|/fluent-bit/secrets/{secret_name}|Stores secrets, eg. TLS files. Specify secrets to mount in [fluentbit.spec.secrets](docs/crd.md#fluentbitspec), then you have access.|
+|/fluent-bit/tail|Stores tail related files, eg. file tracking db. Using [fluentbit.spec.positionDB](docs/fluentbit.md#fluentbitspec) will mount a file `pos.db` under this dir by default.|
+|/fluent-bit/secrets/{secret_name}|Stores secrets, eg. TLS files. Specify secrets to mount in [fluentbit.spec.secrets](docs/fluentbit.md#fluentbitspec), then you have access.|
 |/fluent-bit/config|Stores the main config file and user-defined parser config file.|
 
 > Note that ServiceAccount files are mounted at `/var/run/secrets/kubernetes.io/serviceaccount`.
@@ -377,7 +484,7 @@ Check out the demo in the folder `/manifests/regex-parser` for how to use a cust
 
 ### Documentation
 
-[API Doc](docs/crd.md) is generated automatically. To modify it, edit the comment above struct fields, then run `go run cmd/doc-gen/main.go`.
+[API Doc](docs/fluentbit.md) is generated automatically. To modify it, edit the comment above struct fields, then run `go run cmd/doc-gen/main.go`.
 
 ### Manifests
 

diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/plugins/secret_types.go b/apis/fluentbit/v1alpha2/plugins/secret_types.go
@@ -3,22 +3,22 @@ package plugins
 import (
 	"context"
 	"fmt"
+	"strings"
+
 	"github.com/go-logr/logr"
 	"github.com/go-openapi/errors"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"strings"
 )
 
 // +kubebuilder:object:generate:=true
-
+// Secret defines a value source of a secret.
 type Secret struct {
-	// ValueSource represents a source for the value of a secret.
 	ValueFrom ValueSource `json:"valueFrom,omitempty"`
 }
 
 // +kubebuilder:object:generate:=true
-
+// ValueSource defines a value source of a secret.
 type ValueSource struct {
 	// Selects a key of a secret in the pod's namespace
 	// +optional

diff --git a/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go
diff --git a/apis/fluentd/v1alpha1/plugins/common/buffer_types.go b/apis/fluentd/v1alpha1/plugins/common/buffer_types.go
@@ -10,7 +10,7 @@ import (
 )
 
 // +kubebuilder:object:generate:=true
-// BufferCommon defines parameters for all buffer Plugins
+// BufferCommon defines common parameters for all buffer Plugins
 type BufferCommon struct {
 	// The @id parameter specifies a unique name for the configuration.
 	Id *string `json:"id,omitempty"`
@@ -22,6 +22,7 @@ type BufferCommon struct {
 	LogLevel *string `json:"logLevel,omitempty"`
 }
 
+// Buffer defines all types for all buffer Plugins
 type Buffer struct {
 	BufferCommon `json:",inline,omitempty"`
 	// The file buffer plugin
@@ -139,6 +140,16 @@ type FileSingleBuffer struct {
 	ChunkFormat *string `json:"chunkFormat,omitempty"`
 }
 
+// BufferSection defines the common parameters for buffer sections
+type BufferSection struct {
+	// buffer section
+	Buffer *Buffer `json:"buffer,omitempty"`
+	// format section
+	Format *Format `json:"format,omitempty"`
+	// inject section
+	Inject *Inject `json:"inject,omitempty"`
+}
+
 func (b *Buffer) Name() string {
 	return "buffer"
 }