LogDNA Output support (#530)

* - Logdna support update - rebase * Update * Rebase/generate
kube-logging · Jul 20, 2020 · 991de96 · 991de96
1 parent b461887
commit 991de96
Show file tree

Hide file tree

Showing 29 changed files with 646 additions and 6 deletions.
diff --git a/charts/logging-demo/README.md b/charts/logging-demo/README.md
@@ -73,6 +73,10 @@ The following tables lists the configurable parameters of the logging-demo chart
 | `cloudwatch.aws.region`                                           | AWS CLoudWatch Region                                   | ``             |
 | `cloudwatch.aws.log_group_name`                                   | AWS CLoudWatch Log Group                                | ``             |
 | `cloudwatch.aws.log_stream_name`                                  | AWS CLoudWatch Log Stream                               | ``             |
+| `logdna.enabled`                                                  | Enable LogDNA logging output                            | `false`        |
+| `logdna.api_key`                                                  | LogDNA Api key                                          | ``             |
+| `logdna.hostname`                                                 | Hostname                                                | ``             |
+| `logdna.app`                                                      | Application name                                        | ``             |
 | `logGenerator.enabled`                                            | Enable Demo Log-Gen application                         | `true`         |
 | `loggingOperator.tls.enabled`                                     | Enabled TLS communication between components            | `true`         |
 | `loggingOperator.tls.fluentdSecretName`                           | Specified secret name, which contain tls certs          | This will overwrite automatic Helm certificate generation. |

diff --git a/charts/logging-demo/templates/flow.yaml b/charts/logging-demo/templates/flow.yaml
@@ -20,7 +20,7 @@ spec:
           app.kubernetes.io/instance: {{ .Release.Name }}
   outputRefs:
 
-{{- if or .Values.file.enabled (and (not .Values.cloudwatch.enabled) (not .Values.elasticsearch.enabled) (not .Values.kafka.enabled) (not .Values.loki.enabled) (not .Values.minio.enabled)) }}
+{{- if or .Values.file.enabled (and (not .Values.cloudwatch.enabled) (not .Values.elasticsearch.enabled) (not .Values.kafka.enabled) (not .Values.loki.enabled) (not .Values.minio.enabled) (not .Values.logdna.enabled)) }}
     - {{ include "output.fullname" . }}-file
 {{- end }}
 {{- if .Values.cloudwatch.enabled }}
@@ -38,4 +38,7 @@ spec:
 {{- if .Values.minio.enabled }}
     - {{ include "output.fullname" . }}-minio
 {{- end }}
+{{- if .Values.logdna.enabled }}
+    - {{ include "output.fullname" . }}-logdna
+{{- end }}
 
diff --git a/charts/logging-demo/templates/output_file.yaml b/charts/logging-demo/templates/output_file.yaml
@@ -1,4 +1,4 @@
-{{- if or .Values.file.enabled (and (not .Values.cloudwatch.enabled) (not .Values.elasticsearch.enabled) (not .Values.kafka.enabled) (not .Values.loki.enabled) (not .Values.minio.enabled)) }}
+{{- if or .Values.file.enabled (and (not .Values.cloudwatch.enabled) (not .Values.elasticsearch.enabled) (not .Values.kafka.enabled) (not .Values.loki.enabled) (not .Values.minio.enabled) (not .Values.logdna.enabled)) }}
 apiVersion: logging.banzaicloud.io/v1beta1
 kind: Output
 metadata:

diff --git a/charts/logging-demo/templates/output_logdna.yaml b/charts/logging-demo/templates/output_logdna.yaml
@@ -0,0 +1,15 @@
+{{- if and .Values.logdna.enabled }}
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Output
+metadata:
+  name: {{ include "output.fullname" . }}-logdna
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "logging-demo.labels" . | indent 4 }}
+spec:
+  logdna:
+    api_key: {{ .Values.logdna.api_key }}
+{{/*    hostname: {{ .Values.logdna.hostname }}*/}}
+    hostname: "$.kubernetes.host"
+    app: {{ .Values.logdna.app }}
+{{- end }}
diff --git a/charts/logging-demo/values.yaml b/charts/logging-demo/values.yaml
@@ -79,6 +79,13 @@ cloudwatch:
     log_group_name: ""
     log_stream_name: ""
 
+# Enable LogDNA logging output
+logdna:
+  enabled: false
+  api_key: ""
+  app: ""
+  hostname: ""
+
 # Enable Demo Log-Gen application
 logGenerator:
   enabled: true
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml
@@ -4493,6 +4493,26 @@ spec:
               required:
               - stream_name
               type: object
+            logdna:
+              description: Send your logs to LogDNA
+              properties:
+                api_key:
+                  description: LogDNA Api key
+                  type: string
+                app:
+                  description: Application name
+                  type: string
+                buffer_chunk_limit:
+                  description: Do not increase past 8m (8MB) or your logs will be
+                    rejected by LogDNA server.
+                  type: string
+                hostname:
+                  description: Hostname
+                  type: string
+              required:
+              - api_key
+              - hostname
+              type: object
             loggingRef:
               type: string
             logz:

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml
@@ -4489,6 +4489,26 @@ spec:
               required:
               - stream_name
               type: object
+            logdna:
+              description: Send your logs to LogDNA
+              properties:
+                api_key:
+                  description: LogDNA Api key
+                  type: string
+                app:
+                  description: Application name
+                  type: string
+                buffer_chunk_limit:
+                  description: Do not increase past 8m (8MB) or your logs will be
+                    rejected by LogDNA server.
+                  type: string
+                hostname:
+                  description: Hostname
+                  type: string
+              required:
+              - api_key
+              - hostname
+              type: object
             loggingRef:
               type: string
             logz:

diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml
@@ -4493,6 +4493,26 @@ spec:
               required:
               - stream_name
               type: object
+            logdna:
+              description: Send your logs to LogDNA
+              properties:
+                api_key:
+                  description: LogDNA Api key
+                  type: string
+                app:
+                  description: Application name
+                  type: string
+                buffer_chunk_limit:
+                  description: Do not increase past 8m (8MB) or your logs will be
+                    rejected by LogDNA server.
+                  type: string
+                hostname:
+                  description: Hostname
+                  type: string
+              required:
+              - api_key
+              - hostname
+              type: object
             loggingRef:
               type: string
             logz:

diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml
@@ -4489,6 +4489,26 @@ spec:
               required:
               - stream_name
               type: object
+            logdna:
+              description: Send your logs to LogDNA
+              properties:
+                api_key:
+                  description: LogDNA Api key
+                  type: string
+                app:
+                  description: Application name
+                  type: string
+                buffer_chunk_limit:
+                  description: Do not increase past 8m (8MB) or your logs will be
+                    rejected by LogDNA server.
+                  type: string
+                hostname:
+                  description: Hostname
+                  type: string
+              required:
+              - api_key
+              - hostname
+              type: object
             loggingRef:
               type: string
             logz:

diff --git a/docs/crds/v1beta1/output_types.md b/docs/crds/v1beta1/output_types.md
@@ -25,6 +25,7 @@ generated_file: true
 | kafka | *output.KafkaOutputConfig | No | - |  |
 | cloudwatch | *output.CloudWatchOutput | No | - |  |
 | kinesisStream | *output.KinesisStreamOutputConfig | No | - |  |
+| logdna | *output.LogDNAOutput | No | - |  |
 | newrelic | *output.NewRelicOutputConfig | No | - |  |
 | splunkHec | *output.SplunkHecOutput | No | - |  |
 | http | *output.HTTPOutputConfig | No | - |  |

diff --git a/docs/example-logdna-nginx.md b/docs/example-logdna-nginx.md
@@ -0,0 +1,133 @@
+<p align="center"><img src="./img/logdna.svg" width="340"></p>
+
+# Store Nginx Access Logs in LogDNA with Logging Operator
+
+<p align="center"><img src="./img/nginx-logdna.svg" width="900"></p>
+
+---
+## Contents
+- **Installation**
+  - **Logging Operator**
+    - [Deploy with Helm](#install-with-helm)
+    - [Deploy with Kubernetes Manifests](#install-from-kubernetes-manifests)
+  - **Demo Application**  
+    - [Deploy with Helm](#demo-app-and-logging-definition)
+    - [Deploy with Kubernetes Manifests](#install-from-kubernetes-manifests)
+- **Validation**
+    - [LogDNA Dashboard](#logdna-dashboard)
+---
+<br />
+
+## Deploy Logging-Operator with Demo Application
+
+### Install with Helm 
+#### Add operator chart repository:
+```bash
+helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com
+helm repo update
+```
+#### Logging Operator
+> [How to install Logging-operator with helm](./deploy/README.md#deploy-logging-operator-with-helm)
+
+#### Demo App and Logging Definition
+```bash
+helm install --namespace logging --name logging-demo banzaicloud-stable/logging-demo \
+ --set "logdna.enabled=True" \
+ --set "logdna.api_key=" \
+ --set "logdna.app=" \
+ --set "logdna.hostname=" 
+```
+---
+<br />
+
+
+### Install from Kubernetes manifests
+#### Logging Operator
+> [How to install Logging-operator from manifests](./deploy/README.md#deploy-logging-operator-from-kubernetes-manifests)
+
+#### Create `logging` Namespace
+```bash
+kubectl create ns logging
+```
+
+#### Create `logging` resource
+```bash
+kubectl -n logging apply -f - <<"EOF" 
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Logging
+metadata:
+  name: demo-logging
+spec:
+  fluentd: {}
+  fluentbit: {}
+  controlNamespace: logging
+EOF
+```
+> Note: `ClusterOutput` and `ClusterFlow` resource will only be accepted in the `controlNamespace` 
+
+
+#### Create a LogDNA `Output` Definition 
+```bash
+kubectl -n logging apply -f - <<"EOF" 
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Output
+metadata:
+  name: logdna-output
+  namespace: logging
+spec:
+  logdna:
+    api_key: demoa0315076d75234133124fd2f583demo
+    hostname: logging-operator
+    app: my-app
+EOF
+```
+
+#### Create `flow` resource
+```bash
+kubectl -n logging apply -f - <<"EOF" 
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: nginx-log-to-logdna
+spec:
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        remove_key_name_field: true
+        reserve_data: true
+        parsers:
+          - type: nginx
+  selectors:
+    app.kubernetes.io/name: log-generator
+  outputRefs:
+    - logdna-output
+EOF
+```
+
+#### Install Demo Deployment
+```bash
+kubectl -n logging apply -f - <<"EOF" 
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  name: log-generator
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: log-generator
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: log-generator
+    spec:
+      containers:
+      - name: nginx
+        image: banzaicloud/log-generator:0.3.2
+EOF
+```
+
+## LogDNA Dashboard
+<p align="center"><img src="./img/Screenshot-logdna.png" width="660"></p>
+
+
diff --git a/docs/examples/logging_output_logdna.yaml b/docs/examples/logging_output_logdna.yaml
@@ -0,0 +1,9 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Output
+metadata:
+  name: logdna-output-sample
+spec:
+  logdna:
+    api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxx
+    hostname: logging-operator
+    app: my-app
diff --git a/docs/img/Screenshot-logdna.png b/docs/img/Screenshot-logdna.png
diff --git a/docs/img/logdna.svg b/docs/img/logdna.svg