Add Filebeat as alternative to Fluent Bit

This commit adds Filebeat as an alternative to Fluent Bit. We decided to
try out Filebeat for the collection of logs, because with Fluent Bit not
all our logs seems to be collected.

.github/workflows/docker.yaml

@@ -87,3 +87,43 @@ jobs:
           file: ./cmd/ingester/Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64
           tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-ingester
+
+  filebeat:
+    name: Filebeat
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set Docker Tag
+        id: tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            DOCKER_TAG="${GITHUB_REF:10}"
+          else
+            DOCKER_TAG="main"
+          fi
+          echo ::set-output name=tag::${DOCKER_TAG}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push Docker Image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          file: ./cmd/filebeat/Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64
+          tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-filebeat

.gitignore

@@ -1,3 +1,4 @@
 /out_clickhouse.h
 /out_clickhouse.so
 /ingester
+/filebeat

CHANGELOG.md

@@ -8,6 +8,8 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 ## Unreleased
 
+- [#28](https://github.com/kobsio/klogs/pull/28): Add Filebeat as alternative to Fluent Bit.
+
 ## [v0.8.0](https://github.com/kobsio/klogs/releases/tag/v0.8.0) (2022-03-09)
 
 - [#24](https://github.com/kobsio/klogs/pull/24): Rename repository in `go.mod` file to `github.com/kobsio/klogs`.

Makefile

@@ -23,6 +23,15 @@ build-ingester:
 		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
 		-o ingester ./cmd/ingester;
 
+.PHONY: build-filebeat
+build-filebeat:
+	@go build -ldflags "-X ${REPO}/pkg/version.Version=${VERSION} \
+		-X ${REPO}/pkg/version.Revision=${REVISION} \
+		-X ${REPO}/pkg/version.Branch=${BRANCH} \
+		-X ${REPO}/pkg/version.BuildUser=${BUILDUSER} \
+		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
+		-o filebeat ./cmd/filebeat;
+
 .PHONY: release-major
 release-major:
 	$(eval MAJORVERSION=$(shell git describe --tags --abbrev=0 | sed s/v// | awk -F. '{print "v"$$1+1".0.0"}'))

cluster/filebeat/filebeat-clusterrole.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+      - pods
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list

cluster/filebeat/filebeat-clusterrolebinding.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: filebeat
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io

cluster/filebeat/filebeat-cm.yaml

@@ -0,0 +1,75 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+data:
+  filebeat.yml: |-
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: container
+            close_inactive: 12h
+            paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
+
+    # Internal queue configuration for buffering events to be published.
+    queue:
+      mem:
+        # Max number of events the queue can buffer.
+        events: 100000
+
+        # Hints the minimum number of events stored in the queue,
+        # before providing a batch of events to the outputs.
+        # The default value is set to 2048.
+        # A value of 0 ensures events are immediately available
+        # to be sent to the outputs.
+        flush.min_events: 50000
+
+        # Maximum duration after which events are available to the outputs,
+        # if the number of events stored in the queue is < `flush.min_events`.
+        flush.timeout: 60s
+
+    processors:
+      # The decode_json_fields processor tries to decode a field as json data and insert the data to the target field.
+      # We allow only a depth of 1 to reduce mapping conflicts
+      - decode_json_fields:
+          fields: ["message"]
+          process_array: false
+          max_depth: 3
+          target: "content"
+          overwrite_keys: false
+          add_error_key: true
+
+    output.clickhouse:
+      cluster: ${CLUSTER}
+      address: clickhouse-clickhouse.clickhouse.svc.cluster.local:9000
+      database: logs
+      username: admin
+      password: admin
+      write_timeout: "20"
+      read_timeout: "10"
+      async_insert: true
+      wait_for_async_insert: true
+      force_number_fields:
+        - content.duration
+        - content.upstream_service_time
+
+    # We want to be able to configure the log level with an environment variable. Furthermore we don't want to log
+    # useless metrics output and have the logs as json to be processed in a proper way.
+    # See https://www.elastic.co/guide/en/beats/filebeat/current/configuration-logging.html
+    logging.level: "${LOG_LEVEL:info}"
+    logging.metrics.enabled: false
+    logging.json: true
+
+    # We enabled the http api to get access to the internal metrics, which will be used by the exporter.
+    http:
+      enabled: true
+      host: 0.0.0.0
+      port: 5066

cluster/filebeat/filebeat-ds.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      terminationGracePeriodSeconds: 30
+      containers:
+        - image: localhost:5000/klogs:latest-filebeat
+          imagePullPolicy: Always
+          name: filebeat
+          args:
+            - -c
+            - /etc/filebeat.yml
+            - -e
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CLUSTER
+              value: filebeat-clickhouse
+          ports:
+            - containerPort: 5066
+              name: http
+              protocol: TCP
+          securityContext:
+            runAsUser: 0
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1024Mi
+            requests:
+              cpu: 250m
+              memory: 512Mi
+          volumeMounts:
+            - mountPath: /var/log
+              name: varlog
+              readOnly: true
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+            - mountPath: /etc/filebeat.yml
+              name: config
+              readOnly: true
+              subPath: filebeat.yml
+            - mountPath: /usr/share/filebeat/data
+              name: data
+        - name: filebeat-exporter
+          image: trustpilot/beat-exporter:0.4.0
+          ports:
+            - containerPort: 9479
+              name: http-metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 25m
+              memory: 32Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoExecute
+          operator: Exists
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: data
+          hostPath:
+            path: /var/lib/filebeat-data
+            type: DirectoryOrCreate
+        - name: config
+          configMap:
+            defaultMode: 0640
+            name: filebeat

cluster/filebeat/filebeat-ns.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-sa.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-svc.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  ports:
+    - name: http
+      port: 5066
+      protocol: TCP
+      targetPort: http
+    - name: http-metrics
+      port: 9479
+      protocol: TCP
+      targetPort: http-metrics
+  selector:
+    app: filebeat
+  type: ClusterIP

cmd/filebeat/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.17.3-alpine3.14 as build
+RUN apk update && apk add git make
+WORKDIR /filebeat
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN export CGO_ENABLED=0 && make build-filebeat
+
+FROM alpine:3.14.2
+RUN apk update && apk add --no-cache ca-certificates
+RUN mkdir /filebeat
+COPY --from=build /filebeat/filebeat /filebeat
+WORKDIR /filebeat
+USER nobody
+ENTRYPOINT  [ "/filebeat/filebeat" ]

cmd/filebeat/filebeat.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"os"
+
+	_ "github.com/kobsio/klogs/pkg/filebeat"
+
+	"github.com/elastic/beats/v7/filebeat/cmd"
+	inputs "github.com/elastic/beats/v7/filebeat/input/default-inputs"
+)
+
+func main() {
+	if err := cmd.Filebeat(inputs.Init, cmd.FilebeatSettings()).Execute(); err != nil {
+		os.Exit(1)
+	}
+}