Add Filebeat as alternative to Fluent Bit (#28)

This commit adds Filebeat as an alternative to Fluent Bit. We decided to
try out Filebeat for the collection of logs, because with Fluent Bit not
all our logs seems to be collected.

.github/workflows/docker.yaml

@@ -87,3 +87,43 @@ jobs:
           file: ./cmd/ingester/Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64
           tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-ingester
+
+  filebeat:
+    name: Filebeat
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set Docker Tag
+        id: tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            DOCKER_TAG="${GITHUB_REF:10}"
+          else
+            DOCKER_TAG="main"
+          fi
+          echo ::set-output name=tag::${DOCKER_TAG}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push Docker Image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          file: ./cmd/filebeat/Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64
+          tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-filebeat

.github/workflows/test.yaml

@@ -43,3 +43,7 @@ jobs:
       - name: Build Ingester
         run: |
           make build-ingester
+
+      - name: Build Filebeat
+        run: |
+          make build-filebeat

.gitignore

@@ -1,3 +1,4 @@
 /out_clickhouse.h
 /out_clickhouse.so
 /ingester
+/filebeat

CHANGELOG.md

@@ -8,6 +8,8 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 ## Unreleased
 
+- [#28](https://github.com/kobsio/klogs/pull/28): Add Filebeat as alternative to Fluent Bit.
+
 ## [v0.8.0](https://github.com/kobsio/klogs/releases/tag/v0.8.0) (2022-03-09)
 
 - [#24](https://github.com/kobsio/klogs/pull/24): Rename repository in `go.mod` file to `github.com/kobsio/klogs`.

Makefile

@@ -23,6 +23,15 @@ build-ingester:
 		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
 		-o ingester ./cmd/ingester;
 
+.PHONY: build-filebeat
+build-filebeat:
+	@go build -ldflags "-X ${REPO}/pkg/version.Version=${VERSION} \
+		-X ${REPO}/pkg/version.Revision=${REVISION} \
+		-X ${REPO}/pkg/version.Branch=${BRANCH} \
+		-X ${REPO}/pkg/version.BuildUser=${BUILDUSER} \
+		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
+		-o filebeat ./cmd/filebeat;
+
 .PHONY: release-major
 release-major:
 	$(eval MAJORVERSION=$(shell git describe --tags --abbrev=0 | sed s/v// | awk -F. '{print "v"$$1+1".0.0"}'))

cluster/filebeat/filebeat-clusterrole.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+      - pods
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list

cluster/filebeat/filebeat-clusterrolebinding.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: filebeat
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io

cluster/filebeat/filebeat-cm.yaml

@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+data:
+  filebeat.yml: |-
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: container
+            close_inactive: 12h
+            paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
+
+    queue:
+      mem:
+        events: 100000
+        flush.min_events: 50000
+        flush.timeout: 60s
+
+    processors:
+      - decode_json_fields:
+          fields: ["message"]
+          process_array: false
+          max_depth: 3
+          target: "content"
+          overwrite_keys: false
+          add_error_key: true
+
+    output.clickhouse:
+      cluster: ${CLUSTER}
+      address: clickhouse-clickhouse.clickhouse.svc.cluster.local:9000
+      database: logs
+      username: admin
+      password: admin
+      write_timeout: "20"
+      read_timeout: "10"
+      async_insert: true
+      wait_for_async_insert: true
+      force_number_fields:
+        - content.duration
+        - content.upstream_service_time
+
+    logging.level: "${LOG_LEVEL:info}"
+    logging.metrics.enabled: false
+    logging.json: true
+
+    http:
+      enabled: true
+      host: 0.0.0.0
+      port: 5066

cluster/filebeat/filebeat-ds.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      terminationGracePeriodSeconds: 30
+      containers:
+        - image: localhost:5000/klogs:latest-filebeat
+          imagePullPolicy: Always
+          name: filebeat
+          args:
+            - -c
+            - /etc/filebeat.yml
+            - -e
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CLUSTER
+              value: filebeat-clickhouse
+          ports:
+            - containerPort: 5066
+              name: http
+              protocol: TCP
+          securityContext:
+            runAsUser: 0
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1024Mi
+            requests:
+              cpu: 250m
+              memory: 512Mi
+          volumeMounts:
+            - mountPath: /var/log
+              name: varlog
+              readOnly: true
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+            - mountPath: /etc/filebeat.yml
+              name: config
+              readOnly: true
+              subPath: filebeat.yml
+            - mountPath: /usr/share/filebeat/data
+              name: data
+        - name: filebeat-exporter
+          image: trustpilot/beat-exporter:0.4.0
+          ports:
+            - containerPort: 9479
+              name: http-metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 25m
+              memory: 32Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoExecute
+          operator: Exists
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: data
+          hostPath:
+            path: /var/lib/filebeat-data
+            type: DirectoryOrCreate
+        - name: config
+          configMap:
+            defaultMode: 0640
+            name: filebeat

cluster/filebeat/filebeat-ns.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-sa.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-svc.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  ports:
+    - name: http
+      port: 5066
+      protocol: TCP
+      targetPort: http
+    - name: http-metrics
+      port: 9479
+      protocol: TCP
+      targetPort: http-metrics
+  selector:
+    app: filebeat
+  type: ClusterIP

cmd/filebeat/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.17.3-alpine3.14 as build
+RUN apk update && apk add git make
+WORKDIR /filebeat
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN export CGO_ENABLED=0 && make build-filebeat
+
+FROM alpine:3.14.2
+RUN apk update && apk add --no-cache ca-certificates
+RUN mkdir /filebeat
+COPY --from=build /filebeat/filebeat /filebeat
+WORKDIR /filebeat
+USER nobody
+ENTRYPOINT  [ "/filebeat/filebeat" ]

cmd/filebeat/filebeat.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"os"
+
+	_ "github.com/kobsio/klogs/pkg/filebeat"
+
+	"github.com/elastic/beats/v7/filebeat/cmd"
+	inputs "github.com/elastic/beats/v7/filebeat/input/default-inputs"
+)
+
+func main() {
+	if err := cmd.Filebeat(inputs.Init, cmd.FilebeatSettings()).Execute(); err != nil {
+		os.Exit(1)
+	}
+}