Add Filebeat as alternative to Fluent Bit

.github/workflows/docker.yaml

@@ -87,3 +87,43 @@ jobs:
           file: ./cmd/ingester/Dockerfile
           platforms: linux/amd64,linux/arm,linux/arm64
           tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-ingester
+
+  filebeat:
+    name: Filebeat
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set Docker Tag
+        id: tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            DOCKER_TAG="${GITHUB_REF:10}"
+          else
+            DOCKER_TAG="main"
+          fi
+          echo ::set-output name=tag::${DOCKER_TAG}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push Docker Image
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          context: .
+          file: ./cmd/filebeat/Dockerfile
+          platforms: linux/amd64,linux/arm,linux/arm64
+          tags: kobsio/klogs:${{ steps.tag.outputs.tag }}-filebeat

.github/workflows/test.yaml

@@ -43,3 +43,7 @@ jobs:
       - name: Build Ingester
         run: |
           make build-ingester
+
+      - name: Build Filebeat
+        run: |
+          make build-filebeat

.gitignore

@@ -1,3 +1,4 @@
 /out_clickhouse.h
 /out_clickhouse.so
 /ingester
+/filebeat

CHANGELOG.md

@@ -8,6 +8,8 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 ## Unreleased
 
+- [#28](https://github.com/kobsio/klogs/pull/28): Add Filebeat as alternative to Fluent Bit.
+
 ## [v0.8.0](https://github.com/kobsio/klogs/releases/tag/v0.8.0) (2022-03-09)
 
 - [#24](https://github.com/kobsio/klogs/pull/24): Rename repository in `go.mod` file to `github.com/kobsio/klogs`.

Makefile

@@ -23,6 +23,15 @@ build-ingester:
 		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
 		-o ingester ./cmd/ingester;
 
+.PHONY: build-filebeat
+build-filebeat:
+	@go build -ldflags "-X ${REPO}/pkg/version.Version=${VERSION} \
+		-X ${REPO}/pkg/version.Revision=${REVISION} \
+		-X ${REPO}/pkg/version.Branch=${BRANCH} \
+		-X ${REPO}/pkg/version.BuildUser=${BUILDUSER} \
+		-X ${REPO}/pkg/version.BuildDate=${BUILDTIME}" \
+		-o filebeat ./cmd/filebeat;
+
 .PHONY: release-major
 release-major:
 	$(eval MAJORVERSION=$(shell git describe --tags --abbrev=0 | sed s/v// | awk -F. '{print "v"$$1+1".0.0"}'))

cluster/filebeat/filebeat-clusterrole.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+      - pods
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list

cluster/filebeat/filebeat-clusterrolebinding.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: filebeat
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io

cluster/filebeat/filebeat-cm.yaml

@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+data:
+  filebeat.yml: |-
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: container
+            close_inactive: 12h
+            paths:
+              - /var/log/containers/*${data.kubernetes.container.id}.log
+
+    queue:
+      mem:
+        events: 100000
+        flush.min_events: 50000
+        flush.timeout: 60s
+
+    processors:
+      - decode_json_fields:
+          fields: ["message"]
+          process_array: false
+          max_depth: 3
+          target: "content"
+          overwrite_keys: false
+          add_error_key: true
+
+    output.clickhouse:
+      cluster: ${CLUSTER}
+      address: clickhouse-clickhouse.clickhouse.svc.cluster.local:9000
+      database: logs
+      username: admin
+      password: admin
+      write_timeout: "20"
+      read_timeout: "10"
+      async_insert: true
+      wait_for_async_insert: true
+      force_number_fields:
+        - content.duration
+        - content.upstream_service_time
+
+    logging.level: "${LOG_LEVEL:info}"
+    logging.metrics.enabled: false
+    logging.json: true
+
+    http:
+      enabled: true
+      host: 0.0.0.0
+      port: 5066

cluster/filebeat/filebeat-ds.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      terminationGracePeriodSeconds: 30
+      containers:
+        - image: localhost:5000/klogs:latest-filebeat
+          imagePullPolicy: Always
+          name: filebeat
+          args:
+            - -c
+            - /etc/filebeat.yml
+            - -e
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CLUSTER
+              value: filebeat-clickhouse
+          ports:
+            - containerPort: 5066
+              name: http
+              protocol: TCP
+          securityContext:
+            runAsUser: 0
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1024Mi
+            requests:
+              cpu: 250m
+              memory: 512Mi
+          volumeMounts:
+            - mountPath: /var/log
+              name: varlog
+              readOnly: true
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+            - mountPath: /etc/filebeat.yml
+              name: config
+              readOnly: true
+              subPath: filebeat.yml
+            - mountPath: /usr/share/filebeat/data
+              name: data
+        - name: filebeat-exporter
+          image: trustpilot/beat-exporter:0.4.0
+          ports:
+            - containerPort: 9479
+              name: http-metrics
+              protocol: TCP
+          resources:
+            limits:
+              cpu: 25m
+              memory: 32Mi
+            requests:
+              cpu: 25m
+              memory: 32Mi
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+        - effect: NoExecute
+          operator: Exists
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: data
+          hostPath:
+            path: /var/lib/filebeat-data
+            type: DirectoryOrCreate
+        - name: config
+          configMap:
+            defaultMode: 0640
+            name: filebeat

cluster/filebeat/filebeat-ns.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-sa.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat

cluster/filebeat/filebeat-svc.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: filebeat
+  namespace: filebeat
+  labels:
+    app: filebeat
+spec:
+  ports:
+    - name: http
+      port: 5066
+      protocol: TCP
+      targetPort: http
+    - name: http-metrics
+      port: 9479
+      protocol: TCP
+      targetPort: http-metrics
+  selector:
+    app: filebeat
+  type: ClusterIP

cmd/filebeat/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.17.3-alpine3.14 as build
+RUN apk update && apk add git make
+WORKDIR /filebeat
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN export CGO_ENABLED=0 && make build-filebeat
+
+FROM alpine:3.14.2
+RUN apk update && apk add --no-cache ca-certificates
+RUN mkdir /filebeat
+COPY --from=build /filebeat/filebeat /filebeat
+WORKDIR /filebeat
+USER nobody
+ENTRYPOINT  [ "/filebeat/filebeat" ]

cmd/filebeat/filebeat.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"os"
+
+	_ "github.com/kobsio/klogs/pkg/filebeat"
+
+	"github.com/elastic/beats/v7/filebeat/cmd"
+	inputs "github.com/elastic/beats/v7/filebeat/input/default-inputs"
+)
+
+func main() {
+	if err := cmd.Filebeat(inputs.Init, cmd.FilebeatSettings()).Execute(); err != nil {
+		os.Exit(1)
+	}
+}