use buildx to build multi arches

.github/workflows/build-fb-image.yaml

@@ -9,6 +9,7 @@ on:
     paths:
       - ".github/workflows/build-fb-image.yaml"
       - "cmd/fluent-watcher/fluentbit/**"
+      - "cmd/fluent-watcher/hooks/**"
       - "pkg/filenotify/**"
 
 env:
@@ -18,7 +19,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    name: Build Image for Fluent Bit and Fluentd
+    name: Build Image for Fluent Bit
     steps:
       - name: Install Go
         uses: actions/setup-go@v2
@@ -41,6 +42,10 @@ jobs:
           username: ${{ secrets.REGISTRY_USER }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
 
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Build and Push Image for Fluent Bit
         run: |
-          make build-fb-amd64 -e FB_IMG=${{ env.FB_IMG }} && docker push ${{ env.FB_IMG }}
+          make build-fb -e FB_IMG=${{ env.FB_IMG }}

.github/workflows/build-fd-image.yaml

@@ -9,16 +9,18 @@ on:
     paths:
       - ".github/workflows/build-fd-image.yaml"
       - "cmd/fluent-watcher/fluentd/**"
+      - "cmd/fluent-watcher/hooks/**"
       - "pkg/filenotify/**"
 
 env:
   FD_IMG: 'kubesphere/fluentd:v1.14.4'
+  ARCH: '-arm64'
 
 jobs:
-  build:
+  build-amd64:
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    name: Build Image for Fluent Bit and Fluentd
+    name: Build amd64 Image for Fluentd
     steps:
       - name: Install Go
         uses: actions/setup-go@v2
@@ -41,6 +43,41 @@ jobs:
           username: ${{ secrets.REGISTRY_USER }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
 
-      - name: Build and Push Image for Fluentd
+      - name: Build and Push amd64 Image for Fluentd
         run: |
-          make build-fd-amd64 -e FD_IMG=${{ env.FD_IMG }} && docker push ${{ env.FD_IMG }}
+          make build-fd-amd64 -e FD_IMG=${{ env.FD_IMG }}
+          docker push ${{ env.FD_IMG }}
+
+  build-arm64:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    name: Build arm64 Image for Fluentd
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and Push arm64 Image for Fluentd
+        run: |
+          make build-fd-arm64 -e FD_IMG=${{ env.FD_IMG }} -e ARCH=${{ env.ARCH }}

.github/workflows/build-op-image.yaml

@@ -9,7 +9,6 @@ on:
     paths:
       - ".github/workflows/build-op-image.yaml"
       - "apis/**"
-      - "charts/**"
       - "cmd/fluent-manager/**"
       - "controllers/**"
       - "hack/**"

Makefile

@@ -4,6 +4,8 @@ FB_IMG ?= kubesphere/fluent-bit:v1.8.11
 FD_IMG ?= kubesphere/fluentd:v1.14.4
 FO_IMG ?= kubesphere/fluent-operator:$(VERSION)
 
+ARCH ?= arm64
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
 
@@ -89,13 +91,9 @@ build-op:
 	docker buildx build --push --platform linux/amd64,linux/arm64 -f cmd/fluent-manager/Dockerfile . -t ${FO_IMG}
 
 # Build amd64/arm64 Fluent Bit container image
-build-fb:
+build-fb: prepare-build
 	docker buildx build --push --platform linux/amd64,linux/arm64 -f cmd/fluent-watcher/fluentbit/Dockerfile . -t ${FB_IMG}
 
-# Build amd64/arm64 Fluentd container image
-build-fd:
-	docker buildx build --push --platform linux/amd64,linux/arm64 -f cmd/fluent-watcher/fluentd/Dockerfile.complete . -t ${FD_IMG}
-
 # Build all amd64 docker images
 build-amd64: build-op-amd64 build-fb-amd64 build-fd-amd64
 
@@ -107,9 +105,17 @@ build-op-amd64:
 build-fb-amd64:
 	docker build -f cmd/fluent-watcher/fluentbit/Dockerfile . -t ${FB_IMG}
 
-# Build amd64 Fluent Bit container image
+# Build amd64 Fluentd container image
 build-fd-amd64:
-	docker build -f cmd/fluent-watcher/fluentd/Dockerfile.complete . -t ${FD_IMG}
+	docker build -f cmd/fluent-watcher/fluentd/Dockerfile.amd64 . -t ${FD_IMG}
+
+# Use docker buildx to build arm64 Fluentd container image
+build-fd-arm64: prepare-build
+	docker buildx build --push --platform linux/arm64 -f cmd/fluent-watcher/fluentd/Dockerfile.arm64.quick . -t ${FD_IMG}${ARCH}
+
+# Prepare for arm64 building
+prepare-build:
+	chmod +x cmd/fluent-watcher/hooks/post-hook.sh && bash cmd/fluent-watcher/hooks/post-hook.sh
 
 # Push the amd64 docker image
 push-amd64:

cmd/fluent-watcher/fluentd/Dockerfile.arm64

@@ -0,0 +1,87 @@
+# Fluentd watcher agent
+FROM golang:1.16.6-alpine3.13 as buildergo
+RUN mkdir -p /fluentd
+RUN mkdir -p /code
+COPY . /code/
+WORKDIR /code
+RUN echo $(ls -al /code)
+RUN CGO_ENABLED=0 go build -i -ldflags '-w -s' -o /fluentd/fluentd-watcher /code/cmd/fluent-watcher/fluentd/main.go
+
+# To set multiarch build for Docker hub automated build.
+FROM golang:alpine AS builderqemu
+WORKDIR /go
+ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8
+RUN apk add curl --no-cache
+RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static .
+
+# Fluentd main image
+FROM arm64v8/ruby:2.7-slim-bullseye
+COPY --from=builderqemu /go/qemu-aarch64-static /usr/bin/
+LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.14.4"
+ENV TINI_VERSION=0.18.0
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apt-get purge' has no effect
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+            ca-certificates \
+ && buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 gnupg dirmngr \
+    " \
+ && apt-get install -y --no-install-recommends $buildDeps \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ && gem install oj -v 3.10.18 \
+ && gem install json -v 2.4.1 \
+ && gem install async-http -v 0.54.0 \
+ && gem install ext_monitor -v 0.1.2 \
+ && gem install fluentd -v 1.14.4 \
+ && gem install bigdecimal -v 1.4.4 \
+ && gem install resolv -v 0.2.1 \
+ && gem install elasticsearch -v 7.13.3 \
+ && gem install elasticsearch-xpack -v 7.13.3 \
+ && gem install fluent-plugin-detect-exceptions -v 0.0.14 \
+ && gem install \
+         fluent-plugin-s3 \
+         fluent-plugin-rewrite-tag-filter \
+         fluent-plugin-oss \
+         fluent-plugin-dedot_filter \
+         fluent-plugin-sumologic_output \
+         fluent-plugin-kafka \
+         fluent-plugin-label-router \
+         fluent-plugin-record-modifier \
+         fluent-plugin-multi-format-parser \
+         fluent-plugin-aws-elasticsearch-service \
+ && echo "plugin installed." \
+ && wget -O /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
+ && cd /tmp && tar -xjf jemalloc-4.5.0.tar.bz2 && cd jemalloc-4.5.0/ \
+ && ./configure && make \
+ && mv lib/libjemalloc.so.2 /usr/lib \
+ && apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/2.*/gems/fluentd-*/test
+
+RUN groupadd -r fluent && useradd -r -g fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd
+
+ENV FLUENTD_CONF="fluent.conf"
+
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+EXPOSE 24224 5140
+
+USER fluent
+
+COPY ./cmd/fluent-watcher/fluentd/base/fluent.conf /fluentd/etc/
+COPY ./cmd/fluent-watcher/fluentd/app/app.conf /fluentd/etc/
+
+# copy the output in buildergo
+COPY --from=buildergo /fluentd/fluentd-watcher /fluentd/bin/fluentd-watcher
+# Entry point
+ENTRYPOINT ["/fluentd/bin/fluentd-watcher"]

cmd/fluent-watcher/fluentd/Dockerfile.arm64.base

@@ -0,0 +1,50 @@
+# To set multiarch build for Docker hub automated build.
+FROM golang:alpine AS builderqemu
+WORKDIR /go
+ENV QEMU_DOWNLOAD_SHA256 5db25cccb40ac7b1ca857653b883376b931d91b06ff34ffe70dcf6180bd07bb8
+RUN apk add curl --no-cache
+RUN curl -sL -o qemu-6.0.0.balena1-aarch64.tar.gz https://github.com/balena-io/qemu/releases/download/v6.0.0%2Bbalena1/qemu-6.0.0.balena1-aarch64.tar.gz && echo "$QEMU_DOWNLOAD_SHA256 *qemu-6.0.0.balena1-aarch64.tar.gz" | sha256sum -c - | tar zxvf qemu-6.0.0.balena1-aarch64.tar.gz -C . && mv qemu-6.0.0+balena1-aarch64/qemu-aarch64-static .
+
+# Fluentd main image
+FROM arm64v8/ruby:2.7-slim-bullseye
+COPY --from=builderqemu /go/qemu-aarch64-static /usr/bin/
+LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.14.4"
+ENV TINI_VERSION=0.18.0
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apt-get purge' has no effect
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+            ca-certificates \
+ && buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 gnupg dirmngr\
+    " \
+ && apt-get install -y --no-install-recommends $buildDeps \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ && gem install oj -v 3.10.18 \
+ && gem install json -v 2.4.1 \
+ && gem install async-http -v 0.54.0 \
+ && gem install ext_monitor -v 0.1.2 \
+ && gem install bigdecimal -v 1.4.4 \
+ && gem install resolv -v 0.2.1 \
+ && gem install elasticsearch -v 7.13.3 \
+ && gem install elasticsearch-xpack -v 7.13.3 \
+ && gem install fluent-plugin-detect-exceptions -v 0.0.14 \
+ && gem install \
+         fluent-plugin-s3 \
+         fluent-plugin-rewrite-tag-filter \
+         fluent-plugin-oss \
+         fluent-plugin-dedot_filter \
+         fluent-plugin-sumologic_output \
+         fluent-plugin-kafka \
+         fluent-plugin-label-router \
+         fluent-plugin-record-modifier \
+         fluent-plugin-multi-format-parser \
+         fluent-plugin-aws-elasticsearch-service \
+ && wget -O /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
+ && cd /tmp && tar -xjf jemalloc-4.5.0.tar.bz2 && cd jemalloc-4.5.0/ \
+ && ./configure && make \
+ && mv lib/libjemalloc.so.2 /usr/lib \
+ && echo "plugins installed."

cmd/fluent-watcher/fluentd/Dockerfile.arm64.quick

@@ -0,0 +1,49 @@
+# Fluentd watcher agent
+FROM golang:1.16.6-alpine3.13 as buildergo
+RUN mkdir -p /fluentd
+RUN mkdir -p /code
+COPY . /code/
+WORKDIR /code
+RUN echo $(ls -al /code)
+RUN CGO_ENABLED=0 go build -i -ldflags '-w -s' -o /fluentd/fluentd-watcher /code/cmd/fluent-watcher/fluentd/main.go
+
+# Fluentd main image
+FROM kubesphere/fluentd:v1.14.4-arm64-base
+LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.14.4"
+ENV TINI_VERSION=0.18.0
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apt-get purge' has no effect
+RUN buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 gnupg dirmngr\
+    " \
+ && gem install fluentd -v 1.14.4 \
+ && apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/2.*/gems/fluentd-*/test
+
+RUN groupadd -r fluent && useradd -r -g fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd
+
+ENV FLUENTD_CONF="fluent.conf"
+
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+EXPOSE 24224 5140
+
+USER fluent
+
+COPY ./cmd/fluent-watcher/fluentd/base/fluent.conf /fluentd/etc/
+COPY ./cmd/fluent-watcher/fluentd/app/app.conf /fluentd/etc/
+
+# copy the output in buildergo
+COPY --from=buildergo /fluentd/fluentd-watcher /fluentd/bin/fluentd-watcher
+# Entry point
+ENTRYPOINT ["/fluentd/bin/fluentd-watcher"]

cmd/fluent-watcher/hooks/post-hook.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /post_checkout.erb
+
+
+set -e
+
+HOST_ARCH=$(uname -m)
+
+if [ x"${HOST_ARCH}" == x"aarch64" ]; then
+    echo "Building arm64 image natively"
+    exit
+fi
+
+# Enable cross-platform builds https://github.com/multiarch/qemu-user-static
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes