From 1c5900f8e43ba9145590d7477908392fb47528d9 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 15:30:41 +0000 Subject: [PATCH 1/8] feat: directly cross compile using go --- .github/workflows/beats.yml | 16 ++++------------ build.sh | 17 ++++++++++------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/beats.yml b/.github/workflows/beats.yml index 7436dc4..673b8e9 100644 --- a/.github/workflows/beats.yml +++ b/.github/workflows/beats.yml @@ -19,12 +19,10 @@ jobs: strategy: matrix: - arch: [armv6, armv7] + arch: [armv7] beat: [filebeat, metricbeat, packetbeat, heartbeat] - distro: [buster] - go: [1.17.5] - runs-on: ubuntu-latest + runs-on: golang:1.20.11 name: ${{ matrix.beat }} ${{ matrix.arch }} @@ -40,21 +38,15 @@ jobs: id: build-beat with: arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} githubToken: ${{ github.token }} - install: | - apt-get update -q -y - apt-get install -q -y wget jq curl git build-essential libsystemd-dev - run: | - ./install-go.sh ${{ matrix.go }} armv6l - ./build.sh ${{ matrix.beat }} ${{ steps.vars.outputs.tag }} + ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} ${{ steps.vars.outputs.tag }} - uses: actions/upload-artifact@v2 with: - name: ${{ matrix.beat }}-${{ matrix.arch }}-${{ matrix.distro }} + name: ${{ matrix.beat }}-${{ matrix.arch }} path: ${{ matrix.beat }}-*.tar.gz - name: Release diff --git a/build.sh b/build.sh index 502a2e3..2ce9182 100755 --- a/build.sh +++ b/build.sh @@ -2,15 +2,15 @@ #set -e BEAT="${1:-filebeat}" -TAG="${2}" +ARCH="${2:-armv7l}" +TAG="${3}" if [[ -z ${TAG} || ${TAG} == "master" ]] ; then - TAG="v7.16.2" + TAG="v8.11.4" echo "Using default tag(${TAG})" fi VERSION="${TAG:1}" -ARCH=$(uname -m) SRC=$(pwd) echo "Setup" @@ -19,7 +19,9 @@ unset GOROOT export GOPATH=$(pwd)/go/project export PATH="/usr/local/go/bin:${GOPATH}/bin:${PATH}" mkdir -p "${GOPATH}" + go version +go tool dist list | grep linux if [[ ! -d beats ]] ; then echo "Cloning elastic/beats git tag(${TAG})" @@ -29,11 +31,13 @@ if [[ ! -d beats ]] ; then git describe --tags fi +set -e + echo "Building(${BEAT})" rm -rf "${GOPATH}/bin" cd "${SRC}/beats/${BEAT}" -go get -v -a ./... +env GOOS=linux GOARCH=arm go install -v -a ./... echo "Patching package" @@ -46,6 +50,5 @@ cd "${SRC}" wget -q -N "https://artifacts.elastic.co/downloads/beats/${BEAT}/${SRC_ARCHIVE}.tar.gz" tar -xf "${SRC_ARCHIVE}.tar.gz" mv "${SRC_ARCHIVE}" "${DEST_ARCHIVE}" -cp -f "${GOPATH}/bin/${BEAT}" "${DEST_ARCHIVE}/${BEAT}" -tar -czf "${DEST_ARCHIVE}.tar.gz" "${DEST_ARCHIVE}" -rm "${SRC_ARCHIVE}.tar.gz" \ No newline at end of file +cp -f "${GOPATH}/bin/linux_arm/${BEAT}" "${DEST_ARCHIVE}/${BEAT}" +tar -czf "${DEST_ARCHIVE}.tar.gz" "${DEST_ARCHIVE}" \ No newline at end of file From d19bb68744c561a959e3c9d1efe5e2cfd0758154 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 15:35:11 +0000 Subject: [PATCH 2/8] feat: CI using golang:1.20.11 docker image --- .github/workflows/beats.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/beats.yml b/.github/workflows/beats.yml index 673b8e9..2016055 100644 --- a/.github/workflows/beats.yml +++ b/.github/workflows/beats.yml @@ -22,7 +22,9 @@ jobs: arch: [armv7] beat: [filebeat, metricbeat, packetbeat, heartbeat] - runs-on: golang:1.20.11 + runs-on: ubuntu-latest + container: + image: golang:1.20.11 name: ${{ matrix.beat }} ${{ matrix.arch }} @@ -33,14 +35,11 @@ jobs: id: vars run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} - - uses: uraimo/run-on-arch-action@v2.1.1 - name: Build beat (on arch) + - name: Build beat (on arch) id: build-beat with: arch: ${{ matrix.arch }} - githubToken: ${{ github.token }} - run: | ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} ${{ steps.vars.outputs.tag }} From 6e59de76d52bc3aaf2af8894ce1e72007ae7868d Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 15:36:30 +0000 Subject: [PATCH 3/8] feat: fix run insead of use to build --- .github/workflows/beats.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/beats.yml b/.github/workflows/beats.yml index 2016055..da66a1c 100644 --- a/.github/workflows/beats.yml +++ b/.github/workflows/beats.yml @@ -35,13 +35,9 @@ jobs: id: vars run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} - - name: Build beat (on arch) + - name: Build beat id: build-beat - with: - arch: ${{ matrix.arch }} - githubToken: ${{ github.token }} - run: | - ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} ${{ steps.vars.outputs.tag }} + run: ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} ${{ steps.vars.outputs.tag }} - uses: actions/upload-artifact@v2 with: From c19c9a090edb5f2124e183f8ccbbb031739552f2 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 17:12:53 +0000 Subject: [PATCH 4/8] feat: is not a git tag build master --- build.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 2ce9182..e8b8b9b 100755 --- a/build.sh +++ b/build.sh @@ -3,14 +3,16 @@ #set -e BEAT="${1:-filebeat}" ARCH="${2:-armv7l}" -TAG="${3}" +BRANCH="${3}" -if [[ -z ${TAG} || ${TAG} == "master" ]] ; then - TAG="v8.11.4" - echo "Using default tag(${TAG})" +TAG=$(git describe --tags --exact-match 2> /dev/null) + +if [[ -z ${TAG} ]] ; then + BRANCH="master" + echo "Using default branch(${BRANCH})" fi -VERSION="${TAG:1}" +VERSION="${BRANCH:1}" SRC=$(pwd) echo "Setup" @@ -24,10 +26,10 @@ go version go tool dist list | grep linux if [[ ! -d beats ]] ; then - echo "Cloning elastic/beats git tag(${TAG})" - git clone --quiet --single-branch --branch="${TAG}" --depth=1 "https://github.com/elastic/beats.git" + echo "Cloning elastic/beats git branch(${BRANCH})" + git clone --quiet --single-branch --branch="${BRANCH}" --depth=1 "https://github.com/elastic/beats.git" cd "${SRC}/beats" - git checkout --quiet --detach "${TAG}" + git checkout --quiet --detach "${BRANCH}" git describe --tags fi From c9437287dd3d29dbab71d73f11fbc3990cd3a334 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 17:22:33 +0000 Subject: [PATCH 5/8] feat: skip pakcage build is not a git tag --- README.md | 16 +++++++--------- build.sh | 9 +++++++-- install-go.sh | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1edec93..c4846ad 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![](https://badgen.net/github/release/vrince/arm-beats) [![CI](https://github.com/vrince/arm-beats/actions/workflows/beats.yml/badge.svg)](https://github.com/vrince/arm-beats/actions/workflows/beats.yml) -Automated Elastic `filebeat`, `metricbeat`, `heartbeat` & `packagebeat` builds for **linux/armv7 & linux/armv6 32bits**. +Automated Elastic `filebeat`, `metricbeat`, `heartbeat` & `packagebeat` builds for **linux/armv7 32bits**. ## Goal @@ -16,9 +16,9 @@ Currently elasticsearch team do not provide any **arm 32bits** (armv7 / armv6) b ## Run `filebeat` on a raspberry pi ```bash -wget https://github.com/vrince/arm-beats/releases/download/v8.3.3/filebeat-8.3.3-linux-armv7l.tar.gz -tar -xf filebeat-8.3.3-linux-armv7l.tar.gz -cd filebeat-8.3.3-linux-armv7l +wget https://github.com/vrince/arm-beats/releases/download/v8.11.4/filebeat-8.11.4-linux-armv7l.tar.gz +tar -xf filebeat-8.11.4-linux-armv7l.tar.gz +cd filebeat-8.11.4-linux-armv7l file ./filebeat ``` @@ -33,9 +33,9 @@ For more information about how to configure `filebeat` --> [documentation](https ## Run `metricbeat` on raspberry pi ```bash -wget https://github.com/vrince/arm-beats/releases/download/v8.3.3/metricbeat-8.3.3-linux-armv7l.tar.gz -tar -xf metricbeat-8.3.3-linux-armv7l.tar.gz -cd metricbeat-8.3.3-linux-armv7l +wget https://github.com/vrince/arm-beats/releases/download/v8.11.4/metricbeat-8.11.4-linux-armv7l.tar.gz +tar -xf metricbeat-8.11.4-linux-armv7l.tar.gz +cd metricbeat-8.11.4-linux-armv7l file ./metricbeat ``` @@ -51,8 +51,6 @@ For more information about how to configure `metricbeat` --> [documentation](htt [![CI](https://github.com/vrince/arm-beats/actions/workflows/beats.yml/badge.svg)](https://github.com/vrince/arm-beats/actions/workflows/beats.yml) -This automation script uses emulated (`qemu`) multi-arch capability of docker. Performance is really not that great, so we only perform `go get` to build necessary binary file. Finally we patch official elasticsearch (arm64) beat package swapping the main binary. - ## Reference * Elastic beat : https://www.elastic.co/beats/ diff --git a/build.sh b/build.sh index e8b8b9b..5930f32 100755 --- a/build.sh +++ b/build.sh @@ -41,7 +41,12 @@ rm -rf "${GOPATH}/bin" cd "${SRC}/beats/${BEAT}" env GOOS=linux GOARCH=arm go install -v -a ./... -echo "Patching package" +if [[ -z ${TAG} ]] ; then + ls -al ${GOPATH}/bin/linux_arm/${BEAT} + exit 0 +fi + +echo "Patching package version(${VERSION})" SRC_ARCHIVE="${BEAT}-${VERSION}-linux-arm64" DEST_ARCHIVE="${BEAT}-${VERSION}-linux-${ARCH}" @@ -49,7 +54,7 @@ DEST_ARCHIVE="${BEAT}-${VERSION}-linux-${ARCH}" echo "${SRC_ARCHIVE} --> ${DEST_ARCHIVE}" cd "${SRC}" -wget -q -N "https://artifacts.elastic.co/downloads/beats/${BEAT}/${SRC_ARCHIVE}.tar.gz" +wget "https://artifacts.elastic.co/downloads/beats/${BEAT}/${SRC_ARCHIVE}.tar.gz" tar -xf "${SRC_ARCHIVE}.tar.gz" mv "${SRC_ARCHIVE}" "${DEST_ARCHIVE}" cp -f "${GOPATH}/bin/linux_arm/${BEAT}" "${DEST_ARCHIVE}/${BEAT}" diff --git a/install-go.sh b/install-go.sh index 2cfbacf..9bed6e2 100755 --- a/install-go.sh +++ b/install-go.sh @@ -1,5 +1,5 @@ #!/bin/bash -GO_VERSION=${1:-1.17.5} +GO_VERSION=${1:-1.20.11} GO_ARCH=${2:amd64} FORCE=${3} From 807561e664c5737af3887bf0c63e0bbb7b819491 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 17:27:13 +0000 Subject: [PATCH 6/8] feat: use new GITHUB_STATE --- .github/workflows/beats.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/beats.yml b/.github/workflows/beats.yml index da66a1c..89700b5 100644 --- a/.github/workflows/beats.yml +++ b/.github/workflows/beats.yml @@ -32,14 +32,14 @@ jobs: - uses: actions/checkout@v2 - name: Set tag - id: vars - run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} + run: echo "git_tag=:${GITHUB_REF#refs/*/}" >> $GITHUB_STATE - name: Build beat id: build-beat - run: ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} ${{ steps.vars.outputs.tag }} + run: ./build.sh ${{ matrix.beat }} ${{ matrix.arch }} $git_tag - uses: actions/upload-artifact@v2 + if: startsWith(github.ref, 'refs/tags/') with: name: ${{ matrix.beat }}-${{ matrix.arch }} path: ${{ matrix.beat }}-*.tar.gz From e5ac02c604d1e04bb7a5744294af280aac5eae04 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 17:30:03 +0000 Subject: [PATCH 7/8] feat: enable CGO_ENABLED=1 --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 5930f32..8ae311c 100755 --- a/build.sh +++ b/build.sh @@ -39,7 +39,7 @@ echo "Building(${BEAT})" rm -rf "${GOPATH}/bin" cd "${SRC}/beats/${BEAT}" -env GOOS=linux GOARCH=arm go install -v -a ./... +env GOOS=linux GOARCH=arm CGO_ENABLED=1 go install -v -a ./... if [[ -z ${TAG} ]] ; then ls -al ${GOPATH}/bin/linux_arm/${BEAT} From 93b215e5280b4f448272b279fdfdaf0a644352f1 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Fri, 26 Apr 2024 17:32:39 +0000 Subject: [PATCH 8/8] feat: remove packetbeat --- .github/workflows/beats.yml | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beats.yml b/.github/workflows/beats.yml index 89700b5..9641cea 100644 --- a/.github/workflows/beats.yml +++ b/.github/workflows/beats.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: arch: [armv7] - beat: [filebeat, metricbeat, packetbeat, heartbeat] + beat: [filebeat, metricbeat, heartbeat] runs-on: ubuntu-latest container: diff --git a/build.sh b/build.sh index 8ae311c..5930f32 100755 --- a/build.sh +++ b/build.sh @@ -39,7 +39,7 @@ echo "Building(${BEAT})" rm -rf "${GOPATH}/bin" cd "${SRC}/beats/${BEAT}" -env GOOS=linux GOARCH=arm CGO_ENABLED=1 go install -v -a ./... +env GOOS=linux GOARCH=arm go install -v -a ./... if [[ -z ${TAG} ]] ; then ls -al ${GOPATH}/bin/linux_arm/${BEAT}