elastic · mergify · Oct 2, 2024
diff --git a/Makefile.common b/Makefile.common
@@ -4,7 +4,18 @@ VCS_URL       := https://github.com/elastic/golang-crossbuild
 BUILD_DATE    := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 .DEFAULT_GOAL := build
 
+<<<<<<< HEAD
 # Requires login at https://docker.elastic.co:7000/.
+=======
+# Requires login at google storage.
+copy-sdks:
+ifeq ($(CI),true)
+	@gcloud storage cp gs://ingest-buildkite-ci/sdks . --recursive
+else
+	@echo 'Only available if running in the CI'
+endif
+
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))
 push:
 	echo ">> Pushing $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"
 	@docker push "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"

diff --git a/README.md b/README.md
@@ -111,10 +111,85 @@ The osxcross repository used to cross compile for MacOSX has [instructions for p
 
 The instructions for packaging the SDK on a Linux instance are:
 
+<<<<<<< HEAD
 1. Clone the [osxcross](https://github.com/tpoechtrager/osxcross) repo.
 1. Install `clang`, `make`, `libssl-dev`, `lzma-dev`, `libxml2-dev`, `libbz2-dev`.
 1. Download [Xcode from Apple](Download Xcode: https://developer.apple.com/download/more]).
 1. Run `./tools/gen_sdk_package_pbzx.sh <xcode>.xip`.
+=======
+## go/base-arm Docker image
+
+The `base-arm` image is the base image to crossbuild `linux/arm64` binaries on `arm` hosts, it is build for Debian 7+.
+This image is user to crosscompile in `linux/arm`.
+It was replaced by the `base` image for Debian 9+ when we started to build multiarchitecture Docker images.
+It could be removed in the future, it allow to `crosscompile` arm binaries in `linux/arm64` machines, that it is not needed because the native architecture is the same than the target architecture.
+It was added to golang-crossbuild due limitations of the build system.
+
+## go/main Docker image
+
+The `main` image is the base image for the `amd64` architecture, it is build for Debian 7+.
+It is used to cross compile for `linux/amd`, `linux/amd64`, `win/amd`, and `win/amd64`.
+This Docker immage add two libraries to the `base` image, `libpcap` and `WpdPack` to be able to capture network packages on diferent OS.
+Thes two libraries are precompiled and stored at https://storage.googleapis.com/ingest-buildkite-ci/sdks.
+
+## go/darwin Docker image
+
+The `darwin` image is the base image for the MacOSX cross compilation, it is build for Debian 8+.
+It can compiles for `darwin/amd` (Debian 10+), `darwin/amd64`, `darwin/arm64`, `darwin/arm64e`, and universal binaries.
+This Docker image is based on the `base` image.
+It uses [osxcross](https://codeload.github.com/tpoechtrager/osxcross) to configure the crosscompile for MacOSX.
+This image require a MacOSX SDK to be installed in the Docker image,
+for the instructions to package the MacOS SDK see the [Packaging MacOS SDK](#packaging-macos-sdk) section.
+
+## go/darwin-arm64 Docker image
+
+The `darwin-arm64` image is the base image for the MacOSX cross compilation, it is build for Debian 10+.
+It can compiles for `darwin/amd64`, `darwin/arm64`, and `darwin/arm64e`, and universal binaries.
+This Docker image is based on the `base` image.
+It uses the `llvm-apple` image to cross compile for MacOSX.
+The `darwin-arm64` can replace the `darwin` image in the future, it is faster to build and it does not need to build [osxcross](https://codeload.github.com/tpoechtrager/osxcross) and uses the official LLVM fork from Apple so it support enhacement for `darwin` architectures.
+
+## go/arm Docker image
+
+The `arm` image is the base image for the `arm64` architecture, it is build for Debian 9+.
+It is used to cross compile for `linux/arm64`. This Docker image is based on the `base` image.
+
+## go/armel Docker image
+
+The `armel` image is the base image for the `armel` architecture, it is build for Debian 12+.
+It is used to cross compile for `linux/armel`. This Docker image is based on the `base` image.
+
+## go/armhf Docker image
+
+The `armhf` image is the base image for the `armhf` architecture, it is build for Debian 9+.
+It is used to cross compile for `linux/armhf`. This Docker image is based on the `base` image.
+
+## go/mips Docker image
+
+The `mips` image is the base image for the `mips` architecture, it is build for Debian 12+.
+It is used to cross compile for `linux/mips`. This Docker image is based on the `base` image.
+
+## go/mips32 Docker image
+
+The `mips32` image is the base image for the `mips32` architecture, it is build for Debian 11+.
+It is used to cross compile for `linux/mips32`. This Docker image is based on the `base` image.
+
+## go/ppc Docker image
+
+The `ppc` image is the base image for the `ppc` architecture, it is build for Debian 12+.
+It is used to cross compile for `linux/ppc`. This Docker image is based on the `base` image.
+
+## go/s390x Docker image
+
+The `s390x` image is the base image for the `s390x` architecture, it is build for Debian 12+.
+It is used to cross compile for `linux/s390x`. This Docker image is based on the `base` image.
+
+## go/npcap Docker image
+
+The `npcap` image is a placeholder for the `npcap` library, see [npcap](./NPCAP.md) for more information.
+
+## Troubleshooting
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))
 
 ### bzip2 issues
 
@@ -142,5 +217,5 @@ XCODEDIR=osxcross/build/tmp_<X> ./tools/gen_sdk_package.sh
 The SDK should be in the working directory.
 The tmp dir can be safely deleted after this.
 
-The SDKs should be uploaded into the `gs://obs-ci-cache` bucket on GCP (Google Cloud Platform).
-This is accessible to authorized users in the `elastic-observability` project [here](https://console.cloud.google.com/storage/browser/obs-ci-cache).
+The SDKs should be uploaded into the `gs://ingest-buildkite-ci/sdks` bucket on GCP (Google Cloud Platform).
+This is accessible to authorized users in the `platform-ingest` project [here](https://console.cloud.google.com/storage/browser/ingest-buildkite-ci/sdks).
diff --git a/go/Makefile.common b/go/Makefile.common
@@ -9,7 +9,13 @@ TAG_EXTENSION  ?=
 
 export DEBIAN_VERSION TAG_EXTENSION
 
+<<<<<<< HEAD
 build:
+=======
+DOCKER_CMD := docker build
+
+build: copy-npcap copy-sdks
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))
 	@echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"
 	@go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile
 	@docker build -t "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" \
@@ -23,7 +29,7 @@ build:
 	--build-arg BUILD_DATE="$(BUILD_DATE)" \
 	.
 
-build-arm:
+build-arm: copy-sdks
 	@echo $(SELF_DIR)
 	@echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"
 	@go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile

diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl
@@ -63,9 +63,9 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \
     && go build -o /crossbuild /entrypoint.go \
     && rm -rf /go/* /root/.cache/* /entrypoint.go
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \
-  && mkdir /libpcap \
-	&& tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
+COPY sdks/libpcap-1.8.1.tar.gz . 
+RUN mkdir /libpcap \
+  && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
   && cd /libpcap/libpcap-1.8.1 \
   && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no --host=aarch64-unknown-linux-gnu --with-pcap=linux \
   && make

diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl
@@ -55,11 +55,10 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \
     && go build -o /crossbuild /entrypoint.go \
     && rm -rf /go/* /root/.cache/* /entrypoint.go
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \
-  && echo "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e  libpcap-1.8.1.tar.gz" | sha256sum -c - \
-  && mkdir /libpcap \
-	&& tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
-  && rm libpcap-1.8.1.tar.gz
+COPY sdks/libpcap-1.8.1.tar.gz . 
+RUN mkdir /libpcap \
+    && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
+    && rm libpcap-1.8.1.tar.gz
 
 
 ENV GOLANG_CROSSBUILD=1

diff --git a/go/darwin/Dockerfile.tmpl b/go/darwin/Dockerfile.tmpl
@@ -20,6 +20,7 @@ RUN \
         uuid-dev \
     && rm -rf /var/lib/apt/lists/*
 
+<<<<<<< HEAD
 {{if eq .DEBIAN_VERSION "10"}}
 ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.14.sdk.tar.xz
 ARG OSXCROSS_PATH=/usr/osxcross
@@ -29,18 +30,30 @@ ARG DARWIN_VERSION=17
 ARG OSX_VERSION_MIN=10.10
 {{ else }}
 ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.11.sdk.tar.xz
+=======
+{{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12") }}
+ARG OSXCROSS_PATH=/usr/local/osxcross
+COPY --from=build-llvm-apple /osxcross.tar.gz /tmp/osxcross.tar.gz
+RUN tar -xzf /tmp/osxcross.tar.gz -C / \
+  && rm /tmp/osxcross.tar.gz
+{{- else }}
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))
 ARG OSXCROSS_PATH=/usr/osxcross
 ARG OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa
 ARG SDK_VERSION=10.11
 ARG DARWIN_VERSION=15
 ARG OSX_VERSION_MIN=10.10
+<<<<<<< HEAD
 {{ end }}
 
+=======
+
+COPY sdks/MacOSX${SDK_VERSION}.sdk.tar.xz /tmp/osxcross/tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))
 RUN \
-    mkdir -p /tmp/osxcross && cd /tmp/osxcross \
+    cd /tmp/osxcross \
     && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \
         | tar -C /tmp/osxcross --strip=1 -xzf - \
-    && curl -sSLo "tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz" "${OSXCROSS_SDK_URL}" \
     && UNATTENDED=yes ./build.sh >/dev/null \
     && mv target "${OSXCROSS_PATH}" \
     && rm -rf /tmp/osxcross "/usr/osxcross/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"

diff --git a/go/llvm-apple/Dockerfile.tmpl b/go/llvm-apple/Dockerfile.tmpl
@@ -0,0 +1,88 @@
+ARG DEBIAN_VERSION
+FROM debian:${DEBIAN_VERSION} as BUILD_LLVM_APPLE
+
+RUN apt-get -o Acquire::Check-Valid-Until=false update -y --no-install-recommends --allow-unauthenticated \
+        && apt-get upgrade -y --no-install-recommends --allow-unauthenticated \
+        && apt-get dist-upgrade -y --no-install-recommends --allow-unauthenticated \
+        && apt-get install -y --no-install-recommends --allow-unauthenticated \
+            build-essential \
+            ca-certificates \
+            curl \
+            git \
+            gnupg \
+            make \
+            file \
+            flex \
+            bison \
+            cmake \
+            patch \
+            libssl-dev \
+            libxml2-dev \
+            lzma-dev \
+            uuid-dev \
+{{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}}
+            binutils-multiarch \
+            binutils-multiarch-dev \
+            python3-venv \
+            python3-pip \
+            python3 \
+{{- end }}
+        && rm -rf /var/lib/apt/lists/*
+
+#https://www.llvm.org/docs/CMake.html
+#https://github.com/apple/llvm-project
+RUN mkdir -p /tmp/llvm-project && cd /tmp/llvm-project \
+    && curl -sSL "https://github.com/apple/llvm-project/archive/refs/tags/swift-5.6.3-RELEASE.tar.gz" \
+      | tar -C /tmp/llvm-project --strip=1 -xzf - \
+    && mkdir build && cd build \
+    && cmake -DLLVM_ENABLE_PROJECTS="clang" \
+      -DCMAKE_BUILD_TYPE="Release" \
+      -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \
+      -DLLVM_INCLUDE_TESTS=OFF \
+      -DLLVM_INCLUDE_EXAMPLES=Off \
+      -DLLVM_ENABLE_BACKTRACES=Off \
+      -DLLVM_BUILD_32_BITS=OFF \
+      /tmp/llvm-project/llvm \
+    && cmake --build . \
+    && cmake --build . --target package \
+    && tar -xzf LLVM-*-Linux.tar.gz --strip 1 -C /usr/local
+
+# TODO think about replace this with the Docker image already created at https://github.com/crazy-max/docker-osxcross
+ARG OSX_SDK="MacOSX11.3.sdk"
+# Packaging the SDK on macOS https://github.com/tpoechtrager/osxcross#packaging-the-sdk
+# alternative SDK already generated
+ARG OSXCROSS_PATH=/usr/local/osxcross
+ARG OSXCROSS_REV=062922bbb81ac52787d8e53fa4af190acb552ec7
+ARG OSX_VERSION_MIN=10.10
+
+RUN mkdir -p /tmp/osxcross
+# https://raw.githubusercontent.com/crazy-max/docker-osxcross/main/patches/lcxx.patch
+COPY lcxx.patch /tmp/osxcross/lcxx.patch
+
+COPY sdks/${OSX_SDK}.tar.xz tarballs/${OSX_SDK}.tar.xz
+RUN \
+    cd /tmp/osxcross \
+    && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \
+        | tar -C /tmp/osxcross --strip=1 -xzf - \
+    && patch -p1 < lcxx.patch \
+    && UNATTENDED=1 ENABLE_COMPILER_RT_INSTALL=1 OSX_VERSION_MIN=${OSX_VERSION_MIN} TARGET_DIR=${OSXCROSS_PATH} ./build.sh >/dev/null \
+    && rm -rf "/usr/osxcross/SDK/${OSX_SDK}/usr/share/man" \
+    && tar -czf osxcross.tar.gz ${OSXCROSS_PATH}
+
+FROM debian:stable-slim
+
+COPY --from=BUILD_LLVM_APPLE /tmp/llvm-project/build/LLVM-*-Linux.tar.gz /llvm-apple-Linux.tar.gz
+COPY --from=BUILD_LLVM_APPLE /tmp/osxcross/osxcross.tar.gz /osxcross.tar.gz
+
+# Build-time metadata as defined at http://label-schema.org.
+ARG BUILD_DATE
+ARG IMAGE
+ARG VCS_REF
+ARG VCS_URL
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name=$IMAGE \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.vcs-url=$VCS_URL \
+      org.label-schema.schema-version="1.0"
+
+ENTRYPOINT /bin/bash
diff --git a/go/main/Dockerfile.tmpl b/go/main/Dockerfile.tmpl
@@ -54,8 +54,8 @@ RUN cd /libpcap/libpcap-1.8.1 \
   && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no \
   && make
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/WpdPack_4_1_2.zip \
-  && mkdir -p /libpcap/win \
+COPY sdks/WpdPack_4_1_2.zip . 
+RUN mkdir -p /libpcap/win \
   && echo "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd  WpdPack_4_1_2.zip" | sha256sum -c - \
   && unzip WpdPack_4_1_2.zip -d /libpcap/win \
   && rm WpdPack_4_1_2.zip \
@@ -76,7 +76,14 @@ ARG IMAGE
 ARG VCS_REF
 ARG VCS_URL
 LABEL org.label-schema.build-date=$BUILD_DATE \
+<<<<<<< HEAD
       org.label-schema.name=$IMAGE \
       org.label-schema.vcs-ref=$VCS_REF \
       org.label-schema.vcs-url=$VCS_URL \
       org.label-schema.schema-version="1.0"
+=======
+  org.label-schema.name=$IMAGE \
+  org.label-schema.vcs-ref=$VCS_REF \
+  org.label-schema.vcs-url=$VCS_URL \
+  org.label-schema.schema-version="1.0"
+>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))