diff --git a/Makefile.common b/Makefile.common index 9a29610d..fb98ecad 100644 --- a/Makefile.common +++ b/Makefile.common @@ -4,7 +4,18 @@ VCS_URL := https://github.com/elastic/golang-crossbuild BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") .DEFAULT_GOAL := build +<<<<<<< HEAD # Requires login at https://docker.elastic.co:7000/. +======= +# Requires login at google storage. +copy-sdks: +ifeq ($(CI),true) + @gcloud storage cp gs://ingest-buildkite-ci/sdks . --recursive +else + @echo 'Only available if running in the CI' +endif + +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454)) push: echo ">> Pushing $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" @docker push "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" diff --git a/README.md b/README.md index 30516a30..d60e8757 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,85 @@ The osxcross repository used to cross compile for MacOSX has [instructions for p The instructions for packaging the SDK on a Linux instance are: +<<<<<<< HEAD 1. Clone the [osxcross](https://github.com/tpoechtrager/osxcross) repo. 1. Install `clang`, `make`, `libssl-dev`, `lzma-dev`, `libxml2-dev`, `libbz2-dev`. 1. Download [Xcode from Apple](Download Xcode: https://developer.apple.com/download/more]). 1. Run `./tools/gen_sdk_package_pbzx.sh .xip`. +======= +## go/base-arm Docker image + +The `base-arm` image is the base image to crossbuild `linux/arm64` binaries on `arm` hosts, it is build for Debian 7+. +This image is user to crosscompile in `linux/arm`. +It was replaced by the `base` image for Debian 9+ when we started to build multiarchitecture Docker images. +It could be removed in the future, it allow to `crosscompile` arm binaries in `linux/arm64` machines, that it is not needed because the native architecture is the same than the target architecture. +It was added to golang-crossbuild due limitations of the build system. + +## go/main Docker image + +The `main` image is the base image for the `amd64` architecture, it is build for Debian 7+. +It is used to cross compile for `linux/amd`, `linux/amd64`, `win/amd`, and `win/amd64`. +This Docker immage add two libraries to the `base` image, `libpcap` and `WpdPack` to be able to capture network packages on diferent OS. +Thes two libraries are precompiled and stored at https://storage.googleapis.com/ingest-buildkite-ci/sdks. + +## go/darwin Docker image + +The `darwin` image is the base image for the MacOSX cross compilation, it is build for Debian 8+. +It can compiles for `darwin/amd` (Debian 10+), `darwin/amd64`, `darwin/arm64`, `darwin/arm64e`, and universal binaries. +This Docker image is based on the `base` image. +It uses [osxcross](https://codeload.github.com/tpoechtrager/osxcross) to configure the crosscompile for MacOSX. +This image require a MacOSX SDK to be installed in the Docker image, +for the instructions to package the MacOS SDK see the [Packaging MacOS SDK](#packaging-macos-sdk) section. + +## go/darwin-arm64 Docker image + +The `darwin-arm64` image is the base image for the MacOSX cross compilation, it is build for Debian 10+. +It can compiles for `darwin/amd64`, `darwin/arm64`, and `darwin/arm64e`, and universal binaries. +This Docker image is based on the `base` image. +It uses the `llvm-apple` image to cross compile for MacOSX. +The `darwin-arm64` can replace the `darwin` image in the future, it is faster to build and it does not need to build [osxcross](https://codeload.github.com/tpoechtrager/osxcross) and uses the official LLVM fork from Apple so it support enhacement for `darwin` architectures. + +## go/arm Docker image + +The `arm` image is the base image for the `arm64` architecture, it is build for Debian 9+. +It is used to cross compile for `linux/arm64`. This Docker image is based on the `base` image. + +## go/armel Docker image + +The `armel` image is the base image for the `armel` architecture, it is build for Debian 12+. +It is used to cross compile for `linux/armel`. This Docker image is based on the `base` image. + +## go/armhf Docker image + +The `armhf` image is the base image for the `armhf` architecture, it is build for Debian 9+. +It is used to cross compile for `linux/armhf`. This Docker image is based on the `base` image. + +## go/mips Docker image + +The `mips` image is the base image for the `mips` architecture, it is build for Debian 12+. +It is used to cross compile for `linux/mips`. This Docker image is based on the `base` image. + +## go/mips32 Docker image + +The `mips32` image is the base image for the `mips32` architecture, it is build for Debian 11+. +It is used to cross compile for `linux/mips32`. This Docker image is based on the `base` image. + +## go/ppc Docker image + +The `ppc` image is the base image for the `ppc` architecture, it is build for Debian 12+. +It is used to cross compile for `linux/ppc`. This Docker image is based on the `base` image. + +## go/s390x Docker image + +The `s390x` image is the base image for the `s390x` architecture, it is build for Debian 12+. +It is used to cross compile for `linux/s390x`. This Docker image is based on the `base` image. + +## go/npcap Docker image + +The `npcap` image is a placeholder for the `npcap` library, see [npcap](./NPCAP.md) for more information. + +## Troubleshooting +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454)) ### bzip2 issues @@ -142,5 +217,5 @@ XCODEDIR=osxcross/build/tmp_ ./tools/gen_sdk_package.sh The SDK should be in the working directory. The tmp dir can be safely deleted after this. -The SDKs should be uploaded into the `gs://obs-ci-cache` bucket on GCP (Google Cloud Platform). -This is accessible to authorized users in the `elastic-observability` project [here](https://console.cloud.google.com/storage/browser/obs-ci-cache). +The SDKs should be uploaded into the `gs://ingest-buildkite-ci/sdks` bucket on GCP (Google Cloud Platform). +This is accessible to authorized users in the `platform-ingest` project [here](https://console.cloud.google.com/storage/browser/ingest-buildkite-ci/sdks). diff --git a/go/Makefile.common b/go/Makefile.common index ec38da8f..2aadabb4 100644 --- a/go/Makefile.common +++ b/go/Makefile.common @@ -9,7 +9,13 @@ TAG_EXTENSION ?= export DEBIAN_VERSION TAG_EXTENSION +<<<<<<< HEAD build: +======= +DOCKER_CMD := docker build + +build: copy-npcap copy-sdks +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454)) @echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" @go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile @docker build -t "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" \ @@ -23,7 +29,7 @@ build: --build-arg BUILD_DATE="$(BUILD_DATE)" \ . -build-arm: +build-arm: copy-sdks @echo $(SELF_DIR) @echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" @go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 769c9be6..b3512afd 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -63,9 +63,9 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \ - && mkdir /libpcap \ - && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ +COPY sdks/libpcap-1.8.1.tar.gz . +RUN mkdir /libpcap \ + && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no --host=aarch64-unknown-linux-gnu --with-pcap=linux \ && make diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl index 4489a344..c1d377ae 100644 --- a/go/base/Dockerfile.tmpl +++ b/go/base/Dockerfile.tmpl @@ -55,11 +55,10 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \ && go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \ - && echo "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e libpcap-1.8.1.tar.gz" | sha256sum -c - \ - && mkdir /libpcap \ - && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ - && rm libpcap-1.8.1.tar.gz +COPY sdks/libpcap-1.8.1.tar.gz . +RUN mkdir /libpcap \ + && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ + && rm libpcap-1.8.1.tar.gz ENV GOLANG_CROSSBUILD=1 diff --git a/go/darwin/Dockerfile.tmpl b/go/darwin/Dockerfile.tmpl index 976cf9b1..693a316e 100644 --- a/go/darwin/Dockerfile.tmpl +++ b/go/darwin/Dockerfile.tmpl @@ -20,6 +20,7 @@ RUN \ uuid-dev \ && rm -rf /var/lib/apt/lists/* +<<<<<<< HEAD {{if eq .DEBIAN_VERSION "10"}} ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.14.sdk.tar.xz ARG OSXCROSS_PATH=/usr/osxcross @@ -29,18 +30,30 @@ ARG DARWIN_VERSION=17 ARG OSX_VERSION_MIN=10.10 {{ else }} ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.11.sdk.tar.xz +======= +{{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12") }} +ARG OSXCROSS_PATH=/usr/local/osxcross +COPY --from=build-llvm-apple /osxcross.tar.gz /tmp/osxcross.tar.gz +RUN tar -xzf /tmp/osxcross.tar.gz -C / \ + && rm /tmp/osxcross.tar.gz +{{- else }} +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454)) ARG OSXCROSS_PATH=/usr/osxcross ARG OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa ARG SDK_VERSION=10.11 ARG DARWIN_VERSION=15 ARG OSX_VERSION_MIN=10.10 +<<<<<<< HEAD {{ end }} +======= + +COPY sdks/MacOSX${SDK_VERSION}.sdk.tar.xz /tmp/osxcross/tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454)) RUN \ - mkdir -p /tmp/osxcross && cd /tmp/osxcross \ + cd /tmp/osxcross \ && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \ | tar -C /tmp/osxcross --strip=1 -xzf - \ - && curl -sSLo "tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz" "${OSXCROSS_SDK_URL}" \ && UNATTENDED=yes ./build.sh >/dev/null \ && mv target "${OSXCROSS_PATH}" \ && rm -rf /tmp/osxcross "/usr/osxcross/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man" diff --git a/go/llvm-apple/Dockerfile.tmpl b/go/llvm-apple/Dockerfile.tmpl new file mode 100644 index 00000000..a015a5e1 --- /dev/null +++ b/go/llvm-apple/Dockerfile.tmpl @@ -0,0 +1,88 @@ +ARG DEBIAN_VERSION +FROM debian:${DEBIAN_VERSION} as BUILD_LLVM_APPLE + +RUN apt-get -o Acquire::Check-Valid-Until=false update -y --no-install-recommends --allow-unauthenticated \ + && apt-get upgrade -y --no-install-recommends --allow-unauthenticated \ + && apt-get dist-upgrade -y --no-install-recommends --allow-unauthenticated \ + && apt-get install -y --no-install-recommends --allow-unauthenticated \ + build-essential \ + ca-certificates \ + curl \ + git \ + gnupg \ + make \ + file \ + flex \ + bison \ + cmake \ + patch \ + libssl-dev \ + libxml2-dev \ + lzma-dev \ + uuid-dev \ +{{- if or (eq .DEBIAN_VERSION "10") (eq .DEBIAN_VERSION "11") (eq .DEBIAN_VERSION "12")}} + binutils-multiarch \ + binutils-multiarch-dev \ + python3-venv \ + python3-pip \ + python3 \ +{{- end }} + && rm -rf /var/lib/apt/lists/* + +#https://www.llvm.org/docs/CMake.html +#https://github.com/apple/llvm-project +RUN mkdir -p /tmp/llvm-project && cd /tmp/llvm-project \ + && curl -sSL "https://github.com/apple/llvm-project/archive/refs/tags/swift-5.6.3-RELEASE.tar.gz" \ + | tar -C /tmp/llvm-project --strip=1 -xzf - \ + && mkdir build && cd build \ + && cmake -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE="Release" \ + -DLLVM_TARGETS_TO_BUILD="X86;AArch64" \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=Off \ + -DLLVM_ENABLE_BACKTRACES=Off \ + -DLLVM_BUILD_32_BITS=OFF \ + /tmp/llvm-project/llvm \ + && cmake --build . \ + && cmake --build . --target package \ + && tar -xzf LLVM-*-Linux.tar.gz --strip 1 -C /usr/local + +# TODO think about replace this with the Docker image already created at https://github.com/crazy-max/docker-osxcross +ARG OSX_SDK="MacOSX11.3.sdk" +# Packaging the SDK on macOS https://github.com/tpoechtrager/osxcross#packaging-the-sdk +# alternative SDK already generated +ARG OSXCROSS_PATH=/usr/local/osxcross +ARG OSXCROSS_REV=062922bbb81ac52787d8e53fa4af190acb552ec7 +ARG OSX_VERSION_MIN=10.10 + +RUN mkdir -p /tmp/osxcross +# https://raw.githubusercontent.com/crazy-max/docker-osxcross/main/patches/lcxx.patch +COPY lcxx.patch /tmp/osxcross/lcxx.patch + +COPY sdks/${OSX_SDK}.tar.xz tarballs/${OSX_SDK}.tar.xz +RUN \ + cd /tmp/osxcross \ + && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \ + | tar -C /tmp/osxcross --strip=1 -xzf - \ + && patch -p1 < lcxx.patch \ + && UNATTENDED=1 ENABLE_COMPILER_RT_INSTALL=1 OSX_VERSION_MIN=${OSX_VERSION_MIN} TARGET_DIR=${OSXCROSS_PATH} ./build.sh >/dev/null \ + && rm -rf "/usr/osxcross/SDK/${OSX_SDK}/usr/share/man" \ + && tar -czf osxcross.tar.gz ${OSXCROSS_PATH} + +FROM debian:stable-slim + +COPY --from=BUILD_LLVM_APPLE /tmp/llvm-project/build/LLVM-*-Linux.tar.gz /llvm-apple-Linux.tar.gz +COPY --from=BUILD_LLVM_APPLE /tmp/osxcross/osxcross.tar.gz /osxcross.tar.gz + +# Build-time metadata as defined at http://label-schema.org. +ARG BUILD_DATE +ARG IMAGE +ARG VCS_REF +ARG VCS_URL +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name=$IMAGE \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.schema-version="1.0" + +ENTRYPOINT /bin/bash diff --git a/go/main/Dockerfile.tmpl b/go/main/Dockerfile.tmpl index a2db9cbc..35d5aaeb 100644 --- a/go/main/Dockerfile.tmpl +++ b/go/main/Dockerfile.tmpl @@ -54,8 +54,8 @@ RUN cd /libpcap/libpcap-1.8.1 \ && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no \ && make -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/WpdPack_4_1_2.zip \ - && mkdir -p /libpcap/win \ +COPY sdks/WpdPack_4_1_2.zip . +RUN mkdir -p /libpcap/win \ && echo "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd WpdPack_4_1_2.zip" | sha256sum -c - \ && unzip WpdPack_4_1_2.zip -d /libpcap/win \ && rm WpdPack_4_1_2.zip \ @@ -76,7 +76,14 @@ ARG IMAGE ARG VCS_REF ARG VCS_URL LABEL org.label-schema.build-date=$BUILD_DATE \ +<<<<<<< HEAD org.label-schema.name=$IMAGE \ org.label-schema.vcs-ref=$VCS_REF \ org.label-schema.vcs-url=$VCS_URL \ org.label-schema.schema-version="1.0" +======= + org.label-schema.name=$IMAGE \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url=$VCS_URL \ + org.label-schema.schema-version="1.0" +>>>>>>> 206cfe2 (Updating bucket references from O11y to Ingest path (#454))