diff --git a/Makefile.common b/Makefile.common index 6edb2447..cf231fc1 100644 --- a/Makefile.common +++ b/Makefile.common @@ -24,6 +24,14 @@ else @echo 'Only available if running in the CI' endif +# Requires login at google storage. +copy-sdks: +ifeq ($(CI),true) + @gcloud storage cp gs://ingest-buildkite-ci/sdks . --recursive +else + @echo 'Only available if running in the CI' +endif + push: $(MAKE) atomic-push diff --git a/README.md b/README.md index 36a84590..536e48a2 100644 --- a/README.md +++ b/README.md @@ -292,7 +292,7 @@ It was added to golang-crossbuild due limitations of the build system. The `main` image is the base image for the `amd64` architecture, it is build for Debian 7+. It is used to cross compile for `linux/amd`, `linux/amd64`, `win/amd`, and `win/amd64`. This Docker immage add two libraries to the `base` image, `libpcap` and `WpdPack` to be able to capture network packages on diferent OS. -Thes two libraries are precompiled and stores at https://storage.googleapis.com/obs-ci-cache. +Thes two libraries are precompiled and stored at https://storage.googleapis.com/ingest-buildkite-ci/sdks. ## go/darwin Docker image @@ -379,5 +379,5 @@ XCODEDIR=osxcross/build/tmp_ ./tools/gen_sdk_package.sh The SDK should be in the working directory. The tmp dir can be safely deleted after this. -The SDKs should be uploaded into the `gs://obs-ci-cache` bucket on GCP (Google Cloud Platform). -This is accessible to authorized users in the `elastic-observability` project [here](https://console.cloud.google.com/storage/browser/obs-ci-cache). +The SDKs should be uploaded into the `gs://ingest-buildkite-ci/sdks` bucket on GCP (Google Cloud Platform). +This is accessible to authorized users in the `platform-ingest` project [here](https://console.cloud.google.com/storage/browser/ingest-buildkite-ci/sdks). diff --git a/go/Makefile.common b/go/Makefile.common index 97086237..0d4ced6c 100644 --- a/go/Makefile.common +++ b/go/Makefile.common @@ -11,7 +11,7 @@ export DEBIAN_VERSION TAG_EXTENSION NPCAP_FILE DOCKER_CMD := docker build -build: copy-npcap +build: copy-npcap copy-sdks @echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" @go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile @$(DOCKER_CMD) -t "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" \ @@ -25,7 +25,7 @@ build: copy-npcap --build-arg BUILD_DATE="$(BUILD_DATE)" \ . -build-arm: +build-arm: copy-sdks @echo $(SELF_DIR) @echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" @go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 5466863d..3d689005 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -68,9 +68,9 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \ - && mkdir /libpcap \ - && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ +COPY sdks/libpcap-1.8.1.tar.gz . +RUN mkdir /libpcap \ + && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no --host=aarch64-unknown-linux-gnu --with-pcap=linux \ && make diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl index eb75a866..f06de35b 100644 --- a/go/base/Dockerfile.tmpl +++ b/go/base/Dockerfile.tmpl @@ -61,11 +61,10 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \ && CGO_ENABLED=0 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \ - && echo "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e libpcap-1.8.1.tar.gz" | sha256sum -c - \ - && mkdir /libpcap \ - && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ - && rm libpcap-1.8.1.tar.gz +COPY sdks/libpcap-1.8.1.tar.gz . +RUN mkdir /libpcap \ + && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ + && rm libpcap-1.8.1.tar.gz ENV GOLANG_CROSSBUILD=1 diff --git a/go/darwin/Dockerfile.tmpl b/go/darwin/Dockerfile.tmpl index 6d221e88..143579f5 100644 --- a/go/darwin/Dockerfile.tmpl +++ b/go/darwin/Dockerfile.tmpl @@ -38,17 +38,17 @@ COPY --from=build-llvm-apple /osxcross.tar.gz /tmp/osxcross.tar.gz RUN tar -xzf /tmp/osxcross.tar.gz -C / \ && rm /tmp/osxcross.tar.gz {{- else }} -ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.11.sdk.tar.xz ARG OSXCROSS_PATH=/usr/osxcross ARG OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa ARG SDK_VERSION=10.11 ARG DARWIN_VERSION=15 ARG OSX_VERSION_MIN=10.10 + +COPY sdks/MacOSX${SDK_VERSION}.sdk.tar.xz /tmp/osxcross/tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz RUN \ - mkdir -p /tmp/osxcross && cd /tmp/osxcross \ + cd /tmp/osxcross \ && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \ | tar -C /tmp/osxcross --strip=1 -xzf - \ - && curl -sSLo "tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz" "${OSXCROSS_SDK_URL}" \ && UNATTENDED=yes ./build.sh >/dev/null \ && mv target "${OSXCROSS_PATH}" \ && rm -rf /tmp/osxcross "/usr/osxcross/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man" diff --git a/go/llvm-apple/Dockerfile.tmpl b/go/llvm-apple/Dockerfile.tmpl index 3fac7e93..a015a5e1 100644 --- a/go/llvm-apple/Dockerfile.tmpl +++ b/go/llvm-apple/Dockerfile.tmpl @@ -51,8 +51,6 @@ RUN mkdir -p /tmp/llvm-project && cd /tmp/llvm-project \ ARG OSX_SDK="MacOSX11.3.sdk" # Packaging the SDK on macOS https://github.com/tpoechtrager/osxcross#packaging-the-sdk # alternative SDK already generated -#ARG OSXCROSS_SDK_URL=https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/${OSX_SDK}.tar.xz -ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/${OSX_SDK}.tar.xz ARG OSXCROSS_PATH=/usr/local/osxcross ARG OSXCROSS_REV=062922bbb81ac52787d8e53fa4af190acb552ec7 ARG OSX_VERSION_MIN=10.10 @@ -61,11 +59,11 @@ RUN mkdir -p /tmp/osxcross # https://raw.githubusercontent.com/crazy-max/docker-osxcross/main/patches/lcxx.patch COPY lcxx.patch /tmp/osxcross/lcxx.patch +COPY sdks/${OSX_SDK}.tar.xz tarballs/${OSX_SDK}.tar.xz RUN \ cd /tmp/osxcross \ && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \ | tar -C /tmp/osxcross --strip=1 -xzf - \ - && curl -sSLo "tarballs/${OSX_SDK}.tar.xz" "${OSXCROSS_SDK_URL}" \ && patch -p1 < lcxx.patch \ && UNATTENDED=1 ENABLE_COMPILER_RT_INSTALL=1 OSX_VERSION_MIN=${OSX_VERSION_MIN} TARGET_DIR=${OSXCROSS_PATH} ./build.sh >/dev/null \ && rm -rf "/usr/osxcross/SDK/${OSX_SDK}/usr/share/man" \ diff --git a/go/main/Dockerfile.tmpl b/go/main/Dockerfile.tmpl index 12192837..3cda882b 100644 --- a/go/main/Dockerfile.tmpl +++ b/go/main/Dockerfile.tmpl @@ -59,8 +59,8 @@ RUN cd /libpcap/libpcap-1.8.1 \ && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no \ && make -RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/WpdPack_4_1_2.zip \ - && mkdir -p /libpcap/win \ +COPY sdks/WpdPack_4_1_2.zip . +RUN mkdir -p /libpcap/win \ && echo "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd WpdPack_4_1_2.zip" | sha256sum -c - \ && unzip WpdPack_4_1_2.zip -d /libpcap/win \ && rm WpdPack_4_1_2.zip \ @@ -84,4 +84,4 @@ LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.name=$IMAGE \ org.label-schema.vcs-ref=$VCS_REF \ org.label-schema.vcs-url=$VCS_URL \ - org.label-schema.schema-version="1.0" \ No newline at end of file + org.label-schema.schema-version="1.0"