[1.20](backport #454) Updating bucket references from O11y to Ingest path

Makefile.common

@@ -26,6 +26,14 @@ else
 	@echo 'Only available if running in the CI'
 endif
 
+# Requires login at google storage.
+copy-sdks:
+ifeq ($(CI),true)
+	@gcloud storage cp gs://ingest-buildkite-ci/sdks . --recursive
+else
+	@echo 'Only available if running in the CI'
+endif
+
 push:
 	$(MAKE) atomic-push
 

README.md

@@ -291,7 +291,7 @@ It was added to golang-crossbuild due limitations of the build system.
 The `main` image is the base image for the `amd64` architecture, it is build for Debian 7+.
 It is used to cross compile for `linux/amd`, `linux/amd64`, `win/amd`, and `win/amd64`.
 This Docker immage add two libraries to the `base` image, `libpcap` and `WpdPack` to be able to capture network packages on diferent OS.
-Thes two libraries are precompiled and stores at https://storage.googleapis.com/obs-ci-cache.
+Thes two libraries are precompiled and stored at https://storage.googleapis.com/ingest-buildkite-ci/sdks.
 
 ## go/darwin Docker image
 
@@ -378,5 +378,5 @@ XCODEDIR=osxcross/build/tmp_<X> ./tools/gen_sdk_package.sh
 The SDK should be in the working directory.
 The tmp dir can be safely deleted after this.
 
-The SDKs should be uploaded into the `gs://obs-ci-cache` bucket on GCP (Google Cloud Platform).
-This is accessible to authorized users in the `elastic-observability` project [here](https://console.cloud.google.com/storage/browser/obs-ci-cache).
+The SDKs should be uploaded into the `gs://ingest-buildkite-ci/sdks` bucket on GCP (Google Cloud Platform).
+This is accessible to authorized users in the `platform-ingest` project [here](https://console.cloud.google.com/storage/browser/ingest-buildkite-ci/sdks).

go/Makefile.common

@@ -11,7 +11,7 @@ export DEBIAN_VERSION TAG_EXTENSION
 
 DOCKER_CMD := docker build
 
-build: copy-npcap
+build: copy-npcap copy-sdks
 	@echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"
 	@go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile
 	@$(DOCKER_CMD) -t "$(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)" \
@@ -25,7 +25,7 @@ build: copy-npcap
 	--build-arg BUILD_DATE="$(BUILD_DATE)" \
 	.
 
-build-arm:
+build-arm: copy-sdks
 	@echo $(SELF_DIR)
 	@echo ">> Building $(REPOSITORY)/$(NAME):$(VERSION)$(SUFFIX)$(TAG_EXTENSION)"
 	@go run $(SELF_DIR)/../template.go -t Dockerfile.tmpl -o Dockerfile

go/base-arm/Dockerfile.tmpl

@@ -63,9 +63,9 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \
     && go build -o /crossbuild /entrypoint.go \
     && rm -rf /go/* /root/.cache/* /entrypoint.go
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \
-  && mkdir /libpcap \
-	&& tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
+COPY sdks/libpcap-1.8.1.tar.gz . 
+RUN mkdir /libpcap \
+  && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
   && cd /libpcap/libpcap-1.8.1 \
   && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no --host=aarch64-unknown-linux-gnu --with-pcap=linux \
   && make

go/base/Dockerfile.tmpl

@@ -53,11 +53,10 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \
     && go build -o /crossbuild /entrypoint.go \
     && rm -rf /go/* /root/.cache/* /entrypoint.go
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/libpcap-1.8.1.tar.gz \
-  && echo "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e  libpcap-1.8.1.tar.gz" | sha256sum -c - \
-  && mkdir /libpcap \
-	&& tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
-  && rm libpcap-1.8.1.tar.gz
+COPY sdks/libpcap-1.8.1.tar.gz . 
+RUN mkdir /libpcap \
+    && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \
+    && rm libpcap-1.8.1.tar.gz
 
 
 ENV GOLANG_CROSSBUILD=1

go/darwin/Dockerfile.tmpl

@@ -38,17 +38,17 @@ COPY --from=build-llvm-apple /osxcross.tar.gz /tmp/osxcross.tar.gz
 RUN tar -xzf /tmp/osxcross.tar.gz -C / \
   && rm /tmp/osxcross.tar.gz
 {{- else }}
-ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/MacOSX10.11.sdk.tar.xz
 ARG OSXCROSS_PATH=/usr/osxcross
 ARG OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa
 ARG SDK_VERSION=10.11
 ARG DARWIN_VERSION=15
 ARG OSX_VERSION_MIN=10.10
+
+COPY sdks/MacOSX${SDK_VERSION}.sdk.tar.xz /tmp/osxcross/tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz
 RUN \
-    mkdir -p /tmp/osxcross && cd /tmp/osxcross \
+    cd /tmp/osxcross \
     && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \
         | tar -C /tmp/osxcross --strip=1 -xzf - \
-    && curl -sSLo "tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz" "${OSXCROSS_SDK_URL}" \
     && UNATTENDED=yes ./build.sh >/dev/null \
     && mv target "${OSXCROSS_PATH}" \
     && rm -rf /tmp/osxcross "/usr/osxcross/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"

go/llvm-apple/Dockerfile.tmpl

@@ -51,8 +51,6 @@ RUN mkdir -p /tmp/llvm-project && cd /tmp/llvm-project \
 ARG OSX_SDK="MacOSX11.3.sdk"
 # Packaging the SDK on macOS https://github.com/tpoechtrager/osxcross#packaging-the-sdk
 # alternative SDK already generated
-#ARG OSXCROSS_SDK_URL=https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/${OSX_SDK}.tar.xz
-ARG OSXCROSS_SDK_URL=https://storage.googleapis.com/obs-ci-cache/beats/${OSX_SDK}.tar.xz
 ARG OSXCROSS_PATH=/usr/local/osxcross
 ARG OSXCROSS_REV=062922bbb81ac52787d8e53fa4af190acb552ec7
 ARG OSX_VERSION_MIN=10.10
@@ -61,11 +59,11 @@ RUN mkdir -p /tmp/osxcross
 # https://raw.githubusercontent.com/crazy-max/docker-osxcross/main/patches/lcxx.patch
 COPY lcxx.patch /tmp/osxcross/lcxx.patch
 
+COPY sdks/${OSX_SDK}.tar.xz tarballs/${OSX_SDK}.tar.xz
 RUN \
     cd /tmp/osxcross \
     && curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" \
         | tar -C /tmp/osxcross --strip=1 -xzf - \
-    && curl -sSLo "tarballs/${OSX_SDK}.tar.xz" "${OSXCROSS_SDK_URL}" \
     && patch -p1 < lcxx.patch \
     && UNATTENDED=1 ENABLE_COMPILER_RT_INSTALL=1 OSX_VERSION_MIN=${OSX_VERSION_MIN} TARGET_DIR=${OSXCROSS_PATH} ./build.sh >/dev/null \
     && rm -rf "/usr/osxcross/SDK/${OSX_SDK}/usr/share/man" \

go/main/Dockerfile.tmpl

@@ -61,8 +61,8 @@ RUN cd /libpcap/libpcap-1.8.1 \
   && ./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no \
   && make
 
-RUN curl -sSLO https://storage.googleapis.com/obs-ci-cache/beats/WpdPack_4_1_2.zip \
-  && mkdir -p /libpcap/win \
+COPY sdks/WpdPack_4_1_2.zip . 
+RUN mkdir -p /libpcap/win \
   && echo "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd  WpdPack_4_1_2.zip" | sha256sum -c - \
   && unzip WpdPack_4_1_2.zip -d /libpcap/win \
   && rm WpdPack_4_1_2.zip \
@@ -86,4 +86,4 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
   org.label-schema.name=$IMAGE \
   org.label-schema.vcs-ref=$VCS_REF \
   org.label-schema.vcs-url=$VCS_URL \
-  org.label-schema.schema-version="1.0"
+  org.label-schema.schema-version="1.0"