Merge remote-tracking branch 'origin/topic/awelzel/2674-arm64-contain…

…ers-on-cirrus'

* origin/topic/awelzel/2674-arm64-containers-on-cirrus:
  cirrus/containers: Do not install btest pcaps
  cirrus: Replace curl with _cache
  cirrus: Add cluster testing
  cirrus: Polish container_image_manifest_docker_builder
  cirrus: Use ccache for faster container builds
  cirrus: Add docker_builder tasks to build and push images
  github: No more docker workflow

.cirrus.yml

@@ -382,3 +382,171 @@ windows_task:
     CTEST_OUTPUT_ON_FAILURE: 1
   << : *BRANCH_WHITELIST
   << : *SKIP_TASK_ON_PR
+
+
+# Container images
+#
+# Use two separate tasks to build images for amd64 and arm64.
+# Use use a third docker_builder task to collect the produced images
+# (through CIRRUS_HTTP_CACHE) and push them to the registry as
+# zeek/zeek:v1.2.3-<arch> or zeek/zeek-dev:latest-<arch> tags. Once
+# pushed, create a manifest for zeek/zeek:v1.2.3 or zeek/zeek-dev:latest
+# that includes the just pushed architecture specific images.
+#
+# We've previously tried using docker buildx with QEMU using GitHub
+# actions. The emulated arm64 build on the amd64 VMs they provide took
+# more than 6 hours and timed out. Using separate builders on Cirrus allows
+# us build natively and much faster at the expense of the docker manifest
+# wrangling (and not being able to use the nice GitHub actions).
+docker_build_template: &DOCKER_BUILD_TEMPLATE
+  cpu: *CPUS
+  memory: *MEMORY
+  set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV
+
+  env:
+    ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps
+    CIRRUS_LOG_TIMESTAMP: true
+    BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache
+    ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH}
+
+  always:
+    ccache_cache:
+      folder: /tmp/ccache
+      fingerprint_script: echo ccache-$CIRRUS_TASK_NAME-$CIRRUS_OS
+      reupload_on_changes: true
+
+    builder_image_cache:
+      folder: /tmp/builder-image-cache
+      fingerprint_script: echo builder-image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS
+      reupload_on_changes: true
+
+    zeek_image_cache:
+      folder: /tmp/zeek-image-cache-${CIRRUS_ARCH}
+      fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-${CIRRUS_ARCH}
+      reupload_on_changes: true
+
+  sync_submodules_script: git submodule update --recursive --init --recommend-shallow -j $(nproc)
+
+  prepare_builder_script:
+    - set -x
+    - mkdir -p ${BUILDER_IMAGE_CACHE_DIR} ${ZEEK_IMAGE_CACHE_DIR}
+    - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/builder.zst | docker load; fi
+    - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/final.zst | docker load; fi
+    - cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile .
+    - docker save zeek-builder:latest | zstd > ${BUILDER_IMAGE_CACHE_DIR}/builder.zst
+  build_zeek_script:
+    - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -e CCACHE_NOSTATS=1 -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install"
+    # The "zeek-build" tag is used within final.Dockerfile using COPY --from=...
+    - docker commit zeek-builder-container zeek-build
+  build_final_script:
+    - cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile .
+    - docker save ${IMAGE_TAG} | zstd > ${ZEEK_IMAGE_CACHE_DIR}/final.zst
+
+arm64_container_image_docker_builder:
+  env:
+    CIRRUS_ARCH: arm64
+  << : *DOCKER_BUILD_TEMPLATE
+
+amd64_container_image_docker_builder:
+  env:
+    CIRRUS_ARCH: amd64
+  << : *DOCKER_BUILD_TEMPLATE
+
+container_image_manifest_docker_builder:
+  cpu: 1
+  # Push master builds to zeek/zeek-dev, or tagged release branches to zeek/zeek
+  only_if: >
+    ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' &&
+      ( $CIRRUS_BRANCH == 'master' ||
+        ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '')
+      )
+    )
+  env:
+    DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!]
+    DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!]
+    AWS_ECR_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!]
+    AWS_ECR_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!]
+  login_script: |
+    docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
+    AWS_ACCESS_KEY_ID=$AWS_ECR_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_ECR_SECRET_ACCESS_KEY aws ecr-public get-login-password --region us-east-1 | \
+        docker login --username AWS $AWS_ECR_USERNAME --password-stdin public.ecr.aws
+  set_image_tag_script: |
+    # If we have a CIRRUS_TAG, use the value in VERSION to push the multiarch
+    # images, otherwise use latest. Basically we push the arch images as
+    #    zeek/zeek:1.2.3-<amd64|arm64> or
+    #    zeek/zeek-dev:latest-<amd64|arm64>
+    # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG}
+    # for tags, or zeek/zeek-dev:latest for pushes to master.
+    if [ -n "${CIRRUS_TAG}" ]; then
+      echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV
+      echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_NAME=zeek"  >> $CIRRUS_ENV
+    elif [ "${CIRRUS_BRANCH}" = "master" ]; then
+      echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV
+      echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_NAME=zeek-dev"  >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
+    # Hunk for testing and pushing into zeek/zeek-next. Make sure
+    # to allow the branch in the above only_if attribute of this task.
+    # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then
+    #  echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV
+    #  echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+    #  echo "ARCH_IMAGE_NAME=zeek-next"  >> $CIRRUS_ENV
+    #  echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
+    else
+      echo "Bad tag/branch for container_image_manifest"
+      env
+      exit 1
+    fi
+
+  # These should've been populated by the previous jobs
+  zeek_image_arm64_cache:
+    folder: /tmp/zeek-image-cache-arm64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-arm64
+
+  zeek_image_amd64_cache:
+    folder: /tmp/zeek-image-cache-amd64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64
+
+  load_image_script:
+    - set -x;
+    - ls -lha /tmp/zeek-image-cache-*/
+    - zstd -d < /tmp/zeek-image-cache-arm64/final.zst | docker load
+    - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load
+  tag_push_script:
+    # Tag images and push to Docker Hub and AWS ECR
+    - ./ci/container-images-tag-and-push.sh
+    - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh
+  depends_on:
+    - arm64_container_image
+    - amd64_container_image
+
+cluster_testing_docker_builder:
+  cpu: *CPUS
+  memory: *MEMORY
+  only_if: $CIRRUS_REPO_FULL_NAME == 'zeek/zeek'
+  env:
+    CIRRUS_LOG_TIMESTAMP: true
+    # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to
+    # see if it should rebuild the Zeek image or not.
+    GITHUB_ACTION: fake
+  install_deps_script:
+    # The cluster tests depend on jq and docker_builder doesn't have that :-(
+    - apt-get -q update && apt-get install -y --no-install-recommends jq
+  sync_btest_script: git submodule update --init ./auxil/btest/
+  checkout_script:
+    - set -x; cd testing/external/ && git clone https://github.com/zeek/zeek-testing-cluster.git && make checkout-repos
+  zeek_image_amd64_cache:
+    folder: /tmp/zeek-image-cache-amd64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64
+  load_image_script:
+    - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load
+    - docker tag zeek/zeek-multiarch:amd64 zeektest:latest
+  test_script:
+    - cd testing/external/zeek-testing-cluster && make
+  on_failure:
+    upload_cluster_testing_artifacts:
+      path: "testing/external/zeek-testing-cluster/.tmp/**"
+  depends_on:
+    - amd64_container_image

CHANGES

@@ -1,3 +1,19 @@
+5.2.0-dev.589 | 2023-02-01 09:02:00 +0100
+
+  * cirrus/containers: Do not install btest pcaps (Arne Welzel, Corelight)
+
+  * cirrus: Replace curl with _cache (Arne Welzel, Corelight)
+
+  * cirrus: Add cluster testing (Arne Welzel, Corelight)
+
+  * cirrus: Polish container_image_manifest_docker_builder (Arne Welzel, Corelight)
+
+  * cirrus: Use ccache for faster container builds (Arne Welzel, Corelight)
+
+  * cirrus: Add docker_builder tasks to build and push images (Arne Welzel, Corelight)
+
+  * github: No more docker workflow (Arne Welzel, Corelight)
+
 5.2.0-dev.579 | 2023-01-31 14:49:29 +0100
 
   * mysql: Recognize when client/server negotiate SSL (Arne Welzel, Corelight)