Test dra snapshot no python 7 17

.buildkite/scripts/dra/generatesteps.py

@@ -19,8 +19,8 @@ def package_x86_step(branch, workflow_type):
   key: "logstash_build_packages_dra"
   agents:
     provider: gcp
-    imageProject: elastic-images-prod
-    image: family/platform-ingest-logstash-ubuntu-2004
+    imageProject: elastic-images-qa
+    image: platform-ingest-logstash-ubuntu-2004-1720001094  # see https://buildkite.com/elastic/vm-images-platform-ingest/builds/233#0190780d-e98e-412d-b0a2-39ff0379e975 and https://github.com/elastic/ci-agent-images/pull/575#issuecomment-2205648206
     machineType: "n2-standard-16"
     diskSizeGb: 200
   command: |
@@ -38,8 +38,8 @@ def package_x86_docker_step(branch, workflow_type):
   key: "logstash_build_x86_64_docker_dra"
   agents:
     provider: gcp
-    imageProject: elastic-images-prod
-    image: family/platform-ingest-logstash-ubuntu-2004
+    imageProject: elastic-images-qa
+    image: platform-ingest-logstash-ubuntu-2004-1720001094  # see https://buildkite.com/elastic/vm-images-platform-ingest/builds/233#0190780d-e98e-412d-b0a2-39ff0379e975 and https://github.com/elastic/ci-agent-images/pull/575#issuecomment-2205648206
     machineType: "n2-standard-16"
     diskSizeGb: 200
   command: |
@@ -58,7 +58,7 @@ def package_aarch64_docker_step(branch, workflow_type):
   key: "logstash_build_aarch64_docker_dra"
   agents:
     provider: aws
-    imagePrefix: platform-ingest-logstash-ubuntu-2004-aarch64
+    imagePrefix: test-platform-ingest-logstash-ubuntu-2004-aarch64  # see https://buildkite.com/elastic/vm-images-platform-ingest/builds/233#0190780d-e995-4112-b76d-ba4054137942/523-543 and https://github.com/elastic/ci-agent-images/pull/575#issuecomment-2205648206
     instanceType: "m6g.4xlarge"
     diskSizeGb: 200
   command: |
@@ -78,8 +78,8 @@ def publish_dra_step(branch, workflow_type, depends_on):
   depends_on: "{depends_on}"
   agents:
     provider: gcp
-    imageProject: elastic-images-prod
-    image: family/platform-ingest-logstash-ubuntu-2004
+    imageProject: elastic-images-qa
+    image: platform-ingest-logstash-ubuntu-2004-1720001094  # see https://buildkite.com/elastic/vm-images-platform-ingest/builds/233#0190780d-e98e-412d-b0a2-39ff0379e975 and https://github.com/elastic/ci-agent-images/pull/575#issuecomment-2205648206
     machineType: "n2-standard-16"
     diskSizeGb: 200
   command: |

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -61,7 +61,7 @@ def compat_step(imagesuffix: str, command: LiteralScalarString) -> dict[str, typ
     else:
        step["agents"] = {
             "provider": "gcp",
-            "imageProject": "elastic-images-prod",
+            "imageProject": "elastic-images-qa",
             "image": f"family/{VM_IMAGE_PREFIX}{imagesuffix}",
             "machineType": "n2-standard-4",
             "diskSizeGb": 200,
@@ -93,10 +93,10 @@ def aws_agent(vm_name: str, instance_type: str, image_prefix: str = "platform-in
         "diskSizeGb": disk_size_gb,
     }
 
-def gcp_agent(vm_name: str, instance_type: str = "n2-standard-4", image_prefix: str = "family/platform-ingest-logstash-multi-jdk", disk_size_gb: int = 200) -> dict[str, typing.Any]:
+def gcp_agent(vm_name: str, instance_type: str = "n2-standard-4", image_prefix: str = "family/platform-ingest-logstash-multi-jdk", image_project="elastic-images-qa", disk_size_gb: int = 200) -> dict[str, typing.Any]:
     return {
         "provider": "gcp",
-        "imageProject": "elastic-images-prod",
+        "imageProject": image_project,
         "image": f"{image_prefix}-{vm_name}",
         "machineType": instance_type,
         "diskSizeGb": disk_size_gb,

Gemfile.template

@@ -13,6 +13,7 @@ gem "ruby-progressbar", "~> 1"
 gem "logstash-output-elasticsearch", ">= 10.4.2"
 gem "childprocess", "~> 4", :group => :build
 gem "fpm", "~> 1", ">= 1.14.1", :group => :build # compound due to bugfix https://github.com/jordansissel/fpm/pull/1856
+gem "erb", "~> 2", :group => :build
 gem "gems", "~> 1", :group => :build
 gem "octokit", "~> 4", :group => :build
 gem "rubyzip", "~> 1", :group => :build

docker/Makefile

@@ -1,7 +1,5 @@
 SHELL=/bin/bash
 ELASTIC_REGISTRY ?= docker.elastic.co
-PY_VERSION ?= 3.6.13
-export PATH := ./bin:$(HOME)/.pyenv/bin:$(HOME)/.pyenv/shims:./venv/bin:$(PATH)
 
 # Determine the version to build.
 ELASTIC_VERSION := $(shell ../vendor/jruby/bin/jruby bin/elastic-version)
@@ -24,42 +22,34 @@ DEFAULT_IMAGE_FLAVOR ?= full
 IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash
 HTTPD ?= logstash-docker-artifact-server
 
-FIGLET := pyfiglet -w 160 -f puffy
-
 all: build-from-local-artifacts build-from-local-oss-artifacts public-dockerfiles
 
-lint: venv
-	flake8 tests
-
 # Build from artifacts on the local filesystem, using an http server (running
 # in a container) to provide the artifacts to the Dockerfile.
-build-from-local-full-artifacts: venv dockerfile env2yaml
+build-from-local-full-artifacts: dockerfile env2yaml
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	pyfiglet -f puffy -w 160 "Building: full"; \
 	docker build --network=host -t $(IMAGE_TAG)-full:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-full data/logstash || \
 	  (docker kill $(HTTPD); false); \
 	docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG);
 	docker kill $(HTTPD)
 
-build-from-local-oss-artifacts: venv dockerfile env2yaml
+build-from-local-oss-artifacts: dockerfile env2yaml
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	pyfiglet -f puffy -w 160 "Building: oss"; \
 	docker build --network=host -t $(IMAGE_TAG)-oss:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-oss data/logstash || \
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
-build-from-local-ubi8-artifacts: venv dockerfile env2yaml
+build-from-local-ubi8-artifacts: dockerfile env2yaml
 	docker run --rm -d --name=$(HTTPD) \
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	pyfiglet -f puffy -w 160 "Building: ubi8"; \
 	docker build --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
@@ -111,61 +101,61 @@ ironbank_docker_paths:
 
 public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_ironbank
 
-public-dockerfiles_full: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES)
-	jinja2 \
-	  -D created_date='$(BUILD_DATE)' \
-	  -D elastic_version='$(ELASTIC_VERSION)' \
-	  -D arch='${ARCHITECTURE}' \
-	  -D version_tag='$(VERSION_TAG)' \
-	  -D image_flavor='full' \
-	  -D local_artifacts='false' \
-	  -D release='$(RELEASE)' \
-	  templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-full && \
+public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
+	../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+	  release="${RELEASE}" \
+		image_flavor="full" \
+		local_artifacts="false" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
 	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-public-dockerfiles_oss: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES)
-	jinja2 \
-	  -D created_date='$(BUILD_DATE)' \
-	  -D elastic_version='$(ELASTIC_VERSION)' \
-	  -D arch='${ARCHITECTURE}' \
-	  -D version_tag='$(VERSION_TAG)' \
-	  -D image_flavor='oss' \
-	  -D local_artifacts='false' \
-	  -D release='$(RELEASE)' \
-	  templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-oss && \
+public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
+	../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+	  release="${RELEASE}" \
+		image_flavor="oss" \
+		local_artifacts="false" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
 	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-public-dockerfiles_ubi8: venv templates/Dockerfile.j2 docker_paths $(COPY_FILES)
-	jinja2 \
-	  -D created_date='$(BUILD_DATE)' \
-	  -D elastic_version='$(ELASTIC_VERSION)' \
-	  -D arch='${ARCHITECTURE}' \
-	  -D version_tag='$(VERSION_TAG)' \
-	  -D image_flavor='ubi8' \
-	  -D local_artifacts='false' \
-	  -D release='$(RELEASE)' \
-	  templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-ubi8 && \
+public-dockerfiles_ubi8: templates/Dockerfile.erb docker_paths $(COPY_FILES)
+	../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+	  release="${RELEASE}" \
+		image_flavor="ubi8" \
+		local_artifacts="false" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ubi8" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \
 	tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-public-dockerfiles_ironbank: templates/hardening_manifest.yaml templates/Dockerfile.j2 ironbank_docker_paths $(COPY_IRONBANK_FILES)
-	jinja2 \
-	  -D elastic_version='$(ELASTIC_VERSION)' \
-	  templates/hardening_manifest.yaml > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \
-	jinja2 \
-	  -D created_date='$(BUILD_DATE)' \
-	  -D elastic_version='$(ELASTIC_VERSION)' \
-	  -D arch='${ARCHITECTURE}' \
-	  -D version_tag='$(VERSION_TAG)' \
-	  -D image_flavor='ironbank' \
-	  -D local_artifacts='false' \
-	  -D release='$(RELEASE)' \
-	  templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-ironbank && \
+public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Dockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
+	../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+	  elastic_version="${ELASTIC_VERSION}" \
+	  templates/hardening_manifest.yaml.erb > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \
+	../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+	  release="${RELEASE}" \
+		image_flavor="ironbank" \
+		local_artifacts="false" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
 	cd $(ARTIFACTS_DIR)/ironbank && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \
 	tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md
@@ -182,39 +172,28 @@ push:
 	docker push push.$(IMAGE_TAG):$(VERSION_TAG);
 	docker rmi push.$(IMAGE_TAG):$(VERSION_TAG);
 
-# The tests are written in Python. Make a virtualenv to handle the dependencies.
-venv: requirements.txt
-	LOCAL_PY_VER=`python3 --version 2>&1`&&\
-	echo "Was using $$LOCAL_PY_VER" &&\
-	eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" &&\
-	pyenv install --skip-existing $(PY_VERSION) &&\
-	pyenv local $(PY_VERSION) &&\
-	python3 -mvenv venv && \
-	for i in 0 1 2 3 4 5; do sleep "$i"; pip install -r requirements.txt && break; done &&\
-	touch venv
-
 # Compile "env2yaml", the helper for configuring logstash.yml via environment
 # variables.
 env2yaml:
 	docker run --rm \
 	  -v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-		-w /usr/src/env2yaml golang:1 go build
+	  -w /usr/src/env2yaml golang:1 go build
 
-# Generate the Dockerfiles from Jinja2 templates.
-dockerfile: venv templates/Dockerfile.j2
+# Generate the Dockerfiles from ERB templates.
+dockerfile: templates/Dockerfile.erb
 	$(foreach FLAVOR, $(IMAGE_FLAVORS), \
-	  jinja2 \
-	    -D created_date='$(BUILD_DATE)' \
-	    -D elastic_version='$(ELASTIC_VERSION)' \
-	    -D arch='${ARCHITECTURE}' \
-	    -D version_tag='$(VERSION_TAG)' \
-	    -D image_flavor='$(FLAVOR)' \
-	    -D local_artifacts='true' \
-	    templates/Dockerfile.j2 > $(ARTIFACTS_DIR)/Dockerfile-$(FLAVOR); \
+		../vendor/jruby/bin/jruby -S ../bin/bundle exec erb -T "-" \
+			created_date="${BUILD_DATE}" \
+			elastic_version="${ELASTIC_VERSION}" \
+			arch="${ARCHITECTURE}" \
+			version_tag="${VERSION_TAG}" \
+			image_flavor="${FLAVOR}" \
+			local_artifacts="true" \
+			templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-${FLAVOR}" ; \
 	)
 
 clean:
-	rm -f ${ARTIFACTS_DIR}/env2yaml/env2yaml ${ARTIFACTS_DIR}/Dockerfile
-	rm -rf venv
+	rm -f ${ARTIFACTS_DIR}/env2yaml/env2yaml-* ${ARTIFACTS_DIR}/Dockerfile
+
 
 .PHONY: clean push