refactor(testing-tools): split into multiple images, switch to UBI, remove unnecessary components

CHANGELOG.md

@@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file.
 
 - airflow: Extend list of providers for 3.0.6 ([#1336])
 - airflow: Bump celery version to 5.5.3 for Airflow 3.x ([#1343]).
+- testing-tools: refactoring: Split image into multiple images, remove unnecessary components and switch to UBI as base image ([#1354]).
 
 ### Removed
 
@@ -22,6 +23,7 @@ All notable changes to this project will be documented in this file.
 [#1337]: https://github.com/stackabletech/docker-images/pull/1337
 [#1343]: https://github.com/stackabletech/docker-images/pull/1343
 [#1340]: https://github.com/stackabletech/docker-images/pull/1340
+[#1354]: https://github.com/stackabletech/docker-images/pull/1354
 
 ## [25.11.0] - 2025-11-07
 

testing-tools/Dockerfile

@@ -1,15 +1,14 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
-# Manifest list digest because of multi architecture builds ( https://www.redhat.com/architect/pull-container-image#:~:text=A%20manifest%20list%20exists%20to,system%20on%20a%20specific%20architecture )
-# https://hub.docker.com/_/python/tags
-# In Docker Hub, open up the tag and look for Index Digest. Otherwise do:
-# docker pull python:3.12-slim-bullseye and see the digest that appears in the output.
-FROM python:3.12-slim-bullseye@sha256:411fa4dcfdce7e7a3057c45662beba9dcd4fa36b2e50a2bfcd6c9333e59bf0db
+# Find the latest version at https://catalog.redhat.com/en/software/containers/ubi10/ubi-minimal/66f1504a379b9c2cf23e145c#get-this-image
+# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures
+# rather than just the "Image Digest" that references the image for the selected architecture.
+FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:28ec2f4662bdc4b0d4893ef0d8aebf36a5165dfb1d1dc9f46319bd8a03ed3365
 
 ARG PRODUCT_VERSION
+ARG PYTHON_VERSION
 ARG RELEASE_VERSION
-ARG KEYCLOAK_VERSION
 ARG STACKABLE_USER_UID
 ARG STACKABLE_USER_GID
 ARG STACKABLE_USER_NAME
@@ -25,50 +24,54 @@ LABEL name="Stackable Testing Tools" \
 # https://github.com/hadolint/hadolint/wiki/DL4006
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-# This is needed so that krb5-user installs without prompting for a realm.
-ENV DEBIAN_FRONTEND=noninteractive
 
+# We configure microdnf to not install weak dependencies in this file
+# Not doing this caused the content of images to become unpredictable because
+# based on which packages get updated by `microdnf update` new weak dependencies
+# might be installed that were not present earlier (the ubi base image doesn't
+# seem to install weak dependencies)
+# This also affects the packages that are installed in our Dockerfiles (java as prime
+# example).
+# https://github.com/stackabletech/docker-images/pull/533
+COPY stackable-base/stackable/dnf.conf /etc/dnf/dnf.conf
 
-COPY testing-tools/python /stackable/python
-COPY testing-tools/licenses /licenses
+# Default curl configuration to avoid forgetting settings and to declutter the Dockerfiles
+COPY stackable-base/stackable/curlrc /root/.curlrc
 
+# Base requirements for all testing-tools images
+COPY testing-tools/requirements.txt /stackable/requirements.txt
+COPY testing-tools/licenses /licenses
 
 RUN <<EOF
-apt-get update
-apt-get install -y --no-install-recommends \
-  build-essential \
-  ca-certificates \
-  curl \
-  gzip \
-  jq \
-  `# krb5-user/libkrb5-dev are needed for Kerberos support. ` \
-  krb5-user \
-  libkrb5-dev \
-  kubernetes-client \
-  libssl-dev \
-  libxml2-dev \
-  libxslt1-dev \
+microdnf update
+
+microdnf install \
+  gcc \
+  make \
   pkg-config \
-  python3-certifi \
-  python3-idna \
-  python3-semver \
-  python3-thrift \
-  python3-toml \
-  python3-urllib3 \
+  openssl-devel \
+  libxml2-devel \
+  libxslt-devel \
+  python${PYTHON_VERSION} \
+  python${PYTHON_VERSION}-devel \
+  python${PYTHON_VERSION}-pip \
+  jq \
+  gzip \
+  curl \
   tar \
   zip \
-  unzip \
-  `# Java 11 seems like the best middle-ground for all tools` \
-  openjdk-11-jdk-headless
+  unzip
 
-apt-get clean
-rm -rf /var/lib/apt/lists/*
+microdnf clean all
+rm -rf /var/cache/yum
 
-curl --fail -L https://repo.stackable.tech/repository/packages/keycloak/keycloak-${KEYCLOAK_VERSION}.tar.gz | tar -xzC /stackable
-ln -s /stackable/keycloak-${KEYCLOAK_VERSION} /stackable/keycloak
+python3.12 -m pip install --no-cache-dir --upgrade pip
+python3.12 -m pip install --no-cache-dir -r /stackable/requirements.txt
 
-pip install --no-cache-dir --upgrade pip
-pip install --no-cache-dir -r /stackable/python/requirements.txt
+ln -s /usr/bin/python3.12 /usr/bin/python
+
+# Added only temporarily to create the user and group, removed again below
+microdnf install shadow-utils
 
 groupadd --gid ${STACKABLE_USER_GID} --system ${STACKABLE_USER_NAME}
 useradd \
@@ -80,11 +83,12 @@ useradd \
   --home-dir /stackable \
    ${STACKABLE_USER_NAME}
 
+microdnf remove shadow-utils
+microdnf clean all
+
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 EOF
 
-ENV PATH=/stackable/keycloak/bin:$PATH
-
 USER ${STACKABLE_USER_UID}
 
 ENV STACKABLE_PRODUCT_VERSION=${PRODUCT_VERSION}

testing-tools/boil-config.toml

@@ -1,2 +1,2 @@
-[versions."0.2.0".build-arguments]
-keycloak-version = "26.3.5"
+[versions."0.3.0".build-arguments]
+python-version = "3.12"

testing-tools/hive/Dockerfile

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
+# check=error=true
+
+FROM local-image/testing-tools
+
+ARG PRODUCT_VERSION
+ARG RELEASE_VERSION
+ARG STACKABLE_USER_UID
+
+LABEL name="Stackable Testing Tools - Hive" \
+      maintainer="info@stackable.tech" \
+      vendor="Stackable GmbH" \
+      version="${PRODUCT_VERSION}" \
+      release="${RELEASE_VERSION}" \
+      summary="Stackable tools for Hive integration tests." \
+      description="Hive-specific testing tools image based on testing-tools base image."
+
+# https://github.com/hadolint/hadolint/wiki/DL4006
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+USER root
+
+COPY testing-tools/hive/requirements.txt /stackable/hive/requirements.txt
+
+RUN <<EOF
+microdnf update
+
+microdnf install \
+  krb5-workstation \
+  krb5-devel
+
+microdnf clean all
+rm -rf /var/cache/yum
+
+pip install --no-cache-dir -r /stackable/hive/requirements.txt
+EOF
+
+USER ${STACKABLE_USER_UID}
+
+ENV STACKABLE_PRODUCT_VERSION=${PRODUCT_VERSION}
+
+WORKDIR /stackable

testing-tools/hive/boil-config.toml

@@ -0,0 +1,2 @@
+[versions."0.3.0".local-images]
+testing-tools = "0.3.0"

testing-tools/hive/requirements.txt

@@ -0,0 +1,7 @@
+hive-metastore-client==1.0.9
+thrift==0.13.0
+
+# gssapi/kerberos/pure-sasl are needed for using the python metastore client with kerberos.
+gssapi==1.10.1
+kerberos==1.3.1
+pure-sasl==0.6.2