[release-4.17] OCPBUGS-42410: Install ansible using pip instead of rpm

.ci-operator.yaml

@@ -1,4 +1,4 @@
 build_root_image:
   name: release
   namespace: openshift
-  tag: rhel-8-release-golang-1.22-openshift-4.17
+  tag: rhel-9-release-golang-1.22-openshift-4.17

images/ansible-operator/Pipfile

@@ -7,8 +7,10 @@ name = "pypi"
 ansible-runner = "~=2.3.3"
 ansible-runner-http = "~=1.0.0"
 ansible-core = "~=2.15.9"
-urllib3 = "~=1.26.17"
+urllib3 = "~=1.26.19"
 kubernetes = "==29.0.0"
+requests = "==2.31.0"
+setuptools = "==70.0.0"
 
 [dev-packages]
 

openshift/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.22-openshift-4.17 AS builder
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-openshift-4.17 AS builder
 
 ENV GO111MODULE=on \
     GOFLAGS=-mod=vendor
@@ -7,7 +7,7 @@ COPY . /go/src/github.com/openshift/ansible-operator-plugins
 RUN cd /go/src/github.com/openshift/ansible-operator-plugins \
  && make build
 
-FROM registry.ci.openshift.org/ocp/4.17:base
+FROM registry.ci.openshift.org/ocp/4.17:base-rhel9
 
 RUN mkdir -p /etc/ansible \
     && echo "localhost ansible_connection=local" > /etc/ansible/hosts \
@@ -19,32 +19,39 @@ ENV HOME=/opt/ansible \
     USER_NAME=ansible \
     USER_UID=1001
 
-RUN yum install -y \
-      ansible \
-      ansible-runner \
-      ansible-runner-http \
-      python3-kubernetes \
-      python3-openshift \
-      python3-oauthlib \
-      python3-requests-oauthlib \
-      python3-jmespath \
-      tini \
- && yum clean all \
- && rm -rf /var/cache/yum
+# cryptography packages cannot be installed in the OSBS environment using pip.
+# Thus installing it via rpm.
+RUN yum clean all && rm -rf /var/cache/yum/* \
+    && yum update -y \
+    && yum install -y catatonit python3-cryptography python3-devel gcc python3-pip
+
+# pip install will install a higher version of the requests module. However, pip fails
+# to uninstall the existing request module as it is installed from rpm. Thus, removing
+# the module before using pip install.
+RUN yum remove -y python3-requests
+
+# Add steps for cachito
+ENV REMOTE_SOURCES=${REMOTE_SOURCES:-"./openshift/"}
+ENV REMOTE_SOURCES_DIR=${REMOTE_SOURCES_DIR:-"/cachito"}
+COPY ${REMOTE_SOURCES} ${REMOTE_SOURCES_DIR}
+COPY openshift/install-ansible.sh .
+RUN chmod +x /install-ansible.sh \
+    && ./install-ansible.sh
+
+# Remove packages which are only needed for cachito
+RUN yum remove -y python3-devel gcc \
+    && yum clean all \
+    && rm -rf /var/cache/yum
 
 COPY --from=builder /go/src/github.com/openshift/ansible-operator-plugins/ansible-operator /usr/local/bin/ansible-operator
 COPY openshift/release/ansible/ansible_collections ${HOME}/.ansible/collections/ansible_collections
 
-COPY openshift/release/ansible/bin /usr/local/bin
-
-RUN chmod +x /usr/local/bin/user_setup && \
-    ./usr/local/bin/user_setup
-
 # Ensure directory permissions are properly set
-RUN mkdir -p ${HOME}/.ansible/tmp \
- && chown -R ${USER_UID}:0 ${HOME} \
- && chmod -R ug+rwx ${HOME}
+RUN echo "${USER_NAME}:x:${USER_UID}:0:${USER_NAME} user:${HOME}:/sbin/nologin" >> /etc/passwd \
+  && mkdir -p ${HOME}/.ansible/tmp \
+  && chown -R ${USER_UID}:0 ${HOME} \
+  && chmod -R ug+rwx ${HOME}
 
 WORKDIR ${HOME}
 USER ${USER_UID}
-ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/ansible-operator", "run", "--watches-file=./watches.yaml"]
+ENTRYPOINT ["/usr/libexec/catatonit/catatonit", "--", "/usr/local/bin/ansible-operator", "run", "--watches-file=./watches.yaml"]

openshift/Dockerfile.requirements

@@ -0,0 +1,69 @@
+FROM registry.ci.openshift.org/ocp/4.17:base-rhel9
+
+RUN yum install -y rust cargo libffi-devel python3-devel gcc python3-pip
+
+COPY ./Pipfile* ./
+
+# The build dependencies are required by cachito. Following script
+# does exactly the same. More info at: https://github.com/containerbuildsystem/cachito/blob/master/docs/pip.md#build-dependencies
+RUN curl -LO https://raw.githubusercontent.com/containerbuildsystem/cachito/master/bin/pip_find_builddeps.py \
+    && chmod +x pip_find_builddeps.py
+
+# Create requirements.in file from the pipenv created using the
+# same Pipfile and Pipfile.lock used for upstream image. Then
+# use pip-compile to generate the requirements.txt file. Copy
+# setuptools into requirements-build.txt as pip-compile will
+# omit it from the requierements.txt file.
+RUN python3 -m pip install pipenv==2023.11.15 \
+    && python3 -m pip install pip-tools \
+    && pipenv install --deploy \
+    # NOTE: This ignored vulnerability (70612) was detected in jinja2, \
+    # but the vulnerability is disputed and may never be fixed. See: \
+    #  - https://github.com/advisories/GHSA-f6pv-j8mr-w6rr \
+    #  - https://github.com/dbt-labs/dbt-core/issues/10250 \
+    #  - https://data.safetycli.com/v/70612/97c/ \
+    # NOTE: This ignored vulnerability (71064) was detected in requests, \
+    # but the upgraded version doesn't support the use case (protocol we are using).\
+    # Ref: https://github.com/operator-framework/ansible-operator-plugins/pull/67#issuecomment-2189164688
+    && pipenv check --ignore 70612 --ignore 71064 \
+    && pipenv run pip freeze --all  > ./requirements.in \
+    # Add wheel and setuptools to requirements-build.in file as these packages
+    # are part of the build dependencies of some packages in requirements.in file.
+    && grep "wheel==" ./requirements.in >> ./requirements-build.in || true \
+    && grep "setuptools==" ./requirements.in >> ./requirements-build.in || true \
+    && pip-compile --output-file=./requirements.txt ./requirements.in --strip-extras \
+    # NOTE: Comment out ansible-runner, ansible-runner-http and python-daemon as
+    # both ansible-runner and ansible-runner-http has python-daemon as dependency.
+    # pip_find_builddeps.py encounters an error when trying to get the build
+    # dependencies of python-daemon==3.0.1.
+    # TODO: Whenever a newer version of python-daemon is released check
+    # whether pip_find_builddeps.py is able to fetch its build dependencies
+    # without any error. If so, then remove the following lines which comments
+    # out ansible-runner, ansible-runner-http and python-daemon.
+    && sed -i '/ansible-runner==/s/^/#/g' ./requirements.txt \
+    && sed -i '/ansible-runner-http==/s/^/#/g' ./requirements.txt \
+    && sed -i '/python-daemon==/s/^/#/g' ./requirements.txt \
+    && ./pip_find_builddeps.py requirements.txt -o requirements-build.in --append \
+    # Uncomment ansible-runner, ansible-runner-http and python-daemon, so that
+    # they are re-enabled in the requirements.txt file.
+    && sed -i '/ansible-runner==/s/^#//g' ./requirements.txt \
+    && sed -i '/ansible-runner-http==/s/^#//g' ./requirements.txt \
+    && sed -i '/python-daemon==/s/^#//g' ./requirements.txt \
+    && pip-compile --output-file=./requirements-build.txt ./requirements-build.in --strip-extras --allow-unsafe \
+    # NOTE: Comment out cryptography and its dependencies from the requirements.txt
+    # and requirements-build.txt files as these packages can't be installed in the
+    # isolated environment of OSBS image build. These packages will be installed
+    # through rpms.
+    && sed -i '/cryptography==/s/^/#/g' ./requirements.txt \
+    && sed -i '/cffi==/s/^/#/g' ./requirements.txt \
+    && sed -i '/pycparser==/s/^/#/g' ./requirements.txt \
+    && sed -i '/cffi==/s/^/#/g' ./requirements-build.txt \
+    && sed -i '/pycparser==/s/^/#/g' ./requirements-build.txt \
+    && sed -i '/maturin==/s/^/#/g' ./requirements-build.txt \
+    # Add flit-core to requirements-pre-build.in file as this package is part of the
+    # build dependencies of some packages in requirements-build.txt file.
+    && grep "flit-core==" ./requirements-build.txt >> ./requirements-pre-build.in || true \
+    && pip-compile --output-file=./requirements-pre-build.txt ./requirements-pre-build.in --strip-extras
+
+VOLUME /tmp/requirements
+ENTRYPOINT ["cp", "./requirements.txt", "./requirements-build.txt", "./requirements-pre-build.txt", "/tmp/requirements/"]

openshift/Makefile

@@ -8,3 +8,23 @@ include $(addprefix $(DIR)/vendor/github.com/openshift/build-machinery-go/make/,
 .PHONY: test-e2e-ansible
 test-e2e-ansible:
 	./openshift/ci/tests/e2e-ansible-scaffolding.sh
+
+CONTAINER_ENGINE ?= docker
+REQUIREMENTS_IMG ?= pip-requirements
+
+.PHONY: build-requirements
+build-requirements:
+	cp ../images/ansible-operator/Pipfile* .
+	$(CONTAINER_ENGINE) build -t $(REQUIREMENTS_IMG) -f Dockerfile.requirements .
+	rm Pipfile*
+
+# Use this target to generate the requirements.txt, requirements-build.txt and requirements-pre-build.txt
+# files using the corresponding Pipfile and Pipfile.lock from the images/ansible-operator directory. The
+# generated files will be used for building the image using cachito in the OSBS environment.
+.PHONY: generate-requirements
+generate-requirements: build-requirements
+	$(CONTAINER_ENGINE) run --rm -it -v .:/tmp/requirements/:Z $(REQUIREMENTS_IMG)
+
+.PHONY: check-requirements
+check-requirements: generate-requirements
+	git diff --exit-code # fast-fail if generate-requirements produced changes

openshift/install-ansible.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+PIP_OPTS="--no-cache-dir"
+# Check if we are building the image in the OSBS environment. If so, source the
+# env vars specific for enabling cachito.
+if [ -d ${REMOTE_SOURCES_DIR}/cachito-gomod-with-deps ]; then
+    source ${REMOTE_SOURCES_DIR}/cachito-gomod-with-deps/cachito.env
+    cd ${REMOTE_SOURCES_DIR}/cachito-gomod-with-deps/app/openshift
+else
+    cd  ${REMOTE_SOURCES_DIR}
+fi
+
+# Install the packages in order of build dependency to avoid issues during installation.
+python3 -m pip install ${PIP_OPTS} -r requirements-pre-build.txt
+python3 -m pip install ${PIP_OPTS} -r requirements-build.txt
+python3 -m pip install ${PIP_OPTS} -r requirements.txt
+
+rm -rf ${REMOTE_SOURCES_DIR}

openshift/requirements-build.txt

@@ -0,0 +1,52 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=./requirements-build.txt --strip-extras ./requirements-build.in
+#
+#cffi==1.17.1
+    # via -r ./requirements-build.in
+cython==3.0.11
+    # via -r ./requirements-build.in
+flit-core==3.9.0
+    # via -r ./requirements-build.in
+#maturin==1.7.1
+    # via -r ./requirements-build.in
+packaging==24.1
+    # via
+    #   -r ./requirements-build.in
+    #   setuptools-scm
+pbr==6.1.0
+    # via -r ./requirements-build.in
+poetry-core==1.9.0
+    # via -r ./requirements-build.in
+#pycparser==2.22
+    # via
+    #   -r ./requirements-build.in
+    #   cffi
+semantic-version==2.10.0
+    # via
+    #   -r ./requirements-build.in
+    #   setuptools-rust
+setuptools-rust==1.10.1
+    # via -r ./requirements-build.in
+setuptools-scm==7.1.0
+    # via -r ./requirements-build.in
+tomli==2.0.1
+    # via
+    #   -r ./requirements-build.in
+    #   maturin
+    #   setuptools-scm
+typing-extensions==4.12.2
+    # via
+    #   -r ./requirements-build.in
+    #   setuptools-scm
+wheel==0.44.0
+    # via -r ./requirements-build.in
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==70.0.0
+    # via
+    #   -r ./requirements-build.in
+    #   setuptools-rust
+    #   setuptools-scm

openshift/requirements-pre-build.txt

@@ -0,0 +1,8 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --output-file=./requirements-pre-build.txt --strip-extras ./requirements-pre-build.in
+#
+flit-core==3.9.0
+    # via -r ./requirements-pre-build.in