From 3e786f2de9b7c3e14a96b5dd12d1699ea01866fe Mon Sep 17 00:00:00 2001 From: Mike DePaulo Date: Tue, 23 Jul 2019 14:15:18 -0400 Subject: [PATCH] containers: Switch to Fedora 30 Includes: A Fix for launching pulp-content on Fedora, where /usr/local/bin/pulp-content is a pip-installed script in the PATH, ahead of /usr/bin/pulp-content . Making the pulp-resource-manager & pulp-worker scripts more consistent fixes #5176 --- containers/images/pulp/Dockerfile | 69 +++++++++++++------ .../images/pulp/container-assets/pulp-api | 12 ++-- .../pulp-common-entrypoint.sh | 8 ++- .../images/pulp/container-assets/pulp-content | 4 +- .../container-assets/pulp-resource-manager | 2 +- .../images/pulp/container-assets/pulp-worker | 2 +- .../wait_on_database_migrations.sh | 2 +- 7 files changed, 65 insertions(+), 34 deletions(-) diff --git a/containers/images/pulp/Dockerfile b/containers/images/pulp/Dockerfile index 160614d9c6..e6ddb235f4 100644 --- a/containers/images/pulp/Dockerfile +++ b/containers/images/pulp/Dockerfile @@ -1,4 +1,4 @@ -FROM centos:7 +FROM fedora:30 ARG PLUGINS="" @@ -6,12 +6,37 @@ ARG PLUGINS="" # otherwise, `mysql_config` is not found # It also needs gcc at least, so we install the dev tools package group. # And it needs Python.h -RUN echo "tsflags=nodocs" >> /etc/yum.conf && \ - yum -y update && \ - yum -y install epel-release centos-release-scl && \ - yum -y install wget git rh-python36-python-pip && \ - yum -y install @development mariadb-devel rh-python36-python-devel && \ - yum clean all +# +# The Fedora 30 image already has tsflags=nodocs set in dnf.conf +# It already has pip +# +# wget & git are presumably needed for purposes like pip installs from git +# +# libxcrypt-compat is needed by psycopg2-binary from PyPI +# +# python3-psycopg2 is installed by ansible-pulp +# +# glibc-langpack-en is needed to provide the en_US.UTF-8 locale, which Pulp +# seems to need. +# +# The development packages are needed for installing nightly versions with pip. +# The development packages group is the equivalent of EL7's "development" group. +# +# Also installing the long list of pulp-rpm deps from +# https://github.com/pulp/pulp-rpm-prerequisites/blob/master/vars/Fedora.yml +# (Including it literally, dups @c-development won't slow down dnf much) +RUN dnf -y update && \ + dnf -y install wget git && \ + dnf -y install libxcrypt-compat && \ + dnf -y install @c-development mariadb-devel python3-devel && \ + dnf clean all + +# Docs suggest RHEL8 uses the alternatives system for /usr/bin/python , +# but Fedora does not. +RUN ln -s /usr/bin/python3 /usr/bin/python +# If pip2 is installed, it will replace /usr/bin/pip . +# /usr/local/bin will be before it in the PATH . +RUN ln -s /usr/bin/pip3 /usr/local/bin/pip ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US.UTF-8 @@ -21,21 +46,21 @@ ENV DJANGO_SETTINGS_MODULE=pulpcore.app.settings RUN mkdir -p /etc/pulp -RUN scl enable rh-python36 'pip install gunicorn' -RUN scl enable rh-python36 'pip install git+https://github.com/pulp/pulpcore.git' -RUN scl enable rh-python36 'pip install git+https://github.com/pulp/pulpcore-plugin.git' -RUN scl enable rh-python36 'pip install git+https://github.com/pulp/pulpcore.git#egg=pulpcore[postgres]' -RUN scl enable rh-python36 'pip install git+https://github.com/pulp/pulpcore.git#egg=pulpcore[mysql]' -RUN scl enable rh-python36 'pip install $PLUGINS' - -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulpcore/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulpcore/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_file/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_ansible/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_cookbook/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_docker/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_maven/app/migrations -RUN mkdir -p /opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/pulp_python/app/migrations +RUN pip install gunicorn +RUN pip install git+https://github.com/pulp/pulpcore.git +RUN pip install git+https://github.com/pulp/pulpcore-plugin.git +RUN pip install git+https://github.com/pulp/pulpcore.git#egg=pulpcore[postgres] +RUN pip install git+https://github.com/pulp/pulpcore.git#egg=pulpcore[mysql] +RUN pip install $PLUGINS + +RUN mkdir -p /usr/lib/python3.7/site-packages/pulpcore/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulpcore/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_file/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_ansible/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_cookbook/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_docker/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_maven/app/migrations +RUN mkdir -p /usr/lib/python3.7/site-packages/pulp_python/app/migrations COPY container-assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py COPY container-assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh diff --git a/containers/images/pulp/container-assets/pulp-api b/containers/images/pulp/container-assets/pulp-api index 127d10b8d2..3c7b25a623 100755 --- a/containers/images/pulp/container-assets/pulp-api +++ b/containers/images/pulp/container-assets/pulp-api @@ -4,15 +4,15 @@ # Generating /var/lib/pulp/static at runtime rather than at container build time # facilitates all of /var/lib/pulp being a separate volume. -scl enable rh-python36 "django-admin collectstatic --noinput" +django-admin collectstatic --noinput #TODO: Determine list of installed plugins by inspecting image contents -scl enable rh-python36 "django-admin makemigrations file ansible cookbook docker maven python" -scl enable rh-python36 "django-admin migrate --noinput" -scl enable rh-python36 "django-admin migrate auth --noinput" +django-admin makemigrations file ansible cookbook docker maven python +django-admin migrate --noinput +django-admin migrate auth --noinput if [ -n "${PULP_ADMIN_PASSWORD}" ]; then - scl enable rh-python36 "django-admin reset-admin-password --password '${PULP_ADMIN_PASSWORD}'" + django-admin reset-admin-password --password "${PULP_ADMIN_PASSWORD}" fi -exec scl enable rh-python36 "gunicorn -b 0.0.0.0:24817 pulpcore.app.wsgi:application" +exec gunicorn -b 0.0.0.0:24817 pulpcore.app.wsgi:application diff --git a/containers/images/pulp/container-assets/pulp-common-entrypoint.sh b/containers/images/pulp/container-assets/pulp-common-entrypoint.sh index 902915cd28..6ea2afd53c 100755 --- a/containers/images/pulp/container-assets/pulp-common-entrypoint.sh +++ b/containers/images/pulp/container-assets/pulp-common-entrypoint.sh @@ -2,4 +2,10 @@ export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -exec "$@" +# Prevent pip-installed /usr/local/bin/pulp-content from getting run instead of +# our /usr/bin/pulp script. +# +# We still want conatiner users to call command names, not paths, so we can +# change our scripts' locations in the future, and call special logic in this +# script based solely on the command name. +exec "/usr/bin/$@" diff --git a/containers/images/pulp/container-assets/pulp-content b/containers/images/pulp/container-assets/pulp-content index d30b9f2083..311f9b1155 100755 --- a/containers/images/pulp/container-assets/pulp-content +++ b/containers/images/pulp/container-assets/pulp-content @@ -3,8 +3,8 @@ /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh -exec scl enable rh-python36 "gunicorn pulpcore.content:server \ +exec gunicorn pulpcore.content:server \ --bind 0.0.0.0:24816 \ --worker-class 'aiohttp.GunicornWebWorker' \ -w 2 \ ---access-logfile -" +--access-logfile - diff --git a/containers/images/pulp/container-assets/pulp-resource-manager b/containers/images/pulp/container-assets/pulp-resource-manager index f1d9f5016d..af647ecd10 100755 --- a/containers/images/pulp/container-assets/pulp-resource-manager +++ b/containers/images/pulp/container-assets/pulp-resource-manager @@ -3,4 +3,4 @@ /usr/bin/wait_on_postgres.py /usr/bin/wait_on_database_migrations.sh -exec scl enable rh-python36 "rq worker --url 'redis://$REDIS_SERVICE_HOST:$REDIS_SERVICE_PORT' -n resource-manager@%h -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig'" +exec rq worker --url "redis://${REDIS_SERVICE_HOST}:${REDIS_SERVICE_PORT}" -n "resource-manager@${HOSTNAME}" -w "pulpcore.tasking.worker.PulpWorker" -c "pulpcore.rqconfig" diff --git a/containers/images/pulp/container-assets/pulp-worker b/containers/images/pulp/container-assets/pulp-worker index 59a5a8f04b..a9bb460faf 100755 --- a/containers/images/pulp/container-assets/pulp-worker +++ b/containers/images/pulp/container-assets/pulp-worker @@ -5,4 +5,4 @@ # TODO: Set ${PULP_WORKER_NUMBER} to the Pod Number # In the meantime, the hostname provides uniqueness. -exec scl enable rh-python36 "rq worker --url 'redis://${REDIS_SERVICE_HOST}:${REDIS_SERVICE_PORT}' -n reserved-resource-worker-${PULP_WORKER_NUMBER}@${HOSTNAME} -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig'" +exec rq worker --url "redis://${REDIS_SERVICE_HOST}:${REDIS_SERVICE_PORT}" -n "reserved-resource-worker-${PULP_WORKER_NUMBER}@${HOSTNAME}" -w "pulpcore.tasking.worker.PulpWorker" -c "pulpcore.rqconfig" diff --git a/containers/images/pulp/container-assets/wait_on_database_migrations.sh b/containers/images/pulp/container-assets/wait_on_database_migrations.sh index 2965c1f851..e6e172b29e 100755 --- a/containers/images/pulp/container-assets/wait_on_database_migrations.sh +++ b/containers/images/pulp/container-assets/wait_on_database_migrations.sh @@ -4,7 +4,7 @@ database_migrated=false echo "Checking for database migrations" while [ $database_migrated = false ]; do - scl enable rh-python36 "django-admin showmigrations | grep '\[ \]'" + django-admin showmigrations | grep '\[ \]' if [ $? -gt 0 ]; then echo "Database migrated!" database_migrated=true