From 7f647498d1c74e02da33ed554bfb6f323940cba2 Mon Sep 17 00:00:00 2001 From: Tatiana Tereshchenko Date: Tue, 12 May 2020 19:50:23 +0200 Subject: [PATCH 1/5] Update travis [noissue] --- .travis.yml | 22 ++-- .travis/.gitignore | 3 + .travis/Containerfile.ci_base | 82 ++++++++++++ .travis/Containerfile.j2 | 22 ++++ .travis/ansible.cfg | 7 ++ .travis/assets/nginx.conf | 85 +++++++++++++ .travis/assets/nginx.run | 2 + .travis/assets/postgres.prep | 115 +++++++++++++++++ .travis/assets/postgres.run | 3 + .travis/assets/pulpcore-api.run | 4 + .travis/assets/pulpcore-content.run | 4 + .travis/assets/pulpcore-resource-manager.run | 4 + .travis/assets/pulpcore-worker@1.run | 5 + .travis/assets/pulpcore-worker@2.run | 5 + .travis/assets/redis.run | 3 + .travis/before_install.sh | 102 +++++++++------ .travis/before_script.sh | 39 ++---- .travis/build_container.yaml | 48 +++++++ .travis/check_commit.sh | 4 +- .travis/check_pulpcore_imports.sh | 20 +++ .travis/cherrypick.py | 12 +- .travis/filter/repr.py | 25 ++++ .travis/install.sh | 124 ++++++++----------- .travis/inventory.yaml | 11 ++ .travis/publish_client_gem.sh | 15 +-- .travis/publish_client_pypi.sh | 13 +- .travis/publish_plugin_pypi.sh | 4 +- .travis/script.sh | 76 ++++-------- .travis/settings.py.j2 | 23 ++++ .travis/smash-config.json | 33 +++++ .travis/start_container.yaml | 85 +++++++++++++ .travis/utils.sh | 13 ++ .travis/validate_commit_message.py | 43 ++++--- template_config.yml | 11 +- 34 files changed, 822 insertions(+), 245 deletions(-) create mode 100644 .travis/.gitignore create mode 100644 .travis/Containerfile.ci_base create mode 100644 .travis/Containerfile.j2 create mode 100644 .travis/ansible.cfg create mode 100644 .travis/assets/nginx.conf create mode 100644 .travis/assets/nginx.run create mode 100644 .travis/assets/postgres.prep create mode 100644 .travis/assets/postgres.run create mode 100644 .travis/assets/pulpcore-api.run create mode 100644 .travis/assets/pulpcore-content.run create mode 100644 .travis/assets/pulpcore-resource-manager.run create mode 100644 .travis/assets/pulpcore-worker@1.run create mode 100644 .travis/assets/pulpcore-worker@2.run create mode 100644 .travis/assets/redis.run create mode 100644 .travis/build_container.yaml create mode 100755 .travis/check_pulpcore_imports.sh create mode 100644 .travis/filter/repr.py create mode 100644 .travis/inventory.yaml create mode 100644 .travis/settings.py.j2 create mode 100644 .travis/smash-config.json create mode 100644 .travis/start_container.yaml create mode 100644 .travis/utils.sh diff --git a/.travis.yml b/.travis.yml index 9a24af9b..6cc5dd74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template --- @@ -34,14 +34,10 @@ install: .travis/install.sh before_script: .travis/before_script.sh script: .travis/script.sh after_failure: - - http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/ - - sudo docker images - - sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000 - - sudo kubectl logs -l name=pulp-operator -c operator --tail=10000 - - sudo kubectl logs -l app=pulp-api --tail=50000 - - sudo kubectl logs -l app=pulp-content --tail=10000 - - sudo kubectl logs -l app=pulp-resource-manager --tail=10000 - - sudo kubectl logs -l app=pulp-worker --tail=10000 + - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ + - docker images + - docker ps -a + - docker logs pulp jobs: include: # So long as we have a build matrix of only 1 build (via only 1 python @@ -83,6 +79,12 @@ jobs: env: - TEST=bindings if: tag IS present + - stage: cherry-pick + if: type = cron + before_install: skip + install: skip + before_script: skip + script: bash .travis/cherrypick.sh notifications: irc: channels: diff --git a/.travis/.gitignore b/.travis/.gitignore new file mode 100644 index 00000000..06644f55 --- /dev/null +++ b/.travis/.gitignore @@ -0,0 +1,3 @@ +Containerfile +settings/ +vars/ diff --git a/.travis/Containerfile.ci_base b/.travis/Containerfile.ci_base new file mode 100644 index 00000000..1a978b21 --- /dev/null +++ b/.travis/Containerfile.ci_base @@ -0,0 +1,82 @@ +FROM fedora:31 +ADD https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-amd64.tar.gz /tmp/ + +RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C / --exclude="./bin" && \ + tar xzf /tmp/s6-overlay-amd64.tar.gz -C /usr ./bin + +# https://superuser.com/questions/959380/how-do-i-install-generate-all-locales-on-fedora +# This may not be necessary anymore because Fedora 30, unlike CentOS 7, has +# glibc subpackages like glibc-langpack-en. +RUN rm /etc/rpm/macros.image-language-conf +RUN echo 'LANG="en_US.UTF-8"' > /etc/locale.conf +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV PYTHONUNBUFFERED=0 +ENV DJANGO_SETTINGS_MODULE=pulpcore.app.settings +ENV PULP_SETTINGS=/etc/pulp/settings.py +ENV _BUILDAH_STARTED_IN_USERNS="" +ENV BUILDAH_ISOLATION=chroot + +# The Fedora 30 image already has tsflags=nodocs set in dnf.conf +# It already has pip +# +# wget & git are presumably needed for purposes like pip installs from git +# +# libxcrypt-compat is needed by psycopg2-binary from PyPI +# +# python3-psycopg2 is installed by ansible-pulp +# +# glibc-langpack-en is needed to provide the en_US.UTF-8 locale, which Pulp +# seems to need. +# +# python3-createrepo_c is needed for pulp_rpm +RUN dnf -y update && \ + dnf -y install wget git && \ + dnf -y install libxcrypt-compat && \ + dnf -y install python3-psycopg2 && \ + dnf -y install glibc-langpack-en && \ + dnf -y install python3-libmodulemd && \ + dnf -y install python3-libcomps && \ + dnf -y install postgresql && \ + dnf -y install postgresql-server && \ + dnf -y install nginx && \ + dnf -y install redis && \ + dnf -y install python3-setuptools && \ + dnf -y install gcc python3-devel openssl-devel && \ + dnf -y install libmodulemd-devel && \ + dnf -y install libcomps-devel && \ + dnf -y install ninja-build && \ + dnf -y install 'dnf-command(builddep)' && \ + dnf -y builddep createrepo_c && \ + dnf clean all + +RUN dnf -y install buildah --exclude container-selinux && \ + dnf clean all + +# Until the update is re-submitted & released +# https://bodhi.fedoraproject.org/updates/FEDORA-2019-0d122cc67a +RUN rpm -q python3-libcomps --queryformat=%{VERSION}-%{RELEASE} | grep -v 0.1.11-1 || dnf upgrade -y --enablerepo=updates-testing python3-libcomps + +RUN sed 's|^#mount_program|mount_program|g' -i /etc/containers/storage.conf + +RUN mkdir -p /etc/pulp + +RUN easy_install pip + +RUN echo "/var/lib/pgsql true postgres 0600 0750" >> /etc/fix-attrs.d/postgres + +RUN mkdir -p /etc/services.d/nginx /etc/services.d/postgresql /etc/services.d/redis /etc/services.d/pulpcore-worker@1 /etc/services.d/pulpcore-worker@2 /etc/services.d/pulpcore-resource-manager /etc/services.d/pulpcore-api /etc/services.d/pulpcore-content /var/lib/pgsql /var/run/pulpcore-worker-1 /var/run/pulpcore-worker-2 /var/run/pulpcore-resource-manager + +COPY assets/pulpcore-content.run /etc/services.d/pulpcore-content/run +COPY assets/postgres.run /etc/services.d/postgresql/run +COPY assets/redis.run /etc/services.d/redis/run +COPY assets/pulpcore-worker@2.run /etc/services.d/pulpcore-worker@2/run +COPY assets/pulpcore-resource-manager.run /etc/services.d/pulpcore-resource-manager/run +COPY assets/pulpcore-api.run /etc/services.d/pulpcore-api/run +COPY assets/postgres.prep /etc/cont-init.d/postgres +COPY assets/pulpcore-worker@1.run /etc/services.d/pulpcore-worker@1/run +COPY assets/nginx.conf /etc/nginx/nginx.conf +COPY assets/nginx.run /etc/services.d/nginx/run + +ENTRYPOINT ["/init"] diff --git a/.travis/Containerfile.j2 b/.travis/Containerfile.j2 new file mode 100644 index 00000000..4b26e7f5 --- /dev/null +++ b/.travis/Containerfile.j2 @@ -0,0 +1,22 @@ +FROM pulp_ci_base:{{ image.tag }} + +{% if s3_test | default(false) %} +# Hacking botocore (https://github.com/boto/botocore/pull/1990) +RUN pip3 install django-storages[boto3] +RUN pip3 install git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue +{% endif %} + +{% for item in plugins %} + +# Install {{ item.name }} from {{ item.source }} +{% if item.name == "pulp-certguard" %} +RUN pip3 install python-dateutil rhsm +{% endif %} +{% if item.source.startswith("./") %} +ADD {{ item.source }} {{ item.source }} +{% endif %} +RUN pip3 install {{ item.source }} +RUN ln /usr/local/lib/python3.7/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/conf.d/{{ item.name }} || true +{% endfor %} + +ENTRYPOINT ["/init"] diff --git a/.travis/ansible.cfg b/.travis/ansible.cfg new file mode 100644 index 00000000..7acf1d9d --- /dev/null +++ b/.travis/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +inventory = inventory.yaml +filter_plugins = filter +retry_files_enabled = False +transport = local +nocows = 1 +stdout_callback = yaml diff --git a/.travis/assets/nginx.conf b/.travis/assets/nginx.conf new file mode 100644 index 00000000..3d41f334 --- /dev/null +++ b/.travis/assets/nginx.conf @@ -0,0 +1,85 @@ +# TODO: Support IPv6. +# TODO: Configure SSL certificates. +# TODO: Maybe serve multiple `location`s, not just one. + +# The "nginx" package on fedora creates this user and group. +user nginx nginx; +# Gunicorn docs suggest this value. +worker_processes 1; +daemon off; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + upstream pulp-content { + server 127.0.0.1:24816; + } + + upstream pulp-api { + server 127.0.0.1:24817; + } + + server { + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + listen 80 default_server deferred; + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + # Path to Pulp's static files. + root /usr/local/lib/pulp/lib/python3.7/site-packages/rest_framework; + + location /pulp/content/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://127.0.0.1:24816; + } + + location /pulp/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://127.0.0.1:24817; + client_max_body_size 10m; + } + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://127.0.0.1:24817; + } + + include conf.d/*; + + location / { + # checks for static file, if not found proxy to app + try_files $uri =404; + } + } +} diff --git a/.travis/assets/nginx.run b/.travis/assets/nginx.run new file mode 100644 index 00000000..d04e248b --- /dev/null +++ b/.travis/assets/nginx.run @@ -0,0 +1,2 @@ +#!/usr/bin/execlineb -P +/usr/sbin/nginx \ No newline at end of file diff --git a/.travis/assets/postgres.prep b/.travis/assets/postgres.prep new file mode 100644 index 00000000..4e75a271 --- /dev/null +++ b/.travis/assets/postgres.prep @@ -0,0 +1,115 @@ +#!/usr/bin/execlineb -S0 + +backtick -n BASENAME { s6-basename ${0} } +importas -u BASENAME BASENAME +define PREFFIX "[cont-init.d] ${BASENAME}:" + +define C000 "\033[0m" +define C030 "\033[0;30m" +define C031 "\033[0;31m" +define C032 "\033[0;32m" +define C033 "\033[0;33m" +define C034 "\033[0;34m" +define C035 "\033[0;35m" +define C036 "\033[0;36m" +define C037 "\033[0;37m" +define C130 "\033[1;30m" +define C131 "\033[1;31m" +define C132 "\033[1;32m" +define C133 "\033[1;33m" +define C134 "\033[1;34m" +define C135 "\033[1;35m" +define C136 "\033[1;36m" +define C137 "\033[1;37m" + +backtick -n ! { pipeline { postgres --version } egrep -o "[0-9]{1,}\.[0-9]{1,}" } +importas -u PGVERSION ! +define PGHOME "/var/lib/pgsql" +define PGDATA "${PGHOME}/data" + +foreground { +ifthenelse { s6-test -d "${PGDATA}/base" -a -f "${PGDATA}/PG_VERSION" } +{ + backtick -n ! { redirfd -r 0 "${PGDATA}/PG_VERSION" s6-cat } + importas -u PGDATA_VERSION ! + ifelse { s6-test "${PGDATA_VERSION}" = "${PGVERSION}" } { + if { s6-echo "${PREFFIX} ${C031}${PGDATA} exists but needs upgrade ${PGDATA_VERSION} != ${PGVERSION}${C000}" } + exit 200 + } +} +{ + if { + if { s6-echo "${PREFFIX} ${C032}initdb --pgdata ${PGDATA}${C000}" } + #redirfd -w 1 /dev/null + fdmove -c 2 1 + s6-setuidgid postgres + backtick -D "C" -n LANG { printcontenv LANG } + initdb --pgdata=${PGDATA} + } + if { + if { s6-echo "${PREFFIX} ${C032}s6-hiercopy /etc/postgresql/ ${PGDATA}/ ${C000}" } + s6-hiercopy /etc/postgresql/ ${PGDATA}/ + } +} +} + +importas -u ? ? +ifelse { s6-test ${?} -ne 0 } { exit ${?} } + +background { + #redirfd -w 1 /dev/null + fdmove -c 2 1 + s6-setuidgid postgres + postgres -D ${PGDATA} +} +importas -u PG_PID ! +if { s6-echo "${PREFFIX} ${C133}started service [${PG_PID}] postgres${C000}" } + +if { s6-echo "${PREFFIX} ${C130}wait for services startup...${C000}" } + +loopwhilex -x 0 foreground { + s6-sleep -m 500 + s6-setuidgid postgres + pg_isready --quiet +} + +backtick -n ! { + s6-setuidgid postgres + psql --no-align --tuples-only --command="SELECT (1) FROM pg_database WHERE datname='pulp' LIMIT 1" +} +importas -u DATABASE_EXISTS ! + +foreground { + if { s6-test ${DATABASE_EXISTS} != "1" } + if { s6-echo "${PREFFIX} ${C032}createuser pulp${C000}" } + #redirfd -w 1 /dev/null + fdmove -c 2 1 + s6-setuidgid postgres + createuser pulp +} + +foreground { + if { s6-test ${DATABASE_EXISTS} != "1" } + if { s6-echo "${PREFFIX} ${C032}createdb --encoding=utf-8 -O pulp pulp${C000}" } + redirfd -w 1 /dev/null + fdmove -c 2 1 + s6-setuidgid postgres + createdb --encoding=utf-8 pulp +} + +foreground { + redirfd -w 1 /dev/null + redirfd -w 2 /dev/null + export DJANGO_SETTINGS_MODULE pulpcore.app.settings + export PULP_CONTENT_ORIGIN localhost + s6-setuidgid postgres + /usr/local/bin/django-admin migrate +} + +foreground { + if { s6-echo "${PREFFIX} ${C133}shutdown service [${PG_PID}] postgres${C000}" } + redirfd -w 1 /dev/null + fdmove -c 2 1 + s6-setuidgid postgres + pg_ctl -D ${PGDATA} stop +} diff --git a/.travis/assets/postgres.run b/.travis/assets/postgres.run new file mode 100644 index 00000000..f25648c1 --- /dev/null +++ b/.travis/assets/postgres.run @@ -0,0 +1,3 @@ +#!/usr/bin/execlineb -P +s6-setuidgid postgres +postgres -D /var/lib/pgsql/data \ No newline at end of file diff --git a/.travis/assets/pulpcore-api.run b/.travis/assets/pulpcore-api.run new file mode 100644 index 00000000..1438ef1d --- /dev/null +++ b/.travis/assets/pulpcore-api.run @@ -0,0 +1,4 @@ +#!/usr/bin/execlineb -P +export DJANGO_SETTINGS_MODULE pulpcore.app.settings +export PULP_SETTINGS /etc/pulp/settings.py +/usr/local/bin/gunicorn pulpcore.app.wsgi:application --bind "127.0.0.1:24817" --access-logfile - \ No newline at end of file diff --git a/.travis/assets/pulpcore-content.run b/.travis/assets/pulpcore-content.run new file mode 100644 index 00000000..a8410bc6 --- /dev/null +++ b/.travis/assets/pulpcore-content.run @@ -0,0 +1,4 @@ +#!/usr/bin/execlineb -P +export DJANGO_SETTINGS_MODULE pulpcore.app.settings +export PULP_SETTINGS /etc/pulp/settings.py +/usr/local/bin/gunicorn pulpcore.content:server --bind "127.0.0.1:24816" --worker-class "aiohttp.GunicornWebWorker" -w 2 --access-logfile - diff --git a/.travis/assets/pulpcore-resource-manager.run b/.travis/assets/pulpcore-resource-manager.run new file mode 100644 index 00000000..df54d66b --- /dev/null +++ b/.travis/assets/pulpcore-resource-manager.run @@ -0,0 +1,4 @@ +#!/usr/bin/execlineb -P +export DJANGO_SETTINGS_MODULE pulpcore.app.settings +export PULP_SETTINGS /etc/pulp/settings.py +/usr/local/bin/rq worker -w pulpcore.tasking.worker.PulpWorker -n resource-manager --pid=/var/run/pulpcore-resource-manager/resource-manager.pid -c "pulpcore.rqconfig" --disable-job-desc-logging \ No newline at end of file diff --git a/.travis/assets/pulpcore-worker@1.run b/.travis/assets/pulpcore-worker@1.run new file mode 100644 index 00000000..4ac04d04 --- /dev/null +++ b/.travis/assets/pulpcore-worker@1.run @@ -0,0 +1,5 @@ +#!/usr/bin/execlineb -P +export DJANGO_SETTINGS_MODULE pulpcore.app.settings +export PULP_SETTINGS /etc/pulp/settings.py +export PATH /usr/local/bin:/usr/bin/ +/usr/local/bin/rq worker -w pulpcore.tasking.worker.PulpWorker --pid=/var/run/pulpcore-worker-1/reserved-resource-worker-1.pid -c "pulpcore.rqconfig" --disable-job-desc-logging \ No newline at end of file diff --git a/.travis/assets/pulpcore-worker@2.run b/.travis/assets/pulpcore-worker@2.run new file mode 100644 index 00000000..6ddb2f0c --- /dev/null +++ b/.travis/assets/pulpcore-worker@2.run @@ -0,0 +1,5 @@ +#!/usr/bin/execlineb -P +export DJANGO_SETTINGS_MODULE pulpcore.app.settings +export PULP_SETTINGS /etc/pulp/settings.py +export PATH /usr/local/bin:/usr/bin/ +/usr/local/bin/rq worker -w pulpcore.tasking.worker.PulpWorker --pid=/var/run/pulpcore-worker-2/reserved-resource-worker-2.pid -c "pulpcore.rqconfig" --disable-job-desc-logging \ No newline at end of file diff --git a/.travis/assets/redis.run b/.travis/assets/redis.run new file mode 100644 index 00000000..a81269bd --- /dev/null +++ b/.travis/assets/redis.run @@ -0,0 +1,3 @@ +#!/usr/bin/execlineb -P +s6-setuidgid redis +/usr/bin/redis-server /etc/redis.conf \ No newline at end of file diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 1f5a666b..48b8e1f7 100755 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -2,8 +2,8 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template @@ -16,7 +16,7 @@ COMMIT_MSG=$(git log --format=%B --no-merges -1) export COMMIT_MSG if [ -f $PRE_BEFORE_INSTALL ]; then - $PRE_BEFORE_INSTALL + source $PRE_BEFORE_INSTALL fi if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then @@ -26,17 +26,19 @@ fi if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master" ] then - export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}') export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}') - export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}') - export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') - export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_FILE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp_file\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_CONTAINER_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp_container\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_RPM_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp_rpm\/pull\/(\d+)' | awk -F'/' '{print $7}') else - export PULP_PR_NUMBER= + export PULPCORE_PR_NUMBER= export PULP_SMASH_PR_NUMBER= - export PULP_ROLES_PR_NUMBER= - export PULP_BINDINGS_PR_NUMBER= - export PULP_OPERATOR_PR_NUMBER= + export PULP_OPENAPI_GENERATOR_PR_NUMBER= + export PULP_FILE_PR_NUMBER= + export PULP_CONTAINER_PR_NUMBER= + export PULP_RPM_PR_NUMBER= fi # dev_requirements contains tools needed for flake8, etc. @@ -51,45 +53,32 @@ pip install -r dev_requirements.txt # Lint code. flake8 --config flake8.cfg -cd .. -git clone --depth=1 https://github.com/pulp/ansible-pulp.git -if [ -n "$PULP_ROLES_PR_NUMBER" ]; then - cd ansible-pulp - git fetch --depth=1 origin pull/$PULP_ROLES_PR_NUMBER/head:$PULP_ROLES_PR_NUMBER - git checkout $PULP_ROLES_PR_NUMBER - cd .. -fi +# check for imports from pulpcore that aren't pulpcore.plugin +./.travis/check_pulpcore_imports.sh - -git clone --depth=1 https://github.com/pulp/pulp-operator.git -if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then - cd pulp-operator - git fetch --depth=1 origin pull/$PULP_OPERATOR_PR_NUMBER/head:$PULP_OPERATOR_PR_NUMBER - git checkout $PULP_OPERATOR_PR_NUMBER - RELEASE_VERSION=v0.9.0 - curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu - chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu - sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest - cd .. -fi +cd .. git clone https://github.com/pulp/pulp-openapi-generator.git -if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then +if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then cd pulp-openapi-generator - git fetch origin pull/$PULP_BINDINGS_PR_NUMBER/head:$PULP_BINDINGS_PR_NUMBER - git checkout $PULP_BINDINGS_PR_NUMBER + git fetch origin pull/$PULP_OPENAPI_GENERATOR_PR_NUMBER/head:$PULP_OPENAPI_GENERATOR_PR_NUMBER + git checkout $PULP_OPENAPI_GENERATOR_PR_NUMBER cd .. fi +cd pulp-openapi-generator +sed -i -e 's/localhost:24817/pulp/g' generate.sh +sed -i -e 's/:24817/pulp/g' generate.sh +cd .. git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master -if [ -n "$PULP_PR_NUMBER" ]; then - cd pulpcore - git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER - git checkout $PULP_PR_NUMBER - cd .. +cd pulpcore +if [ -n "$PULPCORE_PR_NUMBER" ]; then + git fetch --depth=1 origin pull/$PULPCORE_PR_NUMBER/head:$PULPCORE_PR_NUMBER + git checkout $PULPCORE_PR_NUMBER fi +cd .. @@ -112,10 +101,41 @@ if [ -z "$TRAVIS_TAG" ]; then fi -pip install ansible +git clone --depth=1 https://github.com/pulp/pulp_file.git --branch master +if [ -n "$PULP_FILE_PR_NUMBER" ]; then + cd pulp_file + git fetch --depth=1 origin pull/$PULP_FILE_PR_NUMBER/head:$PULP_FILE_PR_NUMBER + git checkout $PULP_FILE_PR_NUMBER + cd .. +fi + +git clone --depth=1 https://github.com/pulp/pulp_container.git --branch master +if [ -n "$PULP_CONTAINER_PR_NUMBER" ]; then + cd pulp_container + git fetch --depth=1 origin pull/$PULP_CONTAINER_PR_NUMBER/head:$PULP_CONTAINER_PR_NUMBER + git checkout $PULP_CONTAINER_PR_NUMBER + cd .. +fi + +git clone --depth=1 https://github.com/pulp/pulp_rpm.git --branch master +if [ -n "$PULP_RPM_PR_NUMBER" ]; then + cd pulp_rpm + git fetch --depth=1 origin pull/$PULP_RPM_PR_NUMBER/head:$PULP_RPM_PR_NUMBER + git checkout $PULP_RPM_PR_NUMBER + cd .. +fi + +# Intall requirements for ansible playbooks +pip install docker netaddr boto3 + +# Install ansible with the boto3 tags to dict fix +# There is a PR for this issue: +# https://github.com/ansible-collections/amazon.aws/pull/37 +# Be aware, that the code will have moved to that collection with upcoming releases of ansible +pip install git+https://github.com/mdellweg/ansible.git@fix_boto3_tags_dict cd pulp-2to3-migration if [ -f $POST_BEFORE_INSTALL ]; then - $POST_BEFORE_INSTALL + source $POST_BEFORE_INSTALL fi diff --git a/.travis/before_script.sh b/.travis/before_script.sh index 33589567..885eafdf 100755 --- a/.travis/before_script.sh +++ b/.travis/before_script.sh @@ -2,46 +2,31 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template set -euv +source .travis/utils.sh + export PRE_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/pre_before_script.sh export POST_BEFORE_SCRIPT=$TRAVIS_BUILD_DIR/.travis/post_before_script.sh -# Aliases for running commands in the pulp-api container. -export PULP_API_POD=$(sudo kubectl get pods | grep -E -o "pulp-api-(\w+)-(\w+)") -# Run a command -export CMD_PREFIX="sudo kubectl exec $PULP_API_POD --" -# Run a command, and pass STDIN -export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" - if [[ -f $PRE_BEFORE_SCRIPT ]]; then - $PRE_BEFORE_SCRIPT -fi - -mkdir -p ~/.config/pulp_smash - -if [[ -f .travis/pulp-smash-config.json ]]; then - sed "s/localhost/$(hostname)/g" .travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json -else - sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json + source $PRE_BEFORE_SCRIPT fi +# Developers often want to know the final pulp config +echo "PULP CONFIG:" +tail -v -n +1 .travis/settings/settings.* ~/.config/pulp_smash/settings.json - -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then - # Many tests require pytest/mock, but users do not need them at runtime - # (or to add plugins on top of pulpcore or pulp container images.) - # So install it here, rather than in the image Dockerfile. - $CMD_PREFIX pip3 install pytest mock - # Many functional tests require these - $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then + # Many functional tests require these + cmd_prefix dnf install -yq lsof which dnf-plugins-core fi if [[ -f $POST_BEFORE_SCRIPT ]]; then - $POST_BEFORE_SCRIPT + source $POST_BEFORE_SCRIPT fi diff --git a/.travis/build_container.yaml b/.travis/build_container.yaml new file mode 100644 index 00000000..21f9007c --- /dev/null +++ b/.travis/build_container.yaml @@ -0,0 +1,48 @@ +# Ansible playbook to create the pulp service containers image +--- +- hosts: localhost + gather_facts: false + vars_files: + - vars/main.yaml + tasks: + - name: "Generate Containerfile from template" + template: + src: Containerfile.j2 + dest: Containerfile + + - name: "Build ci base image" + docker_image: + name: "pulp_ci_base" + tag: "{{ image.tag }}" + build: + path: "." + dockerfile: Containerfile.ci_base + nocache: "{{ not cache | default(true) | bool }}" + pull: true + state: present + source: build + + - name: "Build pulp image" + docker_image: + # We build from the ../.. (parent dir of pulpcore git repo) Docker build + # "context" so that repos like pulp-smash are accessible to Docker + # build. So that PR branches can be used via relative paths. + # + # We default to using the docker build / podman buildah cache, for + # 1-off-builds and Travis CI purposes (which has no cache across CI runs.) + # Run build.yaml with -e cache=false if your builds are using outdated + # layers. + name: "{{ image.name }}" + tag: "{{ image.tag }}" + build: + path: "../.." + dockerfile: "{{ playbook_dir }}/Containerfile" + nocache: "{{ not cache | default(true) | bool }}" + pull: false + state: present + source: build + + - name: "Clean image cache" + docker_prune: + images : true +... diff --git a/.travis/check_commit.sh b/.travis/check_commit.sh index 9bc4aca6..7421d48c 100755 --- a/.travis/check_commit.sh +++ b/.travis/check_commit.sh @@ -2,8 +2,8 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template diff --git a/.travis/check_pulpcore_imports.sh b/.travis/check_pulpcore_imports.sh new file mode 100755 index 00000000..ec2d8225 --- /dev/null +++ b/.travis/check_pulpcore_imports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -uv + +# check for imports not from pulpcore.plugin. exclude tests +MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") + +if [ $? -ne 1 ]; then + printf "\nERROR: Detected bad imports from pulpcore:\n" + echo "$MATCHES" + printf "\nPlugins should import from pulpcore.plugin." + exit 1 +fi diff --git a/.travis/cherrypick.py b/.travis/cherrypick.py index 048ab3bd..2491af33 100644 --- a/.travis/cherrypick.py +++ b/.travis/cherrypick.py @@ -66,7 +66,7 @@ def get_merged_commits(pr): g = Github(GITHUB_TOKEN) grepo = g.get_repo(REPOSITORY) -(label,) = (l for l in grepo.get_labels() if l.name == PR_LABEL) +(label,) = (lbl for lbl in grepo.get_labels() if lbl.name == PR_LABEL) issues = grepo.get_issues(labels=[label], state="all", sort="updated", direction="asc") cherrypicks = [] @@ -91,13 +91,13 @@ def get_merged_commits(pr): if ret.returncode != 0: print(f"Failed to cherry-pick commit {commit.sha}: {ret.stderr.decode('ascii')}") exit(1) - else: - cherrypicks.append(issue) - print(f"Cherry-picked commit {commit.sha}.") + + cherrypicks.append(issue) + print(f"Cherry-picked commit {commit.sha}.") # check if we cherry picked anything if len(cherrypicks) == 0: - print(f"No cherry picks detected.") + print("No cherry picks detected.") exit(0) # push our changes @@ -115,7 +115,7 @@ def get_merged_commits(pr): for cp in cherrypicks: labels = cp.labels labels.remove(label) - cp.edit(labels=labels) + cp.edit(labels=[lbl.name for lbl in labels]) print(f"Removed label '{PR_LABEL}' from PR #{cp.number}.") print("Cherry picking complete.") diff --git a/.travis/filter/repr.py b/.travis/filter/repr.py new file mode 100644 index 00000000..95f17396 --- /dev/null +++ b/.travis/filter/repr.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + + +def _repr_filter(value): + return repr(value) + + +# ---- Ansible filters ---- +class FilterModule(object): + """repr filter""" + + def filters(self): + """Filter associations""" + return { + "repr": _repr_filter, + } diff --git a/.travis/install.sh b/.travis/install.sh index 76997d77..2217b337 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -2,21 +2,21 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template set -euv -if [ "$TEST" = 'docs' ]; then +if [ "$TEST" = "docs" ]; then pip install -r ../pulpcore/doc_requirements.txt pip install -r doc_requirements.txt fi pip install -r functest_requirements.txt -cd $TRAVIS_BUILD_DIR/../pulpcore/containers/ +cd .travis # Although the tag name is not used outside of this script, we might use it # later. And it is nice to have a friendly identifier for it. @@ -37,7 +37,7 @@ elif [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then # For push builds and hopefully cron builds elif [ -n "$TRAVIS_BRANCH" ]; then TAG=$(echo $TRAVIS_BRANCH | tr / _) - if [ "$TAG" = "master" ]; then + if [ "${TAG}" = "master" ]; then TAG=latest fi else @@ -45,7 +45,6 @@ else TAG=$(git rev-parse --abbrev-ref HEAD | tr / _) fi - if [ -e $TRAVIS_BUILD_DIR/../pulp_file ]; then PULP_FILE=./pulp_file else @@ -64,79 +63,60 @@ else PULP_RPM=git+https://github.com/pulp/pulp_rpm.git@master fi +mkdir vars if [ -n "$TRAVIS_TAG" ]; then # Install the plugin only and use published PyPI packages for the rest - cat > vars/vars.yaml << VARSYAML + # Quoting ${TAG} ensures Ansible casts the tag as a string. + cat > vars/main.yaml << VARSYAML --- -images: - - pulp-2to3-migration-${TAG}: - image_name: pulp-2to3-migration - tag: $TAG - pulpcore: pulpcore - plugins: - - ./pulp-2to3-migration - - pulp_file - - pulp_container - - pulp_rpm +image: + name: pulp + tag: "${TAG}" +plugins: + - name: pulpcore + source: pulpcore + - name: pulp-2to3-migration + source: ./pulp-2to3-migration + - name: pulp_file + source: pulp_file + - name: pulp_container + source: pulp_container + - name: pulp_rpm + source: pulp_rpm +services: + - name: pulp + image: "pulp:${TAG}" + volumes: + - ./settings:/etc/pulp VARSYAML else - cat > vars/vars.yaml << VARSYAML + cat > vars/main.yaml << VARSYAML --- -images: - - pulp-2to3-migration-${TAG}: - image_name: pulp-2to3-migration - tag: $TAG - pulpcore: ./pulpcore - plugins: - - ./pulp-2to3-migration - - $PULP_FILE - - $PULP_CONTAINER - - $PULP_RPM +image: + name: pulp + tag: "${TAG}" +plugins: + - name: pulpcore + source: ./pulpcore + - name: pulp-2to3-migration + source: ./pulp-2to3-migration + - name: pulp_file + source: $PULP_FILE + - name: pulp_container + source: $PULP_CONTAINER + - name: pulp_rpm + source: $PULP_RPM +services: + - name: pulp + image: "pulp:${TAG}" + volumes: + - ./settings:/etc/pulp VARSYAML fi -ansible-playbook -v build.yaml - -cd $TRAVIS_BUILD_DIR/../pulp-operator -# Tell pulp-perator to deploy our image -# NOTE: With k3s 1.17, $TAG must be quoted. So that 3.0 does not become 3. -cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML -apiVersion: pulpproject.org/v1alpha1 -kind: Pulp -metadata: - name: example-pulp -spec: - pulp_file_storage: - # k3s local-path requires this - access_mode: "ReadWriteOnce" - # We have a little over 40GB free on Travis VMs/instances - size: "40Gi" - image: pulp-2to3-migration - tag: "${TAG}" - database_connection: - username: pulp - password: pulp - admin_password: pulp -CRYAML - -# Install k3s, lightweight Kubernetes -.travis/k3s-install.sh -# Deploy pulp-operator, with the pulp containers, according to CRYAML -sudo ./up.sh -# Needed for the script below -# Since it is being run during install rather than actual tests (unlike in -# pulp-operator), and therefore does not trigger the equivalent after_failure -# travis commands. -show_logs_and_return_non_zero() { - readonly local rc="$?" - - for containerlog in "pulp-api" "pulp-content" "pulp-resource-manager" "pulp-worker" - do - echo -en "travis_fold:start:$containerlog"'\\r' - sudo kubectl logs -l app=$containerlog --tail=10000 - echo -en "travis_fold:end:$containerlog"'\\r' - done +cat >> vars/main.yaml << VARSYAML +pulp_settings: null +VARSYAML - return "${rc}" -} -.travis/pulp-operator-check-and-wait.sh || show_logs_and_return_non_zero +ansible-playbook build_container.yaml +ansible-playbook start_container.yaml diff --git a/.travis/inventory.yaml b/.travis/inventory.yaml new file mode 100644 index 00000000..c1c9fb6c --- /dev/null +++ b/.travis/inventory.yaml @@ -0,0 +1,11 @@ +--- +all: + children: + containers: + hosts: + pulp: + pulp-fixtures: + minio: + vars: + ansible_connection: docker +... diff --git a/.travis/publish_client_gem.sh b/.travis/publish_client_gem.sh index 258b58d4..c683e949 100755 --- a/.travis/publish_client_gem.sh +++ b/.travis/publish_client_gem.sh @@ -2,8 +2,8 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template @@ -13,11 +13,8 @@ echo "--- :rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials sudo chmod 600 ~/.gem/credentials -django-admin runserver 24817 >> ~/django_runserver.log 2>&1 & -sleep 5 - cd $TRAVIS_BUILD_DIR -export REPORTED_VERSION=$(http :24817/pulp/api/v3/status/ | jq --arg plugin pulp_2to3_migration -r '.versions[] | select(.component == $plugin) | .version') +export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin pulp_2to3_migration -r '.versions[] | select(.component == $plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -38,12 +35,10 @@ then exit fi -cd -git clone https://github.com/pulp/pulp-openapi-generator.git -cd pulp-openapi-generator +cd "${TRAVIS_BUILD_DIR}"/../pulp-openapi-generator ./generate.sh pulp_2to3_migration ruby $VERSION cd pulp_2to3_migration-client gem build pulp_2to3_migration_client -GEM_FILE="$(ls | grep pulp_2to3_migration_client-)" +GEM_FILE="$(ls pulp_2to3_migration_client-*)" gem push ${GEM_FILE} diff --git a/.travis/publish_client_pypi.sh b/.travis/publish_client_pypi.sh index 54d2fd7e..707e7188 100755 --- a/.travis/publish_client_pypi.sh +++ b/.travis/publish_client_pypi.sh @@ -2,8 +2,8 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template @@ -11,11 +11,8 @@ set -euv pip install twine -django-admin runserver 24817 >> ~/django_runserver.log 2>&1 & -sleep 5 - cd "${TRAVIS_BUILD_DIR}" -export REPORTED_VERSION=$(http :24817/pulp/api/v3/status/ | jq --arg plugin pulp_2to3_migration -r '.versions[] | select(.component == $plugin) | .version') +export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin pulp_2to3_migration -r '.versions[] | select(.component == $plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -36,9 +33,7 @@ then exit fi -cd -git clone https://github.com/pulp/pulp-openapi-generator.git -cd pulp-openapi-generator +cd "${TRAVIS_BUILD_DIR}"/../pulp-openapi-generator ./generate.sh pulp_2to3_migration python $VERSION cd pulp_2to3_migration-client diff --git a/.travis/publish_plugin_pypi.sh b/.travis/publish_plugin_pypi.sh index 50cbd1a6..48bd880c 100755 --- a/.travis/publish_plugin_pypi.sh +++ b/.travis/publish_plugin_pypi.sh @@ -2,8 +2,8 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template diff --git a/.travis/script.sh b/.travis/script.sh index 7dc783de..777c148a 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -3,13 +3,15 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template set -mveuo pipefail +source .travis/utils.sh + export POST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/post_script.sh export POST_DOCS_TEST=$TRAVIS_BUILD_DIR/.travis/post_docs_test.sh export FUNC_TEST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/func_test_script.sh @@ -18,34 +20,31 @@ export FUNC_TEST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/func_test_script.sh # Gets set in .travis/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings +export PULP_SETTINGS=$TRAVIS_BUILD_DIR/.travis/settings/settings.py -if [ "$TEST" = 'docs' ]; then - - +if [ "$TEST" = "docs" ]; then cd docs make html cd .. if [ -f $POST_DOCS_TEST ]; then - $POST_DOCS_TEST + source $POST_DOCS_TEST fi exit fi cd ../pulp-openapi-generator -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') - -if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then - git fetch origin pull/$PULP_BINDINGS_PR_NUMBER/head:$PULP_BINDINGS_PR_NUMBER - git checkout $PULP_BINDINGS_PR_NUMBER -fi ./generate.sh pulpcore python pip install ./pulpcore-client - ./generate.sh pulp_2to3_migration python - pip install ./pulp_2to3_migration-client - +./generate.sh pulp_2to3_migration python +pip install ./pulp_2to3_migration-client +./generate.sh pulp_file python +pip install ./pulp_file-client +./generate.sh pulp_container python +pip install ./pulp_container-client +./generate.sh pulp_rpm python +pip install ./pulp_rpm-client cd $TRAVIS_BUILD_DIR if [ "$TEST" = 'bindings' ]; then @@ -58,70 +57,49 @@ if [ "$TEST" = 'bindings' ]; then rm -rf ./pulpcore-client - ./generate.sh pulpcore ruby + ./generate.sh pulpcore ruby 0 cd pulpcore-client gem build pulpcore_client gem install --both ./pulpcore_client-0.gem cd .. - rm -rf ./pulp_2to3_migration-client - ./generate.sh pulp_2to3_migration ruby + ./generate.sh pulp_2to3_migration ruby 0 cd pulp_2to3_migration-client gem build pulp_2to3_migration_client gem install --both ./pulp_2to3_migration_client-0.gem cd .. - ruby $TRAVIS_BUILD_DIR/.travis/test_bindings.rb exit fi -# Aliases for running commands in the pulp-api container. -export PULP_API_POD=$(sudo kubectl get pods | grep -E -o "pulp-api-(\w+)-(\w+)") -# Run a command -export CMD_PREFIX="sudo kubectl exec $PULP_API_POD --" -# Run a command, and pass STDIN -export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" -# The alias does not seem to work in Travis / the scripting framework -#alias pytest="$CMD_PREFIX pytest" - -cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt" -$CMD_PREFIX pip3 install -r /tmp/test_requirements.txt +cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt # Run unit tests. -$CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulp_2to3_migration/tests/unit/" - -# Note: This function is in the process of being merged into after_failure -show_logs_and_return_non_zero() { - readonly local rc="$?" - return "${rc}" -} -export -f show_logs_and_return_non_zero +cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulp_2to3_migration/tests/unit/" # Run functional tests -set +u - -export PYTHONPATH=$TRAVIS_BUILD_DIR:$TRAVIS_BUILD_DIR/../pulpcore:${PYTHONPATH} - -set -u +export PYTHONPATH=$TRAVIS_BUILD_DIR:$TRAVIS_BUILD_DIR/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} if [[ "$TEST" == "performance" ]]; then + wget -qO- https://github.com/crazy-max/travis-wait-enhanced/releases/download/v1.0.0/travis-wait-enhanced_1.0.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -zxvf - travis-wait-enhanced echo "--- Performance Tests ---" if [[ -z ${PERFORMANCE_TEST+x} ]]; then - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance || show_logs_and_return_non_zero + travis-wait-enhanced --interval=1m --timeout=40m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance else - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance.test_$PERFORMANCE_TEST || show_logs_and_return_non_zero + travis-wait-enhanced --interval=1m --timeout=40m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance.test_$PERFORMANCE_TEST fi exit fi if [ -f $FUNC_TEST_SCRIPT ]; then - $FUNC_TEST_SCRIPT + source $FUNC_TEST_SCRIPT else - pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional || show_logs_and_return_non_zero + pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional fi if [ -f $POST_SCRIPT ]; then - $POST_SCRIPT + source $POST_SCRIPT fi diff --git a/.travis/settings.py.j2 b/.travis/settings.py.j2 new file mode 100644 index 00000000..e934727c --- /dev/null +++ b/.travis/settings.py.j2 @@ -0,0 +1,23 @@ +CONTENT_ORIGIN = "http://pulp:80" +ANSIBLE_API_HOSTNAME = "http://pulp:80" +ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" +TOKEN_AUTH_DISABLED = True + +{% if pulp_settings %} +{% for key, value in pulp_settings.items() %} +{{ key | upper }} = {{ value | repr }} +{% endfor %} +{% endif %} + +{% if s3_test | default(false) %} +AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" +AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" +AWS_S3_REGION_NAME = "eu-central-1" +AWS_S3_ADDRESSING_STYLE = "path" +S3_USE_SIGV4 = True +AWS_S3_SIGNATURE_VERSION = "s3v4" +AWS_STORAGE_BUCKET_NAME = "pulp3" +AWS_S3_ENDPOINT_URL = "http://minio:9000" +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" +AWS_DEFAULT_ACL = "@none None" +{% endif %} diff --git a/.travis/smash-config.json b/.travis/smash-config.json new file mode 100644 index 00000000..9af43820 --- /dev/null +++ b/.travis/smash-config.json @@ -0,0 +1,33 @@ +{ + "pulp": { + "auth": [ + "admin", + "password" + ], + "selinux enabled": false, + "version": "3" + }, + "hosts": [ + { + "hostname": "pulp", + "roles": { + "api": { + "port": 80, + "scheme": "http", + "service": "nginx" + }, + "content": { + "port": 80, + "scheme": "http", + "service": "pulp_content_app" + }, + "pulp resource manager": {}, + "pulp workers": {}, + "redis": {}, + "shell": { + "transport": "docker" + } + } + } + ] +} diff --git a/.travis/start_container.yaml b/.travis/start_container.yaml new file mode 100644 index 00000000..d0394846 --- /dev/null +++ b/.travis/start_container.yaml @@ -0,0 +1,85 @@ +# Ansible playbook to start the pulp service container and its supporting services +--- +- hosts: localhost + gather_facts: false + vars_files: + - vars/main.yaml + tasks: + - name: "Create Settings Directories" + file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - settings + - ~/.config/pulp_smash + + - name: "Generate Pulp Settings" + template: + src: settings.py.j2 + dest: settings/settings.py + + - name: "Configure pulp-smash" + copy: + src: smash-config.json + dest: ~/.config/pulp_smash/settings.json + + - name: "Setup docker networking" + docker_network: + name: pulp_ci_bridge + + - name: "Start Service Containers" + docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + auto_remove: true + recreate: true + privileged: false + networks: + - name: pulp_ci_bridge + aliases: "{{ item.name }}" + volumes: "{{ item.volumes | default(omit) }}" + env: "{{ item.env | default(omit) }}" + command: "{{ item.command | default(omit) }}" + state: started + loop: "{{ services | default([]) }}" + + - name: "Retrieve Docker Network Info" + docker_network_info: + name: pulp_ci_bridge + register: pulp_ci_bridge_info + + - name: "Update /etc/hosts" + lineinfile: + path: /etc/hosts + regexp: "\\s{{ item.value.Name }}\\s*$" + line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" + loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" + become: true + + - name: "Create Pulp Bucket" + s3_bucket: + aws_access_key: "{{ minio_access_key }}" + aws_secret_key: "{{ minio_secret_key }}" + s3_url: "http://minio:9000" + region: eu-central-1 + name: pulp3 + state: present + when: s3_test | default(false) + + - name: "Wait for Pulp" + uri: + url: "http://pulp/pulp/api/v3/status/" + follow_redirects: none + register: result + until: result.status == 200 + retries: 6 + delay: 5 + +- hosts: pulp + gather_facts: false + tasks: + - name: "Set pulp admin password" + command: + cmd: "pulpcore-manager reset-admin-password --password password" +... diff --git a/.travis/utils.sh b/.travis/utils.sh new file mode 100644 index 00000000..cae65ada --- /dev/null +++ b/.travis/utils.sh @@ -0,0 +1,13 @@ +# This file is meant to be sourced by ci-scripts + +PULP_CONTAINER=pulp + +# Run a command +cmd_prefix() { + docker exec "$PULP_CONTAINER" "$@" +} + +# Run a command, and pass STDIN +cmd_stdin_prefix() { + docker exec -i "$PULP_CONTAINER" "$@" +} diff --git a/.travis/validate_commit_message.py b/.travis/validate_commit_message.py index a29b7879..c9d27444 100644 --- a/.travis/validate_commit_message.py +++ b/.travis/validate_commit_message.py @@ -1,11 +1,12 @@ # WARNING: DO NOT EDIT! # -# This file was generated by plugin_template, and is managed by bootstrap.py. Please use -# bootstrap.py to update this file. +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --travis pulp_2to3_migration' to update this file. # # For more info visit https://github.com/pulp/plugin_template import glob +import os import re import requests import subprocess @@ -14,13 +15,16 @@ KEYWORDS = ["fixes", "closes", "re", "ref"] NO_ISSUE = "[noissue]" STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"] +REDMINE_URL = "https://pulp.plan.io" +CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] sha = sys.argv[1] +project = "" message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") def __check_status(issue): - response = requests.get("https://pulp.plan.io/issues/{}.json".format(issue)) + response = requests.get(f"{REDMINE_URL}/issues/{issue}.json") response.raise_for_status() bug_json = response.json() status = bug_json["issue"]["status"]["name"] @@ -30,27 +34,38 @@ def __check_status(issue): "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES)) ) + if project: + project_id = bug_json["issue"]["project"]["id"] + project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json() + if project_json["project"]["identifier"] != project: + sys.exit(f"Error: issue {issue} is not in the {project} project.") + def __check_changelog(issue): - if len(glob.glob(f"CHANGES/**/{issue}.*", recursive=True)) < 1: + matches = glob.glob(f"CHANGES/**/{issue}.*", recursive=True) + + if len(matches) < 1: sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.") + for match in matches: + if os.path.splitext(match)[1] not in CHANGELOG_EXTS: + sys.exit(f"Invalid extension for changelog entry '{match}'.") print("Checking commit message for {sha}.".format(sha=sha[0:7])) # validate the issue attached to the commit -if NO_ISSUE in message: - print("Commit {sha} has no issue attached. Skipping issue check".format(sha=sha[0:7])) -else: - regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) - pattern = re.compile(regex) +regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) +pattern = re.compile(regex) - issues = pattern.findall(message) +issues = pattern.findall(message) - if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) +if issues: + for issue in pattern.findall(message): + __check_status(issue) + __check_changelog(issue) +else: + if NO_ISSUE in message: + print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) else: sys.exit( "Error: no attached issues found for {sha}. If this was intentional, add " diff --git a/template_config.yml b/template_config.yml index e61b95ea..3581bec4 100644 --- a/template_config.yml +++ b/template_config.yml @@ -2,9 +2,12 @@ # were not present before running plugin-template have been added with their default values. additional_plugins: -- { name: pulp_file, branch: master } -- { name: pulp_container, branch: master } -- { name: pulp_rpm, branch: master } +- branch: master + name: pulp_file +- branch: master + name: pulp_container +- branch: master + name: pulp_rpm black: false check_commit_message: true cherry_pick_automation: true @@ -30,9 +33,11 @@ pulpcore_branch: master pulpcore_pip_version_specifier: null pydocstyle: false pypi_username: pulp +redmine_project: null stable_branch: '0.1' test_bindings: false test_performance: false +test_s3: false travis_addtl_services: - mongodb travis_notifications: From 97933f234a44d4e882fb9ddc12723b855f6446b8 Mon Sep 17 00:00:00 2001 From: Tatiana Tereshchenko Date: Tue, 12 May 2020 20:29:08 +0200 Subject: [PATCH 2/5] Temporarily comment out the check for imports We need to decide what to do with it for the migraiton plugin. It seem to have some valid special cases when to import it. [noissue] --- .travis/before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 48b8e1f7..ceedb18e 100755 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -54,7 +54,7 @@ pip install -r dev_requirements.txt flake8 --config flake8.cfg # check for imports from pulpcore that aren't pulpcore.plugin -./.travis/check_pulpcore_imports.sh +#./.travis/check_pulpcore_imports.sh cd .. From c6ac4edda32c36a58dff8eb5cfbac1adb36d73b8 Mon Sep 17 00:00:00 2001 From: Tatiana Tereshchenko Date: Tue, 12 May 2020 22:05:12 +0200 Subject: [PATCH 3/5] Change $CMD_PREFIX to cmd_prefix It's in the custom script whcih is not managed by plugin_template [noissue] --- .travis/post_before_script.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis/post_before_script.sh b/.travis/post_before_script.sh index cbf8d2a1..c7f549a7 100755 --- a/.travis/post_before_script.sh +++ b/.travis/post_before_script.sh @@ -5,9 +5,9 @@ set -euv sudo sed -i "s/bindIp: 127.0.0.1/bindIp: 127.0.0.1,$(ip address show dev ens4 | grep -o "inet [0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*")/g" /etc/mongod.conf sudo systemctl restart mongod -$CMD_PREFIX bash -c "git clone https://github.com/pulp/pulp-2to3-migration-test-fixtures" -$CMD_PREFIX bash -c "mv pulp-2to3-migration-test-fixtures/20191031/var/lib/pulp/content /var/lib/pulp/content" -$CMD_PREFIX bash -c "mv pulp-2to3-migration-test-fixtures/20191031/var/lib/pulp/published /var/lib/pulp/published" +cmd_prefix bash -c "git clone https://github.com/pulp/pulp-2to3-migration-test-fixtures" +cmd_prefix bash -c "mv pulp-2to3-migration-test-fixtures/20191031/var/lib/pulp/content /var/lib/pulp/content" +cmd_prefix bash -c "mv pulp-2to3-migration-test-fixtures/20191031/var/lib/pulp/published /var/lib/pulp/published" wget https://github.com/pulp/pulp-2to3-migration-test-fixtures/raw/master/20191031/pulp2filecontent.20191031.archive mongorestore --archive=pulp2filecontent.20191031.archive From 268c8a8b121b95f57ba5377858e0d7f5789d1ba1 Mon Sep 17 00:00:00 2001 From: Tatiana Tereshchenko Date: Wed, 13 May 2020 12:41:35 +0200 Subject: [PATCH 4/5] Update the hostname to `pulp` [noissue] --- pulp_2to3_migration/tests/functional/test_file_migration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulp_2to3_migration/tests/functional/test_file_migration.py b/pulp_2to3_migration/tests/functional/test_file_migration.py index cc329e58..0f97c385 100644 --- a/pulp_2to3_migration/tests/functional/test_file_migration.py +++ b/pulp_2to3_migration/tests/functional/test_file_migration.py @@ -1,5 +1,4 @@ import json -import socket import unittest from pulpcore.client.pulpcore import ( @@ -118,7 +117,7 @@ def setUpClass(cls): configuration = Configuration() configuration.username = 'admin' configuration.password = 'password' - configuration.host = 'http://{}:24817'.format(socket.gethostname()) + configuration.host = 'http://pulp' configuration.safe_chars_for_path_param = '/' core_client = CoreApiClient(configuration) From 4a937253ed0e0a18e05d9446fe8f2c20fd422855 Mon Sep 17 00:00:00 2001 From: Tatiana Tereshchenko Date: Thu, 14 May 2020 12:49:10 +0200 Subject: [PATCH 5/5] Fix mongo connection to mongodb [noissue] --- .travis/post_before_install.sh | 3 ++- .travis/post_before_script.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis/post_before_install.sh b/.travis/post_before_install.sh index 466adc70..5e1ab317 100755 --- a/.travis/post_before_install.sh +++ b/.travis/post_before_install.sh @@ -2,6 +2,7 @@ set -mveuo pipefail -sed -i "s/'seeds': 'localhost:27017'/'seeds': '$(hostname):27017'/g" $TRAVIS_BUILD_DIR/pulp_2to3_migration/app/settings.py +export MONGODB_IP=$(ip address show dev docker0 | grep -o "inet [0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*") +sed -i "s/'seeds': 'localhost:27017'/'seeds': '$MONGODB_IP:27017'/g" $TRAVIS_BUILD_DIR/pulp_2to3_migration/app/settings.py sed -i "s/'username': ''/'username': 'travis'/g" $TRAVIS_BUILD_DIR/pulp_2to3_migration/app/settings.py sed -i "s/'password': ''/'password': 'travis'/g" $TRAVIS_BUILD_DIR/pulp_2to3_migration/app/settings.py diff --git a/.travis/post_before_script.sh b/.travis/post_before_script.sh index c7f549a7..156fd8c8 100755 --- a/.travis/post_before_script.sh +++ b/.travis/post_before_script.sh @@ -2,7 +2,7 @@ set -euv -sudo sed -i "s/bindIp: 127.0.0.1/bindIp: 127.0.0.1,$(ip address show dev ens4 | grep -o "inet [0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*")/g" /etc/mongod.conf +sudo sed -i "s/bindIp: 127.0.0.1/bindIp: 127.0.0.1,$(ip address show dev docker0 | grep -o "inet [0-9]*\.[0-9]*\.[0-9]*\.[0-9]*" | grep -o "[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*")/g" /etc/mongod.conf sudo systemctl restart mongod cmd_prefix bash -c "git clone https://github.com/pulp/pulp-2to3-migration-test-fixtures"