Merge pull request #148 from goosemania/travis-update

Update travis

.travis.yml

@@ -1,7 +1,7 @@
 # WARNING: DO NOT EDIT!
 #
-# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
-# bootstrap.py to update this file.
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --travis pulp_2to3_migration' to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
 ---
@@ -34,14 +34,10 @@ install: .travis/install.sh
 before_script: .travis/before_script.sh
 script: .travis/script.sh
 after_failure:
-  - http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/
-  - sudo docker images
-  - sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000
-  - sudo kubectl logs -l name=pulp-operator -c operator --tail=10000
-  - sudo kubectl logs -l app=pulp-api --tail=50000
-  - sudo kubectl logs -l app=pulp-content --tail=10000
-  - sudo kubectl logs -l app=pulp-resource-manager --tail=10000
-  - sudo kubectl logs -l app=pulp-worker --tail=10000
+  - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/
+  - docker images
+  - docker ps -a
+  - docker logs pulp
 jobs:
   include:
     # So long as we have a build matrix of only 1 build (via only 1 python
@@ -83,6 +79,12 @@ jobs:
       env:
         - TEST=bindings
       if: tag IS present
+    - stage: cherry-pick
+      if: type = cron
+      before_install: skip
+      install: skip
+      before_script: skip
+      script: bash .travis/cherrypick.sh
 notifications:
   irc:
     channels:

.travis/.gitignore

@@ -0,0 +1,3 @@
+Containerfile
+settings/
+vars/

.travis/Containerfile.ci_base

@@ -0,0 +1,82 @@
+FROM fedora:31
+ADD https://github.com/just-containers/s6-overlay/releases/download/v1.22.1.0/s6-overlay-amd64.tar.gz /tmp/
+
+RUN tar xzf /tmp/s6-overlay-amd64.tar.gz -C / --exclude="./bin" && \
+    tar xzf /tmp/s6-overlay-amd64.tar.gz -C /usr ./bin
+
+# https://superuser.com/questions/959380/how-do-i-install-generate-all-locales-on-fedora
+# This may not be necessary anymore because Fedora 30, unlike CentOS 7, has
+# glibc subpackages like glibc-langpack-en.
+RUN rm /etc/rpm/macros.image-language-conf
+RUN echo 'LANG="en_US.UTF-8"' > /etc/locale.conf
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+ENV PYTHONUNBUFFERED=0
+ENV DJANGO_SETTINGS_MODULE=pulpcore.app.settings
+ENV PULP_SETTINGS=/etc/pulp/settings.py
+ENV _BUILDAH_STARTED_IN_USERNS=""
+ENV BUILDAH_ISOLATION=chroot
+
+# The Fedora 30 image already has tsflags=nodocs set in dnf.conf
+# It already has pip
+#
+# wget & git are presumably needed for purposes like pip installs from git
+#
+# libxcrypt-compat is needed by psycopg2-binary from PyPI
+#
+# python3-psycopg2 is installed by ansible-pulp
+#
+# glibc-langpack-en is needed to provide the en_US.UTF-8 locale, which Pulp
+# seems to need.
+#
+# python3-createrepo_c is needed for pulp_rpm
+RUN dnf -y update && \
+    dnf -y install wget git && \
+    dnf -y install libxcrypt-compat && \
+    dnf -y install python3-psycopg2 && \
+    dnf -y install glibc-langpack-en && \
+    dnf -y install python3-libmodulemd && \
+    dnf -y install python3-libcomps && \
+    dnf -y install postgresql && \
+    dnf -y install postgresql-server && \
+    dnf -y install nginx && \
+    dnf -y install redis && \
+    dnf -y install python3-setuptools && \
+    dnf -y install gcc python3-devel openssl-devel && \
+    dnf -y install libmodulemd-devel && \
+    dnf -y install libcomps-devel && \
+    dnf -y install ninja-build && \
+    dnf -y install 'dnf-command(builddep)' && \
+    dnf -y builddep createrepo_c && \
+    dnf clean all
+
+RUN dnf -y install buildah --exclude container-selinux && \
+    dnf clean all
+
+# Until the update is re-submitted & released
+# https://bodhi.fedoraproject.org/updates/FEDORA-2019-0d122cc67a
+RUN rpm -q python3-libcomps --queryformat=%{VERSION}-%{RELEASE} | grep -v 0.1.11-1 || dnf upgrade -y --enablerepo=updates-testing python3-libcomps
+
+RUN sed 's|^#mount_program|mount_program|g' -i /etc/containers/storage.conf
+
+RUN mkdir -p /etc/pulp
+
+RUN easy_install pip
+
+RUN echo "/var/lib/pgsql true postgres 0600 0750" >> /etc/fix-attrs.d/postgres
+
+RUN mkdir -p /etc/services.d/nginx /etc/services.d/postgresql /etc/services.d/redis /etc/services.d/pulpcore-worker@1 /etc/services.d/pulpcore-worker@2 /etc/services.d/pulpcore-resource-manager /etc/services.d/pulpcore-api /etc/services.d/pulpcore-content /var/lib/pgsql /var/run/pulpcore-worker-1 /var/run/pulpcore-worker-2 /var/run/pulpcore-resource-manager
+
+COPY assets/pulpcore-content.run /etc/services.d/pulpcore-content/run
+COPY assets/postgres.run /etc/services.d/postgresql/run
+COPY assets/redis.run /etc/services.d/redis/run
+COPY assets/pulpcore-worker@2.run /etc/services.d/pulpcore-worker@2/run
+COPY assets/pulpcore-resource-manager.run /etc/services.d/pulpcore-resource-manager/run
+COPY assets/pulpcore-api.run /etc/services.d/pulpcore-api/run
+COPY assets/postgres.prep /etc/cont-init.d/postgres
+COPY assets/pulpcore-worker@1.run /etc/services.d/pulpcore-worker@1/run
+COPY assets/nginx.conf /etc/nginx/nginx.conf
+COPY assets/nginx.run /etc/services.d/nginx/run
+
+ENTRYPOINT ["/init"]

.travis/Containerfile.j2

@@ -0,0 +1,22 @@
+FROM pulp_ci_base:{{ image.tag }}
+
+{% if s3_test | default(false) %}
+# Hacking botocore (https://github.com/boto/botocore/pull/1990)
+RUN pip3 install django-storages[boto3]
+RUN pip3 install git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
+{% endif %}
+
+{% for item in plugins %}
+
+# Install {{ item.name }} from {{ item.source }}
+{% if item.name == "pulp-certguard" %}
+RUN pip3 install python-dateutil rhsm
+{% endif %}
+{% if item.source.startswith("./") %}
+ADD {{ item.source }} {{ item.source }}
+{% endif %}
+RUN pip3 install {{ item.source }}
+RUN ln /usr/local/lib/python3.7/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/conf.d/{{ item.name }} || true
+{% endfor %}
+
+ENTRYPOINT ["/init"]

.travis/ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+inventory = inventory.yaml
+filter_plugins = filter
+retry_files_enabled = False
+transport = local
+nocows = 1
+stdout_callback = yaml

.travis/assets/nginx.conf

@@ -0,0 +1,85 @@
+# TODO: Support IPv6.
+# TODO: Configure SSL certificates.
+# TODO: Maybe serve multiple `location`s, not just one.
+
+# The "nginx" package on fedora creates this user and group.
+user nginx nginx;
+# Gunicorn docs suggest this value.
+worker_processes 1;
+daemon off;
+events {
+    worker_connections 1024;  # increase if you have lots of clients
+    accept_mutex off;  # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    upstream pulp-content {
+         server 127.0.0.1:24816;
+    }
+
+    upstream pulp-api {
+         server 127.0.0.1:24817;
+    }
+
+    server {
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        listen 80 default_server deferred;
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        # Path to Pulp's static files.
+        root /usr/local/lib/pulp/lib/python3.7/site-packages/rest_framework;
+
+        location /pulp/content/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://127.0.0.1:24816;
+        }
+
+        location /pulp/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://127.0.0.1:24817;
+            client_max_body_size 10m;
+        }
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://127.0.0.1:24817;
+        }
+
+        include conf.d/*;
+
+        location / {
+            # checks for static file, if not found proxy to app
+            try_files $uri =404;
+        }
+    }
+}

.travis/assets/nginx.run

@@ -0,0 +1,2 @@
+#!/usr/bin/execlineb -P
+/usr/sbin/nginx

.travis/assets/postgres.prep

@@ -0,0 +1,115 @@
+#!/usr/bin/execlineb -S0
+
+backtick -n BASENAME { s6-basename ${0} }
+importas -u BASENAME BASENAME
+define PREFFIX "[cont-init.d] ${BASENAME}:"
+
+define C000 "\033[0m"
+define C030 "\033[0;30m"
+define C031 "\033[0;31m"
+define C032 "\033[0;32m"
+define C033 "\033[0;33m"
+define C034 "\033[0;34m"
+define C035 "\033[0;35m"
+define C036 "\033[0;36m"
+define C037 "\033[0;37m"
+define C130 "\033[1;30m"
+define C131 "\033[1;31m"
+define C132 "\033[1;32m"
+define C133 "\033[1;33m"
+define C134 "\033[1;34m"
+define C135 "\033[1;35m"
+define C136 "\033[1;36m"
+define C137 "\033[1;37m"
+
+backtick -n ! { pipeline { postgres --version } egrep -o "[0-9]{1,}\.[0-9]{1,}" }
+importas -u PGVERSION !
+define PGHOME "/var/lib/pgsql"
+define PGDATA "${PGHOME}/data"
+
+foreground {
+ifthenelse { s6-test -d "${PGDATA}/base" -a -f "${PGDATA}/PG_VERSION" }
+{
+  backtick -n ! { redirfd -r 0 "${PGDATA}/PG_VERSION" s6-cat }
+  importas -u PGDATA_VERSION !
+  ifelse { s6-test "${PGDATA_VERSION}" = "${PGVERSION}" } {
+    if { s6-echo "${PREFFIX} ${C031}${PGDATA} exists but needs upgrade ${PGDATA_VERSION} != ${PGVERSION}${C000}" }
+    exit 200
+  }
+}
+{
+  if {
+    if { s6-echo "${PREFFIX} ${C032}initdb --pgdata ${PGDATA}${C000}" }
+    #redirfd -w 1 /dev/null
+    fdmove -c 2 1
+    s6-setuidgid postgres
+    backtick -D "C" -n LANG { printcontenv LANG }
+    initdb --pgdata=${PGDATA}
+  }
+  if {
+    if { s6-echo "${PREFFIX} ${C032}s6-hiercopy /etc/postgresql/ ${PGDATA}/ ${C000}" }
+    s6-hiercopy /etc/postgresql/ ${PGDATA}/
+  }
+}
+}
+
+importas -u ? ?
+ifelse { s6-test ${?} -ne 0 } { exit ${?} }
+
+background {
+  #redirfd -w 1 /dev/null
+  fdmove -c 2 1
+  s6-setuidgid postgres
+  postgres -D ${PGDATA}
+}
+importas -u PG_PID !
+if { s6-echo "${PREFFIX} ${C133}started service [${PG_PID}] postgres${C000}" }
+
+if { s6-echo "${PREFFIX} ${C130}wait for services startup...${C000}" }
+
+loopwhilex -x 0 foreground {
+  s6-sleep -m 500
+  s6-setuidgid postgres
+  pg_isready --quiet
+}
+
+backtick -n ! {
+  s6-setuidgid postgres
+  psql --no-align --tuples-only --command="SELECT (1) FROM pg_database WHERE datname='pulp' LIMIT 1"
+}
+importas -u DATABASE_EXISTS !
+
+foreground {
+  if { s6-test ${DATABASE_EXISTS} != "1" }
+  if { s6-echo "${PREFFIX} ${C032}createuser pulp${C000}" }
+  #redirfd -w 1 /dev/null
+  fdmove -c 2 1
+  s6-setuidgid postgres
+  createuser pulp
+}
+
+foreground {
+  if { s6-test ${DATABASE_EXISTS} != "1" }
+  if { s6-echo "${PREFFIX} ${C032}createdb --encoding=utf-8 -O pulp pulp${C000}" }
+  redirfd -w 1 /dev/null
+  fdmove -c 2 1
+  s6-setuidgid postgres
+  createdb --encoding=utf-8 pulp
+}
+
+foreground {
+  redirfd -w 1 /dev/null
+  redirfd -w 2 /dev/null
+  export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+  export PULP_CONTENT_ORIGIN localhost
+  s6-setuidgid postgres
+  /usr/local/bin/django-admin migrate
+}
+
+foreground {
+  if { s6-echo "${PREFFIX} ${C133}shutdown service [${PG_PID}] postgres${C000}" }
+  redirfd -w 1 /dev/null
+  fdmove -c 2 1
+  s6-setuidgid postgres
+  pg_ctl -D ${PGDATA} stop
+}

.travis/assets/postgres.run

@@ -0,0 +1,3 @@
+#!/usr/bin/execlineb -P
+s6-setuidgid postgres
+postgres -D /var/lib/pgsql/data

.travis/assets/pulpcore-api.run

@@ -0,0 +1,4 @@
+#!/usr/bin/execlineb -P
+export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+export PULP_SETTINGS /etc/pulp/settings.py
+/usr/local/bin/gunicorn pulpcore.app.wsgi:application --bind "127.0.0.1:24817" --access-logfile -

.travis/assets/pulpcore-content.run

@@ -0,0 +1,4 @@
+#!/usr/bin/execlineb -P
+export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+export PULP_SETTINGS /etc/pulp/settings.py
+/usr/local/bin/gunicorn pulpcore.content:server --bind "127.0.0.1:24816" --worker-class "aiohttp.GunicornWebWorker" -w 2 --access-logfile -

.travis/assets/pulpcore-resource-manager.run

@@ -0,0 +1,4 @@
+#!/usr/bin/execlineb -P
+export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+export PULP_SETTINGS /etc/pulp/settings.py
+/usr/local/bin/rq worker -w pulpcore.tasking.worker.PulpWorker -n resource-manager --pid=/var/run/pulpcore-resource-manager/resource-manager.pid -c "pulpcore.rqconfig" --disable-job-desc-logging

.travis/assets/pulpcore-worker@1.run

@@ -0,0 +1,5 @@
+#!/usr/bin/execlineb -P
+export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+export PULP_SETTINGS /etc/pulp/settings.py
+export PATH /usr/local/bin:/usr/bin/
+/usr/local/bin/rq worker -w pulpcore.tasking.worker.PulpWorker --pid=/var/run/pulpcore-worker-1/reserved-resource-worker-1.pid -c "pulpcore.rqconfig" --disable-job-desc-logging