Problem: pulp not provisioned during gem publish step

Solution: install pulp in the install install script This patch updates how Pulp is installed by Travis. It is now installed using ansible-pulp. The install step now happens as part of the install script. This also adds MariaDB testing. re: #4695 https://pulp.plan.io/issues/4695
pulp · May 1, 2019 · 1c72a15 · 1c72a15
1 parent c366a90
commit 1c72a15
Show file tree

Hide file tree

Showing 9 changed files with 204 additions and 105 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,26 +3,36 @@ sudo: required
 dist: xenial
 language: python
 python:
-    # python versions used in el7 SCL & supported fedora
-    - "3.6"
-    - "3.7"
-addons:
-    # postgres versions provided by el7 RHSCL (lowest supportable version)
-    postgresql: "9.6"
+  - '3.6'
+  - '3.7'
+env:
+  matrix:
+    - DB=postgres TEST=pulp
+    - DB=mariadb TEST=pulp
+    - DB=postgres TEST=bindings
+matrix:
+  exclude:
+    - python: '3.6'
+      env: DB=mariadb TEST=pulp
+    - python: '3.7'
+      env: DB=postgres TEST=bindings
+  fast_finish: true
 services:
-    - postgresql
-    - redis-server
-install: source .travis/install.sh
-before_script: source .travis/before_script.sh
-script: source .travis/script.sh
-after_failure:
-  - sh -c "cat ~/django_runserver.log"
-  - sh -c "cat ~/resource_manager.log"
-  - sh -c "cat ~/reserved_workers-1.log"
-stages:
-- name: test
-- name: deploy
-  if: tag IS present
+  - postgresql
+  - mariadb
+addons:
+  apt:
+    packages:
+      - httpie
+      - jq
+  # postgres versions provided by el7 RHSCL (lowest supportable version)
+  postgresql: '9.6'
+  mariadb: '10.3'
+before_install: .travis/before_install.sh
+install: .travis/install.sh
+before_script: .travis/before_script.sh
+script:
+  - .travis/script.sh
 jobs:
   include:
   - stage: deploy
@@ -35,6 +45,10 @@ jobs:
         secure: pQVElIk7hM0cyyGz8K+0R23OlcSBocYibi7yOPYUvQ/x8oTEuH/WFt18M5i3YK2tPg3Mypnm+uaUg1vxd+QlJRZTTTj+sSIiGflTkTMIWPpoQ9bm8/oo5MlOP630mK3/Lp117BLAqNJfUm6XW8dikhBFiWZ+IVge3bsuaIOTs+8TNmXT6F0aZZ66RQElcfdllfGSmYv39QzH95XL6d/AgSBh93he26mqlOpHf7gcsT/5UwggW8wDjyF53WYBBuYjg3A7umwHzoilLy98E3ortGF1RaMjlu0jSrlsT+pBdu1ZKwgxEfk3zqKaTYNr/0w9AudsRWMXY8F2oAOG1W5qUoabqiFiVc4AvrP2oZ6q3y1ikv1kLSjGpfmPdz524XY25xoXZOp2M5NIuVo36bZZ9tUhNtkGnE0pufzaRYDAAChWW5xvcOU1ngSDfFkogg4cUmWkVrARBhRwaUcbN2fU/lPDpdaPjg0UCLTCmO8VsGfdvX0NAGu8MUQjbOLvPqzAIR+pEDgKiRIqgW8UlByIAmjh7Z0W8NXLGcTNt2vP9qXnSJpjXP6YU7WlfFEQn9sFwIBQEiCiU2vUfV83iNoVDSaN+0zeeJCHmrY5g/FaqHvXw++vu0ZzK43ZLDy6TU58QP2qooZ6pie8/PGYWVr6U3avXtk/epC3hSHwo6tv1GA=
       on:
         tags: true
+    if: tag IS present
   - stage: publish-pulp-file-client-gem
     script: bash .travis/publish_pulp_file_client_gem.sh
+    env:
+      - DB=postgres
+      - TEST=bindings
     if: type != pull_request
diff --git a/.travis/before_install.sh b/.travis/before_install.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env sh
+set -v
+
+COMMIT_MSG=$(git show HEAD^2 -s)
+export COMMIT_MSG
+export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore-plugin\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
+
+# dev_requirements should not be needed for testing; don't install them to make sure
+pip install -r test_requirements.txt
+
+# check the commit message
+./.travis/check_commit.sh
+
+# Lint code.
+flake8 --config flake8.cfg || exit 1
+
+cd ..
+git clone https://github.com/pulp/ansible-pulp.git
+if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
+  pushd ansible-pulp
+  git fetch origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+git clone https://github.com/pulp/pulpcore.git
+
+if [ -n "$PULP_PR_NUMBER" ]; then
+  pushd pulpcore
+  git fetch origin +refs/pull/$PULP_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+
+git clone https://github.com/pulp/pulpcore-plugin.git
+
+if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
+  pushd pulpcore-plugin
+  git fetch origin +refs/pull/$PULP_PLUGIN_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+
+if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
+  git clone https://github.com/PulpQE/pulp-smash.git
+  pushd pulp-smash
+  git fetch origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+if [ "$DB" = 'mariadb' ]; then
+  # working around https://travis-ci.community/t/mariadb-build-error-with-xenial/3160
+  mysql -u root -e "DROP USER IF EXISTS 'travis'@'%';"
+  mysql -u root -e "CREATE USER 'travis'@'%';"
+  mysql -u root -e "CREATE DATABASE pulp;"
+  mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'travis'@'%';";
+else
+  psql -c 'CREATE DATABASE pulp OWNER travis;'
+fi
+
+pip install ansible
+cp pulp_file/.travis/playbook.yml ansible-pulp/playbook.yml
+cp pulp_file/.travis/postgres.yml ansible-pulp/postgres.yml
+cp pulp_file/.travis/mariadb.yml ansible-pulp/mariadb.yml
+
+cd pulp_file
diff --git a/.travis/before_script.sh b/.travis/before_script.sh
@@ -1,16 +1,5 @@
 #!/usr/bin/env sh
 set -v
 
-psql -U postgres -c 'CREATE USER pulp WITH SUPERUSER LOGIN;'
-psql -U postgres -c 'CREATE DATABASE pulp OWNER pulp;'
-
 mkdir -p ~/.config/pulp_smash
 cp ../pulpcore/.travis/pulp-smash-config.json ~/.config/pulp_smash/settings.json
-
-sudo mkdir -p /var/lib/pulp/tmp
-sudo mkdir /var/cache/pulp
-sudo mkdir /etc/pulp/
-sudo chown -R travis:travis /var/lib/pulp
-sudo chown travis:travis /var/cache/pulp
-
-echo "SECRET_KEY: \"$(cat /dev/urandom | tr -dc 'a-z0-9!@#$%^&*(\-_=+)' | head -c 50)\"" | sudo tee -a /etc/pulp/settings.py
diff --git a/.travis/install.sh b/.travis/install.sh
@@ -1,44 +1,10 @@
 #!/usr/bin/env sh
 set -v
 
-export COMMIT_MSG=$(git show HEAD^2 -s)
-export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore-plugin\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+# Run Ansible playbook
+cd ../ansible-pulp
+ansible-galaxy install -r requirements.yml
 
-pip install -r test_requirements.txt
-
-cd .. && git clone https://github.com/pulp/pulpcore.git
-
-if [ -n "$PULP_PR_NUMBER" ]; then
-  pushd pulpcore
-  git fetch origin +refs/pull/$PULP_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-fi
-
-pip install -e ./pulpcore[postgres]
-
-git clone https://github.com/pulp/pulpcore-plugin.git
-
-if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
-  pushd pulpcore-plugin
-  git fetch origin +refs/pull/$PULP_PLUGIN_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-fi
-
-pip install ./pulpcore-plugin
-
-if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
-  pip uninstall -y pulp-smash
-  git clone https://github.com/PulpQE/pulp-smash.git
-  pushd pulp-smash
-  git fetch origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-  pip install -e ./pulp-smash
-fi
-
-cd pulp_file
-pip install -e .
+ansible-playbook --connection=local --inventory 127.0.0.1, playbook.yml --extra-vars \
+  "pulp_python_interpreter=$VIRTUAL_ENV/bin/python, pulp_install_dir=$VIRTUAL_ENV \
+  pulp_db_type=$DB"
diff --git a/.travis/mariadb.yml b/.travis/mariadb.yml
@@ -0,0 +1,2 @@
+---
+pulp_db_backend: django.db.backends.mysql
diff --git a/.travis/playbook.yml b/.travis/playbook.yml
@@ -0,0 +1,29 @@
+---
+- hosts: all
+  pre_tasks:
+    - name: Load DB variables
+      include_vars: '{{ pulp_db_type }}.yml'
+  vars:
+    pulp_default_admin_password: admin
+    pulp_secret_key: secret
+    pulp_source_dir: '/home/travis/build/pulp/pulpcore/'
+    pulp_plugin_source_dir: "/home/travis/build/pulp/pulpcore-plugin"
+    pulp_install_plugins:
+      pulp-file:
+        app_label: "file"
+        source_dir: "/home/travis/build/pulp/pulp_file"
+    ansible_python_interpreter: '/opt/pyenv/shims/python3'
+    pulp_user: 'travis'
+    developer_user: 'travis'
+    pulp_install_db: false
+    pulp_db_user: 'travis'
+    pulp_db_password: ''
+    pulp_preq_packages: []
+  environment:
+    DJANGO_SETTINGS_MODULE: pulpcore.app.settings
+  roles:
+    - pulp-database
+    - pulp-workers
+    - pulp-resource-manager
+    - pulp-webserver
+    - pulp-content
diff --git a/.travis/postgres.yml b/.travis/postgres.yml
@@ -0,0 +1,2 @@
+---
+pulp_db_backend: django.db.backends.postgresql_psycopg2
diff --git a/.travis/publish_pulp_file_client_gem.sh b/.travis/publish_pulp_file_client_gem.sh
@@ -6,7 +6,7 @@ sudo chmod 600 ~/.gem/credentials
 django-admin runserver 24817 >> ~/django_runserver.log 2>&1 &
 sleep 5
 
-cd /home/travis/build/pulp/pulp_file/
+cd /home/vagrant/devel/pulp_file/
 COMMIT_SHA="$(git rev-parse HEAD | cut -c1-8)"
 export COMMIT_SHA
 
@@ -17,7 +17,7 @@ cd pulp-swagger-codegen
 
 
 sudo ./generate.sh pulp_file ruby $COMMIT_SHA
-sudo chown travis:travis pulp_file-client
+sudo chown -R vagrant:vagrant pulp_file-client
 cd pulp_file-client
 gem build pulp_file_client
 GEM_FILE="$(ls | grep pulp_file_client-)"

diff --git a/.travis/script.sh b/.travis/script.sh
@@ -1,30 +1,55 @@
 #!/usr/bin/env bash
 # coding=utf-8
-set -veuo pipefail
 
-# check the commit message
-./.travis/check_commit.sh
+set -mveuo pipefail
 
-# Lint code.
-flake8 --config flake8.cfg || exit 1
-
-# Run migrations.
+# Needed for both starting the service and building the docs.
+# Gets set in .travis/settings.yml, but doesn't seem to inherited by
+# this script.
 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
-export PULP_CONTENT_HOST=localhost:24816
-django-admin makemigrations file
-django-admin migrate --noinput
 
-# Run unit tests.
-(cd ../pulpcore && coverage run manage.py test pulp_file.tests.unit)
+wait_for_pulp() {
+  TIMEOUT=${1:-5}
+  while [ "$TIMEOUT" -gt 0 ]
+  do
+    echo -n .
+    sleep 1
+    TIMEOUT=$(($TIMEOUT - 1))
+    if [ $(http :24817/pulp/api/v3/status/ | jq '.database_connection.connected and .redis_connection.connected') = 'true' ]
+    then
+      echo
+      return
+    fi
+  done
+  echo
+  return 1
+}
+
+if [ "$TEST" = 'bindings' ]; then
+  COMMIT_MSG=$(git show HEAD^2 -s)
+  export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-swagger-codegen\/pull\/(\d+)' | awk -F'/' '{print $7}')
+
+  cd ..
+  git clone https://github.com/pulp/pulp-swagger-codegen.git
+  cd pulp-swagger-codegen
+
+  if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then
+    git fetch origin +refs/pull/$PULP_BINDINGS_PR_NUMBER/merge
+    git checkout FETCH_HEAD
+  fi
 
-# Run functional tests.
-django-admin reset-admin-password --password admin
-django-admin runserver 24817 >> ~/django_runserver.log 2>&1 &
-gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 >> ~/content_app.log 2>&1 &
-rq worker -n 'resource-manager@%h' -w 'pulpcore.tasking.worker.PulpWorker' >> ~/resource_manager.log 2>&1 &
-rq worker -n 'reserved-resource-worker-1@%h' -w 'pulpcore.tasking.worker.PulpWorker' >> ~/reserved_worker-1.log 2>&1 &
-sleep 8
+  sudo ./generate.sh pulpcore python
+  sudo ./generate.sh pulp_file python
+  pip install ./pulpcore-client
+  pip install ./pulp_file-client
+  python test_bindings.py
+  exit
+fi
 
+# Run unit tests.
+django-admin test ./pulp_file/tests/unit/
+
+# Run functional tests, and upload coverage report to codecov.
 show_logs_and_return_non_zero() {
     readonly local rc="$?"
     cat ~/django_runserver.log
@@ -33,18 +58,18 @@ show_logs_and_return_non_zero() {
     cat ~/reserved_worker-1.log
     return "${rc}"
 }
-pytest -v -r sx --color=yes --pyargs pulp_file.tests.functional || show_logs_and_return_non_zero
 
-# test against pulpcore as well since pulpcore has a set of tests which use pulp_file
-cd ../pulpcore
-pytest -v -r sx --color=yes --pyargs pulpcore.tests.functional.api.using_plugin || show_logs_and_return_non_zero
-
-# Travis' scripts use unbound variables. This is problematic, because the
-# changes made to this script's environment appear to persist when Travis'
-# scripts execute. Perhaps this script is sourced by Travis? Regardless of why,
-# we need to reset the environment when this script finishes.
-#
-# We can't use `trap cleanup_function EXIT` or similar, because this script is
-# apparently sourced, and such a trap won't execute until the (buggy!) calling
-# script finishes.
-set +euo pipefail
+# Stop services started by ansible roles
+sudo systemctl stop pulp-worker* pulp-resource-manager pulp-content-app pulp-api
+
+# Start services with logs and coverage
+export PULP_CONTENT_HOST=localhost:24816
+rq worker -n 'resource-manager@%h' -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig' >> ~/resource_manager.log 2>&1 &
+rq worker -n 'reserved-resource-worker-1@%h' -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig' >> ~/reserved_worker-1.log 2>&1 &
+gunicorn pulpcore.tests.functional.content_with_coverage:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 >> ~/content_app.log 2>&1 &
+coverage run $(which django-admin) runserver 24817 --noreload >> ~/django_runserver.log 2>&1 &
+wait_for_pulp 20
+
+# Run functional tests
+pytest -v -r sx --color=yes --pyargs pulpcore.tests.functional || show_logs_and_return_non_zero
+pytest -v -r sx --color=yes --pyargs pulp_file.tests.functional || show_logs_and_return_non_zero