Problem: ansible-pulp is not used to deploy on Travis

Solution: use ansible-pulp when deploying on Travis This patch was generated using the plugin_template boostrap script. If this works, I will add a separate script that will update plugins instead of bootstrapping new ones. re: #4773 https://pulp.plan.io/issues/4773
pulp · May 3, 2019 · 8b0b681 · 8b0b681
1 parent 99e6f2a
commit 8b0b681
Show file tree

Hide file tree

Showing 8 changed files with 208 additions and 109 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,35 +3,43 @@ sudo: required
 dist: xenial
 language: python
 python:
-    # python versions used in el7 SCL & supported fedora
-    - "3.6"
-    - "3.7"
-addons:
-    # postgres versions provided by el7 RHSCL (lowest supportable version)
-    postgresql: "9.6"
+  - '3.6'
+  - '3.7'
+env:
+  matrix:
+    - DB=postgres TEST=pulp
+    - DB=mariadb TEST=pulp
+matrix:
+  exclude:
+    - python: '3.6'
+      env: DB=mariadb TEST=pulp
+  fast_finish: true
 services:
-    - postgresql
-    - redis-server
-install: source .travis/install.sh
-before_script: source .travis/before_script.sh
-script: source .travis/script.sh
-after_failure:
-  - sh -c "cat ~/django_runserver.log"
-  - sh -c "cat ~/resource_manager.log"
-  - sh -c "cat ~/reserved_workers-1.log"
-stages:
-- name: test
-- name: deploy
-  if: tag IS present
-jobs:
-  include:
-  - stage: deploy
-    script: skip
-    deploy:
-      provider: pypi
-      distributions: sdist bdist_wheel
-      user: pulp
-      password:
-        secure: "hUYap9s4eLWShcYDFJSaq3NEnB1y/8tZBFDNerlMKeqAzb0ilzHVN+7UNogSd40wXZMYvrhcs4sumcyEKm8wwx4YUf9Bz00UopMsFfxTJ1rE+cr2krZVFGCwazeUd7JducTGbrK6LVTaB8BdyoDhavaxzqjPIUNaZlFBh9UYDbgkqAzYdAABz6NfF1uzp/xJfwdUZcZQZOfCF6bpxqmnSA6FqnU6foCjal8rMJVhPHvJi/71MNmJSc6iQsPmytSytzTgCRbIDARxrw5C7kxsOJX0wSTynZNgS+64Dl4w1aYmUBiX42ZVGUxYni3ttOZa3FUZdm9DhCBrPw/YKNL9WDxZ5kwfleJI90N0uzbqMrsgEf9kKnGdgeXKFJlqV/kW19w4l7+YbwXVGxMjb3U6Jv99CrIO7em0JqySpF0tKh/xwchsroV6Udw9MII9YmMyRtPS8O/VIqlkMUhE9F43d8MdBrclnxVuNKCR7wBmv/os7d7SXRikyG++iO8dJz9a6D2x/80KdD1tMp72Yj/6O1kjZvIe6BsKTwrU5JM4Pyx/E2n9CK/1Gvy2EnVVNQp9FITynq0Ly+jqqvYVfqt0/zpZ4dOQ1+AHV52RJrNoSKnWWpuZvhheZGNnXc+8f9lxHJuWAtysrwfuG5vl7Nyykzri8ws5XwDqFmMBns3LZLM="
-      on:
-        tags: true
+  - postgresql
+  - mariadb
+addons:
+  apt:
+    packages:
+      - httpie
+      - jq
+  # postgres versions provided by el7 RHSCL (lowest supportable version)
+  postgresql: '9.6'
+  mariadb: '10.3'
+before_install: .travis/before_install.sh
+install: .travis/install.sh
+before_script: .travis/before_script.sh
+script:
+  - .travis/script.sh
+#jobs:
+#  include:
+#  - stage: deploy
+#    script: skip
+#    deploy:
+#      provider: pypi
+#      distributions: sdist bdist_wheel
+#      user: <pypi username>
+#      password:
+#        secure: <encrypted pypi password>
+#      on:
+#        tags: true
+#    if: tag IS present
diff --git a/.travis/before_install.sh b/.travis/before_install.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env sh
+set -v
+
+COMMIT_MSG=$(git show HEAD^2 -s)
+export COMMIT_MSG
+export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore-plugin\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
+
+# dev_requirements should not be needed for testing; don't install them to make sure
+pip install -r test_requirements.txt
+
+# check the commit message
+# uncomment for plugins that use pulp.plan.io as issue tracker
+# ./.travis/check_commit.sh
+
+# Lint code.
+flake8 --config flake8.cfg || exit 1
+
+cd ..
+git clone https://github.com/pulp/ansible-pulp.git
+if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
+  pushd ansible-pulp
+  git fetch origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+git clone https://github.com/pulp/pulpcore.git
+
+if [ -n "$PULP_PR_NUMBER" ]; then
+  pushd pulpcore
+  git fetch origin +refs/pull/$PULP_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+
+git clone https://github.com/pulp/pulpcore-plugin.git
+
+if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
+  pushd pulpcore-plugin
+  git fetch origin +refs/pull/$PULP_PLUGIN_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+
+if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
+  git clone https://github.com/PulpQE/pulp-smash.git
+  pushd pulp-smash
+  git fetch origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  popd
+fi
+
+if [ "$DB" = 'mariadb' ]; then
+  # working around https://travis-ci.community/t/mariadb-build-error-with-xenial/3160
+  mysql -u root -e "DROP USER IF EXISTS 'travis'@'%';"
+  mysql -u root -e "CREATE USER 'travis'@'%';"
+  mysql -u root -e "CREATE DATABASE pulp;"
+  mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'travis'@'%';";
+else
+  psql -c 'CREATE DATABASE pulp OWNER travis;'
+fi
+
+pip install ansible
+cp pulp_maven/.travis/playbook.yml ansible-pulp/playbook.yml
+cp pulp_maven/.travis/postgres.yml ansible-pulp/postgres.yml
+cp pulp_maven/.travis/mariadb.yml ansible-pulp/mariadb.yml
+
+cd pulp_maven
diff --git a/.travis/before_script.sh b/.travis/before_script.sh
@@ -1,14 +1,5 @@
 #!/usr/bin/env sh
 set -v
 
-psql -U postgres -c 'CREATE USER pulp WITH SUPERUSER LOGIN;'
-psql -U postgres -c 'CREATE DATABASE pulp OWNER pulp;'
-
 mkdir -p ~/.config/pulp_smash
 cp ../pulpcore/.travis/pulp-smash-config.json ~/.config/pulp_smash/settings.json
-
-sudo mkdir -p /var/lib/pulp/tmp
-sudo mkdir /etc/pulp/
-sudo chown -R travis:travis /var/lib/pulp
-
-echo "SECRET_KEY: \"$(cat /dev/urandom | tr -dc 'a-z0-9!@#$%^&*(\-_=+)' | head -c 50)\"" | sudo tee -a /etc/pulp/settings.py
diff --git a/.travis/install.sh b/.travis/install.sh
@@ -1,44 +1,10 @@
 #!/usr/bin/env sh
 set -v
 
-export COMMIT_MSG=$(git show HEAD^2 -s)
-export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore-plugin\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+# Run Ansible playbook
+cd ../ansible-pulp
+ansible-galaxy install -r requirements.yml
 
-pip install -r test_requirements.txt
-
-cd .. && git clone https://github.com/pulp/pulpcore.git
-
-if [ -n "$PULP_PR_NUMBER" ]; then
-  pushd pulpcore
-  git fetch origin +refs/pull/$PULP_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-fi
-
-pip install -e ./pulpcore[postgres]
-
-git clone https://github.com/pulp/pulpcore-plugin.git
-
-if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
-  pushd pulpcore-plugin
-  git fetch origin +refs/pull/$PULP_PLUGIN_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-fi
-
-pip install ./pulpcore-plugin
-
-if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
-  pip uninstall -y pulp-smash
-  git clone https://github.com/PulpQE/pulp-smash.git
-  pushd pulp-smash
-  git fetch origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
-  git checkout FETCH_HEAD
-  popd
-  pip install -e ./pulp-smash
-fi
-
-cd pulp_maven
-pip install -e .
+ansible-playbook --connection=local --inventory 127.0.0.1, playbook.yml --extra-vars \
+  "pulp_python_interpreter=$VIRTUAL_ENV/bin/python, pulp_install_dir=$VIRTUAL_ENV \
+  pulp_db_type=$DB"
diff --git a/.travis/mariadb.yml b/.travis/mariadb.yml
@@ -0,0 +1,2 @@
+---
+pulp_db_backend: django.db.backends.mysql
diff --git a/.travis/playbook.yml b/.travis/playbook.yml
@@ -0,0 +1,29 @@
+---
+- hosts: all
+  pre_tasks:
+    - name: Load DB variables
+      include_vars: '{{ pulp_db_type }}.yml'
+  vars:
+    pulp_default_admin_password: admin
+    pulp_secret_key: secret
+    pulp_source_dir: '/home/travis/build/pulp/pulpcore/'
+    pulp_plugin_source_dir: "/home/travis/build/pulp/pulpcore-plugin"
+    pulp_install_plugins:
+      pulp-maven:
+        app_label: "maven"
+        source_dir: "/home/travis/build/pulp/pulp_maven"
+    ansible_python_interpreter: '/opt/pyenv/shims/python3'
+    pulp_user: 'travis'
+    developer_user: 'travis'
+    pulp_install_db: false
+    pulp_db_user: 'travis'
+    pulp_db_password: ''
+    pulp_preq_packages: []
+  environment:
+    DJANGO_SETTINGS_MODULE: pulpcore.app.settings
+  roles:
+    - pulp-database
+    - pulp-workers
+    - pulp-resource-manager
+    - pulp-webserver
+    - pulp-content
diff --git a/.travis/postgres.yml b/.travis/postgres.yml
@@ -0,0 +1,2 @@
+---
+pulp_db_backend: django.db.backends.postgresql_psycopg2
diff --git a/.travis/script.sh b/.travis/script.sh
@@ -1,30 +1,55 @@
 #!/usr/bin/env bash
 # coding=utf-8
-set -veuo pipefail
 
-# check the commit message
-./.travis/check_commit.sh
+set -mveuo pipefail
 
-# Lint code.
-flake8 --config flake8.cfg || exit 1
-
-# Run migrations.
+# Needed for both starting the service and building the docs.
+# Gets set in .travis/settings.yml, but doesn't seem to inherited by
+# this script.
 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
-export PULP_CONTENT_HOST=localhost:24816
-django-admin makemigrations maven
-django-admin migrate --noinput
 
-# Run unit tests.
-(cd ../pulpcore && coverage run manage.py test pulp_maven.tests.unit)
+wait_for_pulp() {
+  TIMEOUT=${1:-5}
+  while [ "$TIMEOUT" -gt 0 ]
+  do
+    echo -n .
+    sleep 1
+    TIMEOUT=$(($TIMEOUT - 1))
+    if [ $(http :24817/pulp/api/v3/status/ | jq '.database_connection.connected and .redis_connection.connected') = 'true' ]
+    then
+      echo
+      return
+    fi
+  done
+  echo
+  return 1
+}
+
+if [ "$TEST" = 'bindings' ]; then
+  COMMIT_MSG=$(git show HEAD^2 -s)
+  export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-swagger-codegen\/pull\/(\d+)' | awk -F'/' '{print $7}')
+
+  cd ..
+  git clone https://github.com/pulp/pulp-swagger-codegen.git
+  cd pulp-swagger-codegen
+
+  if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then
+    git fetch origin +refs/pull/$PULP_BINDINGS_PR_NUMBER/merge
+    git checkout FETCH_HEAD
+  fi
 
-# Run functional tests.
-django-admin reset-admin-password --password admin
-django-admin runserver 24817 >> ~/django_runserver.log 2>&1 &
-gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 >> ~/content_app.log 2>&1 &
-rq worker -n 'resource-manager@%h' -w 'pulpcore.tasking.worker.PulpWorker' >> ~/resource_manager.log 2>&1 &
-rq worker -n 'reserved-resource-worker-1@%h' -w 'pulpcore.tasking.worker.PulpWorker' >> ~/reserved_worker-1.log 2>&1 &
-sleep 8
+  sudo ./generate.sh pulpcore python
+  sudo ./generate.sh pulp_maven python
+  pip install ./pulpcore-client
+  pip install ./pulp_maven-client
+  python test_bindings.py
+  exit
+fi
 
+# Run unit tests.
+django-admin test ./pulp_maven/tests/unit/
+
+# Run functional tests, and upload coverage report to codecov.
 show_logs_and_return_non_zero() {
     readonly local rc="$?"
     cat ~/django_runserver.log
@@ -33,15 +58,18 @@ show_logs_and_return_non_zero() {
     cat ~/reserved_worker-1.log
     return "${rc}"
 }
-pytest -v -r sx --color=yes --pyargs pulp_maven.tests.functional || show_logs_and_return_non_zero
 
-# Travis' scripts use unbound variables. This is problematic, because the
-# changes made to this script's environment appear to persist when Travis'
-# scripts execute. Perhaps this script is sourced by Travis? Regardless of why,
-# we need to reset the environment when this script finishes.
-#
-# We can't use `trap cleanup_function EXIT` or similar, because this script is
-# apparently sourced, and such a trap won't execute until the (buggy!) calling
-# script finishes.
-set +euo pipefail
+# Stop services started by ansible roles
+sudo systemctl stop pulp-worker* pulp-resource-manager pulp-content-app pulp-api
 
+# Start services with logs and coverage
+export PULP_CONTENT_HOST=localhost:24816
+rq worker -n 'resource-manager@%h' -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig' >> ~/resource_manager.log 2>&1 &
+rq worker -n 'reserved-resource-worker-1@%h' -w 'pulpcore.tasking.worker.PulpWorker' -c 'pulpcore.rqconfig' >> ~/reserved_worker-1.log 2>&1 &
+gunicorn pulpcore.tests.functional.content_with_coverage:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 >> ~/content_app.log 2>&1 &
+coverage run $(which django-admin) runserver 24817 --noreload >> ~/django_runserver.log 2>&1 &
+wait_for_pulp 20
+
+# Run functional tests
+pytest -v -r sx --color=yes --pyargs pulpcore.tests.functional || show_logs_and_return_non_zero
+pytest -v -r sx --color=yes --pyargs pulp_maven.tests.functional || show_logs_and_return_non_zero