As a pulp_installer user, I can install Pulp from RPMs

Heavily adapted from "allow installing pulp from packages"

#188

Includes:
1. Dropping the fixed repo from the old PR.
2. Add var pulp_pkg_repo (currently only yum/dnf)
3. Finish cleaning up pulp_webserver_static_dir
4. Many new variables, but still using the overall
pulp_install_plugins.
5. Testing upgrades with new images on quay
(can't truly test until RPM updates are issued)
6. Cron/branch CI: Adding py38 testing
7. PR CI: Move upgrade testing from py37 to py38, and testing
from RPMs on py37.
8. Deduplicating the tasks for checking plugins for nginx/apache
snippets.
9. Using pulp_django_admin_paths list var because RPMs often use
different names for django-admin.
10. Note: Rest of tasks should be package manager independent.
11. pulp_upgraded_manually variable in case users ran `yum update`
12. package only variables in seperate section of the pulp README.md.

fixes: #6793

Includes these subtasks implemented:
re: #6794
re: #6795
re: #6796
(It was easiest to implement them all in one PR.)

.github/workflows/pull_request.yaml

@@ -7,6 +7,8 @@ jobs:
     strategy:
       fail-fast: false
       # For PRs, to save GHA resources, only run on 1 python version
+      # Also, have the pip modes focus on as many different python
+      # versions as possible, package mode can be just 1 version.
       matrix:
         include:
           - env:
@@ -22,11 +24,20 @@ jobs:
               TEST_TYPE: source-dynamic
             python: "3.6"
           - env:
-              TEST_TYPE: release-upgrade
+              TEST_TYPE: packages-static
+            python: "3.7"
+          - env:
+              TEST_TYPE: packages-dynamic
+            python: "3.7"
+          - env:
+              TEST_TYPE: packages-upgrade
             python: "3.7"
+          - env:
+              TEST_TYPE: release-upgrade
+            python: "3.8"
           - env:
               TEST_TYPE: source-upgrade
-            python: 3.7
+            python: "3.8"
     steps:
       - uses: actions/checkout@v2
       - run: |
@@ -61,12 +72,20 @@ jobs:
       - name: Set tox env for python 37
         if: matrix.python == '3.7'
         run: echo "::set-env name=TOXENV::py37-${{ matrix.env.TEST_TYPE }}"
-      - name: Pulling images for upgrade
+      - name: Set tox env for python 38
+        if: matrix.python == '3.8'
+        run: echo "::set-env name=TOXENV::py38-${{ matrix.env.TEST_TYPE }}"
+      - name: Pulling images for PyPI upgrades
         if: matrix.env.TEST_TYPE == 'release-upgrade' || matrix.env.TEST_TYPE == 'source-upgrade'
         run: |
           docker pull quay.io/pulp/pulp-ci-c7:3.0.0
           docker pull quay.io/pulp/pulp-ci-dbuster:3.0.0
           docker pull quay.io/pulp/pulp-ci-f31:3.0.0
+      - name: Pulling images for distro packages upgrades
+        if: matrix.env.TEST_TYPE == 'packages-upgrade'
+        run: |
+          docker pull quay.io/pulp/pulp-ci-pkgs-c7:3.3.1
+          docker pull quay.io/pulp/pulp-ci-pkgs-c8:3.3.1
       - name: Run tox
         run: |
           tox && exit 0 || true

.github/workflows/pulp_ci.yaml

@@ -15,8 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["2.7", "3.6", "3.7"]
-        # TODO: Once we have more than 3x2, use separate variables
+        python: ["2.7", "3.6", "3.7", "3.8"]
+        # TODO: Once we have more than 3x3, use separate variables
         # This Travis syntax will probably work on GHA:
         # https://docs.travis-ci.com/user/environment-variables/#defining-multiple-variables-per-item
         env:
@@ -26,6 +26,9 @@ jobs:
           - TEST_TYPE: source-dynamic
           - TEST_TYPE: source-static
           - TEST_TYPE: source-upgrade
+          - TEST_TYPE: packages-static
+          - TEST_TYPE: packages-dynamic
+          - TEST_TYPE: packages-upgrade
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -45,12 +48,20 @@ jobs:
       - name: Set tox env for python 37
         if: matrix.python == '3.7'
         run: echo "::set-env name=TOXENV::py37-${{ matrix.env.TEST_TYPE }}"
-      - name: Pulling images for upgrade
+      - name: Set tox env for python 38
+        if: matrix.python == '3.8'
+        run: echo "::set-env name=TOXENV::py38-${{ matrix.env.TEST_TYPE }}"
+      - name: Pulling images for PyPI upgrades
         if: matrix.env.TEST_TYPE == 'release-upgrade' || matrix.env.TEST_TYPE == 'source-upgrade'
         run: |
           docker pull quay.io/pulp/pulp-ci-c7:3.0.0
           docker pull quay.io/pulp/pulp-ci-dbuster:3.0.0
           docker pull quay.io/pulp/pulp-ci-f31:3.0.0
+      - name: Pulling images for distro packages upgrades
+        if: matrix.env.TEST_TYPE == 'packages-upgrade'
+        run: |
+          docker pull quay.io/pulp/pulp-ci-pkgs-c7:3.3.1
+          docker pull quay.io/pulp/pulp-ci-pkgs-c8:3.3.1
       - name: Run tox
         run: |
           tox && exit 0 || true

CHANGES/6793.feature

@@ -0,0 +1 @@
+Add the ability to install Pulp from Linux distro (RPM) packages.

CHANGES/6794.feature

@@ -0,0 +1 @@
+Let users specify an RPM repo containing Pulp. Introduces the new variable: `pulp_pkg_repo`

CHANGES/6795.feature

@@ -0,0 +1,2 @@
+Add variables so users can specify the names of each RPM package to install (pulp_pkg_pulpcore_name
+& pulp_install_plugins's pkg_name) or to just override the prefix (pulp_pkg_name_prefix).

CHANGES/6796.misc

@@ -0,0 +1 @@
+Create CI for installing Pulp from RPMs

molecule/packages-dynamic/.ansible-lint

@@ -0,0 +1 @@
+../../.ansible-lint

molecule/packages-dynamic/Dockerfile.j2

@@ -0,0 +1 @@
+../scenario_resources/Dockerfile.j2

molecule/packages-dynamic/converge.yml

@@ -0,0 +1 @@
+../release-dynamic/converge.yml

molecule/packages-dynamic/debian-redis-server.service

@@ -0,0 +1 @@
+../scenario_resources/debian-redis-server.service

molecule/packages-dynamic/group_vars

@@ -0,0 +1 @@
+../packages-static/group_vars

molecule/packages-dynamic/molecule.yml

@@ -0,0 +1,54 @@
+---
+dependency:
+  name: galaxy
+  role-file: requirements.yml
+driver:
+  name: docker
+lint: |
+    yamllint .
+    ansible-lint
+# This is ignored by molecule, but can be reused in yaml
+.platform_base: &platform_base
+  privileged: False
+  volumes:
+    - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  tmpfs:
+    # Fixes issues accessing .so files under /tmp during pip builds.
+    # https://github.com/docker/compose/issues/1339
+    - /tmp:exec,mode=1777
+    - /run
+    - /run/lock
+  capabilities:
+    - NET_ADMIN
+    - NET_RAW
+platforms:
+  - <<: *platform_base
+    name: centos-7
+    image: centos:7
+    command: /sbin/init
+  - <<: *platform_base
+    name: centos-8
+    image: centos:8
+    command: /sbin/init
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      stdout_callback: debug
+  inventory:
+    links:
+      host_vars: ../scenario_resources/host_vars/
+      group_vars: ../scenario_resources/group_vars/
+scenario:
+  test_sequence:
+    - lint
+    - dependency
+    - syntax
+    - create
+    - prepare
+    - converge
+    - idempotence
+    - side_effect
+    - verify
+verifier:
+  name: inspec

molecule/packages-dynamic/prepare.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Hot patch redis-server service for docker operation (Debian)
+      copy:
+        src: debian-redis-server.service
+        dest: /etc/systemd/system/redis-server.service
+      when: ansible_distribution == "Debian"

molecule/packages-dynamic/requirements.yml

@@ -0,0 +1 @@
+../../requirements.yml

molecule/packages-dynamic/roles

@@ -0,0 +1 @@
+../../roles

molecule/packages-static/.ansible-lint

@@ -0,0 +1 @@
+../../.ansible-lint

molecule/packages-static/Dockerfile.j2

@@ -0,0 +1 @@
+../scenario_resources/Dockerfile.j2

molecule/packages-static/converge.yml

@@ -0,0 +1 @@
+../../playbooks/example-use/playbook.yml

molecule/packages-static/debian-redis-server.service

@@ -0,0 +1 @@
+../scenario_resources/debian-redis-server.service

molecule/packages-static/group_vars/all

@@ -0,0 +1,10 @@
+---
+pulp_default_admin_password: password
+pulp_upgrade: true
+pulp_install_source: packages
+pulp_install_plugins:
+  pulp-file:
+pulp_settings:
+  secret_key: secret
+  content_origin: "http://{{ ansible_fqdn }}"
+pulp_pkg_repo: "https://fedorapeople.org/groups/katello/releases/yum/nightly/pulpcore/el{{ ansible_distribution_major_version }}/x86_64/"

molecule/packages-static/molecule.yml

@@ -0,0 +1,54 @@
+---
+dependency:
+  name: galaxy
+  role-file: requirements.yml
+driver:
+  name: docker
+lint: |
+    yamllint .
+    ansible-lint
+# This is ignored by molecule, but can be reused in yaml
+.platform_base: &platform_base
+  privileged: False
+  volumes:
+    - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  tmpfs:
+    # Fixes issues accessing .so files under /tmp during pip builds.
+    # https://github.com/docker/compose/issues/1339
+    - /tmp:exec,mode=1777
+    - /run
+    - /run/lock
+  capabilities:
+    - NET_ADMIN
+    - NET_RAW
+platforms:
+  - <<: *platform_base
+    name: centos-7
+    image: centos:7
+    command: /sbin/init
+  - <<: *platform_base
+    name: centos-8
+    image: centos:8
+    command: /sbin/init
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      stdout_callback: debug
+  inventory:
+    links:
+      host_vars: ../scenario_resources/host_vars/
+      group_vars: ../scenario_resources/group_vars/
+scenario:
+  test_sequence:
+    - lint
+    - dependency
+    - syntax
+    - create
+    - prepare
+    - converge
+    - idempotence
+    - side_effect
+    - verify
+verifier:
+  name: inspec

molecule/packages-static/prepare.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Hot patch redis-server service for docker operation (Debian)
+      copy:
+        src: debian-redis-server.service
+        dest: /etc/systemd/system/redis-server.service
+      when: ansible_distribution == "Debian"

molecule/packages-static/requirements.yml

@@ -0,0 +1 @@
+../../requirements.yml

molecule/packages-static/roles

@@ -0,0 +1 @@
+../../roles

molecule/packages-upgrade/.ansible-lint

@@ -0,0 +1 @@
+../../.ansible-lint

molecule/packages-upgrade/Dockerfile.j2

@@ -0,0 +1 @@
+../scenario_resources/Dockerfile.j2

molecule/packages-upgrade/converge.yml

@@ -0,0 +1 @@
+../../playbooks/example-use/playbook.yml

molecule/packages-upgrade/debian-redis-server.service

@@ -0,0 +1 @@
+../scenario_resources/debian-redis-server.service

molecule/packages-upgrade/group_vars/all

@@ -0,0 +1,11 @@
+---
+pulp_default_admin_password: password
+pulp_upgrade: true
+pulp_install_source: packages
+pulp_install_plugins:
+  pulp-file:
+pulp_settings:
+  secret_key: secret
+  content_origin: "http://{{ ansible_fqdn }}"
+pulp_pkg_repo: "https://fedorapeople.org/groups/katello/releases/yum/nightly/pulpcore/el{{ ansible_distribution_major_version }}/x86_64/"
+pulp_pkg_upgrade_all: true

molecule/packages-upgrade/molecule.yml

@@ -0,0 +1,55 @@
+---
+dependency:
+  name: galaxy
+  role-file: requirements.yml
+driver:
+  name: docker
+lint: |
+    yamllint .
+    ansible-lint
+# This is ignored by molecule, but can be reused in yaml
+.platform_base: &platform_base
+  privileged: False
+  volumes:
+    - /sys/fs/cgroup:/sys/fs/cgroup:ro
+  tmpfs:
+    # Fixes issues accessing .so files under /tmp during pip builds.
+    # https://github.com/docker/compose/issues/1339
+    - /tmp:exec,mode=1777
+    - /run
+    - /run/lock
+  capabilities:
+    - NET_ADMIN
+    - NET_RAW
+platforms:
+  - <<: *platform_base
+    name: centos-7
+    # molecule often fails to pull, so we pull all images in .travis.yml
+    image: quay.io/pulp/pulp-ci-pkgs-c7:3.3.1
+    command: /sbin/init
+  - <<: *platform_base
+    name: centos-8
+    image: quay.io/pulp/pulp-ci-pkgs-c8:3.3.1
+    command: /sbin/init
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      stdout_callback: debug
+  inventory:
+    links:
+      host_vars: ../scenario_resources/host_vars/
+      group_vars: ../scenario_resources/group_vars/
+scenario:
+  test_sequence:
+    - lint
+    - dependency
+    - syntax
+    - create
+    - prepare
+    - converge
+    - idempotence
+    - side_effect
+    - verify
+verifier:
+  name: inspec

molecule/packages-upgrade/prepare.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  tasks:
+    - name: Hot patch redis-server service for docker operation (Debian)
+      copy:
+        src: debian-redis-server.service
+        dest: /etc/systemd/system/redis-server.service
+      when: ansible_distribution == "Debian"

molecule/packages-upgrade/requirements.yml

@@ -0,0 +1 @@
+../../requirements.yml

molecule/packages-upgrade/roles

@@ -0,0 +1 @@
+../../roles