From 184f1235f98687cf5078801d0e0559f74a3eab01 Mon Sep 17 00:00:00 2001 From: Mike DePaulo Date: Thu, 4 Jun 2020 15:38:22 -0400 Subject: [PATCH] As a pulp_installer user, I can install Pulp from RPMs Heavily adapted from "allow installing pulp from packages" https://github.com/pulp/pulp_installer/pull/188 Includes: 1. Dropping the fixed repo from the old PR. 2. Add var pulp_pkg_repo (currently only yum/dnf) 3. Finish cleaning up pulp_webserver_static_dir 4. Many new variables, but still using the overall pulp_install_plugins. 5. Testing upgrades with new images on quay (can't truly test until RPM updates are issued) 6. Cron/branch CI: Adding py38 testing 7. PR CI: Move upgrade testing from py37 to py38, and testing from RPMs on py37. 8. Deduplicating the tasks for checking plugins for nginx/apache snippets. 9. Using pulp_django_admin_paths list var because RPMs often use different names for django-admin. 10. Note: Rest of tasks should be package manager independent. 11. pulp_upgraded_manually variable in case users ran `yum update` 12. package only variables in seperate section of the pulp README.md. fixes: #6793 Includes these subtasks implemented: re: #6794 re: #6795 re: #6796 (It was easiest to implement them all in one PR.) --- .github/workflows/pull_request.yaml | 25 +- .github/workflows/pulp_ci.yaml | 17 +- CHANGES/6793.feature | 1 + CHANGES/6794.feature | 1 + CHANGES/6795.feature | 2 + CHANGES/6796.misc | 1 + molecule/packages-dynamic/.ansible-lint | 1 + molecule/packages-dynamic/Dockerfile.j2 | 1 + molecule/packages-dynamic/converge.yml | 1 + .../debian-redis-server.service | 1 + molecule/packages-dynamic/group_vars | 1 + molecule/packages-dynamic/molecule.yml | 54 +++++ molecule/packages-dynamic/prepare.yml | 8 + molecule/packages-dynamic/requirements.yml | 1 + molecule/packages-dynamic/roles | 1 + molecule/packages-static/.ansible-lint | 1 + molecule/packages-static/Dockerfile.j2 | 1 + molecule/packages-static/converge.yml | 1 + .../debian-redis-server.service | 1 + molecule/packages-static/group_vars/all | 10 + molecule/packages-static/molecule.yml | 54 +++++ molecule/packages-static/prepare.yml | 8 + molecule/packages-static/requirements.yml | 1 + molecule/packages-static/roles | 1 + molecule/packages-upgrade/.ansible-lint | 1 + molecule/packages-upgrade/Dockerfile.j2 | 1 + molecule/packages-upgrade/converge.yml | 1 + .../debian-redis-server.service | 1 + molecule/packages-upgrade/group_vars/all | 11 + molecule/packages-upgrade/molecule.yml | 55 +++++ molecule/packages-upgrade/prepare.yml | 8 + molecule/packages-upgrade/requirements.yml | 1 + molecule/packages-upgrade/roles | 1 + roles/pulp/README.md | 51 ++++ roles/pulp/defaults/main.yml | 16 +- roles/pulp/handlers/main.yml | 2 +- roles/pulp/handlers/restart_all_pulp.yml | 2 +- roles/pulp/tasks/find_django_admin.yml | 10 + roles/pulp/tasks/install.yml | 225 ++---------------- roles/pulp/tasks/install_packages.yml | 51 ++++ roles/pulp/tasks/install_pip.yml | 221 +++++++++++++++++ roles/pulp/tasks/main.yml | 13 + roles/pulp/tasks/repos.yml | 11 + roles/pulp/tasks/wsgi.yml | 9 - roles/pulp_database/README.md | 1 - roles/pulp_database/tasks/main.yml | 18 +- roles/pulp_devel/tasks/main.yml | 2 +- roles/pulp_devel/templates/alias.bashrc.j2 | 8 +- roles/pulp_webserver/README.md | 3 + roles/pulp_webserver/tasks/apache.yml | 24 -- roles/pulp_webserver/tasks/main.yml | 21 +- roles/pulp_webserver/tasks/nginx.yml | 24 -- tox.ini | 2 +- 53 files changed, 685 insertions(+), 303 deletions(-) create mode 100644 CHANGES/6793.feature create mode 100644 CHANGES/6794.feature create mode 100644 CHANGES/6795.feature create mode 100644 CHANGES/6796.misc create mode 120000 molecule/packages-dynamic/.ansible-lint create mode 120000 molecule/packages-dynamic/Dockerfile.j2 create mode 120000 molecule/packages-dynamic/converge.yml create mode 120000 molecule/packages-dynamic/debian-redis-server.service create mode 120000 molecule/packages-dynamic/group_vars create mode 100644 molecule/packages-dynamic/molecule.yml create mode 100644 molecule/packages-dynamic/prepare.yml create mode 120000 molecule/packages-dynamic/requirements.yml create mode 120000 molecule/packages-dynamic/roles create mode 120000 molecule/packages-static/.ansible-lint create mode 120000 molecule/packages-static/Dockerfile.j2 create mode 120000 molecule/packages-static/converge.yml create mode 120000 molecule/packages-static/debian-redis-server.service create mode 100644 molecule/packages-static/group_vars/all create mode 100644 molecule/packages-static/molecule.yml create mode 100644 molecule/packages-static/prepare.yml create mode 120000 molecule/packages-static/requirements.yml create mode 120000 molecule/packages-static/roles create mode 120000 molecule/packages-upgrade/.ansible-lint create mode 120000 molecule/packages-upgrade/Dockerfile.j2 create mode 120000 molecule/packages-upgrade/converge.yml create mode 120000 molecule/packages-upgrade/debian-redis-server.service create mode 100644 molecule/packages-upgrade/group_vars/all create mode 100644 molecule/packages-upgrade/molecule.yml create mode 100644 molecule/packages-upgrade/prepare.yml create mode 120000 molecule/packages-upgrade/requirements.yml create mode 120000 molecule/packages-upgrade/roles create mode 100644 roles/pulp/tasks/find_django_admin.yml create mode 100644 roles/pulp/tasks/install_packages.yml create mode 100644 roles/pulp/tasks/install_pip.yml diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 1dce72ac9..d8728fec7 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -7,6 +7,8 @@ jobs: strategy: fail-fast: false # For PRs, to save GHA resources, only run on 1 python version + # Also, have the pip modes focus on as many different python + # versions as possible, package mode can be just 1 version. matrix: include: - env: @@ -22,11 +24,20 @@ jobs: TEST_TYPE: source-dynamic python: "3.6" - env: - TEST_TYPE: release-upgrade + TEST_TYPE: packages-static + python: "3.7" + - env: + TEST_TYPE: packages-dynamic + python: "3.7" + - env: + TEST_TYPE: packages-upgrade python: "3.7" + - env: + TEST_TYPE: release-upgrade + python: "3.8" - env: TEST_TYPE: source-upgrade - python: 3.7 + python: "3.8" steps: - uses: actions/checkout@v2 - run: | @@ -61,12 +72,20 @@ jobs: - name: Set tox env for python 37 if: matrix.python == '3.7' run: echo "::set-env name=TOXENV::py37-${{ matrix.env.TEST_TYPE }}" - - name: Pulling images for upgrade + - name: Set tox env for python 38 + if: matrix.python == '3.8' + run: echo "::set-env name=TOXENV::py38-${{ matrix.env.TEST_TYPE }}" + - name: Pulling images for PyPI upgrades if: matrix.env.TEST_TYPE == 'release-upgrade' || matrix.env.TEST_TYPE == 'source-upgrade' run: | docker pull quay.io/pulp/pulp-ci-c7:3.0.0 docker pull quay.io/pulp/pulp-ci-dbuster:3.0.0 docker pull quay.io/pulp/pulp-ci-f31:3.0.0 + - name: Pulling images for distro packages upgrades + if: matrix.env.TEST_TYPE == 'packages-upgrade' + run: | + docker pull quay.io/pulp/pulp-ci-pkgs-c7:3.3.1 + docker pull quay.io/pulp/pulp-ci-pkgs-c8:3.3.1 - name: Run tox run: | tox && exit 0 || true diff --git a/.github/workflows/pulp_ci.yaml b/.github/workflows/pulp_ci.yaml index 219f07d99..5fc69296d 100644 --- a/.github/workflows/pulp_ci.yaml +++ b/.github/workflows/pulp_ci.yaml @@ -15,8 +15,8 @@ jobs: strategy: fail-fast: false matrix: - python: ["2.7", "3.6", "3.7"] - # TODO: Once we have more than 3x2, use separate variables + python: ["2.7", "3.6", "3.7", "3.8"] + # TODO: Once we have more than 3x3, use separate variables # This Travis syntax will probably work on GHA: # https://docs.travis-ci.com/user/environment-variables/#defining-multiple-variables-per-item env: @@ -26,6 +26,9 @@ jobs: - TEST_TYPE: source-dynamic - TEST_TYPE: source-static - TEST_TYPE: source-upgrade + - TEST_TYPE: packages-static + - TEST_TYPE: packages-dynamic + - TEST_TYPE: packages-upgrade steps: - uses: actions/checkout@v2 - name: Set up Python @@ -45,12 +48,20 @@ jobs: - name: Set tox env for python 37 if: matrix.python == '3.7' run: echo "::set-env name=TOXENV::py37-${{ matrix.env.TEST_TYPE }}" - - name: Pulling images for upgrade + - name: Set tox env for python 38 + if: matrix.python == '3.8' + run: echo "::set-env name=TOXENV::py38-${{ matrix.env.TEST_TYPE }}" + - name: Pulling images for PyPI upgrades if: matrix.env.TEST_TYPE == 'release-upgrade' || matrix.env.TEST_TYPE == 'source-upgrade' run: | docker pull quay.io/pulp/pulp-ci-c7:3.0.0 docker pull quay.io/pulp/pulp-ci-dbuster:3.0.0 docker pull quay.io/pulp/pulp-ci-f31:3.0.0 + - name: Pulling images for distro packages upgrades + if: matrix.env.TEST_TYPE == 'packages-upgrade' + run: | + docker pull quay.io/pulp/pulp-ci-pkgs-c7:3.3.1 + docker pull quay.io/pulp/pulp-ci-pkgs-c8:3.3.1 - name: Run tox run: | tox && exit 0 || true diff --git a/CHANGES/6793.feature b/CHANGES/6793.feature new file mode 100644 index 000000000..c67a73028 --- /dev/null +++ b/CHANGES/6793.feature @@ -0,0 +1 @@ +Add the ability to install Pulp from Linux distro (RPM) packages. diff --git a/CHANGES/6794.feature b/CHANGES/6794.feature new file mode 100644 index 000000000..97300a3c3 --- /dev/null +++ b/CHANGES/6794.feature @@ -0,0 +1 @@ +Let users specify an RPM repo containing Pulp. Introduces the new variable: `pulp_pkg_repo` diff --git a/CHANGES/6795.feature b/CHANGES/6795.feature new file mode 100644 index 000000000..f2c92989f --- /dev/null +++ b/CHANGES/6795.feature @@ -0,0 +1,2 @@ +Add variables so users can specify the names of each RPM package to install (pulp_pkg_pulpcore_name +& pulp_install_plugins's pkg_name) or to just override the prefix (pulp_pkg_name_prefix). diff --git a/CHANGES/6796.misc b/CHANGES/6796.misc new file mode 100644 index 000000000..dcc4cc984 --- /dev/null +++ b/CHANGES/6796.misc @@ -0,0 +1 @@ +Create CI for installing Pulp from RPMs diff --git a/molecule/packages-dynamic/.ansible-lint b/molecule/packages-dynamic/.ansible-lint new file mode 120000 index 000000000..3da8d984e --- /dev/null +++ b/molecule/packages-dynamic/.ansible-lint @@ -0,0 +1 @@ +../../.ansible-lint \ No newline at end of file diff --git a/molecule/packages-dynamic/Dockerfile.j2 b/molecule/packages-dynamic/Dockerfile.j2 new file mode 120000 index 000000000..0104fa8c8 --- /dev/null +++ b/molecule/packages-dynamic/Dockerfile.j2 @@ -0,0 +1 @@ +../scenario_resources/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/packages-dynamic/converge.yml b/molecule/packages-dynamic/converge.yml new file mode 120000 index 000000000..6f4452c48 --- /dev/null +++ b/molecule/packages-dynamic/converge.yml @@ -0,0 +1 @@ +../release-dynamic/converge.yml \ No newline at end of file diff --git a/molecule/packages-dynamic/debian-redis-server.service b/molecule/packages-dynamic/debian-redis-server.service new file mode 120000 index 000000000..f78c356f6 --- /dev/null +++ b/molecule/packages-dynamic/debian-redis-server.service @@ -0,0 +1 @@ +../scenario_resources/debian-redis-server.service \ No newline at end of file diff --git a/molecule/packages-dynamic/group_vars b/molecule/packages-dynamic/group_vars new file mode 120000 index 000000000..90a31405b --- /dev/null +++ b/molecule/packages-dynamic/group_vars @@ -0,0 +1 @@ +../packages-static/group_vars \ No newline at end of file diff --git a/molecule/packages-dynamic/molecule.yml b/molecule/packages-dynamic/molecule.yml new file mode 100644 index 000000000..89617cacb --- /dev/null +++ b/molecule/packages-dynamic/molecule.yml @@ -0,0 +1,54 @@ +--- +dependency: + name: galaxy + role-file: requirements.yml +driver: + name: docker +lint: | + yamllint . + ansible-lint +# This is ignored by molecule, but can be reused in yaml +.platform_base: &platform_base + privileged: False + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + # Fixes issues accessing .so files under /tmp during pip builds. + # https://github.com/docker/compose/issues/1339 + - /tmp:exec,mode=1777 + - /run + - /run/lock + capabilities: + - NET_ADMIN + - NET_RAW +platforms: + - <<: *platform_base + name: centos-7 + image: centos:7 + command: /sbin/init + - <<: *platform_base + name: centos-8 + image: centos:8 + command: /sbin/init +provisioner: + name: ansible + config_options: + defaults: + stdout_callback: debug + inventory: + links: + host_vars: ../scenario_resources/host_vars/ + group_vars: ../scenario_resources/group_vars/ +scenario: + test_sequence: + - lint + - dependency + - syntax + - create + - prepare + - converge + - idempotence + - side_effect + - verify +verifier: + name: inspec diff --git a/molecule/packages-dynamic/prepare.yml b/molecule/packages-dynamic/prepare.yml new file mode 100644 index 000000000..52bad7114 --- /dev/null +++ b/molecule/packages-dynamic/prepare.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + tasks: + - name: Hot patch redis-server service for docker operation (Debian) + copy: + src: debian-redis-server.service + dest: /etc/systemd/system/redis-server.service + when: ansible_distribution == "Debian" diff --git a/molecule/packages-dynamic/requirements.yml b/molecule/packages-dynamic/requirements.yml new file mode 120000 index 000000000..9a736435a --- /dev/null +++ b/molecule/packages-dynamic/requirements.yml @@ -0,0 +1 @@ +../../requirements.yml \ No newline at end of file diff --git a/molecule/packages-dynamic/roles b/molecule/packages-dynamic/roles new file mode 120000 index 000000000..b741aa3db --- /dev/null +++ b/molecule/packages-dynamic/roles @@ -0,0 +1 @@ +../../roles \ No newline at end of file diff --git a/molecule/packages-static/.ansible-lint b/molecule/packages-static/.ansible-lint new file mode 120000 index 000000000..3da8d984e --- /dev/null +++ b/molecule/packages-static/.ansible-lint @@ -0,0 +1 @@ +../../.ansible-lint \ No newline at end of file diff --git a/molecule/packages-static/Dockerfile.j2 b/molecule/packages-static/Dockerfile.j2 new file mode 120000 index 000000000..0104fa8c8 --- /dev/null +++ b/molecule/packages-static/Dockerfile.j2 @@ -0,0 +1 @@ +../scenario_resources/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/packages-static/converge.yml b/molecule/packages-static/converge.yml new file mode 120000 index 000000000..f68c2c56d --- /dev/null +++ b/molecule/packages-static/converge.yml @@ -0,0 +1 @@ +../../playbooks/example-use/playbook.yml \ No newline at end of file diff --git a/molecule/packages-static/debian-redis-server.service b/molecule/packages-static/debian-redis-server.service new file mode 120000 index 000000000..f78c356f6 --- /dev/null +++ b/molecule/packages-static/debian-redis-server.service @@ -0,0 +1 @@ +../scenario_resources/debian-redis-server.service \ No newline at end of file diff --git a/molecule/packages-static/group_vars/all b/molecule/packages-static/group_vars/all new file mode 100644 index 000000000..4a69fd0e7 --- /dev/null +++ b/molecule/packages-static/group_vars/all @@ -0,0 +1,10 @@ +--- +pulp_default_admin_password: password +pulp_upgrade: true +pulp_install_source: packages +pulp_install_plugins: + pulp-file: +pulp_settings: + secret_key: secret + content_origin: "http://{{ ansible_fqdn }}" +pulp_pkg_repo: "https://fedorapeople.org/groups/katello/releases/yum/nightly/pulpcore/el{{ ansible_distribution_major_version }}/x86_64/" diff --git a/molecule/packages-static/molecule.yml b/molecule/packages-static/molecule.yml new file mode 100644 index 000000000..89617cacb --- /dev/null +++ b/molecule/packages-static/molecule.yml @@ -0,0 +1,54 @@ +--- +dependency: + name: galaxy + role-file: requirements.yml +driver: + name: docker +lint: | + yamllint . + ansible-lint +# This is ignored by molecule, but can be reused in yaml +.platform_base: &platform_base + privileged: False + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + # Fixes issues accessing .so files under /tmp during pip builds. + # https://github.com/docker/compose/issues/1339 + - /tmp:exec,mode=1777 + - /run + - /run/lock + capabilities: + - NET_ADMIN + - NET_RAW +platforms: + - <<: *platform_base + name: centos-7 + image: centos:7 + command: /sbin/init + - <<: *platform_base + name: centos-8 + image: centos:8 + command: /sbin/init +provisioner: + name: ansible + config_options: + defaults: + stdout_callback: debug + inventory: + links: + host_vars: ../scenario_resources/host_vars/ + group_vars: ../scenario_resources/group_vars/ +scenario: + test_sequence: + - lint + - dependency + - syntax + - create + - prepare + - converge + - idempotence + - side_effect + - verify +verifier: + name: inspec diff --git a/molecule/packages-static/prepare.yml b/molecule/packages-static/prepare.yml new file mode 100644 index 000000000..52bad7114 --- /dev/null +++ b/molecule/packages-static/prepare.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + tasks: + - name: Hot patch redis-server service for docker operation (Debian) + copy: + src: debian-redis-server.service + dest: /etc/systemd/system/redis-server.service + when: ansible_distribution == "Debian" diff --git a/molecule/packages-static/requirements.yml b/molecule/packages-static/requirements.yml new file mode 120000 index 000000000..9a736435a --- /dev/null +++ b/molecule/packages-static/requirements.yml @@ -0,0 +1 @@ +../../requirements.yml \ No newline at end of file diff --git a/molecule/packages-static/roles b/molecule/packages-static/roles new file mode 120000 index 000000000..b741aa3db --- /dev/null +++ b/molecule/packages-static/roles @@ -0,0 +1 @@ +../../roles \ No newline at end of file diff --git a/molecule/packages-upgrade/.ansible-lint b/molecule/packages-upgrade/.ansible-lint new file mode 120000 index 000000000..3da8d984e --- /dev/null +++ b/molecule/packages-upgrade/.ansible-lint @@ -0,0 +1 @@ +../../.ansible-lint \ No newline at end of file diff --git a/molecule/packages-upgrade/Dockerfile.j2 b/molecule/packages-upgrade/Dockerfile.j2 new file mode 120000 index 000000000..0104fa8c8 --- /dev/null +++ b/molecule/packages-upgrade/Dockerfile.j2 @@ -0,0 +1 @@ +../scenario_resources/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/packages-upgrade/converge.yml b/molecule/packages-upgrade/converge.yml new file mode 120000 index 000000000..f68c2c56d --- /dev/null +++ b/molecule/packages-upgrade/converge.yml @@ -0,0 +1 @@ +../../playbooks/example-use/playbook.yml \ No newline at end of file diff --git a/molecule/packages-upgrade/debian-redis-server.service b/molecule/packages-upgrade/debian-redis-server.service new file mode 120000 index 000000000..f78c356f6 --- /dev/null +++ b/molecule/packages-upgrade/debian-redis-server.service @@ -0,0 +1 @@ +../scenario_resources/debian-redis-server.service \ No newline at end of file diff --git a/molecule/packages-upgrade/group_vars/all b/molecule/packages-upgrade/group_vars/all new file mode 100644 index 000000000..d2d139356 --- /dev/null +++ b/molecule/packages-upgrade/group_vars/all @@ -0,0 +1,11 @@ +--- +pulp_default_admin_password: password +pulp_upgrade: true +pulp_install_source: packages +pulp_install_plugins: + pulp-file: +pulp_settings: + secret_key: secret + content_origin: "http://{{ ansible_fqdn }}" +pulp_pkg_repo: "https://fedorapeople.org/groups/katello/releases/yum/nightly/pulpcore/el{{ ansible_distribution_major_version }}/x86_64/" +pulp_pkg_upgrade_all: true diff --git a/molecule/packages-upgrade/molecule.yml b/molecule/packages-upgrade/molecule.yml new file mode 100644 index 000000000..e48a11efc --- /dev/null +++ b/molecule/packages-upgrade/molecule.yml @@ -0,0 +1,55 @@ +--- +dependency: + name: galaxy + role-file: requirements.yml +driver: + name: docker +lint: | + yamllint . + ansible-lint +# This is ignored by molecule, but can be reused in yaml +.platform_base: &platform_base + privileged: False + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + # Fixes issues accessing .so files under /tmp during pip builds. + # https://github.com/docker/compose/issues/1339 + - /tmp:exec,mode=1777 + - /run + - /run/lock + capabilities: + - NET_ADMIN + - NET_RAW +platforms: + - <<: *platform_base + name: centos-7 + # molecule often fails to pull, so we pull all images in .travis.yml + image: quay.io/pulp/pulp-ci-pkgs-c7:3.3.1 + command: /sbin/init + - <<: *platform_base + name: centos-8 + image: quay.io/pulp/pulp-ci-pkgs-c8:3.3.1 + command: /sbin/init +provisioner: + name: ansible + config_options: + defaults: + stdout_callback: debug + inventory: + links: + host_vars: ../scenario_resources/host_vars/ + group_vars: ../scenario_resources/group_vars/ +scenario: + test_sequence: + - lint + - dependency + - syntax + - create + - prepare + - converge + - idempotence + - side_effect + - verify +verifier: + name: inspec diff --git a/molecule/packages-upgrade/prepare.yml b/molecule/packages-upgrade/prepare.yml new file mode 100644 index 000000000..52bad7114 --- /dev/null +++ b/molecule/packages-upgrade/prepare.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + tasks: + - name: Hot patch redis-server service for docker operation (Debian) + copy: + src: debian-redis-server.service + dest: /etc/systemd/system/redis-server.service + when: ansible_distribution == "Debian" diff --git a/molecule/packages-upgrade/requirements.yml b/molecule/packages-upgrade/requirements.yml new file mode 120000 index 000000000..9a736435a --- /dev/null +++ b/molecule/packages-upgrade/requirements.yml @@ -0,0 +1 @@ +../../requirements.yml \ No newline at end of file diff --git a/molecule/packages-upgrade/roles b/molecule/packages-upgrade/roles new file mode 120000 index 000000000..b741aa3db --- /dev/null +++ b/molecule/packages-upgrade/roles @@ -0,0 +1 @@ +../../roles \ No newline at end of file diff --git a/roles/pulp/README.md b/roles/pulp/README.md index 7c05687fd..759e696c2 100644 --- a/roles/pulp/README.md +++ b/roles/pulp/README.md @@ -22,6 +22,7 @@ Role Variables See `prereq_pip_packages` also. * **Example**: ```yaml + pulp_install_source: pip pulp_install_plugins: pulp-zero: {} pulp-one: # plugin name (pulp-ansible, pulp-container, pulp-rpm, ...) @@ -33,6 +34,8 @@ Role Variables pulp-four: prereq_role: "pulp.pulp_three_role" # role to run immediately before the venv is created ``` +* `pulp_install_api_service`: Whether to create systemd service files for + pulpcore-api. Defaults to "true". * `pulp_cache_dir`: Location of Pulp cache. Defaults to "/var/lib/pulp/tmp". * `pulp_config_dir`: Directory which will contain Pulp configuration files. Defaults to "/etc/pulp". @@ -42,6 +45,7 @@ Role Variables * `pulp_user_home`: equivalent to `MEDIA_ROOT` from `pulpcore` i.e. absolute path for pulp user home. * `pulp_install_api_service`: Whether to create systemd service files for pulpcore-api. Defaults to "true". +* `pulp_source_dir`: Optional. Absolute path to pulpcore source code. If present, pulpcore will be installed from source in editable mode. Also accepts a pip VCS URL, to (for example) install the master branch. * `pulp_user`: User that owns and runs Pulp. Defaults to "pulp". @@ -95,6 +99,52 @@ Role Variables Also accepts a single string or empty string. Only affects RHEL7 (RHEL8 no longer has an optional repo.) +Role Variables if installing from RPMs +-------------------------------------- +Normally, Pulp is installed from Python pip packages (from PyPI.) pulp_installer can install Pulp from +RPM packages instead if this variable is set. Other distro packaging formats may work as well: + +* `pulp_install_source`: Whether to install from "pip" (PyPI, python packages) or the Linux distro's + (RPM) "packages". + Defaults to "pip". + +If it is set to "packages", the following variables are used, or behave *differently* from above: +* `pulp_install_plugins`: A nested dictionary of plugin configuration options. + Defaults to "{}", which will not install any plugins. + * *Dictionary Key*: The plugin name. **Required**. + * `pkg_name`: If this is left undefined, each Linux distro package will be installed by the name `pulp_pkg_name_prefix` + with the Dictionary Key appended to it. `pulp_pkg_name_prefix` defaults to "python3-", so if the Dictionary key is, "pulp-file", + the package `python3-pulp-file` will be installed. This variable overrides the entire package name. + * **Example**: + ```yaml + pulp_install_source: packages + pulp_install_plugins: + pulp-zero: {} # Effectively python3-pulp-zero + pulp-one: {} + pkg_name: python3-pulp-one-ng + pulp-two: + pkg_name: pulp_two_underscores + ``` +* `pulp_install_dir`: Location of the filesystem prefix where package installed python programs + (gunicorn & rq) are looked for on the filesystem. Defaults to "/usr" (such as for "/usr/bin/gunicorn"). +* pulp_django_admin_paths: A list of possible filepaths to the `django-admin` + command. The first one found will be used for pulp_installer's + internal usage. See `defaults/main.yml` for default values. +* pulp_pkg_name_prefix: The beginning of the Linux distro (RPM) package names for pulp, that is + appended to in order to install "pulpcore" and the plugins. Defaults to "python3-". +* pulp_pkg_pulpcore_name: The entire name of the Linux distro (RPM) package for pulpcore. + Defaults to: "python3-pulpcore" +* pulp_pkg_repo: yum/dnf package repo to add to the system before installing Pulp + Consists simply of the URL to the repo. Defaults to nothing. Does not support any other repo + type yet. +* pulp_pkg_undeclared_deps: Additional Linux distro (RPM) packages to install before installing pulpcore. + See `defaults/main.yml` for default values. +* pulp_pkg_upgrade_all: Whether to upgrade all Pulp Linux distro (RPM) packages (including the `pulp_pkg_undeclared_deps` + packages.) +* pulp_upgraded_manually: Set this to `true` if you updated/upgraded Pulp manually beforehand, + without using the installer. (e.g., you ran `yum update` and your Pulp installation is broken. Re-running the + installer will fix it.) + Defaults to `false`. Shared Variables ---------------- @@ -121,6 +171,7 @@ Idempotency ----------- This role is idempotent by default. It is dependent on these settings remaining `false`: * Every `upgrade` under `pulp_install_plugins` +* pulp_upgraded_manually License ------- diff --git a/roles/pulp/defaults/main.yml b/roles/pulp/defaults/main.yml index aa8e5eb71..df8a11cd2 100644 --- a/roles/pulp/defaults/main.yml +++ b/roles/pulp/defaults/main.yml @@ -15,7 +15,8 @@ pulp_settings_db_defaults: NAME: pulp USER: pulp PASSWORD: pulp -pulp_install_dir: '/usr/local/lib/pulp' +pulp_install_source: pip +pulp_install_dir: '{{ (pulp_install_source == "packages") | ternary("/usr", "/usr/local/lib/pulp") }}' pulp_install_plugins: {} pulp_install_api_service: true # Deprecated unused. Variables for dependency upgrades are TBD @@ -53,3 +54,16 @@ epel_release_packages: # set it, and even then we may just help them to append to a list. pulp_irregularly_named_plugins: - galaxy-ng +pulp_django_admin_paths: + - "{{ pulp_install_dir }}/bin/django-admin" + - "{{ pulp_install_dir }}/bin/python3-django-admin" +pulp_pkg_name_prefix: "python3-" +pulp_pkg_pulpcore_name: "{{ pulp_pkg_name_prefix }}pulpcore" +pulp_pkg_repo: +pulp_pkg_undeclared_deps: + - python3-psycopg2 + - pulpcore-selinux + - python3-djangorestframework + - python3-djangorestframework-queryfields +pulp_pkg_upgrade_all: false +pulp_upgraded_manually: false diff --git a/roles/pulp/handlers/main.yml b/roles/pulp/handlers/main.yml index 001556dc8..bdb883f55 100644 --- a/roles/pulp/handlers/main.yml +++ b/roles/pulp/handlers/main.yml @@ -14,7 +14,7 @@ static: no - name: Collect static content - command: '{{ pulp_install_dir }}/bin/django-admin collectstatic --noinput --link' + command: "{{ pulp_django_admin_path }} collectstatic --noinput --link" register: staticresult changed_when: "staticresult.stdout is not search('\n0 static files')" become: true diff --git a/roles/pulp/handlers/restart_all_pulp.yml b/roles/pulp/handlers/restart_all_pulp.yml index 05db3b1ff..b8f9bce91 100644 --- a/roles/pulp/handlers/restart_all_pulp.yml +++ b/roles/pulp/handlers/restart_all_pulp.yml @@ -6,7 +6,7 @@ # pulp_workers is called. But said users' ansible does not know # the number of pulp-workers yet. # -# Later one, the Pulp services *should* be refactored into a +# TODO: The Pulp services *should* be refactored into a # systemd multi-service app. That would mean a single task would # restart all of Pulp. - name: Reload systemd diff --git a/roles/pulp/tasks/find_django_admin.yml b/roles/pulp/tasks/find_django_admin.yml new file mode 100644 index 000000000..cae2ba53b --- /dev/null +++ b/roles/pulp/tasks/find_django_admin.yml @@ -0,0 +1,10 @@ +--- +- name: "check if {{ item }} exists" + stat: + path: "{{ item }}" + register: path + +- name: "Set {{ item }} to be used if it exists" + set_fact: + pulp_django_admin_path: "{{ item }}" + when: path.stat.exists diff --git a/roles/pulp/tasks/install.yml b/roles/pulp/tasks/install.yml index e3b457e5c..3c009197a 100644 --- a/roles/pulp/tasks/install.yml +++ b/roles/pulp/tasks/install.yml @@ -76,13 +76,6 @@ group: '{{ pulp_group }}' mode: "u+rwx,g+rwx,o+rx" - - name: Create pulp install dir - file: - path: '{{ pulp_install_dir }}' - state: directory - owner: '{{ pulp_user }}' - group: '{{ pulp_group }}' - # If so, owner will be apache - name: Check if we have Pulp 2 installed stat: @@ -136,208 +129,18 @@ become: true -- name: Prepare the virtualenv itself - block: - - # Hack for use system-wide packages when enabled by user - Must be first usage of virtualenv - - name: Create a venv with system-wide packages setting if the venv does not exist - pip: - name: pip # specifying pip is a safe hack to create the venv, since pip is automatically in every new venv. - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - virtualenv_site_packages: yes - when: pulp_use_system_wide_pkgs | bool - - - name: Upgrade to a recent edition of pip (supporting manylinux2014) - pip: - name: pip>=20.0 - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - - - name: Update any existing venv to allow system-wide packages - lineinfile: - path: "{{ pulp_install_dir }}/pyvenv.cfg" - regexp: '^include-system-site-packages' - line: "include-system-site-packages = true" - when: pulp_use_system_wide_pkgs | bool - - become: true - become_user: '{{ pulp_user }}' - -- name: Prepare for preflight checks - block: - - - name: Obtain list of packages in the venv to see if any plugins are installed - pip_package_info: - clients: "{{ pulp_install_dir }}/bin/pip" - register: pip_pkgs - - - name: Create requirements.in file to check pulpcore/plugin compatibility - template: - src: templates/requirements.in.j2 - dest: "{{ pulp_install_dir }}/requirements.in" - - - name: Install pip-tools, which provides pip-compile to check version compatibility - pip: - name: pip-tools>=5.2.0 - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - - when: pulp_source_dir is undefined - become: true - become_user: '{{ pulp_user }}' - -# When there are no plugin prereqs roles -- name: Run preflight check (no prereq roles) - include_tasks: preflight_function.yml - when: - - pulp_source_dir is undefined - - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count == 0 - -# When there are plugin prereq roles -# We are not using a block for these next few tasks; see preflight_function.yml -# -# Some plugins have deps that need to be installed for setup.py to load. When an error -# occurs due to those missing deps, the version comparison will not occur. -# So we will ignore all other errors this time. The version comparison occurs before -# that error is raised. We only care about that contain this string. -- name: Run preflight check before prereq roles - include_tasks: preflight_function.yml - vars: - failed_condition: '"Could not find a version" in compatibility.stderr' - # When there is at least 1 prereq role. - when: - - pulp_source_dir is undefined - - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count > 0 - -# Note: We would do a static import like in pulp_database, but -# looping does not work with it, so we do a dynamic include. -- name: Include plugins prereq roles - include_role: - name: "{{ item.value.prereq_role }}" - with_dict: "{{ pulp_install_plugins_normalized }}" - when: - - item.value.prereq_role is defined - -# No loop over pulp_install_plugins_normalized; let's run this only once. -# Also, this time it must succeed, a return code of 0. -- name: Re-run preflight check after plugin prereq roles - include_tasks: preflight_function.yml - vars: - failed_condition: > - (compatibility.rc != 0) and - ("AttributeError: module \'setuptools.build_meta\' has no attribute \'__legacy__\'" not in compatibility.stderr) - # if it didn't succeed last time - when: - - pulp_source_dir is undefined - - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count > 0 - - compatibility is defined - - compatibility.rc is defined - - compatibility.rc != 0 - -- name: Install Pulp via Pip - block: - - - name: Install the prereq_pip_packages - pip: - name: '{{ prereq_pip_packages }}' - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - when: prereq_pip_packages | length > 0 - - - name: Install pulpcore package via PyPI - pip: - name: pulpcore - # pulp_upgrade true/false -> ansible pip module latest/present -> - # pip command "install --upgrade"/"install" - # Commented out because: - # "version is incompatible with state=latest" - # But we still need to handle upgrading dependencies. - # state: "{{ pulp_upgrade | ternary('latest','present') }}" - version: '{{ pulp_version }}' - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - when: pulp_source_dir is undefined - notify: - - Collect static content - - Restart all Pulp services - - - name: Install pulpcore package from source - pip: - name: '{{ pulp_source_dir }}' - editable: '{{ pulp_pip_editable }}' - # For this task, `state` has no effect. It installs or upgrades/ - # downgrades to the exact dir's or VCS URL's version regardless. - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - when: pulp_source_dir is defined - register: result - # This is a hack. Editable pip installs are always changed, which fails molecule's - # idempotence test. - # Unfortunately, this will prevent the handlers from being notified. - changed_when: result.changed and not pulp_pip_editable - notify: - - Collect static content - - Restart all Pulp services - - # We have to do this separate task, even though the pip module should - # register the installed version, because when tested (Ansible 2.9, with no - # changes applied), the registered result from "Install pulpcore package - # from source" left the version field "null". - - name: Obtain list of packages & versions in the venv after pulpcore install - pip_package_info: - clients: "{{ pulp_install_dir }}/bin/pip" - register: pip_pkgs - - - name: Create constraints file to lock the pulpcore version when plugins are installed - copy: - content: "pulpcore=={{ pip_pkgs.packages[pulp_install_dir + '/bin/pip'].pulpcore[0].version }}\n" - dest: "{{ pulp_install_dir }}/pip_constraints_for_plugins.txt" - - - name: Install Pulp plugins via PyPI - pip: - name: '{{ item.key }}' - # item.upgrade true/false -> ansible pip module latest/present -> - # pip command "install --upgrade"/"install" - state: "{{ item.value.upgrade | default(false) | ternary('latest','present','present') }}" - # TODO: Handle the fact that "version is incompatible with state=latest" - # through proper means, rather than just telling users not to set both. - version: '{{ item.value.version | default(omit) }}' - extra_args: "-c {{ pulp_install_dir }}/pip_constraints_for_plugins.txt" - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - with_dict: '{{ pulp_install_plugins_normalized }}' - when: item.value.source_dir is undefined - notify: - - Collect static content - - Restart all Pulp services - - - name: Install Pulp plugins from source - pip: - name: '{{ item.value.source_dir }}' - editable: '{{ pulp_pip_editable }}' - # For this task, `state` has no effect. It installs or upgrades/ - # downgrades to the exact dir's or VCS URL's version regardless. - state: present - extra_args: "-c {{ pulp_install_dir }}/pip_constraints_for_plugins.txt" - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - with_dict: '{{ pulp_install_plugins_normalized }}' - when: item.value.source_dir is defined - register: result - # This is a hack. Editable pip installs are always changed, which fails molecule's - # idempotence test. - # Unfortunately, this will prevent the handlers from being notified. - changed_when: result.changed and not pulp_pip_editable - notify: - - Collect static content - - Restart all Pulp services - - become: true - become_user: '{{ pulp_user }}' +- name: install pulp from {{ pulp_install_source }} + include_tasks: install_{{ pulp_install_source }}.yml + +# Needed for when Pulp was manually upgraded by the user before running +# pulp_installer. Especially `yum update`/`apt upgrade`. +# TODO: For usability purposes, auto-detect by inspecting the system. +- name: Triggers handlers if pulp_upgraded_manually + debug: + msg: Triggering handlers because Pulp was updated manually + notify: + - Collect static content + - Restart all Pulp services + when: pulp_upgraded_manually + changed_when: pulp_upgraded_manually ... diff --git a/roles/pulp/tasks/install_packages.yml b/roles/pulp/tasks/install_packages.yml new file mode 100644 index 000000000..dc195dcac --- /dev/null +++ b/roles/pulp/tasks/install_packages.yml @@ -0,0 +1,51 @@ +--- +- block: + + - name: "Install the Pulp undeclared {{ ansible_pkg_mgr }} package dependencies" + package: + name: "{{ pulp_pkg_undeclared_deps }}" + state: "{{ pulp_pkg_upgrade_all | ternary('latest','present') }}" + notify: + - Collect static content + - Restart all Pulp services + + - name: "Install pulpcore via {{ ansible_pkg_mgr }} packages" + package: + name: "{{ pulp_pkg_pulpcore_name }}" + state: "{{ pulp_pkg_upgrade_all | ternary('latest','present') }}" + notify: + - Collect static content + - Restart all Pulp services + + - name: "Install Pulp plugins via {{ ansible_pkg_mgr }} packages" + package: + name: '{{ pulp_pkg_name_prefix }}{{ item.key }}' + state: "{{ pulp_pkg_upgrade_all | ternary('latest','present') }}" + # We use pulp_install_plugins rather than pulp_install_plugins_normalized + # so that users can specify package names with underscores if distros + # require it. + with_dict: '{{ pulp_install_plugins }}' + when: + - item.value.pkg_name is undefined + notify: + - Collect static content + - Restart all Pulp services + + - name: "Install Pulp plugins (with overriden names) via {{ ansible_pkg_mgr }}" + package: + name: '{{ item.value.pkg_name }}' + state: present + with_dict: '{{ pulp_install_plugins }}' + when: + - item.value.pkg_name is defined + notify: + - Collect static content + - Restart all Pulp services + + - name: "Install gunicorn via {{ ansible_pkg_mgr }} packages" + package: + name: python3-gunicorn + state: present + + become: true +... diff --git a/roles/pulp/tasks/install_pip.yml b/roles/pulp/tasks/install_pip.yml new file mode 100644 index 000000000..3107f36e4 --- /dev/null +++ b/roles/pulp/tasks/install_pip.yml @@ -0,0 +1,221 @@ +--- +- name: Create pulp install dir + file: + path: '{{ pulp_install_dir }}' + state: directory + owner: '{{ pulp_user }}' + group: '{{ pulp_group }}' + become: true + +- name: Prepare the virtualenv itself + block: + + # Hack for use system-wide packages when enabled by user - Must be first usage of virtualenv + - name: Create a venv with system-wide packages setting if the venv does not exist + pip: + name: pip # specifying pip is a safe hack to create the venv, since pip is automatically in every new venv. + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + virtualenv_site_packages: yes + when: pulp_use_system_wide_pkgs | bool + + - name: Upgrade to a recent edition of pip (supporting manylinux2014) + pip: + name: pip>=20.0 + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + + - name: Update any existing venv to allow system-wide packages + lineinfile: + path: "{{ pulp_install_dir }}/pyvenv.cfg" + regexp: '^include-system-site-packages' + line: "include-system-site-packages = true" + when: pulp_use_system_wide_pkgs | bool + + become: true + become_user: '{{ pulp_user }}' + +- name: Prepare for preflight checks + block: + + - name: Obtain list of packages in the venv to see if any plugins are installed + pip_package_info: + clients: "{{ pulp_install_dir }}/bin/pip" + register: pip_pkgs + + - name: Create requirements.in file to check pulpcore/plugin compatibility + template: + src: templates/requirements.in.j2 + dest: "{{ pulp_install_dir }}/requirements.in" + + - name: Install pip-tools, which provides pip-compile to check version compatibility + pip: + name: pip-tools>=5.2.0 + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + + when: pulp_source_dir is undefined + become: true + become_user: '{{ pulp_user }}' + +# When there are no plugin prereqs roles +- name: Run preflight check (no prereq roles) + include_tasks: preflight_function.yml + when: + - pulp_source_dir is undefined + - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count == 0 + +# When there are plugin prereq roles +# We are not using a block for these next few tasks; see preflight_function.yml +# +# Some plugins have deps that need to be installed for setup.py to load. When an error +# occurs due to those missing deps, the version comparison will not occur. +# So we will ignore all other errors this time. The version comparison occurs before +# that error is raised. We only care about that contain this string. +- name: Run preflight check before prereq roles + include_tasks: preflight_function.yml + vars: + failed_condition: '"Could not find a version" in compatibility.stderr' + # When there is at least 1 prereq role. + when: + - pulp_source_dir is undefined + - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count > 0 + +# Note: We would do a static import like in pulp_database, but +# looping does not work with it, so we do a dynamic include. +- name: Include plugins prereq roles + include_role: + name: "{{ item.value.prereq_role }}" + with_dict: "{{ pulp_install_plugins_normalized }}" + when: + - item.value.prereq_role is defined + +# No loop over pulp_install_plugins_normalized; let's run this only once. +# Also, this time it must succeed, a return code of 0. +- name: Re-run preflight check after plugin prereq roles + include_tasks: preflight_function.yml + vars: + failed_condition: > + (compatibility.rc != 0) and + ("AttributeError: module \'setuptools.build_meta\' has no attribute \'__legacy__\'" not in compatibility.stderr) + # if it didn't succeed last time + when: + - pulp_source_dir is undefined + - pulp_install_plugins_normalized | dict2items | selectattr('value.prereq_role', 'defined') | list | count > 0 + - compatibility is defined + - compatibility.rc is defined + - compatibility.rc != 0 + +- name: Install Pulp via Pip + block: + + - name: Install the prereq_pip_packages + pip: + name: '{{ prereq_pip_packages }}' + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + when: prereq_pip_packages | length > 0 + + - name: Install pulpcore via PyPI + pip: + name: pulpcore + # pulp_upgrade true/false -> ansible pip module latest/present -> + # pip command "install --upgrade"/"install" + # Commented out because: + # "version is incompatible with state=latest" + # But we still need to handle upgrading dependencies. + # state: "{{ pulp_upgrade | ternary('latest','present') }}" + version: '{{ pulp_version }}' + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + when: pulp_source_dir is undefined + notify: + - Collect static content + - Restart all Pulp services + + - name: Install pulpcore package from source + pip: + name: '{{ pulp_source_dir }}' + editable: '{{ pulp_pip_editable }}' + # For this task, `state` has no effect. It installs or upgrades/ + # downgrades to the exact dir's or VCS URL's version regardless. + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + when: pulp_source_dir is defined + register: result + # This is a hack. Editable pip installs are always changed, which fails molecule's + # idempotence test. + # Unfortunately, this will prevent the handlers from being notified. + changed_when: result.changed and not pulp_pip_editable + notify: + - Collect static content + - Restart all Pulp services + + # We have to do this separate task, even though the pip module should + # register the installed version, because when tested (Ansible 2.9, with no + # changes applied), the registered result from "Install pulpcore package + # from source" left the version field "null". + - name: Obtain list of packages & versions in the venv after pulpcore install + pip_package_info: + clients: "{{ pulp_install_dir }}/bin/pip" + register: pip_pkgs + + - name: Create constraints file to lock the pulpcore version when plugins are installed + copy: + content: "pulpcore=={{ pip_pkgs.packages[pulp_install_dir + '/bin/pip'].pulpcore[0].version }}\n" + dest: "{{ pulp_install_dir }}/pip_constraints_for_plugins.txt" + + - name: Install Pulp plugins via PyPI + pip: + name: '{{ item.key }}' + # item.upgrade true/false -> ansible pip module latest/present -> + # pip command "install --upgrade"/"install" + state: "{{ item.value.upgrade | default(false) | ternary('latest','present','present') }}" + # TODO: Handle the fact that "version is incompatible with state=latest" + # through proper means, rather than just telling users not to set both. + version: '{{ item.value.version | default(omit) }}' + extra_args: "-c {{ pulp_install_dir }}/pip_constraints_for_plugins.txt" + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + with_dict: '{{ pulp_install_plugins_normalized }}' + when: item.value.source_dir is undefined + notify: + - Collect static content + - Restart all Pulp services + + - name: Install Pulp plugins from source + pip: + name: '{{ item.value.source_dir }}' + editable: '{{ pulp_pip_editable }}' + # For this task, `state` has no effect. It installs or upgrades/ + # downgrades to the exact dir's or VCS URL's version regardless. + state: present + extra_args: "-c {{ pulp_install_dir }}/pip_constraints_for_plugins.txt" + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + with_dict: '{{ pulp_install_plugins_normalized }}' + when: item.value.source_dir is defined + register: result + # This is a hack. Editable pip installs are always changed, which fails molecule's + # idempotence test. + # Unfortunately, this will prevent the handlers from being notified. + changed_when: result.changed and not pulp_pip_editable + notify: + - Collect static content + - Restart all Pulp services + + - name: Install gunicorn via PyPI + pip: + name: gunicorn + state: present + virtualenv: '{{ pulp_install_dir }}' + virtualenv_command: '{{ pulp_python_interpreter }} -m venv' + + become: true + become_user: '{{ pulp_user }}' +... diff --git a/roles/pulp/tasks/main.yml b/roles/pulp/tasks/main.yml index 30f893eb4..2372ab20c 100644 --- a/roles/pulp/tasks/main.yml +++ b/roles/pulp/tasks/main.yml @@ -53,7 +53,20 @@ - name: set the default system PATH as a fact set_fact: default_bin_path: "{{ systemd_show_env_path.stdout }}" + - import_tasks: repos.yml - import_tasks: install.yml + +# This task is after install.yml since it gets installed then. +- name: Find and use the first found possible path to django-admin + include_tasks: find_django_admin.yml + with_items: "{{ pulp_django_admin_paths }}" + when: + # Works for both strings and lists to make sure not empty + - pulp_django_admin_paths is not none + - pulp_django_admin_paths | length > 0 + # Prevents running again once completed. + - pulp_django_admin_path is not defined + - import_tasks: configure.yml - import_tasks: wsgi.yml diff --git a/roles/pulp/tasks/repos.yml b/roles/pulp/tasks/repos.yml index 3228be915..d1a9aeae6 100644 --- a/roles/pulp/tasks/repos.yml +++ b/roles/pulp/tasks/repos.yml @@ -112,3 +112,14 @@ when: - ansible_distribution == "CentOS" - ansible_distribution_major_version|int >= 8 + +- name: Add pulpcore RPM repositories + yum_repository: + name: pulpcore + description: pulpcore + baseurl: "{{ pulp_pkg_repo }}" + gpgcheck: false + enabled: true + when: + - ansible_os_family == "RedHat" + - pulp_pkg_repo is not none diff --git a/roles/pulp/tasks/wsgi.yml b/roles/pulp/tasks/wsgi.yml index 5e5cf21e5..7e7fab03f 100644 --- a/roles/pulp/tasks/wsgi.yml +++ b/roles/pulp/tasks/wsgi.yml @@ -1,13 +1,4 @@ --- -- name: Install gunicorn - pip: - name: gunicorn - state: present - virtualenv: '{{ pulp_install_dir }}' - virtualenv_command: '{{ pulp_python_interpreter }} -m venv' - become: true - become_user: '{{ pulp_user }}' - - block: - name: Install pulpcore-api service files diff --git a/roles/pulp_database/README.md b/roles/pulp_database/README.md index 322475d76..aa44110d8 100644 --- a/roles/pulp_database/README.md +++ b/roles/pulp_database/README.md @@ -33,7 +33,6 @@ variables which are documented in that role: * `pulp_user` * `pulp_install_dir` -* `pulp_install_plugins` * `pulp_default_admin_password` * `pulp_settings` diff --git a/roles/pulp_database/tasks/main.yml b/roles/pulp_database/tasks/main.yml index 81f4860c0..4d5fbda6e 100644 --- a/roles/pulp_database/tasks/main.yml +++ b/roles/pulp_database/tasks/main.yml @@ -19,32 +19,20 @@ - meta: flush_handlers -- name: Get the exact pulpcore install dir - find: - paths: '{{ pulp_install_dir }}/lib/' - pattern: '^python.\..$' - use_regex: true - file_type: directory - register: result - -- name: Assert the pulpcore install dir was found - assert: - that: '{{ result.matched }} == 1' - - block: - name: Run database auth migrations - command: '{{ pulp_install_dir }}/bin/django-admin migrate auth --no-input' + command: '{{ pulp_django_admin_path }} migrate auth --no-input' register: migrate_auth changed_when: "'No migrations to apply' not in migrate_auth.stdout" - name: Run database migrations - command: '{{ pulp_install_dir }}/bin/django-admin migrate --no-input' + command: '{{ pulp_django_admin_path }} migrate --no-input' register: result changed_when: "'No migrations to apply' not in result.stdout" - name: Set the Pulp admin user's password - command: '{{ pulp_install_dir }}/bin/django-admin reset-admin-password --password {{ pulp_default_admin_password }}' + command: '{{ pulp_django_admin_path }} reset-admin-password --password {{ pulp_default_admin_password }}' no_log: true when: pulp_default_admin_password is defined and migrate_auth.changed diff --git a/roles/pulp_devel/tasks/main.yml b/roles/pulp_devel/tasks/main.yml index afaf46bb1..894df13c9 100644 --- a/roles/pulp_devel/tasks/main.yml +++ b/roles/pulp_devel/tasks/main.yml @@ -1,6 +1,6 @@ --- - import_tasks: galaxy_ui.yml - when: pulp_install_plugins['galaxy-ng'] is defined + when: pulp_install_plugins_normalized['galaxy-ng'] is defined become: true - import_tasks: install_basic_packages.yml diff --git a/roles/pulp_devel/templates/alias.bashrc.j2 b/roles/pulp_devel/templates/alias.bashrc.j2 index 97b9fe5f2..d0c207eaf 100644 --- a/roles/pulp_devel/templates/alias.bashrc.j2 +++ b/roles/pulp_devel/templates/alias.bashrc.j2 @@ -31,9 +31,9 @@ _pstatus_help="Report the status of all pulp-related services" pdbreset() { workon pulp pstop - django-admin reset_db --noinput - django-admin migrate - django-admin reset-admin-password --password {{ pulp_default_admin_password }} + {{ pulp_django_admin_path }} reset_db --noinput + {{ pulp_django_admin_path }} migrate + {{ pulp_django_admin_path }} reset-admin-password --password {{ pulp_default_admin_password }} } _pdbreset_help="Reset the Pulp database" # can get away with not resetting terminal settings here since it gets reset in phelp @@ -42,7 +42,7 @@ _pdbreset_help="$_dbreset_help - `setterm -foreground red -bold on`THIS DESTROYS pclean() { pdbreset sudo rm -rf {{ pulp_user_home }}/artifact - django-admin collectstatic --noinput --link + {{ pulp_django_admin_path }} collectstatic --noinput --link } _pclean_help="Restore pulp to a clean-installed state" # can get away with not resetting terminal settings here since it gets reset in phelp diff --git a/roles/pulp_webserver/README.md b/roles/pulp_webserver/README.md index 0780ff43f..4763e1e5e 100644 --- a/roles/pulp_webserver/README.md +++ b/roles/pulp_webserver/README.md @@ -68,5 +68,8 @@ role. * `pulp_install_dir`: Location of a virtual environment for Pulp and its Python dependencies. **Required** if used in a separate play from the `pulp` role. Value must match the value used in the `pulp` role. +* `pulp_install_plugins` (technically `pulp_install_plugins_normalized`). The list + of plugins to install is used to inform pulp_webserver which webserver snippets + to look for and use. * `pulp_user_home`: equivalent to `MEDIA_ROOT` from `pulpcore` i.e. absolute path for pulp user home. diff --git a/roles/pulp_webserver/tasks/apache.yml b/roles/pulp_webserver/tasks/apache.yml index 30d8c7063..3c4d6cc18 100644 --- a/roles/pulp_webserver/tasks/apache.yml +++ b/roles/pulp_webserver/tasks/apache.yml @@ -1,16 +1,4 @@ --- -- name: Find site-packages directory - find: - paths: '{{ pulp_install_dir }}' - recurse: true - file_type: directory - patterns: site-packages - register: result - -- name: Assert one directory has been found - assert: - that: '{{ result.matched }} == 1' - - block: - name: Install Apache @@ -42,18 +30,6 @@ path: "{{ pulp_webserver_apache_snippets_dir }}" state: directory - - name: Check installed plugins for Apache snippets - script: - cmd: check_snippet.py {{ item.key | regex_replace("-", "_") | quote }} apache.conf - args: - executable: "{{ pulp_install_dir }}/bin/python" - register: snippets - with_dict: '{{ pulp_install_plugins_normalized }}' - failed_when: > - snippets.stderr is search("env: 'python': No such file or directory") - changed_when: false - check_mode: false - # FIXME: The ansible output here over the results is very ugly. # It may be a lot cleaner if we were to use the command module, # above and/or create a temporary data structure. diff --git a/roles/pulp_webserver/tasks/main.yml b/roles/pulp_webserver/tasks/main.yml index 848babdac..f04be80b4 100644 --- a/roles/pulp_webserver/tasks/main.yml +++ b/roles/pulp_webserver/tasks/main.yml @@ -13,11 +13,24 @@ tags: - always -- import_tasks: nginx.yml - when: pulp_webserver_server == 'nginx' +- debug: + msg: > + {{ (pulp_install_source == "packages") | ternary(pulp_python_interpreter, pulp_install_dir ~ "/bin/python") }} + verbosity: 1 -- import_tasks: apache.yml - when: pulp_webserver_server == 'apache' +- name: "Check installed plugins for {{ pulp_webserver_server }} snippets" + script: + cmd: check_snippet.py {{ item.key | regex_replace("-", "_") | quote }} {{ pulp_webserver_server }}.conf + args: + executable: '{{ (pulp_install_source == "packages") | ternary(pulp_python_interpreter, pulp_install_dir ~ "/bin/python") }}' + register: snippets + with_dict: '{{ pulp_install_plugins_normalized }}' + failed_when: > + snippets.stderr is search("env: 'python': No such file or directory") + changed_when: false + check_mode: false + +- import_tasks: "{{ pulp_webserver_server }}.yml" - import_tasks: firewalld.yml when: pulp_configure_firewall in ['firewalld', 'auto'] diff --git a/roles/pulp_webserver/tasks/nginx.yml b/roles/pulp_webserver/tasks/nginx.yml index 00ea72ca7..da2327599 100644 --- a/roles/pulp_webserver/tasks/nginx.yml +++ b/roles/pulp_webserver/tasks/nginx.yml @@ -7,18 +7,6 @@ when: ansible_distribution == 'Ubuntu' become: yes -- name: Find site-packages directory - find: - paths: '{{ pulp_install_dir }}' - recurse: true - file_type: directory - patterns: site-packages - register: result - -- name: Assert one directory has been found - assert: - that: '{{ result.matched }} == 1' - - block: - name: Install Nginx @@ -52,18 +40,6 @@ path: "/etc/nginx/pulp/" state: directory - - name: Check installed plugins for nginx snippets - script: - cmd: check_snippet.py {{ item.key | regex_replace("-", "_") | quote }} nginx.conf - args: - executable: "{{ pulp_install_dir }}/bin/python" - register: snippets - with_dict: '{{ pulp_install_plugins_normalized }}' - failed_when: > - snippets.stderr is search("env: 'python': No such file or directory") - changed_when: false - check_mode: false - # FIXME: The ansible output here over the results is very ugly. # It may be a lot cleaner if we were to use the command module, # above and/or create a temporary data structure. diff --git a/tox.ini b/tox.ini index 157793891..73fa9cac1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,36,37}-{release, source}-{static, dynamic, upgrade} +envlist = py{27,36,37,38}-{release, source, packages}-{static, dynamic, upgrade} # This repo isn't a Python project, so it makes no sense to install an sdist # into the virtualenvs tox creates. skipsdist = true