diff --git a/.ci/scripts/release.py b/.ci/scripts/release.py index 9b1a6affa..6039125b9 100644 --- a/.ci/scripts/release.py +++ b/.ci/scripts/release.py @@ -107,8 +107,8 @@ def validate_redmine_data(redmine_query_url, redmine_issues): # Second commit: release with open("roles/pulp_common/vars/main.yml") as f: - __pulp_version = yaml.safe_load(f)["__pulp_version"] - previous_pulpcore_version = __pulp_version.split('"')[1] + __pulpcore_version = yaml.safe_load(f)["__pulpcore_version"] + previous_pulpcore_version = __pulpcore_version.split('"')[1] with open("roles/pulp_common/defaults/main.yml") as f: previous_selinux_version = yaml.safe_load(f)["__pulp_selinux_version"] @@ -119,7 +119,7 @@ def validate_redmine_data(redmine_query_url, redmine_issues): sed( [ "-i", - f"/^__pulp_version:/s/{previous_pulpcore_version}/{pulpcore_version}/", + f"/^__pulpcore_version:/s/{previous_pulpcore_version}/{pulpcore_version}/", "roles/pulp_common/vars/main.yml", ] ) diff --git a/CHANGES/8847.feature b/CHANGES/8847.feature new file mode 100644 index 000000000..7930c2c9f --- /dev/null +++ b/CHANGES/8847.feature @@ -0,0 +1,2 @@ +Added support for specifying a minor version for `pulpcore_version`. A single version of pulp_installer can be used to +install all bug fix releases within a minor release of pulpcore. diff --git a/CHANGES/8847.removal b/CHANGES/8847.removal new file mode 100644 index 000000000..d9423a7ba --- /dev/null +++ b/CHANGES/8847.removal @@ -0,0 +1 @@ +`pulp_version` variable has been replaced with `pulpcore_version`. diff --git a/docs/contributing.md b/docs/contributing.md index ab04e7127..d3e09a15a 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -120,6 +120,9 @@ There are other (intentional) differences between tests: only did, `dynamic` because they will become containers soon anyway to test cluster installs.) 1. `dynamic` - Due to a limitation of Ansible 2.8 with collections, these are not tested with Ansible 2.8. +1. The `release-upgrade` scenario uses its own `converge.yml` playbook instead of the default one + used by all other scenarios. This playbook upgrades a Pulp installation multiple times to ensure + that all upgrade scenarios work correctly. To test both webserver solutions we testing `apache` as webserver with diff --git a/docs/customizing.md b/docs/customizing.md index b8d2d8bad..21d58dd1f 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -66,6 +66,7 @@ In the following example, a multitude of variables are set: ``` vars: + pulpcore_update: true pulp_workers: 4 pulp_install_plugins: pulp-container: {} @@ -86,6 +87,8 @@ In the following example, a multitude of variables are set: default_file_storage: 'storages.backends.azure_storage.AzureStorage' ``` +`pulpcore_update` is set to `true` which means that if pulpcore 3.16 is already installed, it will be upgraded to the latest patch release 3.16.z. + `pulp_workers`, is set to `4`, which means that 4 worker processes will be started for processing jobs on a multi-core system. It replaces the default value of 2. `pulp_install_plugins` does the following: @@ -138,7 +141,7 @@ Recommended Workflows for Pulpcore & Plugin Versioning Initial installation: 1. Make sure you are running the latest version of the installer, which installs pulpcore's - latest version. + latest 3.y release. 1. Confirm that all the latest stable releases of your desired plugins are compatible with pulpcore's latest version , such as by reading the release announcement email thread for pulpcore's latest version, reading the plugins README, or as a last resort, reading their `setup.py`. @@ -164,6 +167,8 @@ Upgrading your installation: 1. If they are not all compatible yet, **wait** for the plugins to be updated for compatibility. 1. Upgrade `pulp_installer` to the latest version. +1. Set `pulpcore_update` to `true`. This will ensure that even if you are on the same minor release, the latest + patch release will be installed. 1. Set `pulp_install_plugins` with each plugin listed as a key, and with each plugin having a key under it called `upgrade` set to the value `true`. 1. Re-run `pulp_installer`. 1. Make sure to save your variables/playbook for later usage. @@ -171,6 +176,7 @@ Upgrading your installation: Example `pulp_install_plugins`: ``` vars: + pulpcore_update: true pulp_install_plugins: pulp-container: upgrade: true @@ -180,6 +186,58 @@ Example `pulp_install_plugins`: upgrade: true ``` +### Latest patch releases for minor releases of pulpcore and plugins + +Initial installation: + +1. Make sure you are running the latest version of the installer, which installs pulpcore's + latest 3.y release. +1. Confirm that all the latest stable releases of your desired plugins are compatible with + pulpcore's latest version , such as by reading the release announcement email thread for + pulpcore's latest version, reading the plugins README, or as a last resort, reading their `setup.py`. +1. Set `pulp_install_plugins` with each plugin listed as a key. For each each plugin add a key under it + called `version` set to the disired minor version of a plugin. +1. Run `pulp_installer`. +1. Make sure to save your variables/playbook for later usage. + +Example `pulp_install_plugins`: +``` + vars: + pulp_install_plugins: + pulp-container: + version: "2.8" + pulp-file: + version: "1.9" + pulp-rpm: + version: "3.15" +``` + +Upgrading your installation: + +1. Use the same version of `pulp_installer` that was used for the initial installation. +1. Set `pulpcore_update` to `true`. This will upgrade your `pulpcore` package to the latest patch release. +1. Set `pulp_install_plugins` with each plugin listed as a key. For each each plugin add a key under it + called `version` set to the currently install `x.y` plugin version. Add a key called `upgrade` set to + the value `true`. +1. Re-run `pulp_installer`. +1. Make sure to save your variables/playbook for later usage. + +Example `pulp_install_plugins`: +``` + vars: + pulpcore_update: true + pulp_install_plugins: + pulp-container: + version: "2.8" + upgrade: true + pulp-file: + version: "1.9" + upgrade: true + pulp-rpm: + version: "3.15" + upgrade: true +``` + ### Specifying Exact Versions with Reproducibility: Initial installation: diff --git a/docs/index.md b/docs/index.md index 7c3a1c13b..1f2bb1b6f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,9 +5,11 @@ The Pulp 3 Ansible installer is a collection of Ansible roles that you can use t Each Ansible role installs and configures a component of Pulp. -This version of the installer, 3.15.2-2, installs Pulp 3.15.2 specifically. +This version of the installer, 3.15, installs the latest release of pulpcore 3.15.z. -If run against an older version of Pulp 3, it will upgrade it to 3.15.2. +If run against an older version of Pulp 3, it will upgrade it to 3.15.z. + +If run again, the installed version of pulpcore 3.15 is not updated unless `pulpcore_update` is set to `true`. --- **Didn't find what you need to get started?** diff --git a/molecule/release-upgrade/converge.yml b/molecule/release-upgrade/converge.yml deleted file mode 120000 index f68c2c56d..000000000 --- a/molecule/release-upgrade/converge.yml +++ /dev/null @@ -1 +0,0 @@ -../../playbooks/example-use/playbook.yml \ No newline at end of file diff --git a/molecule/release-upgrade/converge.yml b/molecule/release-upgrade/converge.yml new file mode 100644 index 000000000..3f23598d5 --- /dev/null +++ b/molecule/release-upgrade/converge.yml @@ -0,0 +1,144 @@ +--- +- hosts: all + tasks: + - set_fact: + pulpcore_version: "3.14.5" + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8.0" + pulp_rpm: + version: "3.13.0" + when: ansible_facts.os_family == 'RedHat' + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8.0" + when: ansible_facts.os_family == 'Debian' + - include_role: + name: pulp_all_services + - name: Obtain list of packages & versions in the venv after install + pip_package_info: + clients: "/usr/local/lib/pulp/bin/pip" + register: pip_pkgs + - name: Assert pulpcore==3.14.5, pulp-file=1.8.0 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulpcore'][0].version is version("3.14.5", "=") + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-file'][0].version is version("1.8.0", "=") + - name: Assert pulp-rpm==3.13.0 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-rpm'][0].version is version("3.13.0", "=") + when: ansible_facts.os_family == 'RedHat' + +- hosts: all + tasks: + - set_fact: + pulpcore_version: "3.14" + pulpcore_update: false + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8" + upgrade: false + pulp_rpm: + version: "3.13" + upgrade: false + when: ansible_facts.os_family == 'RedHat' + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8" + upgrade: fasle + when: ansible_facts.os_family == 'Debian' + - include_role: + name: pulp_common + - name: Obtain list of packages & versions in the venv after install + pip_package_info: + clients: "/usr/local/lib/pulp/bin/pip" + register: pip_pkgs + - name: Assert pulpcore==3.14.5, pulp-file=1.8.0 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulpcore'][0].version is version("3.14.5", "=") + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-file'][0].version is version("1.8.0", "=") + - name: Assert pulp-rpm==3.13.0 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-rpm'][0].version is version("3.13.0", "=") + when: ansible_facts.os_family == 'RedHat' + +- hosts: all + tasks: + - set_fact: + pulpcore_version: "3.14" + pulpcore_update: true + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8" + upgrade: true + pulp_rpm: + version: "3.13" + upgrade: true + when: ansible_facts.os_family == 'RedHat' + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.8" + upgrade: true + when: ansible_facts.os_family == 'Debian' + - include_role: + name: pulp_common + - name: Obtain list of packages & versions in the venv after install + pip_package_info: + clients: "/usr/local/lib/pulp/bin/pip" + register: pip_pkgs + - name: Assert pulpcore==3.14.6, pulp-file==1.8.2 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulpcore'][0].version is version("3.14.6", ">=") + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-file'][0].version is version("1.8.2", ">=") + - name: Assert pulp-rpm==3.13.3 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-rpm'][0].version is version("3.13.3", ">=") + when: ansible_facts.os_family == 'RedHat' + +- hosts: all + tasks: + - set_fact: + pulpcore_version: "3.15" + pulpcore_update: false + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.9" + upgrade: false + pulp_rpm: + version: "3.15" + upgrade: false + when: ansible_facts.os_family == 'RedHat' + - set_fact: + pulp_install_plugins: + pulp_file: + version: "1.9" + upgrade: fasle + when: ansible_facts.os_family == 'Debian' + - include_role: + name: pulp_common + - name: Obtain list of packages & versions in the venv after install + pip_package_info: + clients: "/usr/local/lib/pulp/bin/pip" + register: pip_pkgs + - name: Assert pulpcore==3.15.2, pulp-file=1.9.1 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulpcore'][0].version is version("3.15.2", ">=") + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-file'][0].version is version("1.9.1", ">=") + - name: Assert pulp-rpm==3.15.0 + assert: + that: + - pip_pkgs.packages['/usr/local/lib/pulp/bin/pip']['pulp-rpm'][0].version is version("3.15.0", ">=") + when: ansible_facts.os_family == 'RedHat' diff --git a/molecule/release-upgrade/group_vars/all b/molecule/release-upgrade/group_vars/all index c36a2dd6b..4ce8fde6f 100644 --- a/molecule/release-upgrade/group_vars/all +++ b/molecule/release-upgrade/group_vars/all @@ -1,12 +1,6 @@ --- __pulp_database_config_run_once: false pulp_default_admin_password: password -pulp_upgrade: true -pulp_install_plugins: - pulp-file: - upgrade: true - pulp-rpm: - upgrade: true pulp_settings: secret_key: secret content_origin: "https://{{ ansible_fqdn }}" diff --git a/molecule/release-upgrade/host_vars/debian-11 b/molecule/release-upgrade/host_vars/debian-11 index eac926e90..f1a3ed49c 100644 --- a/molecule/release-upgrade/host_vars/debian-11 +++ b/molecule/release-upgrade/host_vars/debian-11 @@ -1,6 +1 @@ ansible_python_interpreter: /usr/bin/python3 -pulp_install_plugins: - pulp-file: - upgrade: true - # pulp-rpm: - # upgrade: true diff --git a/molecule/release-upgrade/molecule.yml b/molecule/release-upgrade/molecule.yml index 6b9c0cbe8..4ec84fe4c 100644 --- a/molecule/release-upgrade/molecule.yml +++ b/molecule/release-upgrade/molecule.yml @@ -58,8 +58,6 @@ scenario: - create - prepare - converge - - idempotence - - check - side_effect - verify verifier: diff --git a/roles/pulp_common/README.md b/roles/pulp_common/README.md index 3a3a1cec0..3fb9706ac 100644 --- a/roles/pulp_common/README.md +++ b/roles/pulp_common/README.md @@ -121,6 +121,9 @@ Role Variables * `pulp_certs_dir`: Path where to generate or drop the TLS certificates & keys for authentication tokens. Not used directly by pulp_common, but by roles that depend on it. Defaults to '{{ pulp_config_dir }}/certs' . +* `pulpcore_update`: Boolean that specifies whether the pulpcore package should be updated to the + latest bug fix release within the minor release specified by `pulpcore_version`. Defaults + to `false`. Role Variables if installing from RPMs -------------------------------------- @@ -172,9 +175,9 @@ If it is set to "packages", the following variables are used, or behave *differe Role Variables for advanced usage --------------------------------- -* `pulpcore_version`: Specify a specific version of pulpcore one would like to install or upgrade to. - By default the installer will do the right thing by using the version of pulpcore it is designed - for and tested with. It is strongly advised against setting. +* `pulpcore_version`: Specify a minor version of pulpcore (e.g.: `3.15`) one would like to install or upgrade to. + By default the installer will do the right thing by using the minor version of pulpcore it is designed + for and tested with. This can also be a specific patch release (e.g.: `3.15.2`). * `pulp_service_timeout`: Set timeout value for pulp services. Defaults to 90. Shared Variables diff --git a/roles/pulp_common/defaults/main.yml b/roles/pulp_common/defaults/main.yml index 3c0a0f6e4..31cf6383b 100644 --- a/roles/pulp_common/defaults/main.yml +++ b/roles/pulp_common/defaults/main.yml @@ -21,6 +21,7 @@ pulp_user_home: '/var/lib/pulp' pulp_media_root: '/var/lib/pulp/media' pulp_pip_editable: yes pulp_use_system_wide_pkgs: false +pulpcore_update: false prereq_pip_packages: - Jinja2 - pygments diff --git a/roles/pulp_common/tasks/install_packages.yml b/roles/pulp_common/tasks/install_packages.yml index 33e0c5983..54b7c9c2e 100644 --- a/roles/pulp_common/tasks/install_packages.yml +++ b/roles/pulp_common/tasks/install_packages.yml @@ -140,8 +140,8 @@ package_facts: manager: "auto" -- name: Set __pulp_version to the RPM that was installed +- name: Set __pulpcore_version to the RPM that was installed set_fact: - __pulp_version: "{{ ansible_facts.packages[pulp_pkg_pulpcore_name].0.version }}" + __pulpcore_version: "{{ ansible_facts.packages[pulp_pkg_pulpcore_name].0.version }}" when: pulpcore_version | default('') | length == 0 ... diff --git a/roles/pulp_common/tasks/install_pip.yml b/roles/pulp_common/tasks/install_pip.yml index d5c67cb1d..d165351c6 100644 --- a/roles/pulp_common/tasks/install_pip.yml +++ b/roles/pulp_common/tasks/install_pip.yml @@ -209,14 +209,10 @@ - name: Install pulpcore via PyPI pip: - name: "pulpcore{{ '['+ pulp_install_object_storage +']' if pulp_install_object_storage != 'filesystem' else '' }}" + name: "pulpcore{{ '['+ pulp_install_object_storage +']' if pulp_install_object_storage != 'filesystem' else '' }}{% if __pulpcore_version | wordcount >2 %}=={{ __pulpcore_version }}{% else %}~={{ __pulpcore_version }}.0{% endif %}" # pulp_upgrade true/false -> ansible pip module latest/present -> # pip command "install --upgrade"/"install" - # Commented out because: - # "version is incompatible with state=latest" - # But we still need to handle upgrading dependencies. - # state: "{{ pulp_upgrade | ternary('latest','present') }}" - version: '{{ __pulp_version }}' + state: '{{ (pulpcore_update == true) | ternary("latest", "present") }}' virtualenv: '{{ pulp_install_dir }}' virtualenv_command: '{{ pulp_python_interpreter }} -m venv' when: pulp_source_dir is undefined @@ -269,9 +265,10 @@ clients: "{{ pulp_install_dir }}/bin/pip" register: pip_pkgs - - name: Set __pulp_version to the pulpcore that was installed + - name: Set __pulpcore_version to the pulpcore that was installed set_fact: - __pulp_version: "{{ pip_pkgs.packages[pulp_install_dir + '/bin/pip'].pulpcore[0].version }}" + __pulpcore_version: "{{ pip_pkgs.packages[pulp_install_dir + '/bin/pip'].pulpcore[0].version }}" + when: pulp_source_dir is defined - name: Patch md5usedforsecurity in Django (devel only) shell: | @@ -300,13 +297,10 @@ - name: Install Pulp plugins via PyPI pip: - name: '{{ item.key }}' - # item.upgrade true/false -> ansible pip module latest/present -> - # pip command "install --upgrade"/"install" - state: "{{ item.value.upgrade | default(false) | ternary('latest','present','present') }}" - # TODO: Handle the fact that "version is incompatible with state=latest" - # through proper means, rather than just telling users not to set both. - version: '{{ item.value.version | default(omit) }}' + name: "{{ item.key }}{% if item.value.version is defined %}{% if item.value.version|wordcount > 2 %}=={{ item.value.version }}{% else %}~={{ item.value.version }}.0{% endif %}{% endif %}" + # item.upgrade true/false -> ansible pip module latest / present + # pip command "install --upgrade install" + state: "{{ (item.value.upgrade | default(false) == true) | ternary('latest', 'present') }}" extra_args: "-c {{ pulp_install_dir }}/pip_constraints_for_plugins.txt" virtualenv: '{{ pulp_install_dir }}' virtualenv_command: '{{ pulp_python_interpreter }} -m venv' diff --git a/roles/pulp_common/tasks/preflight_function.yml b/roles/pulp_common/tasks/preflight_function.yml index 00c4fdc2e..795e801c9 100644 --- a/roles/pulp_common/tasks/preflight_function.yml +++ b/roles/pulp_common/tasks/preflight_function.yml @@ -34,6 +34,7 @@ - name: Run pip-compile to check pulpcore/plugin compatibility command: '{{ pulp_install_dir }}/bin/pip-compile +{% if pulpcore_update|default(false, true) %} -P pulpcore {% endif %} {% for plugin, value in pulp_install_plugins_normalized.items() %} {% if value["upgrade"]|default(false, true) %} -P {{ plugin }} {% endif %} {% endfor %}' diff --git a/roles/pulp_common/templates/requirements.in.j2 b/roles/pulp_common/templates/requirements.in.j2 index b7928d2d8..118c79823 100644 --- a/roles/pulp_common/templates/requirements.in.j2 +++ b/roles/pulp_common/templates/requirements.in.j2 @@ -1,11 +1,14 @@ -pulpcore=={{ __pulp_version }} +pulpcore{% if __pulpcore_version | wordcount >2 %}=={{ __pulpcore_version }} +{% else %}~={{ __pulpcore_version }}.0 +{% endif %} {% for plugin, value in pulp_install_plugins_normalized.items() %} {% if value['source_dir'] is defined and value['git_url'] is defined-%} git+{{ value['git_url']}}{{ value['git_revision'] | ternary('@', '') }}{{ value['git_revision'] }}#egg={{ plugin }} {% elif value['source_dir'] is defined -%} -e {{ value['source_dir'] }} {% else -%} -{{ plugin }}{% if value['version'] is defined and value['version']|length %}=={{ value['version'] }}{% endif %} +{{ plugin }}{% if value['version'] is defined %}{% if value['version']|wordcount > 2 %}=={{ value['version'] }}{% else %}~={{ value['version'] }}.0 +{% endif %}{% endif %} {% endif %} {% endfor %} diff --git a/roles/pulp_common/vars/main.yml b/roles/pulp_common/vars/main.yml index ce3822f91..9d80ca1ee 100644 --- a/roles/pulp_common/vars/main.yml +++ b/roles/pulp_common/vars/main.yml @@ -36,7 +36,7 @@ __pulp_common_pulp_settings_defaults: __pulp_common_merged_pulp_settings: "{{ __pulp_common_pulp_settings_defaults | combine(pulp_settings, recursive=True) }}" -__pulp_version: '{{ pulpcore_version | default("3.15.2") }}' +__pulpcore_version: '{{ pulpcore_version | default("3.15") }}' __pulp_old_artifact_dir: "{{ pulp_user_home | regex_replace('\\/$', '') }}/artifact" __pulp_artifact_dir: "{{ pulp_media_root | regex_replace('\\/$', '') }}/artifact" diff --git a/roles/pulp_workers/tasks/main.yml b/roles/pulp_workers/tasks/main.yml index 7c853536a..94ed763f4 100644 --- a/roles/pulp_workers/tasks/main.yml +++ b/roles/pulp_workers/tasks/main.yml @@ -30,7 +30,7 @@ become: true when: - ansible_facts.services['pulpcore-resource-manager.service'] is defined - - __pulp_version is version('3.14', '>=') + - __pulpcore_version is version('3.14', '>=') - name: Get already existing workers set_fact: diff --git a/roles/pulp_workers/templates/pulpcore-worker@.service.j2 b/roles/pulp_workers/templates/pulpcore-worker@.service.j2 index 2542067ac..322b84f6f 100644 --- a/roles/pulp_workers/templates/pulpcore-worker@.service.j2 +++ b/roles/pulp_workers/templates/pulpcore-worker@.service.j2 @@ -20,7 +20,7 @@ User={{ pulp_user }} Group={{ pulp_group }} WorkingDirectory=/var/run/pulpcore-worker-%i/ RuntimeDirectory=pulpcore-worker-%i -{% if __pulp_version is version('3.14', '>=') -%} +{% if __pulpcore_version is version('3.14', '>=') -%} ExecStart={{ __pulp_daemons_dir }}/pulpcore-worker {%- else -%} ExecStart={{ __pulp_daemons_dir }}/rq worker \