add a flag to ansible-galaxy to update collections

lib/ansible/cli/galaxy.py

@@ -398,6 +398,8 @@ def add_install_options(self, parser, parents=None):
                                         help='A file containing a list of collections to be installed.')
             install_parser.add_argument('--pre', dest='allow_pre_release', action='store_true',
                                         help='Include pre-release versions. Semantic versioning pre-releases are ignored by default')
+            install_parser.add_argument('-U', '--upgrade', dest='upgrade', action='store_true', default=False,
+                                        help='Upgrade installed collection artifacts. This will also update dependencies unless --no-deps is provided')
         else:
             install_parser.add_argument('-r', '--role-file', dest='requirements',
                                         help='A file containing a list of roles to be installed.')
@@ -1174,7 +1176,9 @@ def _execute_install_collection(
         ignore_errors = context.CLIARGS['ignore_errors']
         no_deps = context.CLIARGS['no_deps']
         force_with_deps = context.CLIARGS['force_with_deps']
+        # If `ansible-galaxy install` is used, collection-only options aren't available to the user and won't be in context.CLIARGS
         allow_pre_release = context.CLIARGS['allow_pre_release'] if 'allow_pre_release' in context.CLIARGS else False
+        upgrade = context.CLIARGS['upgrade'] if 'upgrade' in context.CLIARGS else False
 
         collections_path = C.COLLECTIONS_PATHS
         if len([p for p in collections_path if p.startswith(path)]) == 0:
@@ -1192,6 +1196,7 @@ def _execute_install_collection(
             no_deps, force, force_with_deps,
             allow_pre_release=allow_pre_release,
             artifacts_manager=artifacts_manager,
+            upgrade=upgrade,
         )
 
         return 0

lib/ansible/galaxy/collection/__init__.py

@@ -286,6 +286,7 @@ def download_collections(
             concrete_artifacts_manager=artifacts_manager,
             no_deps=no_deps,
             allow_pre_release=allow_pre_release,
+            upgrade=False,
         )
 
     b_output_path = to_bytes(output_path, errors='surrogate_or_strict')
@@ -407,6 +408,7 @@ def install_collections(
         force_deps,  # type: bool
         allow_pre_release,  # type: bool
         artifacts_manager,  # type: ConcreteArtifactsManager
+        upgrade=False,  # type: bool
 ):  # type: (...) -> None
     """Install Ansible collections to the path specified.
 
@@ -449,7 +451,7 @@ def install_collections(
         if req.fqcn == exs.fqcn and meets_requirements(exs.ver, req.ver)
     }
 
-    if not unsatisfied_requirements:
+    if not unsatisfied_requirements and not upgrade:
         display.display(
             'Nothing to do. All requested collections are already '
             'installed. If you want to reinstall them, '
@@ -482,6 +484,7 @@ def install_collections(
                 concrete_artifacts_manager=artifacts_manager,
                 no_deps=no_deps,
                 allow_pre_release=allow_pre_release,
+                upgrade=upgrade,
             )
         except InconsistentCandidate as inconsistent_candidate_exc:
             # FIXME: Processing this error is hacky and should be removed along
@@ -1279,6 +1282,7 @@ def _resolve_depenency_map(
         preferred_candidates,  # type: Optional[Iterable[Candidate]]
         no_deps,  # type: bool
         allow_pre_release,  # type: bool
+        upgrade=False,  # type: bool
 ):  # type: (...) -> Dict[str, Candidate]
     """Return the resolved dependency map."""
     collection_dep_resolver = build_collection_dependency_resolver(
@@ -1287,6 +1291,7 @@ def _resolve_depenency_map(
         preferred_candidates=preferred_candidates,
         with_deps=not no_deps,
         with_pre_releases=allow_pre_release,
+        upgrade=upgrade,
     )
     try:
         return collection_dep_resolver.resolve(

lib/ansible/galaxy/dependency_resolution/__init__.py

@@ -31,6 +31,7 @@ def build_collection_dependency_resolver(
         preferred_candidates=None,  # type: Iterable[Candidate]
         with_deps=True,  # type: bool
         with_pre_releases=False,  # type: bool
+        upgrade=False,  # type: bool
 ):  # type: (...) -> CollectionDependencyResolver
     """Return a collection dependency resolver.
 
@@ -44,6 +45,7 @@ def build_collection_dependency_resolver(
             preferred_candidates=preferred_candidates,
             with_deps=with_deps,
             with_pre_releases=with_pre_releases,
+            upgrade=upgrade,
         ),
         CollectionDependencyReporter(),
     )

lib/ansible/galaxy/dependency_resolution/providers.py

@@ -43,6 +43,7 @@ def __init__(
             preferred_candidates=None,  # type: Iterable[Candidate]
             with_deps=True,  # type: bool
             with_pre_releases=False,  # type: bool
+            upgrade=False,  # type: bool
     ):  # type: (...) -> None
         r"""Initialize helper attributes.
 
@@ -67,6 +68,7 @@ def __init__(
         self._preferred_candidates = set(preferred_candidates or ())
         self._with_deps = with_deps
         self._with_pre_releases = with_pre_releases
+        self._upgrade = upgrade
 
     def identify(self, requirement_or_candidate):
         # type: (Union[Candidate, Requirement]) -> str
@@ -183,7 +185,7 @@ def find_matches(self, requirements):
             if candidate.fqcn == fqcn
         }
 
-        return list(preinstalled_candidates) + sorted(
+        latest_matches = sorted(
             {
                 candidate for candidate in (
                     Candidate(fqcn, version, src_server, 'galaxy')
@@ -202,6 +204,16 @@ def find_matches(self, requirements):
             reverse=True,  # prefer newer versions over older ones
         )
 
+        if self._upgrade:
+            for latest in latest_matches:
+                for preinstalled in set(preinstalled_candidates):
+                    if latest.fqcn != preinstalled.fqcn:
+                        continue
+                    if SemanticVersion(latest.ver) > SemanticVersion(preinstalled.ver):
+                        preinstalled_candidates.remove(preinstalled)
+
+        return list(preinstalled_candidates) + latest_matches
+
     def is_satisfied_by(self, requirement, candidate):
         # type: (Requirement, Candidate) -> bool
         r"""Whether the given requirement is satisfiable by a candidate.

test/integration/targets/ansible-galaxy-collection/tasks/main.yml

@@ -179,3 +179,94 @@
     test_name: 'galaxy_ng'
     test_server: '{{ galaxy_ng_server }}'
     vX: "v3/"
+
+- name: setup collections for ansible-galaxy collection upgrade tests for {{ test_name }}
+  environment:
+    ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
+  block:
+    - command: ansible-galaxy collection init {{ item }} {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - namespace1.collection1
+        - namespace1.collection2
+
+    - name: add the dependency to the other collection
+      lineinfile:
+        path: '{{ galaxy_dir }}/scratch/namespace1/collection1/galaxy.yml'
+        regexp: '^dependencies'
+        line: 'dependencies: {"namespace1.collection2": ">=1.0.0,<2.0.0"}'
+
+    - command: ansible-galaxy collection build {{ item }} {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - 'namespace1/collection1'
+        - 'namespace1/collection2'
+
+    - command: ansible-galaxy collection publish {{ item }} -s pulp_v3 {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - namespace1-collection1-1.0.0.tar.gz
+        - namespace1-collection2-1.0.0.tar.gz
+
+    - name: update the version to build + publish again to give the dep an upgrade
+      lineinfile:
+        path: '{{ galaxy_dir }}/scratch/namespace1/collection2/galaxy.yml'
+        regexp: '^version'
+        line: 'version: 1.1.0'
+
+    - command: ansible-galaxy collection build namespace1/collection2 {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+
+    - command: ansible-galaxy collection publish {{ item }} -s pulp_v3 {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - namespace1-collection2-1.1.0.tar.gz
+
+    - name: update the version to build + publish again
+      lineinfile:
+        path: "{{ item }}"
+        regexp: '^version'
+        line: 'version: 2.0.0'
+      loop:
+        - '{{ galaxy_dir }}/scratch/namespace1/collection1/galaxy.yml'
+        - '{{ galaxy_dir }}/scratch/namespace1/collection2/galaxy.yml'
+
+    - name: update the dep version constraint
+      lineinfile:
+        path: '{{ galaxy_dir }}/scratch/namespace1/collection1/galaxy.yml'
+        regexp: '^dependencies'
+        line: 'dependencies: {"namespace1.collection2": ">=1.0.0,<=3.0.0"}'
+
+    - command: ansible-galaxy collection build {{ item }} {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - 'namespace1/collection1'
+        - 'namespace1/collection2'
+
+    - command: ansible-galaxy collection publish {{ item }} -s pulp_v3 {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      loop:
+        - namespace1-collection1-2.0.0.tar.gz
+        - namespace1-collection2-2.0.0.tar.gz
+
+    - name: initialize the installation directory - {{ test_name }}
+      file:
+        path: '{{ galaxy_dir }}/ansible_collections'
+        state: '{{ item }}'
+      loop:
+        - absent
+        - directory
+
+    - include_tasks: upgrade.yml
+      args:
+        apply:
+          environment:
+            ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
+            ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'

test/integration/targets/ansible-galaxy-collection/tasks/upgrade.yml

@@ -0,0 +1,86 @@
+# regular expression taken from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+- set_fact:
+    version_re: '((?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)'
+
+- name: test upgrading a collection and its dependencies
+  block:
+    - name: set up an upgradable collection and dependency
+      command: ansible-galaxy collection install namespace1.collection1:1.0.0 namespace1.collection2:1.0.0 --no-deps --force -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+      failed_when:
+        - '"namespace1.collection1:1.0.0 was installed successfully" not in result.stdout_lines'
+        - '"namespace1.collection2:1.0.0 was installed successfully" not in result.stdout_lines'
+
+    - name: upgrade the collection and any dependencies
+      command: ansible-galaxy collection install namespace1.collection1 --upgrade -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+
+    - assert:
+        that:
+          - "collection_line.endswith('was installed successfully')"
+          - "dep_line.endswith('was installed successfully')"
+          - "collection_version == '2.0.0'"
+          - "dep_version == '2.0.0'"
+      vars:
+        # Hack to replace unicode characters for JMESPath filter https://github.com/ansible/ansible/issues/20379
+        collection_line: "{{ result.stdout_lines | to_json | from_json | json_query(\"[?starts_with(@, 'namespace1.collection1')]\") | first }}"
+        collection_version: "{{ collection_line | regex_search(version_re) }}"
+
+        dep_line: "{{ result.stdout_lines | to_json | from_json | json_query(\"[?starts_with(@, 'namespace1.collection2')]\") | first }}"
+        dep_version: "{{ dep_line | regex_search(version_re) }}"
+
+    - name: upgrading again should make no changes
+      command: ansible-galaxy collection install namespace1.collection1:>1.0.0 -U -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+
+    - assert:
+        that:
+          - "\"Skipping 'namespace1.collection1:2.0.0' as it is already installed\" in result.stdout_lines"
+          - "\"Skipping 'namespace1.collection2:2.0.0' as it is already installed\" in result.stdout_lines"
+
+- name: test upgrading a collection with an inexact requirement and --no-deps
+  block:
+    - name: set up an upgradable collection and dependency
+      command: ansible-galaxy collection install namespace1.collection1:1.0.0 namespace1.collection2:1.0.0 --no-deps --force -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+      failed_when:
+        - '"namespace1.collection1:1.0.0 was installed successfully" not in result.stdout_lines'
+        - '"namespace1.collection2:1.0.0 was installed successfully" not in result.stdout_lines'
+
+    - name: upgrade the collection with an inexact version and --no-deps
+      command: ansible-galaxy collection install namespace1.collection1:<=2.0.0 --no-deps --upgrade -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+
+    - assert:
+        that:
+          - '"namespace1.collection1:2.0.0 was installed successfully" in result.stdout_lines'
+          - '"namespace1.collection2:2.0.0 was installed successfully" not in result.stdout_lines'
+          # shouldn't have even been examined
+          - "\"Skipping 'namespace1.collection2:1.0.0' as it is already installed\" not in result.stdout_lines"
+
+- name: test upgrading a dep of a collection matching the latest version
+  block:
+    - name: set up an upgradable collection and dependency
+      command: ansible-galaxy collection install namespace1.collection1:1.0.0 namespace1.collection2:1.0.0 --no-deps --force -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+      failed_when:
+        - '"namespace1.collection1:1.0.0 was installed successfully" not in result.stdout_lines'
+        - '"namespace1.collection2:1.0.0 was installed successfully" not in result.stdout_lines'
+
+    - name: upgrade the dependency of a collection
+      command: ansible-galaxy collection install namespace1.collection1:==1.0.0 --upgrade -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+
+    - assert:
+        that:
+          - "\"Skipping 'namespace1.collection1:1.0.0' as it is already installed\" in result.stdout_lines"
+          - '"namespace1.collection2:1.1.0 was installed successfully" in result.stdout_lines'
+
+    - name: upgrading again should make no changes
+      command: ansible-galaxy collection install namespace1.collection1:==1.0.0 --upgrade -s pulp_v3 {{ galaxy_verbosity }}
+      register: result
+
+    - assert:
+        that:
+          - "\"Skipping 'namespace1.collection1:1.0.0' as it is already installed\" in result.stdout_lines"
+          - "\"Skipping 'namespace1.collection2:1.1.0' as it is already installed\" in result.stdout_lines"