Limit Galaxy API calls during ansible-galaxy dependency resolution (a…

…nsible#77468)

* Limit Galaxy API calls during ansible-galaxy collection dependency resolution when possible

Installing a tarfile with a dependency from a Galaxy server (e.g. dependencies: {'ns.coll': '>=1.0.0'}) does not get the available versions of the dependency from the galaxy server if a sufficient version is already installed.

Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>

changelogs/fragments/77468-ansible-galaxy-remove-unnecessary-api-call.yml

@@ -0,0 +1,5 @@
+bugfixes:
+- >-
+  ``ansible-galaxy`` - remove extra server api call during dependency resolution
+  for requirements and dependencies that are already satisfied
+  (https://github.com/ansible/ansible/issues/77443).

lib/ansible/galaxy/dependency_resolution/providers.py

@@ -14,6 +14,7 @@
         ConcreteArtifactsManager,
     )
     from ansible.galaxy.collection.galaxy_api_proxy import MultiGalaxyAPIProxy
+    from ansible.galaxy.api import GalaxyAPI
 
 from ansible.galaxy.collection.gpg import get_signature_from_source
 from ansible.galaxy.dependency_resolution.dataclasses import (
@@ -316,8 +317,18 @@ def _find_matches(self, requirements):
         # The fqcn is guaranteed to be the same
         version_req = "A SemVer-compliant version or '*' is required. See https://semver.org to learn how to compose it correctly. "
         version_req += "This is an issue with the collection."
+
+        # If we're upgrading collections, we can't calculate preinstalled_candidates until the latest matches are found.
+        # Otherwise, we can potentially avoid a Galaxy API call by doing this first.
+        preinstalled_candidates = set()
+        if not self._upgrade and first_req.type == 'galaxy':
+            preinstalled_candidates = {
+                candidate for candidate in self._preferred_candidates
+                if candidate.fqcn == fqcn and
+                all(self.is_satisfied_by(requirement, candidate) for requirement in requirements)
+            }
         try:
-            coll_versions = self._api_proxy.get_collection_versions(first_req)
+            coll_versions = [] if preinstalled_candidates else self._api_proxy.get_collection_versions(first_req)  # type: t.Iterable[t.Tuple[str, GalaxyAPI]]
         except TypeError as exc:
             if first_req.is_concrete_artifact:
                 # Non hashable versions will cause a TypeError
@@ -395,19 +406,20 @@ def _find_matches(self, requirements):
             reverse=True,  # prefer newer versions over older ones
         )
 
-        preinstalled_candidates = {
-            candidate for candidate in self._preferred_candidates
-            if candidate.fqcn == fqcn and
-            (
-                # check if an upgrade is necessary
-                all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) and
+        if not preinstalled_candidates:
+            preinstalled_candidates = {
+                candidate for candidate in self._preferred_candidates
+                if candidate.fqcn == fqcn and
                 (
-                    not self._upgrade or
-                    # check if an upgrade is preferred
-                    all(SemanticVersion(latest.ver) <= SemanticVersion(candidate.ver) for latest in latest_matches)
+                    # check if an upgrade is necessary
+                    all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) and
+                    (
+                        not self._upgrade or
+                        # check if an upgrade is preferred
+                        all(SemanticVersion(latest.ver) <= SemanticVersion(candidate.ver) for latest in latest_matches)
+                    )
                 )
-            )
-        }
+            }
 
         return list(preinstalled_candidates) + latest_matches
 

test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml

@@ -0,0 +1,51 @@
+- set_fact:
+    init_dir: "{{ galaxy_dir }}/offline/setup"
+    build_dir: "{{ galaxy_dir }}/offline/build"
+    install_dir: "{{ galaxy_dir }}/offline/collections"
+
+- name: create test directories
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ init_dir }}"
+    - "{{ build_dir }}"
+    - "{{ install_dir }}"
+
+- name: test installing a tarfile with an installed dependency offline
+  block:
+    - name: init two collections
+      command: ansible-galaxy collection init ns.{{ item }} --init-path {{ init_dir }}
+      loop:
+        - coll1
+        - coll2
+
+    - name: add one collection as the dependency of the other
+      lineinfile:
+        path: "{{ galaxy_dir }}/offline/setup/ns/coll1/galaxy.yml"
+        regexp: "^dependencies:*"
+        line: "dependencies: {'ns.coll2': '1.0.0'}"
+
+    - name: build both collections
+      command: ansible-galaxy collection build {{ init_dir }}/ns/{{ item }}
+      args:
+        chdir: "{{ build_dir }}"
+      loop:
+        - coll1
+        - coll2
+
+    - name: install the dependency from the tarfile
+      command: ansible-galaxy collection install {{ build_dir }}/ns-coll2-1.0.0.tar.gz -p {{ install_dir }} -s offline
+
+    - name: install the tarfile with the installed dependency
+      command: ansible-galaxy collection install {{ build_dir }}/ns-coll1-1.0.0.tar.gz -p {{ install_dir }} -s offline
+
+  always:
+    - name: clean up test directories
+      file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ init_dir }}"
+        - "{{ build_dir }}"
+        - "{{ install_dir }}"

test/integration/targets/ansible-galaxy-collection/tasks/main.yml

@@ -56,6 +56,13 @@
   loop_control:
     loop_var: resolvelib_version
 
+- name: run ansible-galaxy collection offline installation tests
+  include_tasks: install_offline.yml
+  args:
+    apply:
+      environment:
+        ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
+
 - name: run ansible-galaxy collection publish tests for {{ test_name }}
   include_tasks: publish.yml
   args: