implement download concurrency checks (conan-io#3806)

conans/client/graph/graph_binaries.py

@@ -26,7 +26,7 @@ def _get_package_info(self, package_ref, remote):
         except (NotFoundException, NoRemoteAvailable):  # 404 or no remote
             return False
 
-    def _check_update(self, package_folder, package_ref, remote, output):
+    def _check_update(self, package_folder, package_ref, remote, output, node):
         try:  # get_conan_digest can fail, not in server
             # FIXME: This can iterate remotes to get and associate in registry
             upstream_manifest = self._remote_manager.get_package_manifest(package_ref, remote)
@@ -39,6 +39,7 @@ def _check_update(self, package_folder, package_ref, remote, output):
             if upstream_manifest != read_manifest:
                 if upstream_manifest.time > read_manifest.time:
                     output.warn("Current package is older than remote upstream one")
+                    node.update_manifest = upstream_manifest
                     return True
                 else:
                     output.warn("Current package is newer than remote upstream one")
@@ -86,7 +87,7 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_
         if os.path.exists(package_folder):
             if update:
                 if remote:
-                    if self._check_update(package_folder, package_ref, remote, output):
+                    if self._check_update(package_folder, package_ref, remote, output, node):
                         node.binary = BINARY_UPDATE
                         if build_mode.outdated:
                             package_hash = self._get_package_info(package_ref, remote).recipe_hash

conans/client/installer.py

@@ -31,6 +31,7 @@
 from conans.util.log import logger
 from conans.util.tracer import log_package_built, \
     log_package_got_from_local_cache
+from conans.model.manifest import FileTreeManifest
 
 
 def build_id(conan_file):
@@ -302,6 +303,14 @@ def _build(self, nodes_by_level, deps_graph, keep_build, root_node):
         # Finally, propagate information to root node (conan_ref=None)
         self._propagate_info(root_node, inverse_levels, deps_graph, self._out)
 
+    def _node_concurrently_installed(self, node, package_folder):
+        if node.binary == BINARY_DOWNLOAD and os.path.exists(package_folder):
+            return True
+        elif node.binary == BINARY_UPDATE:
+            read_manifest = FileTreeManifest.load(package_folder)
+            if node.update_manifest == read_manifest:
+                return True
+
     def _handle_node_cache(self, node, package_ref, keep_build, processed_package_references):
         conan_ref, conan_file = node.conan_ref, node.conanfile
         output = ScopedOutput(str(conan_ref), self._out)
@@ -314,9 +323,14 @@ def _handle_node_cache(self, node, package_ref, keep_build, processed_package_re
                 if node.binary == BINARY_BUILD:
                     self._build_package(node, package_ref, output, keep_build)
                 elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD):
-                    self._remote_manager.get_package(package_ref, package_folder,
-                                                     node.binary_remote, output, self._recorder)
-                    self._registry.prefs.set(package_ref, node.binary_remote.name)
+                    if not self._node_concurrently_installed(node, package_folder):
+                        self._remote_manager.get_package(package_ref, package_folder,
+                                                         node.binary_remote, output, self._recorder)
+                        self._registry.prefs.set(package_ref, node.binary_remote.name)
+                    else:
+                        output.success('Download skipped. Probable concurrent download')
+                        log_package_got_from_local_cache(package_ref)
+                        self._recorder.package_fetched_from_cache(package_ref)
                 elif node.binary == BINARY_CACHE:
                     output.success('Already installed!')
                     log_package_got_from_local_cache(package_ref)