From b0067b68eb21d374e4b46eb2cd1881130082fa1f Mon Sep 17 00:00:00 2001 From: James Date: Sat, 20 Oct 2018 10:04:13 +0200 Subject: [PATCH] implement download concurrency checks (#3806) --- conans/client/graph/graph_binaries.py | 5 +++-- conans/client/installer.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index 518c81dff25..18c8008f7d5 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -26,7 +26,7 @@ def _get_package_info(self, package_ref, remote): except (NotFoundException, NoRemoteAvailable): # 404 or no remote return False - def _check_update(self, package_folder, package_ref, remote, output): + def _check_update(self, package_folder, package_ref, remote, output, node): try: # get_conan_digest can fail, not in server # FIXME: This can iterate remotes to get and associate in registry upstream_manifest = self._remote_manager.get_package_manifest(package_ref, remote) @@ -39,6 +39,7 @@ def _check_update(self, package_folder, package_ref, remote, output): if upstream_manifest != read_manifest: if upstream_manifest.time > read_manifest.time: output.warn("Current package is older than remote upstream one") + node.update_manifest = upstream_manifest return True else: output.warn("Current package is newer than remote upstream one") @@ -86,7 +87,7 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_ if os.path.exists(package_folder): if update: if remote: - if self._check_update(package_folder, package_ref, remote, output): + if self._check_update(package_folder, package_ref, remote, output, node): node.binary = BINARY_UPDATE if build_mode.outdated: package_hash = self._get_package_info(package_ref, remote).recipe_hash diff --git a/conans/client/installer.py b/conans/client/installer.py index e7ec4cec8a0..17343bc8da2 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -31,6 +31,7 @@ from conans.util.log import logger from conans.util.tracer import log_package_built, \ log_package_got_from_local_cache +from conans.model.manifest import FileTreeManifest def build_id(conan_file): @@ -302,6 +303,14 @@ def _build(self, nodes_by_level, deps_graph, keep_build, root_node): # Finally, propagate information to root node (conan_ref=None) self._propagate_info(root_node, inverse_levels, deps_graph, self._out) + def _node_concurrently_installed(self, node, package_folder): + if node.binary == BINARY_DOWNLOAD and os.path.exists(package_folder): + return True + elif node.binary == BINARY_UPDATE: + read_manifest = FileTreeManifest.load(package_folder) + if node.update_manifest == read_manifest: + return True + def _handle_node_cache(self, node, package_ref, keep_build, processed_package_references): conan_ref, conan_file = node.conan_ref, node.conanfile output = ScopedOutput(str(conan_ref), self._out) @@ -314,9 +323,14 @@ def _handle_node_cache(self, node, package_ref, keep_build, processed_package_re if node.binary == BINARY_BUILD: self._build_package(node, package_ref, output, keep_build) elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD): - self._remote_manager.get_package(package_ref, package_folder, - node.binary_remote, output, self._recorder) - self._registry.prefs.set(package_ref, node.binary_remote.name) + if not self._node_concurrently_installed(node, package_folder): + self._remote_manager.get_package(package_ref, package_folder, + node.binary_remote, output, self._recorder) + self._registry.prefs.set(package_ref, node.binary_remote.name) + else: + output.success('Download skipped. Probable concurrent download') + log_package_got_from_local_cache(package_ref) + self._recorder.package_fetched_from_cache(package_ref) elif node.binary == BINARY_CACHE: output.success('Already installed!') log_package_got_from_local_cache(package_ref)