#6349 Add support for mirror when downloading files

conans/client/tools/net.py

@@ -3,7 +3,7 @@
 from conans.client.rest.download_cache import CachedFileDownloader
 from conans.client.rest.uploader_downloader import FileDownloader
 from conans.client.tools.files import check_md5, check_sha1, check_sha256, unzip
-from conans.errors import ConanException
+from conans.errors import ConanException, NotFoundException, ConanConnectionError
 from conans.util.fallbacks import default_output, default_requester
 
 
@@ -12,10 +12,13 @@ def get(url, md5='', sha1='', sha256='', destination=".", filename="", keep_perm
         overwrite=False, auth=None, headers=None):
     """ high level downloader + unzipper + (optional hash checker) + delete temporary zip
     """
-    if not filename and ("?" in url or "=" in url):
-        raise ConanException("Cannot deduce file name from url. Use 'filename' parameter.")
 
-    filename = filename or os.path.basename(url)
+    url = [url] if not isinstance(url, list) else url
+    if not filename and ("?" in url[0] or "=" in url[0]):
+            raise ConanException("Cannot deduce file name from the url: '{}'. Use 'filename' "
+                                 "parameter.".format(url[0]))
+    filename = filename or os.path.basename(url[0])
+
     download(url, filename, out=output, requester=requester, verify=verify, retry=retry,
              retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers,
              md5=md5, sha1=sha1, sha256=sha256)
@@ -49,7 +52,30 @@ def ftp_download(ip, filename, login='', password=''):
 
 def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, overwrite=False,
              auth=None, headers=None, requester=None, md5='', sha1='', sha256=''):
+    """Retrieves a file from a given URL into a file with a given filename.
+       It uses certificates from a list of known verifiers for https downloads,
+       but this can be optionally disabled.
 
+    :param url: URL to download. It can be a list, which only the first one will be downloaded, and
+                the follow URLs will be used as mirror in case of download error.
+    :param filename: Name of the file to be created in the local storage
+    :param verify: When False, disables https certificate validation
+    :param out: An object with a write() method can be passed to get the output. stdout will use if
+                not specified
+    :param retry: Number of retries in case of failure. Default is overriden by general.retry in the
+                  conan.conf file or an env variable CONAN_RETRY
+    :param retry_wait: Seconds to wait between download attempts. Default is overriden by
+                       general.retry_wait in the conan.conf file or an env variable CONAN_RETRY_WAIT
+    :param overwrite: When True, Conan will overwrite the destination file if exists. Otherwise it
+                      will raise an exception
+    :param auth: A tuple of user and password to use HTTPBasic authentication
+    :param headers: A dictionary with additional headers
+    :param requester: HTTP requests instance
+    :param md5: MD5 hash code to check the downloaded file
+    :param sha1: SHA-1 hash code to check the downloaded file
+    :param sha256: SHA-256 hash code to check the downloaded file
+    :return: None
+    """
     out = default_output(out, 'conans.client.tools.net.download')
     requester = default_requester(requester, 'conans.client.tools.net.download')
     from conans.tools import _global_config as config
@@ -60,21 +86,36 @@ def download(url, filename, verify=True, out=None, retry=None, retry_wait=None,
     retry_wait = retry_wait if retry_wait is not None else config.retry_wait
     retry_wait = retry_wait if retry_wait is not None else 5
 
-    downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config)
+    url = [url] if not isinstance(url, list) else url
     checksum = sha256 or sha1 or md5
-    # The download cache is only used if a checksum is provided, otherwise, a normal download
+
+    downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config)
     if config and config.download_cache and checksum:
         downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True)
-        downloader.download(url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
-                            auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
-    else:
-        downloader.download(url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
-                            auth=auth, headers=headers)
-        if md5:
-            check_md5(filename, md5)
-        if sha1:
-            check_sha1(filename, sha1)
-        if sha256:
-            check_sha256(filename, sha256)
 
-    out.writeln("")
+    def _download_file(downloader, url):
+        # The download cache is only used if a checksum is provided, otherwise, a normal download
+        if isinstance(downloader, CachedFileDownloader):
+            downloader.download(url, filename, retry=retry, retry_wait=retry_wait,
+                                overwrite=overwrite, auth=auth, headers=headers, md5=md5,
+                                sha1=sha1, sha256=sha256)
+        else:
+            downloader.download(url, filename, retry=retry, retry_wait=retry_wait,
+                                overwrite=overwrite, auth=auth, headers=headers)
+            if md5:
+                check_md5(filename, md5)
+            if sha1:
+                check_sha1(filename, sha1)
+            if sha256:
+                check_sha256(filename, sha256)
+        out.writeln("")
+
+    for index, url_it in enumerate(url):
+        try:
+            _download_file(downloader, url_it)
+            break
+        except (ConanConnectionError, NotFoundException, ConanException):
+            if (index + 1) == len(url):
+                raise
+            out.warn("Could not download from the url {}. Using the next available mirror."
+                     .format(url_it))

conans/test/unittests/util/tools_test.py

@@ -13,6 +13,7 @@
 from mock.mock import mock_open, patch
 from nose.plugins.attrib import attr
 from parameterized import parameterized
+from requests.models import Response
 
 from conans.client import tools
 from conans.client.cache.cache import CONAN_CONF
@@ -661,9 +662,10 @@ def error_url():
 
         out = TestBufferConanOutput()
         # Test: File name cannot be deduced from '?file=1'
-        with six.assertRaisesRegex(self, ConanException,
-                                   "Cannot deduce file name from url. Use 'filename' parameter."):
+        with self.assertRaises(ConanException) as error:
             tools.get("http://localhost:%s/?file=1" % thread.port, output=out)
+        self.assertIn("Cannot deduce file name from the url: 'http://localhost:{}/?file=1'."
+                      " Use 'filename' parameter.".format(thread.port), str(error.exception))
 
         # Test: Works with filename parameter instead of '?file=1'
         with tools.chdir(tools.mkdir_tmp()):
@@ -729,6 +731,58 @@ def get_file():
 
         thread.stop()
 
+    @attr('slow')
+    @attr('local_bottle')
+    @patch("conans.client.tools.net.unzip")
+    def test_get_mirror(self, unzip_mock):
+        """ tools.get must supports a list of URLs. However, only one must be downloaded.
+        """
+
+        class MockRequester(object):
+            def __init__(self):
+                self.count = 0
+                self.fail_first = False
+                self.fail_all = False
+
+            def get(self, *args, **kwargs):
+                self.count += 1
+                resp = Response()
+                resp._content = b'{"results": []}'
+                resp.headers = {"Content-Type": "application/json"}
+                resp.status_code = 200
+                if (self.fail_first and self.count == 1) or self.fail_all:
+                    resp.status_code = 408
+                return resp
+
+        file = "test.txt.gz"
+        out = TestBufferConanOutput()
+        urls = ["http://localhost:/{}/{}".format(8000 + i, file) for i in range(3)]
+
+        # Only the first file must be downloaded
+        with tools.chdir(tools.mkdir_tmp()):
+            requester = MockRequester()
+            tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0)
+            self.assertEqual(1, requester.count)
+
+        # Fail the first, download only the second
+        with tools.chdir(tools.mkdir_tmp()):
+            requester = MockRequester()
+            requester.fail_first = True
+            tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0)
+            self.assertEqual(2, requester.count)
+            self.assertIn("WARN: Could not download from the url {}."
+                          " Using the next available mirror.".format(urls[0]), out)
+
+        # Fail all downloads
+        with tools.chdir(tools.mkdir_tmp()):
+            requester = MockRequester()
+            requester.fail_all = True
+            with self.assertRaises(ConanException) as error:
+                tools.get(urls, requester=requester, output=out, retry=0, retry_wait=0)
+            self.assertEqual(3, requester.count)
+            self.assertIn("Error 408 downloading file http://localhost:/8002/test.txt.gz",
+                          str(error.exception))
+
     def unix_to_dos_unit_test(self):
 
         def save_file(contents):