From ab5309e9e8467508766aee176dc121672d606524 Mon Sep 17 00:00:00 2001 From: Ximin Luo Date: Mon, 4 Jul 2016 16:37:46 +0200 Subject: [PATCH 1/3] Avoid redundant downloads when bootstrapping If the local file is available, then verify it against the hash we just downloaded, and if it matches then we don't need to download it again. --- src/bootstrap/bootstrap.py | 8 ++++++++ src/etc/get-stage0.py | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 33de8fd010767..49f00bf46f13c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -31,6 +31,14 @@ def get(url, path, verbose=False): try: download(sha_path, sha_url, verbose) + if os.path.exists(path): + try: + verify(path, sha_path, verbose) + print("using already-download file " + path) + return + except Exception as e: + print("failed verification for already-download file " + path) + os.unlink(path) download(temp_path, url, verbose) verify(temp_path, sha_path, verbose) print("moving {} to {}".format(temp_path, path)) diff --git a/src/etc/get-stage0.py b/src/etc/get-stage0.py index 28e3363189a08..127251cc802c9 100644 --- a/src/etc/get-stage0.py +++ b/src/etc/get-stage0.py @@ -31,8 +31,6 @@ def main(triple): filename = 'rustc-{}-{}.tar.gz'.format(channel, triple) url = 'https://static.rust-lang.org/dist/{}/{}'.format(date, filename) dst = dl_dir + '/' + filename - if os.path.exists(dst): - os.unlink(dst) bootstrap.get(url, dst) stage0_dst = triple + '/stage0' From 912a9d0ad8e4c8b8a6c200d104a98982b3a07aeb Mon Sep 17 00:00:00 2001 From: Ximin Luo Date: Wed, 6 Jul 2016 00:07:26 +0200 Subject: [PATCH 2/3] Have verify() return a bool rather than a generic RuntimeError --- src/bootstrap/bootstrap.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 49f00bf46f13c..ce5ab9fab844c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -32,15 +32,15 @@ def get(url, path, verbose=False): try: download(sha_path, sha_url, verbose) if os.path.exists(path): - try: - verify(path, sha_path, verbose) + if verify(path, sha_path, verbose): print("using already-download file " + path) return - except Exception as e: - print("failed verification for already-download file " + path) + else: + print("ignoring already-download file " + path + " due to failed verification") os.unlink(path) download(temp_path, url, verbose) - verify(temp_path, sha_path, verbose) + if not verify(temp_path, sha_path, verbose): + raise RuntimeError("failed verification") print("moving {} to {}".format(temp_path, path)) shutil.move(temp_path, path) finally: @@ -72,13 +72,12 @@ def verify(path, sha_path, verbose): found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() - if found != expected: - err = ("invalid checksum:\n" + verified = found == expected + if not verified and verbose: + print("invalid checksum:\n" " found: {}\n" " expected: {}".format(found, expected)) - if verbose: - raise RuntimeError(err) - sys.exit(err) + return verified def unpack(tarball, dst, verbose=False, match=None): From 933a1036ae12d728b2ccffa560154a8011a4c256 Mon Sep 17 00:00:00 2001 From: Ximin Luo Date: Wed, 6 Jul 2016 00:44:31 +0200 Subject: [PATCH 3/3] Tweak verbosity to hopefully better match intuitive expectations --- src/bootstrap/bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index ce5ab9fab844c..832911beb588c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -32,14 +32,14 @@ def get(url, path, verbose=False): try: download(sha_path, sha_url, verbose) if os.path.exists(path): - if verify(path, sha_path, verbose): + if verify(path, sha_path, False): print("using already-download file " + path) return else: print("ignoring already-download file " + path + " due to failed verification") os.unlink(path) download(temp_path, url, verbose) - if not verify(temp_path, sha_path, verbose): + if not verify(temp_path, sha_path, True): raise RuntimeError("failed verification") print("moving {} to {}".format(temp_path, path)) shutil.move(temp_path, path)