diff --git a/fetch_packages.py b/fetch_packages.py index 2c77721..3d2e676 100644 --- a/fetch_packages.py +++ b/fetch_packages.py @@ -105,7 +105,7 @@ def ApplyPatches(pkg): #def VarSubst(cmdstr, filename): # return re.sub(r'\${filename}', filename, cmdstr) -def DownloadPackage(url, pkg, md5): +def DownloadPackage(urls, pkg, md5): #Check if the package already exists if os.path.isfile(pkg): md5sum = FindMd5sum(pkg) @@ -115,22 +115,29 @@ def DownloadPackage(url, pkg, md5): os.remove(pkg) retry_count = 0 - while True: - subprocess.call([wgettool, '--no-check-certificate', '-O', pkg, url, '--timeout=10']) - md5sum = FindMd5sum(pkg) - if ARGS['verbose']: - print "Calculated md5sum: %s" % md5sum - print "Expected md5sum: %s" % md5 - if md5sum == md5: - return - elif retry_count <= _RETRIES: + while retry_count <= _RETRIES: + for url in urls: + # poor man's templating + url = url.text + if "{{ site_mirror }}" in url: + if not ARGS['site_mirror']: + continue + url = url.replace("{{ site_mirror }}", ARGS['site_mirror']) + subprocess.call([wgettool, '--no-check-certificate', '-O', pkg, url, '--timeout=10']) + md5sum = FindMd5sum(pkg) + if ARGS['verbose']: + print "Calculated md5sum: %s" % md5sum + print "Expected md5sum: %s" % md5 + if md5sum == md5: + return os.remove(pkg) - retry_count += 1 - sleep(1) - continue - else: - raise RuntimeError("MD5sum %s, expected(%s) dosen't match for the " - "downloaded package %s" % (md5sum, md5, pkg)) + retry_count += 1 + # back-off retry timer - worst case scenario we wait for 150 seconds + sleep(10 * retry_count) + + # We couldn't download the package, return the last md5sum + raise RuntimeError("MD5sum %s, expected(%s) dosen't match for the " + "downloaded package %s" % (md5sum, md5, pkg)) def ReconfigurePackageSources(path): @@ -193,11 +200,13 @@ def ProcessPackage(pkg): return print "Processing %s ..." % (pkg['name']) - url = str(pkg['url']) - filename = getFilename(pkg, url) + urls = list(pkg['urls'].iterchildren()) + filename = getFilename(pkg, urls[0].text) ccfile = ARGS['cache_dir'] + '/' + filename - DownloadPackage(url, ccfile, pkg.md5) + DownloadPackage(urls, ccfile, pkg.md5) + cmd1=None + # # Determine the name of the directory created by the package. # unpack-directory means that we 'cd' to the given directory before @@ -343,6 +352,7 @@ def parse_args(): parser.add_argument("--node-module-tmp-dir", default=ARGS['node_modules_tmp_dir']) parser.add_argument("--verbose", default=ARGS['verbose'], action='store_true') parser.add_argument("--dry-run", default=ARGS['dry_run'], action='store_true') + parser.add_argument("--site-mirror", dest="site_mirror", required=False, default=None) ARGS = vars(parser.parse_args()) diff --git a/packages.xml b/packages.xml index 4979e17..7e12466 100644 --- a/packages.xml +++ b/packages.xml @@ -16,7 +16,10 @@ --> Boost C++ libraries version 1.48 - http://sourceforge.net/projects/boost/files/boost/1.48.0/boost_1_48_0.tar.gz + + {{ site_mirror }}/contrail-third-party/b/boost_1_48_0.tar.gz + http://sourceforge.net/projects/boost/files/boost/1.48.0/boost_1_48_0.tar.gz + tgz 313a11e97eb56eb7efd18325354631be @@ -55,7 +58,10 @@ curl 7.24 - http://curl.haxx.se/download/curl-7.24.0.tar.gz + + {{ site_mirror }}/contrail-third-party/c/curl-7.24.0.tar.gz + http://curl.haxx.se/download/curl-7.24.0.tar.gz + tgz b93420f80a2baaa61a0f45214eddc2ba @@ -88,7 +94,10 @@ Google C++ Testing Framework 1.6.0 - https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googletest-release-1.6.0.zip + + {{ site_mirror }}/contrail-third-party/g/googletest-release-1.6.0.zip + https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googletest-release-1.6.0.zip + gtest-1.6.0 gtest-1.6.0_patch1.diff @@ -114,7 +123,10 @@ Google C++ Mocking Framework 1.6.0 - https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googlemock-release-1.6.0.zip + + {{ site_mirror }}/contrail-third-party/g/googlemock-release-1.6.0.zip + https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googlemock-release-1.6.0.zip + zip gmock-1.6.0 b51cc3e69a8ef16ac54cee9889b79646 @@ -137,7 +149,10 @@ Google perftools - https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/gperftools-2.1.tar.gz + + {{ site_mirror }}/contrail-third-party/g/gperftools-2.1.tar.gz + https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/gperftools-2.1.tar.gz + tgz 5e5a981caf9baa9b4afe90a82dcf9882 @@ -159,7 +174,10 @@ Minimalistic C client for Redis v0.11 - https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-0.11.0.tar.gz + + {{ site_mirror }}/contrail-third-party/h/hiredis-v0.11.0.tar.gz + https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-0.11.0.tar.gz + hiredis-v0.11.0.tar.gz tgz e2ac29509823ccc96990b6fe765b5d46 @@ -169,7 +187,10 @@ Log4Cplus 1.1.1 - http://downloads.sourceforge.net/project/log4cplus/log4cplus-stable/1.1.1/log4cplus-1.1.1.tar.bz2 + + {{ site_mirror }}/contrail-third-party/l/log4cplus-1.1.1.tar.bz2 + http://downloads.sourceforge.net/project/log4cplus/log4cplus-stable/1.1.1/log4cplus-1.1.1.tar.bz2 + tbz a6fbba1e4b021f301b98c7bc68a068cb @@ -194,7 +215,10 @@ Thread Building Blocks 4.0 Update 2 - http://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb40_20111130oss_src.tgz + + {{ site_mirror }}/contrail-third-party/t/tbb40_20111130oss_src.tgz + http://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb40_20111130oss_src.tgz + tgz 1e6926b21e865e79772119cd44fc3ad8 @@ -223,14 +247,20 @@ Pugi XML 1.2 - https://github.com/zeux/pugixml/releases/download/v1.2/pugixml-1.2.tar.gz + + {{ site_mirror }}/contrail-third-party/p/pugixml-1.2.tar.gz + https://github.com/zeux/pugixml/releases/download/v1.2/pugixml-1.2.tar.gz + pugixml tgz 477f4a7d75af0383f52ee6622b3f6035 Apache Thrift 0.8 - http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz + + {{ site_mirror }}/contrail-third-party/t/thrift-0.8.0.tar.gz + http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz + thrift_patch1.diff thrift_autoconf.patch @@ -241,7 +271,10 @@ hiredis boostasio-adapter - https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-boostasio-adapter-master.zip + + {{ site_mirror }}/contrail-third-party/h/hiredis-boostasio-adapter-master.zip + https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-boostasio-adapter-master.zip + hiredis-master.zip zip c3ba292cb8a6d84da3aae774a33d4356 @@ -252,14 +285,20 @@ Joyent HTTP parser - https://github.com/joyent/http-parser/archive/v2.1.tar.gz + + {{ site_mirror }}/contrail-third-party/h/http-parser-v2.1.tar.gz + https://github.com/joyent/http-parser/archive/v2.1.tar.gz + http-parser-v2.1.tar.gz tgz 93d89867996b8077e0140692c55e997c Simple AMQP Client - https://github.com/alanxz/SimpleAmqpClient/archive/v2.4.0.tar.gz + + {{ site_mirror }}/contrail-third-party/S/SimpleAmqpClient-2.4.0.tar.gz + https://github.com/alanxz/SimpleAmqpClient/archive/v2.4.0.tar.gz + SimpleAmqpClient-2.4.0.tar.gz tgz SimpleAmqpClient @@ -267,7 +306,10 @@ RabbitMQ C - https://github.com/alanxz/rabbitmq-c/releases/download/v0.8.0/rabbitmq-c-0.8.0.tar.gz + + {{ site_mirror }}/contrail-third-party/r/rabbitmq-c-0.8.0.tar.gz + https://github.com/alanxz/rabbitmq-c/releases/download/v0.8.0/rabbitmq-c-0.8.0.tar.gz + rabbitmq-c-0.8.0.tar.gz rabbitmq-c tgz @@ -278,7 +320,10 @@ Bottle Python Web Framework - https://pypi.python.org/packages/source/b/bottle/bottle-0.11.6.tar.gz + + {{ site_mirror }}/contrail-third-party/b/bottle-0.11.6.tar.gz + https://pypi.python.org/packages/source/b/bottle/bottle-0.11.6.tar.gz + tgz 0bafdc4e13ea2b1a3bddf36b5af108c4 @@ -299,13 +344,19 @@ pYthOn Finite State Machine - https://pypi.python.org/packages/source/f/fysom/fysom-1.0.8.tar.gz + + {{ site_mirror }}/contrail-third-party/f/fysom-1.0.8.tar.gz + https://pypi.python.org/packages/source/f/fysom/fysom-1.0.8.tar.gz + tgz 5c8532ebdbc151eb01531ab2c7404297 Gnome XML C parser and toolkit - http://xmlsoft.org/sources/libxml2-2.8.0.tar.gz + + {{ site_mirror }}/contrail-third-party/l/libxml2-2.8.0.tar.gz + http://xmlsoft.org/sources/libxml2-2.8.0.tar.gz + tgz c62106f02ee00b6437f0fb9d370c1093 @@ -335,13 +386,19 @@ JSON parser/generator for C++ - https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip + + {{ site_mirror }}/contrail-third-party/r/rapidjson-v1.1.0.zip + https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip + zip 4f82cc9f41b7d39afeeec10ff8b581b9 ISC Bind - http://www.mirrorservice.org/sites/ftp.isc.org/isc/bind9/9.10.4-P2/bind-9.10.4-P2.tar.gz + + {{ site_mirror }}/contrail-third-party/b/bind-9.10.4-P2.tar.gz + http://www.mirrorservice.org/sites/ftp.isc.org/isc/bind9/9.10.4-P2/bind-9.10.4-P2.tar.gz + tgz 50163d229020c48929c84bec5b0068d7 @@ -351,7 +408,10 @@ ICU - International Components for Unicode - http://download.icu-project.org/files/icu4c/52.1/icu4c-52_1-src.tgz + + {{ site_mirror }}/contrail-third-party/i/icu4c-52_1-src.tgz + http://download.icu-project.org/files/icu4c/52.1/icu4c-52_1-src.tgz + tgz 9e96ed4c1d99c0d14ac03c140f9f346c @@ -381,7 +441,10 @@ vijava - https://github.com/Juniper/contrail-third-party-cache/raw/master/vmware/vijava55b20130927src.jar + + {{ site_mirror }}/contrail-third-party/v/vijava55b20130927src.jar + https://github.com/Juniper/contrail-third-party-cache/raw/master/vmware/vijava55b20130927src.jar + vijava55b20130927src.jar zip 3993c5bf9323e55fe9e6c14c974ea155 @@ -392,7 +455,10 @@ Open vSwitch version 2.3.0 - http://openvswitch.org/releases/openvswitch-2.3.0.tar.gz + + {{ site_mirror }}/contrail-third-party/o/openvswitch-2.3.0.tar.gz + http://openvswitch.org/releases/openvswitch-2.3.0.tar.gz + openvswitch-2.3.0.tar.gz tgz 9c4d1471a56718132e0157af1bfc9310 @@ -402,14 +468,20 @@ Go 1.8 - https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz + + {{ site_mirror }}/contrail-third-party/g/go1.8.linux-amd64.tar.gz + https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz + go1.8.linux-amd64.tar.gz tgz d0f6de92ccef6489a0d9b8efbb8f577f Cni Go Deps - https://github.com/Juniper/contrail-third-party-cache/raw/master/cni/cni_go_deps.tar.gz + + {{ site_mirror }}/contrail-third-party/c/cni_go_deps.tar.gz + https://github.com/Juniper/contrail-third-party-cache/raw/master/cni/cni_go_deps.tar.gz + tgz 823fcd824557fa0b6506e29d7591f76d diff --git a/populate_cache.py b/populate_cache.py new file mode 100644 index 0000000..4584371 --- /dev/null +++ b/populate_cache.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# +# This script creates a local mirror for third-party packages used +# in Contrail builds. +# +# Usage: populate_cache.py target_directory/ + +import logging +import argparse +import os +import hashlib +import shutil +import sys +import urllib2 + +from urlparse import urlparse +from lxml import objectify + +LOG = logging.getLogger("populate_cache") +LOG.setLevel(logging.DEBUG) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +LOG.addHandler(ch) + +def parse_packages_xml(path): + return objectify.parse(path) + +def get_package_list(xml): + root = xml.getroot() + return root.findall("package") + +def get_package_details(element): + md5sum = element['md5'] + canonical_url = element['urls'].findall("url[@canonical='true']") + assert len(canonical_url) == 1 + + try: + filename = element['local-filename'].text + except AttributeError: + parsed_url = urlparse(canonical_url[0].text) + filename = os.path.basename(parsed_url.path) + + return (canonical_url[0].text, filename, md5sum) + +def cached_file_exists(dest, filename, md5sum): + parent_dir = os.path.join(dest, filename[0]) + local_path = os.path.join(parent_dir, filename) + + if not os.path.exists(local_path): + LOG.info("File %s missing, will download", filename) + return False + + with open(local_path, 'rb') as fh: + calculated_md5 = hashlib.md5(fh.read()).hexdigest() + if calculated_md5 != md5sum: + LOG.warn("File %s found, but checksum mismatch " + "(found: '%s' expected: '%s')", + filename, calculated_md5, md5sum) + return False + + LOG.info("File %s found, checksum matches", filename) + return True + +def download_package(canonical_url, dest, filename, md5sum): + parent_dir = os.path.join(dest, filename[0]) + local_path = os.path.join(parent_dir, filename) + temp_path = local_path + '.tmp' + + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + chunk = 16 * 1024 + req = urllib2.urlopen(canonical_url) + with open(temp_path, 'wb') as fh: + shutil.copyfileobj(req, fh, chunk) + + with open(temp_path, 'rb') as fh: + calculated_md5 = hashlib.md5(fh.read()).hexdigest() + if calculated_md5 != md5sum: + LOG.error("File %s checksum error" + "(found: '%s' expected: '%s')", + filename, calculated_md5, md5sum) + return False + + LOG.info("File %s downloaded, checksum matches", filename) + + if os.path.exists(local_path): + os.unlink(local_path) + os.rename(temp_path, local_path) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("destination", nargs=1) + args = parser.parse_args(sys.argv[1:]) + + dest = args.destination[0] + + xml = parse_packages_xml("packages.xml") + packages = get_package_list(xml) + for package in packages: + canonical_url, filename, md5sum = get_package_details(package) + if cached_file_exists(dest, filename, md5sum): + continue + download_package(canonical_url, dest, filename, md5sum) + +if __name__ == "__main__": + main() + diff --git a/windows_packages.xml b/windows_packages.xml index 74785b2..2384399 100755 --- a/windows_packages.xml +++ b/windows_packages.xml @@ -16,7 +16,10 @@ --> Windows Apache Thrift 0.8 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/thrift-0.8.0.zip + + {{ site_mirror }}/contrail-third-party/t/thrift-0.8.0.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/thrift-0.8.0.zip + windows/thrift.patch @@ -26,7 +29,10 @@ log4cplus-1.1.1 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/log4cplus-1.1.1.zip + + {{ site_mirror }}/contrail-third-party/l/log4cplus-1.1.1.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/log4cplus-1.1.1.zip + log4cplus-1.1.1 zip 1d6aab5aa81bfbe2edaa97995b808279 @@ -34,7 +40,10 @@ googletest-release-1.8.0 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/googletest-release-1.8.0.zip + + {{ site_mirror }}/contrail-third-party/g/googletest-release-1.8.0.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/googletest-release-1.8.0.zip + googletest-release-1.8.0 zip adfafc8512ab65fd3cf7955ef0100ff5 @@ -42,7 +51,10 @@ rapidjson-1.1.0 - https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip + + {{ site_mirror }}/contrail-third-party/r/rapidjson-v1.1.0.zip + https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip + rapidjson zip 4f82cc9f41b7d39afeeec10ff8b581b9 @@ -50,28 +62,40 @@ boost 1.62.0 - https://sourceforge.net/projects/boost/files/boost/1.62.0/boost_1_62_0.zip + + {{ site_mirror }}/contrail-third-party/b/boost_1_62_0.zip + https://sourceforge.net/projects/boost/files/boost/1.62.0/boost_1_62_0.zip + boost_1_62_0 zip 9495bfde25f46698edb8b8eb02e55eab openvswitch-2.6.0 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openvswitch-2.6.0.tar.gz + + {{ site_mirror }}/contrail-third-party/o/openvswitch-2.6.0.tar.gz + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openvswitch-2.6.0.tar.gz + openvswitch-2.6.0 tgz 7d73a3e48aa60a807663e690c3928d72 http-parser-2.1 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/http-parser-2.1.zip + + {{ site_mirror }}/contrail-third-party/h/http-parser-2.1.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/http-parser-2.1.zip + http-parser-2.1 zip ebed8ebabaaeff86d75bd2a7bc6ee0ba pugixml-1.2 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/pugixml-1.2.tar.gz + + {{ site_mirror }}/contrail-third-party/p/pugixml-1.2.tar.gz + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/pugixml-1.2.tar.gz + pugixml pugixml tgz @@ -79,7 +103,10 @@ curl-7.52.0 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/curl-curl-7_52_0.zip + + {{ site_mirror }}/contrail-third-party/c/curl-curl-7_52_0.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/curl-curl-7_52_0.zip + curl-curl-7_52_0 curl-7.52.0 zip @@ -87,7 +114,10 @@ dirent-1.21 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/dirent-1.21.zip + + {{ site_mirror }}/contrail-third-party/d/dirent-1.21.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/dirent-1.21.zip + dirent dirent-1.21 zip @@ -95,14 +125,20 @@ openssl-1.0.2k - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openssl-1.0.2k.tar.gz + + {{ site_mirror }}/contrail-third-party/o/openssl-1.0.2k.tar.gz + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openssl-1.0.2k.tar.gz + openssl-1.0.2k tgz f965fc0bf01bf882b31314b61391ae65 libevent-2.0.22 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libevent-2.0.22.tar.gz + + {{ site_mirror }}/contrail-third-party/l/libevent-2.0.22.tar.gz + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libevent-2.0.22.tar.gz + libevent-2.0.22-stable libevent-2.0.22 tgz @@ -110,14 +146,20 @@ libxml2-2.9.4 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libxml2-2.9.4.tar.gz + + {{ site_mirror }}/contrail-third-party/l/libxml2-2.9.4.tar.gz + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libxml2-2.9.4.tar.gz + libxml2 tgz ae249165c173b1ff386ee8ad676815f5 tbb-2017_U7 - https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/tbb-2017_U7.zip + + {{ site_mirror }}/contrail-third-party/t/tbb-2017_U7.zip + https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/tbb-2017_U7.zip + tbb-2017_U7 zip 885d480693287a093d230460a067afb4