diff --git a/fetch_packages.py b/fetch_packages.py
index 2c77721..3d2e676 100644
--- a/fetch_packages.py
+++ b/fetch_packages.py
@@ -105,7 +105,7 @@ def ApplyPatches(pkg):
#def VarSubst(cmdstr, filename):
# return re.sub(r'\${filename}', filename, cmdstr)
-def DownloadPackage(url, pkg, md5):
+def DownloadPackage(urls, pkg, md5):
#Check if the package already exists
if os.path.isfile(pkg):
md5sum = FindMd5sum(pkg)
@@ -115,22 +115,29 @@ def DownloadPackage(url, pkg, md5):
os.remove(pkg)
retry_count = 0
- while True:
- subprocess.call([wgettool, '--no-check-certificate', '-O', pkg, url, '--timeout=10'])
- md5sum = FindMd5sum(pkg)
- if ARGS['verbose']:
- print "Calculated md5sum: %s" % md5sum
- print "Expected md5sum: %s" % md5
- if md5sum == md5:
- return
- elif retry_count <= _RETRIES:
+ while retry_count <= _RETRIES:
+ for url in urls:
+ # poor man's templating
+ url = url.text
+ if "{{ site_mirror }}" in url:
+ if not ARGS['site_mirror']:
+ continue
+ url = url.replace("{{ site_mirror }}", ARGS['site_mirror'])
+ subprocess.call([wgettool, '--no-check-certificate', '-O', pkg, url, '--timeout=10'])
+ md5sum = FindMd5sum(pkg)
+ if ARGS['verbose']:
+ print "Calculated md5sum: %s" % md5sum
+ print "Expected md5sum: %s" % md5
+ if md5sum == md5:
+ return
os.remove(pkg)
- retry_count += 1
- sleep(1)
- continue
- else:
- raise RuntimeError("MD5sum %s, expected(%s) dosen't match for the "
- "downloaded package %s" % (md5sum, md5, pkg))
+ retry_count += 1
+ # back-off retry timer - worst case scenario we wait for 150 seconds
+ sleep(10 * retry_count)
+
+ # We couldn't download the package, return the last md5sum
+ raise RuntimeError("MD5sum %s, expected(%s) dosen't match for the "
+ "downloaded package %s" % (md5sum, md5, pkg))
def ReconfigurePackageSources(path):
@@ -193,11 +200,13 @@ def ProcessPackage(pkg):
return
print "Processing %s ..." % (pkg['name'])
- url = str(pkg['url'])
- filename = getFilename(pkg, url)
+ urls = list(pkg['urls'].iterchildren())
+ filename = getFilename(pkg, urls[0].text)
ccfile = ARGS['cache_dir'] + '/' + filename
- DownloadPackage(url, ccfile, pkg.md5)
+ DownloadPackage(urls, ccfile, pkg.md5)
+
cmd1=None
+
#
# Determine the name of the directory created by the package.
# unpack-directory means that we 'cd' to the given directory before
@@ -343,6 +352,7 @@ def parse_args():
parser.add_argument("--node-module-tmp-dir", default=ARGS['node_modules_tmp_dir'])
parser.add_argument("--verbose", default=ARGS['verbose'], action='store_true')
parser.add_argument("--dry-run", default=ARGS['dry_run'], action='store_true')
+ parser.add_argument("--site-mirror", dest="site_mirror", required=False, default=None)
ARGS = vars(parser.parse_args())
diff --git a/packages.xml b/packages.xml
index 4979e17..7e12466 100644
--- a/packages.xml
+++ b/packages.xml
@@ -16,7 +16,10 @@
-->
Boost C++ libraries version 1.48
- http://sourceforge.net/projects/boost/files/boost/1.48.0/boost_1_48_0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/b/boost_1_48_0.tar.gz
+ http://sourceforge.net/projects/boost/files/boost/1.48.0/boost_1_48_0.tar.gz
+
tgz
313a11e97eb56eb7efd18325354631be
@@ -55,7 +58,10 @@
curl 7.24
- http://curl.haxx.se/download/curl-7.24.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/c/curl-7.24.0.tar.gz
+ http://curl.haxx.se/download/curl-7.24.0.tar.gz
+
tgz
b93420f80a2baaa61a0f45214eddc2ba
@@ -88,7 +94,10 @@
Google C++ Testing Framework 1.6.0
- https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googletest-release-1.6.0.zip
+
+ {{ site_mirror }}/contrail-third-party/g/googletest-release-1.6.0.zip
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googletest-release-1.6.0.zip
+
gtest-1.6.0
gtest-1.6.0_patch1.diff
@@ -114,7 +123,10 @@
Google C++ Mocking Framework 1.6.0
- https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googlemock-release-1.6.0.zip
+
+ {{ site_mirror }}/contrail-third-party/g/googlemock-release-1.6.0.zip
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/googlemock-release-1.6.0.zip
+
zip
gmock-1.6.0
b51cc3e69a8ef16ac54cee9889b79646
@@ -137,7 +149,10 @@
Google perftools
- https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/gperftools-2.1.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/g/gperftools-2.1.tar.gz
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/gperftools-2.1.tar.gz
+
tgz
5e5a981caf9baa9b4afe90a82dcf9882
@@ -159,7 +174,10 @@
Minimalistic C client for Redis v0.11
- https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-0.11.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/h/hiredis-v0.11.0.tar.gz
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-0.11.0.tar.gz
+
hiredis-v0.11.0.tar.gz
tgz
e2ac29509823ccc96990b6fe765b5d46
@@ -169,7 +187,10 @@
Log4Cplus 1.1.1
- http://downloads.sourceforge.net/project/log4cplus/log4cplus-stable/1.1.1/log4cplus-1.1.1.tar.bz2
+
+ {{ site_mirror }}/contrail-third-party/l/log4cplus-1.1.1.tar.bz2
+ http://downloads.sourceforge.net/project/log4cplus/log4cplus-stable/1.1.1/log4cplus-1.1.1.tar.bz2
+
tbz
a6fbba1e4b021f301b98c7bc68a068cb
@@ -194,7 +215,10 @@
Thread Building Blocks 4.0 Update 2
- http://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb40_20111130oss_src.tgz
+
+ {{ site_mirror }}/contrail-third-party/t/tbb40_20111130oss_src.tgz
+ http://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb40_20111130oss_src.tgz
+
tgz
1e6926b21e865e79772119cd44fc3ad8
@@ -223,14 +247,20 @@
Pugi XML 1.2
- https://github.com/zeux/pugixml/releases/download/v1.2/pugixml-1.2.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/p/pugixml-1.2.tar.gz
+ https://github.com/zeux/pugixml/releases/download/v1.2/pugixml-1.2.tar.gz
+
pugixml
tgz
477f4a7d75af0383f52ee6622b3f6035
Apache Thrift 0.8
- http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/t/thrift-0.8.0.tar.gz
+ http://archive.apache.org/dist/thrift/0.8.0/thrift-0.8.0.tar.gz
+
thrift_patch1.diff
thrift_autoconf.patch
@@ -241,7 +271,10 @@
hiredis boostasio-adapter
- https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-boostasio-adapter-master.zip
+
+ {{ site_mirror }}/contrail-third-party/h/hiredis-boostasio-adapter-master.zip
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/hiredis/hiredis-boostasio-adapter-master.zip
+
hiredis-master.zip
zip
c3ba292cb8a6d84da3aae774a33d4356
@@ -252,14 +285,20 @@
Joyent HTTP parser
- https://github.com/joyent/http-parser/archive/v2.1.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/h/http-parser-v2.1.tar.gz
+ https://github.com/joyent/http-parser/archive/v2.1.tar.gz
+
http-parser-v2.1.tar.gz
tgz
93d89867996b8077e0140692c55e997c
Simple AMQP Client
- https://github.com/alanxz/SimpleAmqpClient/archive/v2.4.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/S/SimpleAmqpClient-2.4.0.tar.gz
+ https://github.com/alanxz/SimpleAmqpClient/archive/v2.4.0.tar.gz
+
SimpleAmqpClient-2.4.0.tar.gz
tgz
SimpleAmqpClient
@@ -267,7 +306,10 @@
RabbitMQ C
- https://github.com/alanxz/rabbitmq-c/releases/download/v0.8.0/rabbitmq-c-0.8.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/r/rabbitmq-c-0.8.0.tar.gz
+ https://github.com/alanxz/rabbitmq-c/releases/download/v0.8.0/rabbitmq-c-0.8.0.tar.gz
+
rabbitmq-c-0.8.0.tar.gz
rabbitmq-c
tgz
@@ -278,7 +320,10 @@
Bottle Python Web Framework
- https://pypi.python.org/packages/source/b/bottle/bottle-0.11.6.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/b/bottle-0.11.6.tar.gz
+ https://pypi.python.org/packages/source/b/bottle/bottle-0.11.6.tar.gz
+
tgz
0bafdc4e13ea2b1a3bddf36b5af108c4
@@ -299,13 +344,19 @@
pYthOn Finite State Machine
- https://pypi.python.org/packages/source/f/fysom/fysom-1.0.8.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/f/fysom-1.0.8.tar.gz
+ https://pypi.python.org/packages/source/f/fysom/fysom-1.0.8.tar.gz
+
tgz
5c8532ebdbc151eb01531ab2c7404297
Gnome XML C parser and toolkit
- http://xmlsoft.org/sources/libxml2-2.8.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/l/libxml2-2.8.0.tar.gz
+ http://xmlsoft.org/sources/libxml2-2.8.0.tar.gz
+
tgz
c62106f02ee00b6437f0fb9d370c1093
@@ -335,13 +386,19 @@
JSON parser/generator for C++
- https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip
+
+ {{ site_mirror }}/contrail-third-party/r/rapidjson-v1.1.0.zip
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip
+
zip
4f82cc9f41b7d39afeeec10ff8b581b9
ISC Bind
- http://www.mirrorservice.org/sites/ftp.isc.org/isc/bind9/9.10.4-P2/bind-9.10.4-P2.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/b/bind-9.10.4-P2.tar.gz
+ http://www.mirrorservice.org/sites/ftp.isc.org/isc/bind9/9.10.4-P2/bind-9.10.4-P2.tar.gz
+
tgz
50163d229020c48929c84bec5b0068d7
@@ -351,7 +408,10 @@
ICU - International Components for Unicode
- http://download.icu-project.org/files/icu4c/52.1/icu4c-52_1-src.tgz
+
+ {{ site_mirror }}/contrail-third-party/i/icu4c-52_1-src.tgz
+ http://download.icu-project.org/files/icu4c/52.1/icu4c-52_1-src.tgz
+
tgz
9e96ed4c1d99c0d14ac03c140f9f346c
@@ -381,7 +441,10 @@
vijava
- https://github.com/Juniper/contrail-third-party-cache/raw/master/vmware/vijava55b20130927src.jar
+
+ {{ site_mirror }}/contrail-third-party/v/vijava55b20130927src.jar
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/vmware/vijava55b20130927src.jar
+
vijava55b20130927src.jar
zip
3993c5bf9323e55fe9e6c14c974ea155
@@ -392,7 +455,10 @@
Open vSwitch version 2.3.0
- http://openvswitch.org/releases/openvswitch-2.3.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/o/openvswitch-2.3.0.tar.gz
+ http://openvswitch.org/releases/openvswitch-2.3.0.tar.gz
+
openvswitch-2.3.0.tar.gz
tgz
9c4d1471a56718132e0157af1bfc9310
@@ -402,14 +468,20 @@
Go 1.8
- https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/g/go1.8.linux-amd64.tar.gz
+ https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz
+
go1.8.linux-amd64.tar.gz
tgz
d0f6de92ccef6489a0d9b8efbb8f577f
Cni Go Deps
- https://github.com/Juniper/contrail-third-party-cache/raw/master/cni/cni_go_deps.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/c/cni_go_deps.tar.gz
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/cni/cni_go_deps.tar.gz
+
tgz
823fcd824557fa0b6506e29d7591f76d
diff --git a/populate_cache.py b/populate_cache.py
new file mode 100644
index 0000000..4584371
--- /dev/null
+++ b/populate_cache.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+#
+# This script creates a local mirror for third-party packages used
+# in Contrail builds.
+#
+# Usage: populate_cache.py target_directory/
+
+import logging
+import argparse
+import os
+import hashlib
+import shutil
+import sys
+import urllib2
+
+from urlparse import urlparse
+from lxml import objectify
+
+LOG = logging.getLogger("populate_cache")
+LOG.setLevel(logging.DEBUG)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+
+LOG.addHandler(ch)
+
+def parse_packages_xml(path):
+ return objectify.parse(path)
+
+def get_package_list(xml):
+ root = xml.getroot()
+ return root.findall("package")
+
+def get_package_details(element):
+ md5sum = element['md5']
+ canonical_url = element['urls'].findall("url[@canonical='true']")
+ assert len(canonical_url) == 1
+
+ try:
+ filename = element['local-filename'].text
+ except AttributeError:
+ parsed_url = urlparse(canonical_url[0].text)
+ filename = os.path.basename(parsed_url.path)
+
+ return (canonical_url[0].text, filename, md5sum)
+
+def cached_file_exists(dest, filename, md5sum):
+ parent_dir = os.path.join(dest, filename[0])
+ local_path = os.path.join(parent_dir, filename)
+
+ if not os.path.exists(local_path):
+ LOG.info("File %s missing, will download", filename)
+ return False
+
+ with open(local_path, 'rb') as fh:
+ calculated_md5 = hashlib.md5(fh.read()).hexdigest()
+ if calculated_md5 != md5sum:
+ LOG.warn("File %s found, but checksum mismatch "
+ "(found: '%s' expected: '%s')",
+ filename, calculated_md5, md5sum)
+ return False
+
+ LOG.info("File %s found, checksum matches", filename)
+ return True
+
+def download_package(canonical_url, dest, filename, md5sum):
+ parent_dir = os.path.join(dest, filename[0])
+ local_path = os.path.join(parent_dir, filename)
+ temp_path = local_path + '.tmp'
+
+ if not os.path.exists(parent_dir):
+ os.makedirs(parent_dir)
+
+ chunk = 16 * 1024
+ req = urllib2.urlopen(canonical_url)
+ with open(temp_path, 'wb') as fh:
+ shutil.copyfileobj(req, fh, chunk)
+
+ with open(temp_path, 'rb') as fh:
+ calculated_md5 = hashlib.md5(fh.read()).hexdigest()
+ if calculated_md5 != md5sum:
+ LOG.error("File %s checksum error"
+ "(found: '%s' expected: '%s')",
+ filename, calculated_md5, md5sum)
+ return False
+
+ LOG.info("File %s downloaded, checksum matches", filename)
+
+ if os.path.exists(local_path):
+ os.unlink(local_path)
+ os.rename(temp_path, local_path)
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("destination", nargs=1)
+ args = parser.parse_args(sys.argv[1:])
+
+ dest = args.destination[0]
+
+ xml = parse_packages_xml("packages.xml")
+ packages = get_package_list(xml)
+ for package in packages:
+ canonical_url, filename, md5sum = get_package_details(package)
+ if cached_file_exists(dest, filename, md5sum):
+ continue
+ download_package(canonical_url, dest, filename, md5sum)
+
+if __name__ == "__main__":
+ main()
+
diff --git a/windows_packages.xml b/windows_packages.xml
index 74785b2..2384399 100755
--- a/windows_packages.xml
+++ b/windows_packages.xml
@@ -16,7 +16,10 @@
-->
Windows Apache Thrift 0.8
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/thrift-0.8.0.zip
+
+ {{ site_mirror }}/contrail-third-party/t/thrift-0.8.0.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/thrift-0.8.0.zip
+
windows/thrift.patch
@@ -26,7 +29,10 @@
log4cplus-1.1.1
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/log4cplus-1.1.1.zip
+
+ {{ site_mirror }}/contrail-third-party/l/log4cplus-1.1.1.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/log4cplus-1.1.1.zip
+
log4cplus-1.1.1
zip
1d6aab5aa81bfbe2edaa97995b808279
@@ -34,7 +40,10 @@
googletest-release-1.8.0
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/googletest-release-1.8.0.zip
+
+ {{ site_mirror }}/contrail-third-party/g/googletest-release-1.8.0.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/googletest-release-1.8.0.zip
+
googletest-release-1.8.0
zip
adfafc8512ab65fd3cf7955ef0100ff5
@@ -42,7 +51,10 @@
rapidjson-1.1.0
- https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip
+
+ {{ site_mirror }}/contrail-third-party/r/rapidjson-v1.1.0.zip
+ https://github.com/Juniper/contrail-third-party-cache/raw/master/googlecode/rapidjson-v1.1.0.zip
+
rapidjson
zip
4f82cc9f41b7d39afeeec10ff8b581b9
@@ -50,28 +62,40 @@
boost 1.62.0
- https://sourceforge.net/projects/boost/files/boost/1.62.0/boost_1_62_0.zip
+
+ {{ site_mirror }}/contrail-third-party/b/boost_1_62_0.zip
+ https://sourceforge.net/projects/boost/files/boost/1.62.0/boost_1_62_0.zip
+
boost_1_62_0
zip
9495bfde25f46698edb8b8eb02e55eab
openvswitch-2.6.0
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openvswitch-2.6.0.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/o/openvswitch-2.6.0.tar.gz
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openvswitch-2.6.0.tar.gz
+
openvswitch-2.6.0
tgz
7d73a3e48aa60a807663e690c3928d72
http-parser-2.1
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/http-parser-2.1.zip
+
+ {{ site_mirror }}/contrail-third-party/h/http-parser-2.1.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/http-parser-2.1.zip
+
http-parser-2.1
zip
ebed8ebabaaeff86d75bd2a7bc6ee0ba
pugixml-1.2
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/pugixml-1.2.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/p/pugixml-1.2.tar.gz
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/pugixml-1.2.tar.gz
+
pugixml
pugixml
tgz
@@ -79,7 +103,10 @@
curl-7.52.0
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/curl-curl-7_52_0.zip
+
+ {{ site_mirror }}/contrail-third-party/c/curl-curl-7_52_0.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/curl-curl-7_52_0.zip
+
curl-curl-7_52_0
curl-7.52.0
zip
@@ -87,7 +114,10 @@
dirent-1.21
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/dirent-1.21.zip
+
+ {{ site_mirror }}/contrail-third-party/d/dirent-1.21.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/dirent-1.21.zip
+
dirent
dirent-1.21
zip
@@ -95,14 +125,20 @@
openssl-1.0.2k
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openssl-1.0.2k.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/o/openssl-1.0.2k.tar.gz
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/openssl-1.0.2k.tar.gz
+
openssl-1.0.2k
tgz
f965fc0bf01bf882b31314b61391ae65
libevent-2.0.22
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libevent-2.0.22.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/l/libevent-2.0.22.tar.gz
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libevent-2.0.22.tar.gz
+
libevent-2.0.22-stable
libevent-2.0.22
tgz
@@ -110,14 +146,20 @@
libxml2-2.9.4
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libxml2-2.9.4.tar.gz
+
+ {{ site_mirror }}/contrail-third-party/l/libxml2-2.9.4.tar.gz
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/libxml2-2.9.4.tar.gz
+
libxml2
tgz
ae249165c173b1ff386ee8ad676815f5
tbb-2017_U7
- https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/tbb-2017_U7.zip
+
+ {{ site_mirror }}/contrail-third-party/t/tbb-2017_U7.zip
+ https://github.com/sagarkchitnis/windows/raw/master/windows/thirdpartycache/tbb-2017_U7.zip
+
tbb-2017_U7
zip
885d480693287a093d230460a067afb4