Move hal zipfile downloading to an importable module

hal-roborio/hal_impl/distutils.py

@@ -0,0 +1,63 @@
+#
+# Utilities for compiling things that link against HAL or need the HAL
+# headers
+#
+# Note: this file must not depend on anything else in the library, so we
+# can import it from setup.py. Also used by the wpilib tests
+#
+
+
+hal_version = '2017.1.1'
+
+hal_site = 'http://first.wpi.edu/FRC/roborio/maven/release/edu/wpi/first/wpilib/athena-runtime'
+hal_zip = 'athena-runtime-%s.zip' % hal_version
+
+def _download_halzip():
+    '''
+        Downloads the HAL zipfile to a temporary directory
+    '''
+
+    import atexit
+    from urllib.request import urlretrieve, urlcleanup
+    import sys
+
+    print("Downloading", hal_zip)
+
+    def _reporthook(count, blocksize, totalsize):
+        percent = int(count*blocksize*100/totalsize)
+        sys.stdout.write("\r%02d%%" % percent)
+        sys.stdout.flush()
+
+    filename, _ = urlretrieve("%s/%s/%s" % (hal_site, hal_version, hal_zip),
+                                    reporthook=_reporthook)
+    atexit.register(urlcleanup)
+    return filename
+
+def extract_halzip(to=None):
+    '''
+        Downloads the HAL zipfile and extracts it to a specified location
+    
+        :param to: is either a string or a dict of {src: dst}
+    '''
+
+    import atexit
+    import shutil
+    import tempfile
+    import zipfile
+
+    if to is None:
+        # generate temporary directory
+        tod = tempfile.TemporaryDirectory()
+        to = tod.name
+        atexit.register(tod.cleanup)
+
+    hal_zipfile = _download_halzip()
+    with zipfile.ZipFile(hal_zipfile) as z:
+        if isinstance(to, str):
+            z.extractall(to)
+            return to
+        else:
+            for src, dst in to.items():
+                with z.open(src, 'r') as zfp:
+                    with open(dst, 'wb') as fp:
+                        shutil.copyfileobj(zfp, fp)

hal-roborio/setup.py

@@ -1,34 +1,30 @@
 #!/usr/bin/env python3
 
-import atexit
+import importlib.machinery
 from os.path import dirname, exists, join
-import shutil
 import subprocess
 import sys
-from urllib.request import urlretrieve, urlcleanup
 
 from setuptools import setup
 
 setup_dir = dirname(__file__)
 git_dir = join(setup_dir, '..', '.git')
 base_package = 'hal_impl'
 version_file = join(setup_dir, base_package, 'version.py')
-hal_version = '2017.1.1'
+hal_distutils_file = join(setup_dir, base_package, 'distutils.py')
 
-#
-# Code for downloading/extracting HAL shared library files
-# - Used by wpilib tests
-#
-
-hal_site = 'http://first.wpi.edu/FRC/roborio/maven/release/edu/wpi/first/wpilib/athena-runtime'
-hal_zip = 'athena-runtime-%s.zip' % hal_version
+hal_distutils = importlib.machinery.SourceFileLoader('hal_distutils',
+                                                     hal_distutils_file).load_module()
 
 hal_base_files = [
     'libHALAthena.so',
     'libwpiutil.so',
 ]
 
-hal_files = [join(setup_dir, base_package, f) for f in hal_base_files]
+hal_files = {
+    join('lib', f): join(setup_dir, base_package, f)
+                        for f in hal_base_files
+}
 
 __version__ = "master"
 __hal_version__ = None
@@ -38,31 +34,9 @@
     with open(version_file, 'r') as fp:
         exec(fp.read(), globals())
 
-def download_halzip():
-    print("Downloading", hal_zip)
-
-    def _reporthook(count, blocksize, totalsize):
-        percent = int(count*blocksize*100/totalsize)
-        sys.stdout.write("\r%02d%%" % percent)
-        sys.stdout.flush()
-
-    filename, headers = urlretrieve("%s/%s/%s" % (hal_site, hal_version, hal_zip),
-                                    reporthook=_reporthook)
-    atexit.register(urlcleanup)
-    return filename
-
-hal_download_zip = None
-
 # Download the HAL if required
-if not all(map(exists, hal_files)) or __hal_version__ != hal_version:
-    import zipfile
-
-    hal_download_zip = download_halzip()
-    with zipfile.ZipFile(hal_download_zip) as z:
-        for bf, f in zip(hal_base_files, hal_files):
-            with z.open(join('lib', bf), 'r') as zfp:
-                with open(f, 'wb') as fp:
-                    shutil.copyfileobj(zfp, fp)
+if not all(map(exists, hal_files.values())) or __hal_version__ != hal_distutils.hal_version:
+    hal_distutils.extract_halzip(hal_files)
 
 # Automatically generate a version.py based on the git version
 if exists(git_dir):
@@ -84,9 +58,9 @@ def _reporthook(count, blocksize, totalsize):
     version = __version__
 
 # Generate a new version.py if required
-if not exists(version_file) or __version__ != version or __hal_version__ != hal_version:
+if not exists(version_file) or __version__ != version or __hal_version__ != hal_distutils.hal_version:
     with open(version_file, 'w') as fp:
-        fp.write("# Autogenerated by setup.py\n__version__ = '{0}'\n__hal_version__ = '{1}'".format(version, hal_version))
+        fp.write("# Autogenerated by setup.py\n__version__ = '{0}'\n__hal_version__ = '{1}'".format(version, hal_distutils.hal_version))
 
 
 with open(join(setup_dir, 'README.rst'), 'r') as readme_file:

wpilib/tests/test_specification.py

@@ -11,41 +11,35 @@
 hal_dir = abspath(join(dirname(__file__), '__hal__'))
 hal_version_file = join(hal_dir, 'version')
 
-hal_setup_py = abspath(join(dirname(__file__), '..', '..', 'hal-roborio', 'setup.py'))
+hal_distutils_file = abspath(join(dirname(__file__),
+                             '..', '..',
+                             'hal-roborio', 'hal_impl', 'distutils.py'))
 
 def _download_hal_includes():
 
-    hal_setup = importlib.machinery.SourceFileLoader('hal_setup_py',
-                                                     hal_setup_py).load_module()
+    hal_distutils = importlib.machinery.SourceFileLoader('hal_setup_py',
+                                                         hal_distutils_file).load_module()
+    hal_version = hal_distutils.hal_version
 
-    print("Using HAL", hal_setup.hal_version)
+    print("Using HAL", hal_distutils.hal_version)
     print()
 
     if exists(hal_dir):
         # find the version
         if exists(hal_version_file):
             with open(hal_version_file) as fp:
-                hal_version = fp.read().strip()
-
-            if hal_version == hal_setup.hal_version:
-                return
-
+                if hal_version == fp.read().strip():
+                    return
+
         # Nope, gotta download a new distribution
         shutil.rmtree(hal_dir)
 
     # Download the hal zipfile
-    hal_download_zip = hal_setup.hal_download_zip
-    if hal_download_zip is None:
-        hal_download_zip = hal_setup.download_halzip()
-
-    os.mkdir(hal_dir)
-
-    with zipfile.ZipFile(hal_download_zip) as z:
-        z.extractall(hal_dir)
+    hal_distutils.extract_halzip(hal_dir)
 
     # write the version to a file
     with open(hal_version_file, 'w') as fp:
-        fp.write(hal_setup.hal_version + '\n')
+        fp.write(hal_version + '\n')
 
 
 def test_check_hal_api(hal):