From 28405e54056b52fcd04fc71c6c2d72f3dc8c31cf Mon Sep 17 00:00:00 2001 From: Brent Stapleton Date: Wed, 20 Dec 2017 14:27:05 -0800 Subject: [PATCH] utils: refactor uhd_images_downloader Changing structure of the files.ettus.com images directory, and updating uhd_images_downloader correspondingly. The images downloader now downloads a number of smaller archives of images, instead of one large archive containing all of our images. Furthermore, the images downloader maintains an inventory of currently installed images, so that images are not redownloaded unnecessarily. When called with no arguments, behavior should not change. CMake variables are used to populate a number of fields in the images downloader, including the contents of the manifest file. Reviewed-by: Martin Braun Reviewed-by: Ashish Chaudhari --- host/utils/CMakeLists.txt | 1 + host/utils/uhd_images_downloader.py.in | 657 ++++++++++++------------- images/CMakeLists.txt | 1 + images/manifest.txt | 44 ++ 4 files changed, 370 insertions(+), 333 deletions(-) create mode 100644 images/manifest.txt diff --git a/host/utils/CMakeLists.txt b/host/utils/CMakeLists.txt index 78870f7bc7..6ecfb4ba36 100644 --- a/host/utils/CMakeLists.txt +++ b/host/utils/CMakeLists.txt @@ -121,6 +121,7 @@ IF(ENABLE_X300) ENDIF(ENABLE_X300) #UHD images downloader configuration +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../../images/manifest.txt CMAKE_MANIFEST_CONTENTS) CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/uhd_images_downloader.py.in ${CMAKE_CURRENT_BINARY_DIR}/uhd_images_downloader.py diff --git a/host/utils/uhd_images_downloader.py.in b/host/utils/uhd_images_downloader.py.in index facda4a04d..fccd98b9e3 100644 --- a/host/utils/uhd_images_downloader.py.in +++ b/host/utils/uhd_images_downloader.py.in @@ -1,374 +1,365 @@ #!/usr/bin/env python # -# Copyright 2012-2015 Ettus Research LLC +# Copyright 2018 Ettus Research, a National Instruments Company # # SPDX-License-Identifier: GPL-3.0 # - -import sys -import os -import tempfile +""" +Download image files required for USRPs +""" +from __future__ import print_function +import argparse +import json import math -import traceback +import os +import re import shutil -import hashlib -import requests +import sys +import tempfile import zipfile +import requests +try: + from urllib.parse import urljoin # Python 3 +except ImportError: + from urlparse import urljoin # Python 2 -from optparse import OptionParser -_DEFAULT_BUFFER_SIZE = 8192 +_DEFAULT_TARGET_REGEX = "(fpga|fw|windrv)_default" _BASE_DIR_STRUCTURE_PARTS = ["share", "uhd", "images"] _DEFAULT_INSTALL_PATH = os.path.join("@CMAKE_INSTALL_PREFIX@", *_BASE_DIR_STRUCTURE_PARTS) -_DEFAULT_BASE_URL = "http://files.ettus.com/binaries/images/" -_AUTOGEN_IMAGES_FILENAME = "@UHD_IMAGES_DOWNLOAD_SRC@" -_AUTOGEN_IMAGES_CHECKSUM = "@UHD_IMAGES_MD5SUM@" -_IMAGES_CHECKSUM_TYPE = "md5" +_DEFAULT_BASE_URL = "http://files.ettus.com/binaries/cache/" +_INVENTORY_FILENAME = "inventory.json" _CONTACT = "support@ettus.com" - -def md5Checksum(filePath): - try: - with open(filePath, 'rb') as fh: - m = hashlib.md5() - while True: - data = fh.read(_DEFAULT_BUFFER_SIZE) - if not data: - break - m.update(data) - return m.hexdigest() - except Exception as e: - print("Failed to calculated MD5 sum of: %s (%s)" % (filePath, e)) - raise e - -_checksum_fns = { - 'md5': md5Checksum -} - -class temporary_directory(): +_DEFAULT_BUFFER_SIZE = 8192 +_ARCHIVE_ALGS = ["zip", "targz", "tarxz"] +_ARCHIVE_DEFAULT_TYPE = "zip" +_UHD_VERSION = "@UHD_VERSION@" +# Note: _MANIFEST_CONTENTS are placed at the bottom of this file for aesthetic reasons +_LOG_LEVELS = {"TRACE": 1, + "DEBUG": 2, + "INFO": 3, + "WARN": 4, + "ERROR": 5} +_LOG_LEVEL = _LOG_LEVELS["INFO"] + + +# TODO: Move to a standard logger? +def log(level, message): + """Logging function""" + message_log_level = _LOG_LEVELS.get(level, 0) + if message_log_level >= _LOG_LEVEL: + print("[{level}] {message}".format(level=level, message=message)) + + +def parse_args(): + """Setup argument parser and parse""" + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + # TODO: clean up all the one letter arguments + parser.add_argument('-t', '--types', type=str, default="", + help="RegEx to select image sets from the manifest file.") + parser.add_argument('-i', '--install-location', type=str, default=_DEFAULT_INSTALL_PATH, + help="Set custom install location for images") + parser.add_argument('-m', '--manifest-location', type=str, default="", + help="Set custom location for the manifest file") + parser.add_argument('-I', '--inventory-location', type=str, default="", + help="Set custom location for the inventory file") + parser.add_argument('-l', '--list-targets', action="store_true", default=False, + help="Print targets in the manifest file, and exit.") + parser.add_argument("--buffer-size", type=int, default=_DEFAULT_BUFFER_SIZE, + help="Set download buffer size") + parser.add_argument("-b", "--base-url", type=str, default=_DEFAULT_BASE_URL, + help="Set base URL for images download location") + parser.add_argument("-z", "--archive-type", type=str, default=_ARCHIVE_DEFAULT_TYPE, + help=("Select archiving function (options: {})" + .format(",".join(_ARCHIVE_ALGS)))) + parser.add_argument("-k", "--keep", action="store_true", default=False, + help="Do not clear images directory before extracting new files") + parser.add_argument("--refetch", action="store_true", default=False, + help="Ignore the inventory file and download all images.") + parser.add_argument('-V', '--version', action='version', version=_UHD_VERSION) + parser.add_argument('-q', '--quiet', action='count', default=0, + help="Decrease verbosity level") + parser.add_argument('-v', '--verbose', action='count', default=0, + help="Increase verbosity level") + return parser.parse_args() + + +class TemporaryDirectory: + """Class to create a temporary directory""" def __enter__(self): try: self.name = tempfile.mkdtemp() return self.name - except Exception as e: - print("Failed to create a temporary directory (%s)" % (e)) - raise e + except Exception as ex: + log("ERROR", "Failed to create a temporary directory (%s)" % ex) + raise ex # Can return 'True' to suppress incoming exception - def __exit__(self, type, value, traceback): + def __exit__(self, exc_type, exc_value, traceback): try: shutil.rmtree(self.name) - except Exception as e: - print("Could not delete temporary directory: %s (%s)" % (self.name, e)) + log("TRACE", "Temp directory deleted.") + except Exception as ex: + log("ERROR", "Could not delete temporary directory: %s (%s)" % (self.name, ex)) + return exc_type is None -class uhd_images_downloader(): - def __init__(self): - pass - def download(self, images_url, filename, buffer_size=_DEFAULT_BUFFER_SIZE, print_progress=False): - """ Run the download, show progress """ +def parse_manifest(manifest_contents): + """Parse the manifest file, returns a dictionary of potential targets""" + manifest = {} + for line in manifest_contents.split('\n'): + line_unpacked = line.split() try: - r = requests.get(images_url, stream=True, headers={'User-Agent': 'UHD Images Downloader'}) - except TypeError as te: - ## requests library versions pre-4c3b9df6091b65d8c72763222bd5fdefb7231149 (Dec.'12) workaround - r = requests.get(images_url, prefetch=False, headers={'User-Agent': 'UHD Images Downloader'}) - filesize = float(r.headers['content-length']) - filesize_dl = 0 - with open(filename, "wb") as f: - for buff in r.iter_content(chunk_size=buffer_size): - if buff: - f.write(buff) - filesize_dl += len(buff) - if print_progress: - status = r"%05d kB / %05d kB (%03d%%)" % (int(math.ceil(filesize_dl/1000.)), int(math.ceil(filesize/1000.)), int(math.ceil(filesize_dl*100.)/filesize)) - if os.name == "nt": - status += chr(8)*(len(status)+1) - else: - sys.stdout.write("\x1b[2K\r") #Clear previos line - sys.stdout.write(status) - sys.stdout.flush() - if print_progress: - print('') - return (filesize, filesize_dl) - - def check_directories(self, dirs, print_progress=False): - if dirs is None or dirs == "": - dirs = "." - dirs = os.path.abspath(dirs) - - def _check_part(head, tail=None): - if print_progress: - print("Checking: %s" % (head)) - if tail is not None and tail == "": - return True - if not os.path.exists(head): - if print_progress: - print("Does not exist: %s" % (head)) - return _check_part(*os.path.split(head)) - if not os.path.isdir(head): - if print_progress: - print("Is not a directory: %s" % (head)) - return (False, head) - if not os.access(head, os.W_OK): - if print_progress: - print("Write permission denied on: %s" % (head)) - return (False, head) - if print_progress: - print("Write permission granted on: %s" % (head)) - return (True, head) - - return _check_part(dirs) - - def validate_checksum(self, checksum_fn, file_path, expecting, print_progress=False): - if checksum_fn is None: - return (True, "") - calculated_checksum = checksum_fn(file_path) - if (expecting is not None) and (expecting != "") and calculated_checksum != expecting: - return (False, calculated_checksum) - return (True, calculated_checksum) - - def extract_images_archive(self, archive_path, destination=None, print_progress=False): - if not os.path.exists(archive_path): - if print_progress: - print("Path does not exist: %s" % (archive_path)) - raise Exception("path does not exist: %s" % (archive_path)) - if print_progress: - print("Archive path: %s" % (archive_path)) - (head, tail) = os.path.split(archive_path) - - if not os.access(head, os.W_OK): - if print_progress: - print("Write access denied on: %s" % (head)) - raise Exception("write access denied on: %s" % (head)) - - (root, ext) = os.path.splitext(tail) - temp_dir = os.path.join(head, root) - - if print_progress: - print("Temporary extraction location: %s" % (temp_dir)) - - if os.path.exists(temp_dir): - if print_progress: - print("Deleting existing location: %s" % (temp_dir)) - shutil.rmtree(temp_dir) - - if print_progress: - print("Creating directory: %s" % (temp_dir)) - os.mkdir(temp_dir) - - if print_progress: - print("Extracting archive %s to %s" % (archive_path, temp_dir)) - - images_zip = zipfile.ZipFile(archive_path) - images_zip.extractall(temp_dir) - images_zip.close() - - return temp_dir + # Check that the line isn't empty or a comment + if not line_unpacked or line.strip().startswith('#'): + continue + + target, repo_hash, url, sha_hash = line_unpacked + manifest[target] = {"repo_hash": repo_hash, + "url": url, + "sha_hash": sha_hash, + } + except ValueError: + log("WARN", "Warning: Invalid line in manifest file:\n" + " {}".format(line)) + continue + return manifest + + +def parse_inventory(inventory_fn): + """Parse the inventory file, returns a dictionary of installed files""" + try: + if not os.path.exists(inventory_fn): + log("INFO", "No inventory file found at {}. Creating an empty one.".format(inventory_fn)) + return {} + with open(inventory_fn, 'r') as inventory_file: + # TODO: verify the contents?? + return json.load(inventory_file) + except Exception as ex: + log("WARN", "Error parsing the inventory file. Assuming an empty inventory: {}".format(ex)) + return {} + + +def write_inventory(inventory, inventory_fn): + """Writes the inventory to file""" + try: + with open(inventory_fn, 'w') as inventory_file: + json.dump(inventory, inventory_file) + return True + except Exception as ex: + log("ERROR", "Error writing the inventory file. Contents may be incomplete or corrupted.\n" + "Error message: {}".format(ex)) + return False + + +def lookup_urls(regex, manifest, inventory, refetch=False): + """Takes a list of RegExs to match within the manifest, returns a list of tuples with + (hash, URL) that match the targets and are not in the inventory""" + selected_targets = [] + for target in manifest.keys(): + # Iterate through the possible targets in the manifest. + # If any of them match any of the RegExs supplied, add the URL to the + # return list + if re.findall(regex, target): + log("TRACE", "Selected target: {}".format(target)) + target_info = manifest.get(target) + target_url = target_info.get("url") + target_hash = target_info.get("repo_hash") + # Check if the same filename and hash appear in the inventory + # TODO: Change to the TARGET instead of the filename + filename = os.path.basename(target_url) + if not refetch and inventory.get(filename, {}).get("repo_hash", "") == target_hash: + # We already have this file, we don't need to download it again + log("TRACE", "Selected target already downloaded- skipping.") + else: + # We don't have that exact file, add it to the list + selected_targets.append({"repo_hash": target_hash, "url": target_url}) + return selected_targets - def install_images(self, source, dest, keep=False, print_progress=False): - if not os.path.exists(source): - if print_progress: - print("Source path does not exist: %s" % (source)) - return - if keep: - if print_progress: - print("Not wiping directory tree (existing files will be overwritten): %s" % (dest)) - elif os.path.exists(dest): +def download(images_url, filename, buffer_size=_DEFAULT_BUFFER_SIZE, print_progress=False): + """ Run the download, show progress """ + log("TRACE", "Downloading {} to {}".format(images_url, filename)) + try: + resp = requests.get(images_url, stream=True, + headers={'User-Agent': 'UHD Images Downloader'}) + except TypeError: + # requests library versions pre-4c3b9df6091b65d8c72763222bd5fdefb7231149 + # (Dec.'12) workaround + resp = requests.get(images_url, prefetch=False, + headers={'User-Agent': 'UHD Images Downloader'}) + if resp.status_code != 200: + raise RuntimeError("URL does not exist: {}".format(images_url)) + filesize = float(resp.headers['content-length']) + filesize_dl = 0 + with open(filename, "wb") as temp_file: + for buff in resp.iter_content(chunk_size=buffer_size): + if buff: + temp_file.write(buff) + filesize_dl += len(buff) if print_progress: - print("Deleting directory tree: %s" % (dest)) - shutil.rmtree(dest) - - (head, tail) = os.path.split(source) - - if print_progress: - print("Source install path: %s" % (source)) - - uhd_source = os.path.join(source, tail, *_BASE_DIR_STRUCTURE_PARTS) - - if print_progress: - print("Copying files from: %s" % (uhd_source)) - print("Copying files to: %s" % (dest)) - - if keep: - # mgrant @ http://stackoverflow.com/questions/12683834/how-to-copy-directory-recursively-in-python-and-overwrite-all - def _recursive_overwrite(src, dest, ignore=None): - if os.path.isdir(src): - if not os.path.isdir(dest): - os.makedirs(dest) - files = os.listdir(src) - if ignore is not None: - ignored = ignore(src, files) - else: - ignored = set() - for f in files: - if f not in ignored: - _recursive_overwrite(os.path.join(src, f), os.path.join(dest, f), ignore) + status = r"%05d kB / %05d kB (%03d%%)" % ( + int(math.ceil(filesize_dl / 1000.)), int(math.ceil(filesize / 1000.)), + int(math.ceil(filesize_dl * 100.) / filesize)) + if os.name == "nt": + status += chr(8) * (len(status) + 1) else: - shutil.copyfile(src, dest) + sys.stdout.write("\x1b[2K\r") # Clear previous line + sys.stdout.write(status) + sys.stdout.flush() + if print_progress: + print('') + return filesize, filesize_dl + + +def delete_from_inv(archive_fn, inventory, images_dir): + """Uses the inventory to delete the contents of the archive file specified by `filename`""" + target = inventory.get(archive_fn, {}) + log("TRACE", "Removing contents of {} from inventory ({})".format( + target, target.get("contents", []))) + dirs_to_delete = [] + # Delete all of the files + for image_fn in target.get("contents", []): + image_path = os.path.join(images_dir, image_fn) + if os.path.isfile(image_path): + os.remove(image_path) + log("TRACE", "Deleted {} from inventory".format(image_path)) + elif os.path.isdir(image_path): + dirs_to_delete.append(image_fn) + else: # File doesn't exist + log("WARN", "File {} in inventory does not exist".format(image_path)) + # Then delete all of the (empty) directories + for dir_path in dirs_to_delete: + try: + if os.path.isdir(dir_path): + os.removedirs(dir_path) + except os.error as ex: + log("ERROR", "Failed to delete dir: {}".format(ex)) + inventory.pop(archive_fn, None) + return True + + +def extract(archive_path, images_dir, archive_type): + """Extract the contents of the archive into `images_dir`""" + if archive_type == "zip": + log("TRACE", "Attempting to extracted files from {}".format(archive_path)) + with zipfile.ZipFile(archive_path) as images_zip: + images_zip.extractall(images_dir) + archive_namelist = images_zip.namelist() + log("TRACE", "Extracted files: {}".format(archive_namelist)) + return archive_namelist + else: + raise NotImplementedError("Archive type {} not implemented".format(archive_type)) - _recursive_overwrite(uhd_source, dest) - else: - shutil.copytree(uhd_source, dest) def main(): - ### Set defaults from env variables - if os.environ.get("UHD_IMAGES_DIR") != None and os.environ.get("UHD_IMAGES_DIR") != "": - default_images_dir = os.environ.get("UHD_IMAGES_DIR") - print("UHD_IMAGES_DIR environment variable is set.\nDefault install location: {0}".format(default_images_dir)) - else: - default_images_dir = _DEFAULT_INSTALL_PATH - if os.environ.get("UHD_IMAGES_BASE_URL") != None and os.environ.get("UHD_IMAGES_BASE_URL") != "": - default_base_url = os.environ.get("UHD_IMAGES_BASE_URL") - print("UHD_IMAGES_BASE_URL environment variable is set.\nDefault base URL: {0}".format(default_base_url)) - else: - default_base_url = _DEFAULT_BASE_URL - - ### Setup argument parser and parse - parser = OptionParser() - parser.add_option("-i", "--install-location", type="string", default=default_images_dir, - help="Set custom install location for images [default=%default]") - parser.add_option("--buffer-size", type="int", default=_DEFAULT_BUFFER_SIZE, - help="Set download buffer size [default=%default]") - parser.add_option("-b", "--base-url", type="string", default=default_base_url, - help="Set base URL for images download location [default=%default]") - parser.add_option("-f", "--filename", type="string", default=_AUTOGEN_IMAGES_FILENAME, - help="Set images archive filename [default=%default]") - parser.add_option("-c", "--checksum", type="string", default=_AUTOGEN_IMAGES_CHECKSUM, - help="Validate images archive against this checksum (blank to skip) [default=%default]") - parser.add_option("-t", "--checksum-type", type="string", default=_IMAGES_CHECKSUM_TYPE, - help=("Select checksum hash function (options: %s) [default=%%default]" % (",".join(list(_checksum_fns.keys()))))) - parser.add_option("-k", "--keep", action="store_true", default=False, - help="Do not clear images directory before extracting new files [default=%default]") - parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Enable verbose output [default=%default]") - parser.add_option("--force-delete", action="store_true", default=False, - help="Delete all files in the target images directory without prompting [default=%default]") - (options, args) = parser.parse_args() - if options.buffer_size <= 0: - print("Invalid buffer size: %s" % (options.buffer_size)) + """Download the image files requested by the user""" + args = parse_args() + archive_type = args.archive_type + if archive_type not in _ARCHIVE_ALGS: + log("ERROR", "Selected archive type not supported: {}".format(archive_type)) return 1 + # TODO: keep not implemented + if args.keep: + log("WARN", "--keep not implemented. Downloaded archives will be deleted.") + # Set the verbosity + global _LOG_LEVEL + log("TRACE", "Default log level: {}".format(_LOG_LEVEL)) + _LOG_LEVEL = _LOG_LEVEL - args.verbose + args.quiet + try: + # If we're given a path to a manifest file, use it + if os.path.exists(args.manifest_location): + manifest_fn = args.manifest_location + log("INFO", "Using manifest file at location: {}".format(manifest_fn)) + with open(manifest_fn, 'r') as manifest_file: + manifest_raw = manifest_file.read() + # Otherwise, use the CMake Magic manifest + else: + manifest_raw = _MANIFEST_CONTENTS + log("TRACE", "Raw manifest contents: {}".format(manifest_raw)) + + manifest = parse_manifest(manifest_raw) + if args.list_targets: + char_offset = max(map(len, manifest.keys())) + # Print a couple helpful lines, + # then print each (Target, URL) pair in the manifest + log("INFO", "Potential targets in manifest file:\n" + "{} : {}\n" + "{}".format( + "# TARGET".ljust(char_offset), "RELATIVE_URL", + "\n".join("{} : {}".format(key.ljust(char_offset), value["url"]) + for key, value in sorted(manifest.items())) + )) + return 0 + else: + log("TRACE", "Manifest:\n{}".format( + "\n".join("{}".format(item) for item in manifest.items()) + )) - ### Select checksum algorithm (MD5) - checksum_fn = None - if options.checksum != "": - options.checksum_type = options.checksum_type.lower() - if options.checksum_type not in _checksum_fns: - print("Not a supported checksum function: %s" % (options.checksum_type)) - return 1 - checksum_fn = _checksum_fns[options.checksum_type] - - ### Check if base URL is a local dir or off the webs - images_dir = os.path.abspath(options.install_location) # This will use the current working directory if it's not absolute - images_url = None - if options.base_url.find('http') == 0: - base_url_is_local = False - if options.base_url[-1] != '/': - options.base_url += '/' - images_url = options.base_url + options.filename - else: - base_url_is_local = True - - if options.verbose: - print("Requested install location: %s" % (options.install_location)) - print("Images base URL: %s" % (options.base_url)) - print("Images filename: %s" % (options.filename)) - print("Images checksum: %s (%s)" % (options.checksum, _IMAGES_CHECKSUM_TYPE)) - print("Final install location: %s" % (images_dir)) - print("Copying locally: {0}".format("Yes" if base_url_is_local else "No")) - else: - print("Images destination: %s" % (images_dir)) + # Read the inventory into a dictionary we can perform lookups on + if os.path.isfile(args.inventory_location): + inventory_fn = args.inventory_location + else: + inventory_fn = os.path.join(args.install_location, _INVENTORY_FILENAME) + inventory = parse_inventory(inventory_fn=inventory_fn) + log("TRACE", "Inventory: {}\n{}".format( + os.path.abspath(inventory_fn), + "\n".join("{}".format(item) for item in inventory.items()) + )) + + # Determine the URLs to download based on the input regular expressions + types_regex = _DEFAULT_TARGET_REGEX if args.types == "" else args.types + + + log("TRACE", "RegEx for target selection: {}".format(types_regex)) + targets_info = lookup_urls(types_regex, manifest, inventory, args.refetch) + # Exit early if we don't have anything to download + if targets_info: + target_urls = [info.get("url") for info in targets_info] + log("TRACE", "URLs to download:\n{}".format( + "\n".join("{}".format(item) for item in target_urls) + )) + else: + log("INFO", "No targets selected. Exiting.") + return 0 + + with TemporaryDirectory() as temp_dir: + # Now download all the images archives into a temp directory + images_dir = args.install_location + log("INFO", "Images destination: {}".format(os.path.abspath(images_dir))) + for target_info in targets_info: + target_hash = target_info.get("repo_hash") + target_rel_url = target_info.get("url") + filename = os.path.basename(target_rel_url) + temp_path = os.path.join(temp_dir, filename) + # Add a trailing slash to make sure that urljoin handles things properly + full_url = urljoin(args.base_url+'/', target_rel_url) + _, downloaded_size = download( + images_url=full_url, + filename=temp_path, + buffer_size=args.buffer_size, + print_progress=(_LOG_LEVEL <= _LOG_LEVELS.get("DEBUG", 2)) + ) + # TODO: Check SHA + log("TRACE", "{} successfully downloaded ({} Bytes)" + .format(temp_path, downloaded_size)) - ### Check contradictory arguments - if options.force_delete and options.keep: - print("Error: Keep and force delete options contradict.\n") - parser.print_help() - return 1 + delete_from_inv(filename, inventory, images_dir) + archive_namelist = extract(temp_path, images_dir, archive_type) + inventory[filename] = {"repo_hash": target_hash, "contents": archive_namelist} - ### Prevent accidental file deletion - if options.install_location != default_images_dir and options.force_delete == False and options.keep != True: - print("Custom install location specified, defaulting to overwriting only image files.\n" - "Use \'--force-delete\' to clean the target directory first.") - options.keep = True + write_inventory(inventory, inventory_fn) - ### Download or copy - downloader = uhd_images_downloader() - try: - (access, last_path) = downloader.check_directories(images_dir, print_progress=options.verbose) - if not access: - print("You do not have sufficient permissions to write to: %s" % (last_path)) - print("Are you root?") - return 1 - with temporary_directory() as temp_dir: - if options.verbose: - print("Using temporary directory: %s" % (temp_dir)) - temp_images_dest = os.path.join(temp_dir, options.filename) - if not base_url_is_local: - print("Downloading images from: {0}".format(images_url)) - print("Downloading images to: {0}".format(temp_images_dest)) - (reported_size, downloaded_size) = downloader.download( - images_url=images_url, - filename=temp_images_dest, - buffer_size=options.buffer_size, - print_progress=True - ) - if options.verbose: - print("Downloaded %d of %d bytes" % (downloaded_size, reported_size)) - else: - local_images_pkg = os.path.join(options.base_url, options.filename) - print("Copying images from: {0}".format(local_images_pkg)) - if not os.path.isfile(local_images_pkg): - print("[ERROR] No such file.") - return 1 - shutil.copyfile(local_images_pkg, temp_images_dest) - (checksum_match, calculated_checksum) = downloader.validate_checksum( - checksum_fn, - temp_images_dest, - options.checksum, - print_progress=options.verbose - ) - if options.verbose: - print("Calculated checksum: %s" % (calculated_checksum)) - if checksum_match: - if options.verbose: - if options.checksum == "": - print("Ignoring checksum") - else: - print("Checksum OK") - try: - extract_path = downloader.extract_images_archive(temp_images_dest, print_progress=options.verbose) - if options.verbose: - print("Image archive extracted to: %s" % (extract_path)) - downloader.install_images(extract_path, images_dir, options.keep, print_progress=options.verbose) - if options.verbose: - print("Cleaning up temp location: %s" % (extract_path)) - shutil.rmtree(extract_path) - print("\nImages successfully installed to: %s" % (images_dir)) - except Exception as e: - print("Failed to install image archive: %s" % (e)) - print("This is usually a permissions problem.") - print("Please check your file system access rights and try again.") - if options.verbose: - traceback.print_exc() - else: - print("You can run this again with the '--verbose' flag to see more information") - print("If the problem persists, please email the output to: %s" % (_CONTACT)) - else: - print("Checksum of downloaded file is not correct (not installing - see options to override)") - print("Expected: %s" % (options.checksum)) - print("Calculated: %s" % (calculated_checksum)) - print("Please try downloading again.") - print("If the problem persists, please email the output to: %s" % (_CONTACT)) - except KeyboardInterrupt: - print("\nCancelled at user request") - except Exception as e: - print("Downloader raised an unhandled exception: %s" % (e)) - if options.verbose: - traceback.print_exc() - else: - print("You can run this again with the '--verbose' flag to see more information") - print("If the problem persists, please email the output to: %s" % (_CONTACT)) + except Exception as ex: + log("ERROR", "Downloader raised an unhandled exception: {ex}\n" + "You can run this again with the '--verbose' flag to see more information\n" + "If the problem persists, please email the output to: {contact}" + .format(contact=_CONTACT, ex=ex)) return 1 + log("INFO", "Images download complete.") return 0 +# Placing this near the end of the file so we don't clutter the top +_MANIFEST_CONTENTS = """@CMAKE_MANIFEST_CONTENTS@""" if __name__ == "__main__": sys.exit(main()) diff --git a/images/CMakeLists.txt b/images/CMakeLists.txt index ab2ac81479..229f0c4228 100644 --- a/images/CMakeLists.txt +++ b/images/CMakeLists.txt @@ -49,3 +49,4 @@ MESSAGE(STATUS "Version: ${UHD_VERSION}") FILE(WRITE ${CMAKE_SOURCE_DIR}/images/${UHD_VERSION_MAJOR}.${UHD_VERSION_API}.${UHD_VERSION_ABI}.${UHD_VERSION_PATCH}.tag "${UHD_VERSION}\n${DATETIME_NOW}\n") INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/images DESTINATION share/uhd) INSTALL(FILES ${CMAKE_SOURCE_DIR}/../host/LICENSE DESTINATION share/uhd/images) +INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/manifest.txt DESTINATION share/uhd/images) diff --git a/images/manifest.txt b/images/manifest.txt new file mode 100644 index 0000000000..d65917aa56 --- /dev/null +++ b/images/manifest.txt @@ -0,0 +1,44 @@ +# UHD Image Manifest File +# Target hash url SHA256 +# X300-Series +x3xx_x310_fpga_default fpga-6bea23d x3xx/fpga-6bea23d/x3xx_x310_fpga_default.zip dbda7d0dbaf89f1cb514b8028800bee4bb0c492e85fb1faa771f6e222ba3ee18 +x3xx_x300_fpga_default fpga-6bea23d x3xx/fpga-6bea23d/x3xx_x300_fpga_default.zip 95c84f7d3bc2e6ffdfa261c623ceb3ad1d34177a5a4f411660dc7bc3c21bd164 +# Example daughterboard targets (none currently exist) +#x3xx_twinrx_cpld_default example_target +#dboard_ubx_cpld_default example_target +# E-Series +e3xx_e310_fpga_default fpga-6bea23d e3xx/fpga-6bea23d/e3xx_e310_fpga_default.zip b04c3565f012ea63c20d5f60262c383b0099f39e94d581a60b59f775f86d7a7c +# N300-Series +n3xx_n310_fpga_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_fpga_default.zip 0373ebcefd07c02097c5a6075feaf4022eaf40c9f89727ad3e904b37e6898ef8 +n3xx_n310_fpga_aurora fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_fpga_aurora.zip c5327bb903e0797568e9b773f4d56bae9ce973a3db6e942b8027aa1ac71cf1e1 +#n3xx_n310_cpld_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_cpld_default.zip 0 +# N3XX Mykonos firmware +#n3xx_n310_fw_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_fw_default.zip 0 +# N300-Series Filesystems, etc +#n3xx_common_sdk_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_common_sdk_default.zip 0 +#n3xx_n310_mender_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_mender_default.zip 0 +#n3xx_n300_mender_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n300_mender_default.zip 0 +#n3xx_n310_sdimg_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n310_sdimg_default.zip 0 +#n3xx_n300_sdimg_default fpga-6bea23d n3xx/fpga-6bea23d/n3xx_n300_sdimg_default.zip 0 +# B200-Series +b2xx_b200_fpga_default fpga-6bea23d b2xx/fpga-6bea23d/b2xx_b200_fpga_default.zip f7d0a3d33e026484d89c420df66fe3a698717126f8407ef02240b323d4a12839 +b2xx_b200mini_fpga_default fpga-6bea23d b2xx/fpga-6bea23d/b2xx_b200mini_fpga_default.zip 7fa95b938f0bfbdce821c23950d28ca43e7ef24a7cda39a0b2f09fac84f24aae +b2xx_b210_fpga_default fpga-6bea23d b2xx/fpga-6bea23d/b2xx_b210_fpga_default.zip e08dbdaa6508c1fd480463f40231ef3b221b6f78567fab7db72c1d367d396c6f +b2xx_b205mini_fpga_default fpga-6bea23d b2xx/fpga-6bea23d/b2xx_b205mini_fpga_default.zip a74598cd9ecc71f34e8fba06b31c303d0f1a88532e9689efaff516aa6d5e1ff6 +b2xx_common_fw_default uhd-14000041 b2xx/uhd-14000041/b2xx_common_fw_default.zip 920790744085d8525529c1d0ece8942fef6d29b0a503530a814727fbacd7732c +# USRP2 Devices +usrp2_usrp2_fw_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_usrp2_fw_default.zip d523a18318cb6a7637be40484bf03a6f54766410fee2c1a1f72e8971ea9a9cb6 +usrp2_usrp2_fpga_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_usrp2_fpga_default.zip 505c70aedc8cdfbbfe654bcdbe1ce604c376e733a44cdd1351571f61a7f1cb49 +usrp2_n200_fpga_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_n200_fpga_default.zip 833a0098d66c0c502b9c3975d651a79e125133c507f9f4b2c472f9eb96fdaef8 +usrp2_n200_fw_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_n200_fw_default.zip 3eee2a6195caafe814912167fccf2dfc369f706446f8ecee36e97d2c0830116f +usrp2_n210_fpga_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_n210_fpga_default.zip 5ce68ac539ee6eeb7d04fb3127c1fabcaff442a8edfaaa2f3746590f9df909bd +usrp2_n210_fw_default fpga-6bea23d usrp2/fpga-6bea23d/usrp2_n210_fw_default.zip 3646fcd3fc974d18c621cb10dfe97c4dad6d282036dc63b7379995dfad95fb98 +n230_n230_fpga_default fpga-6bea23d n230/fpga-6bea23d/n230_n230_fpga_default.zip dba4a48d4a6081fa68adb59e42e5b49226a6b43ca1e70827c9b08752b208d4d8 +# USRP1 Devices +usrp1_usrp1_fpga_default fpga-6bea23d usrp1/fpga-6bea23d/usrp1_usrp1_fpga_default.zip 03bf72868c900dd0853bf48e2ede91058d579829b0e70c021e51b0e282d1d5be +usrp1_b100_fpga_default fpga-6bea23d usrp1/fpga-6bea23d/usrp1_b100_fpga_default.zip 7f2306f21e17aa3fae3f966d08c6297d6cf42041974f846ca89f0d633ece8769 +usrp1_b100_fw_default fpga-6bea23d usrp1/fpga-6bea23d/usrp1_b100_fw_default.zip 867f17fac085535dbcb01c226ce87acf49806de6ed0ae9b214d7c8da86e2a71d +# Octoclock +octoclock_octoclock_fw_default uhd-14000041 octoclock/uhd-14000041/octoclock_octoclock_fw_default.zip 8da7f1af8cecb7f6259a237a18c39058ba69a11567fa373cffc9704031a1d053 +# Legacy USB Windows drivers +usb_common_windrv_default uhd-14000041 usb/uhd-14000041/usb_common_windrv_default.zip 835e94b2bdf2312fd3881a1b78e2ec236c1f42b7a5bd3927f85f73cf5e3a5231