Permalink
Cannot retrieve contributors at this time
Fetching contributors…
Cannot retrieve contributors at this time
| #!/usr/bin/python3 | |
| # Copyright (C) 2015 Stefano Verzegnassi <verzegnassi.stefano@gmail.com> | |
| # Copyright (C) 2015 Didier Roche <didrocks@ubuntu.com> | |
| # | |
| # This program is free software: you can redistribute it and/or modify it | |
| # under the terms of the GNU General Public License version 3, as published | |
| # by the Free Software Foundation. | |
| # | |
| # This program is distributed in the hope that it will be useful, but | |
| # WITHOUT ANY WARRANTY; without even the implied warranties of | |
| # MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR | |
| # PURPOSE. See the GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License along | |
| # with this program. If not, see <http://www.gnu.org/licenses/>. | |
| # | |
| # Source code at: | |
| # https://code.launchpad.net/~verzegnassi-stefano/+junk/get-click-deps | |
| # | |
| # A script to automate the fetching all the external dependencies of a | |
| # Click packaged application from internet, and copying them in a given | |
| # folder, so that the click packaging tool can easily build them in the | |
| # package. | |
| # | |
| # usage: get-click-deps [-h] [-f] [-d] [-e] [-c SCRIPT_PATH] | |
| # manifest_path {amd64,i386,armhf} target_path | |
| # | |
| # A tool for adding external libraries to a Ubuntu SDK application or scope. | |
| # | |
| # positional arguments: | |
| # manifest_path path of json file containing the list of packages to | |
| # be downloaded. | |
| # {amd64,i386,armhf} CPU architecture ("amd64", "i386" or "armhf") | |
| # target_path path to the target (a click package or a folder) | |
| # where this tool will include the downloaded binaries. | |
| # If the folder does not exist, it will be created. | |
| # | |
| # optional arguments: | |
| # -h, --help show this help message and exit | |
| # -f, --force-download force a new download of the packages | |
| # -d, --delete-temp delete temp files at the end of the process | |
| # -e, --extract-only only create temp directory and extract the content of | |
| # downloaded packages | |
| # -c SCRIPT_PATH, --custom-script SCRIPT_PATH | |
| # run a custom script after the extraction of Debian | |
| # packages and before copying their content to the | |
| # target destination. The tool will execute the script | |
| # with the path to the packages dump as argument. If | |
| # the '-e' flag has been specified, the script will be | |
| # anyway executed. This option is useful when you need | |
| # to automatically modify the content of temp folder | |
| # (e.g. when you need to fix some path before including | |
| # the files in a click package). | |
| # | |
| # | |
| # USAGE EXAMPLE: | |
| # get-click-deps packages.json armhf <path/to/package.click> | |
| # | |
| # package.json is the package manifest, and contains all the references to the | |
| # .deb packages to be included into the Ubuntu SDK project. | |
| # If you're familiar to the Debian/Ubuntu world, you'll see that it's pretty | |
| # similar to therepository management of these distros: | |
| # see https://wiki.debian.org/SourcesList for further informations. | |
| # | |
| # An example of packages.json file is: | |
| # { | |
| # "armhf": [ | |
| # { | |
| # "url": "http://ports.ubuntu.com/ubuntu-ports/", | |
| # "dist": "vivid", | |
| # "component": "main", | |
| # "packages": [ | |
| # "libgl1-mesa-glx", | |
| # "libxslt1.1", | |
| # "libxcb-glx0", | |
| # "libxcb-dri3-0", | |
| # "libxcb-present0", | |
| # "libxshmfence1", | |
| # "libxxf86vm1" | |
| # ] | |
| # }, | |
| # | |
| # { | |
| # "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu", | |
| # "dist": "vivid", | |
| # "component": "main", | |
| # "packages": [ | |
| # "libreoffice-vanilla" | |
| # ] | |
| # } | |
| # ], | |
| # | |
| # "amd64": [ | |
| # "url": "http://ppa.launchpad.net/canonical-community/ppa/ubuntu", | |
| # "dist": "vivid", | |
| # "component": "main", | |
| # "packages": [ | |
| # "libreoffice-vanilla" | |
| # ] | |
| # } | |
| # ] | |
| # } | |
| # | |
| # Instead of a click package, you can specify the build folder used by Ubuntu | |
| # SDK for compiling the sources of your project. This way the binaries | |
| # downloaded by this tool will be automatically included in the .click package | |
| # the next time you ask Ubuntu SDK to create a new package. | |
| # | |
| # TODO: Complete error handling | |
| # TODO: Make target_path optional if the '-e' flag has been specified. | |
| import sys | |
| import os | |
| import stat | |
| import time | |
| import argparse | |
| import json | |
| import gzip | |
| import subprocess | |
| import urllib.request, urllib.error, urllib.parse | |
| def get_timestamp(): | |
| return time.time() | |
| def check_internet_connection(): | |
| try: | |
| urllib.request.urlopen('http://www.launchpad.net', timeout=20) | |
| return True | |
| except urllib2.error.URLError as err: | |
| pass | |
| return False | |
| def get_arch_triplet(arch): | |
| return subprocess.check_output([ | |
| 'dpkg-architecture', | |
| '-A', | |
| arch, | |
| '-qDEB_TARGET_MULTIARCH']) | |
| def download_file(url, dest, verbose=True): | |
| if verbose: | |
| print ("\nDownloading:\n{}".format(url)) | |
| # TODO: Switch to subprocess | |
| os.system('cd %s && { curl -# -O %s ; cd - ; }' % (dest, url)) | |
| def download_file_and_rename(url, dest, new_filename, verbose=True): | |
| if verbose: | |
| print ("\nDownloading:\n{}".format(url)) | |
| new_path = os.path.join(dest, new_filename) | |
| subprocess.call(['curl', url, '-#', '-o', new_path]) | |
| return new_path | |
| def get_package_download_url(package_name, packages_list, base_url): | |
| pkgs_list = packages_list.decode('utf-8').split('\n') | |
| index = pkgs_list.index('Package: %s' % package_name) | |
| for i in range(index, len(pkgs_list)): | |
| if pkgs_list[i].find('Filename:') > -1: | |
| return "%s/%s" % (base_url, pkgs_list[i].replace('Filename: ', '')) | |
| def get_URLs_for_arch(manifest_path, arch, destpath): | |
| urls = [] | |
| f = open(manifest_path) | |
| content = json.load(f) | |
| f.close() | |
| # Download repository index for each repository in the JSON package | |
| # manifest. | |
| try: | |
| repo_index = 0 | |
| for repo in content[arch]: | |
| repo_index_url = '%s/dists/%s/%s/binary-%s/Packages.gz' % ( | |
| repo['url'], | |
| repo['dist'], | |
| repo['component'], | |
| arch) | |
| print ("\nDownloading repository index at:\n".format(repo_index_url)) | |
| repo_index_zip = download_file_and_rename( | |
| repo_index_url, | |
| destpath, | |
| 'repo-index-%s-%s.gz' % (arch, repo_index), | |
| False) | |
| with gzip.open(repo_index_zip, 'r') as f: | |
| repo_index_content = f.read() | |
| f.close() | |
| # Get the download URL of each package of the repository. | |
| packages = repo['packages'] | |
| for package in packages: | |
| urls.append(get_package_download_url( | |
| package, | |
| repo_index_content, | |
| repo['url'])) | |
| repo_index += 1 | |
| except KeyError: | |
| # Arch not found in the manifest. Exit with no error, since there's no | |
| # need to run the script for this arch. | |
| print ("\n\nRequested arch has been not specified in the manifest. \ | |
| Exiting...") | |
| sys.exit(0) | |
| print ("\nObtained packages informations") | |
| return urls | |
| def check_if_temp_folder_already_exists(path): | |
| return os.path.isdir(path) | |
| def copy_directory_content(sourcepath, destpath): | |
| subprocess.call(['cp', '-r', '%s/.' % sourcepath, destpath]) | |
| def delete_folder(path, recursive=False): | |
| if recursive: | |
| flag = '-rf' | |
| else: | |
| flag = '-f' | |
| subprocess.call(['rm', flag, path]) | |
| def extract_deb_package(deb_path, destpath): | |
| subprocess.call(['dpkg-deb', '-x', deb_path, destpath]) | |
| def extract_click_package(click_path, destpath): | |
| extract_deb_package(click_path, destpath) | |
| manifest = subprocess.check_output(['click', 'info', click_path]) | |
| # The manifest we get has an 'installed-size' key with the value of the | |
| # previous package. Anyway this value will be replaced when we'll run | |
| # 'click build <pkg>', so there's no reason for removing it here. | |
| f = open(os.path.join(destpath, 'manifest.json'), 'w') | |
| f.write(manifest) | |
| f.close() | |
| return destpath | |
| def build_click_package(source_dirpath): | |
| output = subprocess.check_output(['click', 'build', source_dirpath]) | |
| for line in output.split(os.linesep): | |
| if line.find('Successfully built package in ') > -1: | |
| # FIXME: Very ugly. | |
| path = line.replace('Successfully built package in \'', '') | |
| path = path.replace('\'.', '') | |
| return path | |
| def copy_file(sourcepath, destpath, overwrite=False): | |
| flag = '' | |
| if not overwrite: | |
| flag = '-n' | |
| subprocess.call(['cp', flag, sourcepath, destpath]) | |
| # Argument parser | |
| parser = argparse.ArgumentParser( | |
| description="A tool for adding external libraries to a Ubuntu SDK \ | |
| application or scope.") | |
| parser.add_argument( | |
| '-f', | |
| '--force-download', | |
| dest='force_download', | |
| action='store_true', | |
| help='force a new download of the packages') | |
| parser.add_argument( | |
| '-d', | |
| '--delete-temp', | |
| dest='delete_temp', | |
| action='store_true', | |
| help='delete temp files at the end of the process') | |
| parser.add_argument( | |
| '-e', | |
| '--extract-only', | |
| dest='extract_only', | |
| action='store_true', | |
| help='only create temp directory and extract the content of downloaded \ | |
| packages') | |
| parser.add_argument( | |
| '-c', | |
| '--custom-script', | |
| dest='script_path', | |
| type=str, | |
| help='run a custom script after the extraction of Debian packages and \ | |
| before copying their content to the target destination. The tool will \ | |
| execute the script with the path to the packages dump as argument. If \ | |
| the \'-e\' flag has been specified, the script will be anyway \ | |
| executed. This option is useful when you need to automatically modify \ | |
| the content of temp folder (e.g. when you need to fix some path \ | |
| before including the files in a click package).') | |
| parser.add_argument( | |
| 'manifest_path', | |
| type=str, | |
| help='path of json file containing the list of packages to be downloaded.') | |
| parser.add_argument( | |
| 'arch', | |
| type=str, | |
| choices=['amd64', 'i386', 'armhf'], | |
| help='CPU architecture ("amd64", "i386" or "armhf")') | |
| parser.add_argument( | |
| 'target_path', | |
| type=str, | |
| help='path to the target (a click package or a folder) where this \ | |
| tool will include the downloaded binaries. If the folder does not \ | |
| exist, it will be created.') | |
| args = parser.parse_args() | |
| # Variables | |
| manifest_path = args.manifest_path | |
| target_path = args.target_path | |
| manifest_stat = os.stat(manifest_path) | |
| temp_folder = os.path.join( | |
| '/tmp/', | |
| 'tmp-click-deps-%s-%s-%s-%s' % ( | |
| manifest_stat.st_dev, | |
| manifest_stat.st_ino, | |
| manifest_stat.st_size, | |
| manifest_stat.st_mtime)) | |
| temp_arch_folder = os.path.join(temp_folder, args.arch) | |
| is_click_target = os.path.isfile(args.target_path) | |
| # Check command line arguments | |
| if not os.path.isfile(manifest_path): | |
| sys.exit("\n\nERROR: Package manifest is not a valid file. Exit...") | |
| if not os.path.isfile(target_path) and target_path.endswith('.click'): | |
| sys.exit("\n\nERROR: The specified target .click does not exists.") | |
| elif not os.path.isdir(target_path): | |
| print ("\n\nCreating dest folder\n{}".format(target_path)) | |
| os.mkdir(target_path) | |
| # Check if the script exist, if specified any. | |
| if args.script_path and not os.path.exists(args.script_path): | |
| sys.exit("\n\nERROR: The specified script does not exists.") | |
| # If -f argument has been specified, remove all the existing data before | |
| # running this script. | |
| if args.force_download: | |
| if os.path.isdir(temp_folder): | |
| print ("\nRemoving temp data of the previous run, as requested") | |
| delete_folder(temp_folder, True) | |
| # Check if we already have run this script for the same target. | |
| if not check_if_temp_folder_already_exists(temp_arch_folder): | |
| # Check internet connection | |
| if not check_internet_connection(): | |
| sys.exit("\n\nERROR: An internet connection is required in order to \ | |
| download packages from repositories.") | |
| # Create temp folder in /tmp | |
| print ("\nCreating temp folder in {}".format(temp_folder)) | |
| os.mkdir(temp_folder) | |
| os.mkdir(temp_arch_folder) | |
| # Parse the JSON package list and get the download URL of the packages. | |
| debs_url_list = get_URLs_for_arch( | |
| manifest_path, | |
| args.arch, | |
| temp_folder) | |
| # Download packages from web | |
| for url in debs_url_list: | |
| download_file(url, temp_folder) | |
| # Extract DEBs packages | |
| print ("\nExtracting .deb packages to {}".format(temp_arch_folder)) | |
| deb_pkgs_list = [] | |
| for file in os.listdir(temp_folder): | |
| if file.endswith('.deb'): | |
| deb_pkgs_list.append(file) | |
| for deb_pkg in deb_pkgs_list: | |
| extract_deb_package( | |
| os.path.join(temp_folder, deb_pkg), | |
| temp_arch_folder) | |
| # If a script has been specified, run it. | |
| if args.script_path: | |
| print ("\nRunning the script at: {}\n".format(args.script_path)) | |
| # Ensure that we can run the script, otherwise we don't have the | |
| # permission | |
| subprocess.call(['chmod', '+x', args.script_path]) | |
| subprocess.call([ | |
| args.script_path, | |
| temp_arch_folder, | |
| get_arch_triplet(args.arch)]) | |
| else: | |
| print ("\nPackages are already downloaded. Use them...") | |
| # If -e or --extract-only flags have been specified, we have completed our work | |
| if args.extract_only: | |
| print ("\n\nCompleted successfully.") | |
| sys.exit(0) | |
| # Copy temp_arch folder content to its destination | |
| if is_click_target: | |
| print ("\nExtracting target .click package") | |
| temp_click_folder = extract_click_package( | |
| args.target_path, | |
| os.path.join('/tmp/', str(get_timestamp()))) | |
| print ("\nAdding extracted binaries to the package") | |
| copy_directory_content(temp_arch_folder, temp_click_folder) | |
| new_click_package_path = build_click_package(temp_click_folder) | |
| print ("\nCreated new .click package at:\n{}".format(new_click_package_path)) | |
| print ("\nReplacing older .click package") | |
| copy_file(new_click_package_path, args.target_path, True) | |
| delete_folder(temp_click_folder, True) | |
| else: | |
| print ("\nCopying extracted binaries to their destination") | |
| copy_directory_content(temp_arch_folder, args.target_path) | |
| # Delete temp files | |
| if args.delete_temp: | |
| print ("\)nRemoving temp files and directory, as requested") | |
| delete_folder(temp_folder, True) | |
| # Exit | |
| print ("\n\nCompleted successfully.") |