diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index d8480f1..77c233c 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -22,6 +22,7 @@ import time import threading import platform +import subprocess logger = logging.getLogger(constant.LOGGER_NAME) compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"} @@ -64,6 +65,8 @@ def parse_src_link(src_link): if src_link.startswith("git://github.com/"): src_link_changed = change_src_link_to_https(src_link_split[0]) else: + if "rubygems.org" in src_link: + src_info["rubygems"] = True src_link_changed = src_link_split[0] branch_info = [s for s in src_link_split if s.startswith('branch')] @@ -100,7 +103,7 @@ def main(): if args.log_dir: log_dir = args.log_dir - if src_link == "": + if not src_link: print_help_msg_download() else: cli_download_and_extract(src_link, target_dir, log_dir) @@ -114,9 +117,10 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress log_file_name = "fosslight_download_" + \ datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt" logger, log_item = init_log(os.path.join(log_dir, log_file_name)) + link = link.strip() try: - if link == "": + if not link: success = False msg = "Need a link to download." elif os.path.isfile(target_dir): @@ -127,14 +131,19 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress link = src_info.get("url", "") tag = ''.join(src_info.get("tag", "")).split('=')[-1] branch = ''.join(src_info.get("branch", "")).split('=')[-1] + is_rubygems = src_info.get("rubygems", False) - if not download_git_clone(link, target_dir, checkout_to, tag, branch): + # General download (git clone, wget) + if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)): if os.path.isfile(target_dir): shutil.rmtree(target_dir) success, downloaded_file = download_wget(link, target_dir, compressed_only) if success: success = extract_compressed_file(downloaded_file, target_dir, True) + # Download from rubygems.org + elif is_rubygems and shutil.which("gem"): + success = gem_download(link, target_dir, checkout_to) except Exception as error: success = False msg = str(error) @@ -164,10 +173,10 @@ def get_ref_to_checkout(checkout_to, ref_list): def decide_checkout(checkout_to="", tag="", branch=""): - if checkout_to != "": + if checkout_to: ref_to_checkout = checkout_to else: - if branch != "": + if branch: ref_to_checkout = branch else: ref_to_checkout = tag @@ -219,7 +228,7 @@ def download_wget(link, target_dir, compressed_only): Path(target_dir).mkdir(parents=True, exist_ok=True) ret, new_link = get_downloadable_url(link) - if ret and new_link != "": + if ret and new_link: link = new_link if compressed_only: @@ -240,7 +249,7 @@ def download_wget(link, target_dir, compressed_only): else: del alarm - if downloaded_file != "": + if downloaded_file: success = True logger.debug(f"wget - downloaded: {downloaded_file}") except Exception as error: @@ -263,6 +272,7 @@ def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True): def extract_compressed_file(fname, extract_path, remove_after_extract=True): + success = True try: is_compressed_file = True if os.path.isfile(fname): @@ -285,6 +295,7 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True): decompress_bz2(fname, extract_path) else: is_compressed_file = False + success = False logger.warning(f"Unsupported file extension: {fname}") if remove_after_extract and is_compressed_file: @@ -294,8 +305,8 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True): logger.warning(f"Not a file: {fname}") except Exception as error: logger.error(f"Extract - failed: {error}") - return False - return True + success = False + return success def decompress_bz2(source_file, dest_path): @@ -322,5 +333,47 @@ def unzip(source_file, dest_path): return True +def get_gem_version(checkout_to, ver_in_url=""): + gem_ver = "" + ver_found = False + if checkout_to: + ver_found = True + gem_ver = checkout_to + elif ver_in_url: + ver_found = True + gem_ver = ver_in_url + return gem_ver, ver_found + + +def gem_download(link, target_dir, checkout_to): + ver_in_url = "" + success = True + try: + # EX) https://rubygems.org/gems/json/versions/2.6.2 -> get 'json' + gem_name = link.split("rubygems.org/gems/")[-1].split('/')[0] + # Ex) if version info. is included in url, json/versions/2.6.2 -> get '2.6.2' + if 'versions/' in link: + ver_in_url = link.split('versions/')[-1] + gem_ver, ver_found = get_gem_version(checkout_to, ver_in_url) + + # gem fetch + if ver_found: + fetch_cmd = ['gem', 'fetch', gem_name, '-v', gem_ver] + else: + fetch_cmd = ['gem', 'fetch', gem_name] + fetch_result = subprocess.check_output(fetch_cmd, universal_newlines=True) + fetch_result = fetch_result.replace('\n', '').split(' ')[-1] + downloaded_gem = f"{fetch_result}.gem" + + # gem unpack + subprocess.check_output(['gem', 'unpack', downloaded_gem], universal_newlines=True) + # move unpacked file to target directory + shutil.move(fetch_result, target_dir) + except Exception as error: + success = False + logger.warning(f"gem download - failed: {error}") + return success + + if __name__ == '__main__': main()