From 4d113fb93c45abebe3e875401dff68707bd7bc68 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Fri, 24 Feb 2023 12:59:30 +0900 Subject: [PATCH 1/3] Be availble to download src from rubygem --- src/fosslight_util/download.py | 52 +++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index d8480f1..602cf08 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -22,6 +22,7 @@ import time import threading import platform +import subprocess logger = logging.getLogger(constant.LOGGER_NAME) compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"} @@ -64,6 +65,8 @@ def parse_src_link(src_link): if src_link.startswith("git://github.com/"): src_link_changed = change_src_link_to_https(src_link_split[0]) else: + if "rubygems.org" in src_link: + src_info["rubygems"] = True src_link_changed = src_link_split[0] branch_info = [s for s in src_link_split if s.startswith('branch')] @@ -127,14 +130,19 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress link = src_info.get("url", "") tag = ''.join(src_info.get("tag", "")).split('=')[-1] branch = ''.join(src_info.get("branch", "")).split('=')[-1] + is_rubygems = src_info.get("rubygems", False) - if not download_git_clone(link, target_dir, checkout_to, tag, branch): + # General download (git clone, wget) + if (is_rubygems is False) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)): if os.path.isfile(target_dir): shutil.rmtree(target_dir) success, downloaded_file = download_wget(link, target_dir, compressed_only) if success: success = extract_compressed_file(downloaded_file, target_dir, True) + # Download from rubygems.org + elif is_rubygems and shutil.which("gem"): + success = gem_download(link, target_dir, checkout_to) except Exception as error: success = False msg = str(error) @@ -322,5 +330,47 @@ def unzip(source_file, dest_path): return True +def get_gem_version(checkout_to, ver_in_url=""): + gem_ver = "" + ver_found = False + if checkout_to != "": + ver_found = True + gem_ver = checkout_to + elif ver_in_url != "": + ver_found = True + gem_ver = ver_in_url + return gem_ver, ver_found + + +def gem_download(link, target_dir, checkout_to): + ver_in_url = "" + success = True + try: + # EX) https://rubygems.org/gems/json/versions/2.6.2 -> get 'json' + gem_name = link.split("rubygems.org/gems/")[-1].split('/')[0] + # Ex) if version info. is included in url, json/versions/2.6.2 -> get '2.6.2' + if 'versions/' in link: + ver_in_url = link.split('versions/')[-1] + gem_ver, ver_found = get_gem_version(checkout_to, ver_in_url) + + # gem fetch + if ver_found: + fetch_cmd = ['gem', 'fetch', gem_name, '-v', gem_ver] + else: + fetch_cmd = ['gem', 'fetch', gem_name] + fetch_result = subprocess.check_output(fetch_cmd, universal_newlines=True) + fetch_result = fetch_result.replace('\n', '').split(' ')[-1] + downloaded_gem = f"{fetch_result}.gem" + + # gem unpack + subprocess.check_output(['gem', 'unpack', downloaded_gem], universal_newlines=True) + # move unpacked file to target directory + shutil.move(fetch_result, target_dir) + except Exception as error: + success = False + logger.warning(f"gem download - failed: {error}") + return success + + if __name__ == '__main__': main() From 4f555372387c9bdc49f2ca9aa21b1b7b02c68dd1 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Fri, 3 Mar 2023 12:17:02 +0900 Subject: [PATCH 2/3] Set false if downloading is failed --- src/fosslight_util/download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index 602cf08..b4e8833 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -271,6 +271,7 @@ def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True): def extract_compressed_file(fname, extract_path, remove_after_extract=True): + success = True try: is_compressed_file = True if os.path.isfile(fname): @@ -293,6 +294,7 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True): decompress_bz2(fname, extract_path) else: is_compressed_file = False + success = False logger.warning(f"Unsupported file extension: {fname}") if remove_after_extract and is_compressed_file: @@ -303,7 +305,7 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True): except Exception as error: logger.error(f"Extract - failed: {error}") return False - return True + return success def decompress_bz2(source_file, dest_path): From 3a89075cfaeec725e78dfd7f99307c5bd98745b0 Mon Sep 17 00:00:00 2001 From: Jaekwon Bang Date: Tue, 7 Mar 2023 09:09:00 +0900 Subject: [PATCH 3/3] Modify if condition with recommended way --- src/fosslight_util/download.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/fosslight_util/download.py b/src/fosslight_util/download.py index b4e8833..77c233c 100755 --- a/src/fosslight_util/download.py +++ b/src/fosslight_util/download.py @@ -103,7 +103,7 @@ def main(): if args.log_dir: log_dir = args.log_dir - if src_link == "": + if not src_link: print_help_msg_download() else: cli_download_and_extract(src_link, target_dir, log_dir) @@ -117,9 +117,10 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress log_file_name = "fosslight_download_" + \ datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt" logger, log_item = init_log(os.path.join(log_dir, log_file_name)) + link = link.strip() try: - if link == "": + if not link: success = False msg = "Need a link to download." elif os.path.isfile(target_dir): @@ -133,7 +134,7 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress is_rubygems = src_info.get("rubygems", False) # General download (git clone, wget) - if (is_rubygems is False) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)): + if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)): if os.path.isfile(target_dir): shutil.rmtree(target_dir) @@ -172,10 +173,10 @@ def get_ref_to_checkout(checkout_to, ref_list): def decide_checkout(checkout_to="", tag="", branch=""): - if checkout_to != "": + if checkout_to: ref_to_checkout = checkout_to else: - if branch != "": + if branch: ref_to_checkout = branch else: ref_to_checkout = tag @@ -227,7 +228,7 @@ def download_wget(link, target_dir, compressed_only): Path(target_dir).mkdir(parents=True, exist_ok=True) ret, new_link = get_downloadable_url(link) - if ret and new_link != "": + if ret and new_link: link = new_link if compressed_only: @@ -248,7 +249,7 @@ def download_wget(link, target_dir, compressed_only): else: del alarm - if downloaded_file != "": + if downloaded_file: success = True logger.debug(f"wget - downloaded: {downloaded_file}") except Exception as error: @@ -304,7 +305,7 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True): logger.warning(f"Not a file: {fname}") except Exception as error: logger.error(f"Extract - failed: {error}") - return False + success = False return success @@ -335,10 +336,10 @@ def unzip(source_file, dest_path): def get_gem_version(checkout_to, ver_in_url=""): gem_ver = "" ver_found = False - if checkout_to != "": + if checkout_to: ver_found = True gem_ver = checkout_to - elif ver_in_url != "": + elif ver_in_url: ver_found = True gem_ver = ver_in_url return gem_ver, ver_found