diff --git a/setup.py b/setup.py index 30ac208d..a8a6fa51 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ 'console_scripts': [ 'xircuits = xircuits.start_xircuits:main', 'xircuits-examples = xircuits.start_xircuits:download_examples', - 'xircuits-components = xircuits.start_xircuits:download_component_library', + 'xircuits-components = xircuits.start_xircuits:fetch_component_library', 'xircuits-submodules = xircuits.start_xircuits:download_submodule_library', 'xircuits-compile = xircuits.compiler.compiler:main' ]} diff --git a/xircuits/handlers/request_folder.py b/xircuits/handlers/request_folder.py index f939637b..505ae505 100644 --- a/xircuits/handlers/request_folder.py +++ b/xircuits/handlers/request_folder.py @@ -1,8 +1,8 @@ from tqdm import tqdm import os from urllib import request, parse -from github import Github - +from github import Github, GithubException +from .request_submodule import get_submodules def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"): print("Downloading " + folder + " from " + repo_name + " branch " + branch) @@ -11,9 +11,15 @@ def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"): try: repo = g.get_repo(repo_name) contents = repo.get_contents(folder, ref=branch) - except: - print(folder + " from " + repo_name + " branch " + branch + " does not exist!") - return + except GithubException as e: + if e.status == 403: + print("pyGithub API rate limit exceeded. If you're trying to fetch Xircuits components, you can use `xircuits-components`.") + else: + print(folder + " from " + repo_name + " branch " + branch + " does not exist!") + return + except Exception as e: + print("An error occurred: " + str(e)) + return if not os.path.exists(folder): os.mkdir(folder) @@ -34,8 +40,11 @@ def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"): file_url = base_url + "/" + parse.quote(file_content.path) urls.update({file_url: file_content.path}) + submodules = get_submodules(repo, branch) + for url in tqdm(urls): try: request.urlretrieve(url, urls[url]) except: - print("Unable to retrieve " + urls[url] + ". Skipping...") \ No newline at end of file + if urls[url] not in submodules: + print("Unable to retrieve " + urls[url] + ". Skipping...") \ No newline at end of file diff --git a/xircuits/handlers/request_submodule.py b/xircuits/handlers/request_submodule.py index 1696fc32..a5315d15 100644 --- a/xircuits/handlers/request_submodule.py +++ b/xircuits/handlers/request_submodule.py @@ -43,3 +43,18 @@ def request_submodule_library(component_library_query): print("Cloning " + submodule_path + " from " + submodule_url) Repo.clone_from(submodule_url, submodule_path, progress=Progress()) + + +def get_submodules(repo, ref="master"): + try: + gitmodules_content = repo.get_contents(".gitmodules", ref=ref) + gitmodules = gitmodules_content.decoded_content.decode("utf-8") + + submodules = [] + for line in gitmodules.split("\n"): + if "path = " in line: + submodules.append(line.split(" = ")[-1].strip()) + return submodules + + except: + return [] \ No newline at end of file diff --git a/xircuits/start_xircuits.py b/xircuits/start_xircuits.py index f3a1aebd..6904d6ff 100644 --- a/xircuits/start_xircuits.py +++ b/xircuits/start_xircuits.py @@ -12,10 +12,20 @@ def init_xircuits(): + package_name = 'xircuits' + copy_from_installed_wheel(package_name, + resource='.xircuits', + dest_path='.xircuits') + + +def copy_from_installed_wheel(package_name, resource="", dest_path=None): + + if dest_path is None: + dest_path = package_name + + resource_path = pkg_resources.resource_filename(package_name, resource) + shutil.copytree(resource_path, dest_path) - path = ".xircuits" - config_path = pkg_resources.resource_filename('xircuits', '.xircuits') - shutil.copytree(config_path, path) def download_examples(): @@ -28,18 +38,22 @@ def download_examples(): request_folder("datasets", branch=args.branch) -def download_component_library(): - +def fetch_component_library(): parser = argparse.ArgumentParser() + parser.add_argument("--download", default=False, action='store_true') parser.add_argument('--branch', nargs='?', default="master", help='pull files from a xircuits branch') parser.add_argument('--sublib', nargs='*', help='pull component library from a xircuits submodule') - args = parser.parse_args() - if not args.sublib: - request_folder("xai_components", branch=args.branch) + + if args.download: + if not args.sublib: + request_folder("xai_components", branch=args.branch) + else: + for component_lib in args.sublib: + request_submodule_library(component_lib) else: - for component_lib in args.sublib: - request_submodule_library(component_lib) + copy_from_installed_wheel("xai_components") + def download_submodule_library(): @@ -77,8 +91,7 @@ def main(): val = input("Xircuits Component Library is not found. Would you like to load it in the current path (Y/N)? ") if val.lower() == ("y" or "yes"): if args.branch is None: - xai_component_path = pkg_resources.resource_filename('xai_components', '') - shutil.copytree(xai_component_path, "xai_components") + copy_from_installed_wheel('xai_components', '', 'xai_components') else: request_folder("xai_components", branch=args.branch)