🐛 Update xircuits-components to fetch from local

setup.py

@@ -106,7 +106,7 @@
         'console_scripts': [
             'xircuits = xircuits.start_xircuits:main',
             'xircuits-examples = xircuits.start_xircuits:download_examples',
-            'xircuits-components = xircuits.start_xircuits:download_component_library',
+            'xircuits-components = xircuits.start_xircuits:fetch_component_library',
             'xircuits-submodules = xircuits.start_xircuits:download_submodule_library',
             'xircuits-compile = xircuits.compiler.compiler:main'
             ]}

xircuits/handlers/request_folder.py

@@ -1,8 +1,8 @@
 from tqdm import tqdm
 import os
 from urllib import request, parse
-from github import Github
-
+from github import Github, GithubException
+from .request_submodule import get_submodules
 
 def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"):
     print("Downloading " + folder + " from " + repo_name + " branch " + branch)
@@ -11,9 +11,15 @@ def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"):
     try:
         repo = g.get_repo(repo_name)
         contents = repo.get_contents(folder, ref=branch)
-    except:
-       print(folder + " from " + repo_name + " branch " + branch + " does not exist!")
-       return 
+    except GithubException as e:
+        if e.status == 403:
+            print("pyGithub API rate limit exceeded. If you're trying to fetch Xircuits components, you can use `xircuits-components`.")
+        else:
+            print(folder + " from " + repo_name + " branch " + branch + " does not exist!")
+        return 
+    except Exception as e:
+        print("An error occurred: " + str(e))
+        return
 
     if not os.path.exists(folder):
         os.mkdir(folder)
@@ -34,8 +40,11 @@ def request_folder(folder, repo_name="XpressAi/Xircuits", branch="master"):
             file_url = base_url + "/" + parse.quote(file_content.path)
             urls.update({file_url: file_content.path})
 
+    submodules = get_submodules(repo, branch)
+
     for url in tqdm(urls):
         try:
             request.urlretrieve(url, urls[url])
         except:
-            print("Unable to retrieve " + urls[url] + ". Skipping...")
+            if urls[url] not in submodules:
+                print("Unable to retrieve " + urls[url] + ". Skipping...")

xircuits/handlers/request_submodule.py

@@ -43,3 +43,18 @@ def request_submodule_library(component_library_query):
 
     print("Cloning " + submodule_path + " from " + submodule_url)
     Repo.clone_from(submodule_url, submodule_path, progress=Progress())
+
+
+def get_submodules(repo, ref="master"):
+    try:
+        gitmodules_content = repo.get_contents(".gitmodules", ref=ref)
+        gitmodules = gitmodules_content.decoded_content.decode("utf-8")
+
+        submodules = []
+        for line in gitmodules.split("\n"):
+            if "path = " in line:
+                submodules.append(line.split(" = ")[-1].strip())
+        return submodules
+
+    except:
+        return []

xircuits/start_xircuits.py

@@ -12,10 +12,20 @@
 
 def init_xircuits():
 
+    package_name = 'xircuits'
+    copy_from_installed_wheel(package_name, 
+                              resource='.xircuits', 
+                              dest_path='.xircuits')
+
+
+def copy_from_installed_wheel(package_name, resource="", dest_path=None):
+
+    if dest_path is None:
+        dest_path = package_name
+
+    resource_path = pkg_resources.resource_filename(package_name, resource)
+    shutil.copytree(resource_path, dest_path)
 
-    path = ".xircuits"
-    config_path = pkg_resources.resource_filename('xircuits', '.xircuits')
-    shutil.copytree(config_path, path)
 
 def download_examples():
 
@@ -28,18 +38,22 @@ def download_examples():
     request_folder("datasets", branch=args.branch)
 
 
-def download_component_library():
-
+def fetch_component_library():
     parser = argparse.ArgumentParser()
+    parser.add_argument("--download", default=False, action='store_true')
     parser.add_argument('--branch', nargs='?', default="master", help='pull files from a xircuits branch')
     parser.add_argument('--sublib', nargs='*', help='pull component library from a xircuits submodule')
-
     args = parser.parse_args()
-    if not args.sublib:
-        request_folder("xai_components", branch=args.branch)
+
+    if args.download:
+        if not args.sublib:
+            request_folder("xai_components", branch=args.branch)
+        else:
+            for component_lib in args.sublib:
+                request_submodule_library(component_lib)
     else:
-        for component_lib in args.sublib:
-            request_submodule_library(component_lib)
+        copy_from_installed_wheel("xai_components")
+
 
 def download_submodule_library():
 
@@ -77,8 +91,7 @@ def main():
         val = input("Xircuits Component Library is not found. Would you like to load it in the current path (Y/N)? ")
         if val.lower() == ("y" or "yes"):
             if args.branch is None:
-                xai_component_path = pkg_resources.resource_filename('xai_components', '')
-                shutil.copytree(xai_component_path, "xai_components")
+                copy_from_installed_wheel('xai_components', '', 'xai_components')
 
             else:
                 request_folder("xai_components", branch=args.branch)