Merge caa3046 into f67ca25

README.md

@@ -31,6 +31,9 @@ $ pip install meticulous
 
 ## Prerequisites:
 - PyGithub
+- plumbum
+- spelling
+- PyInquirer
 
 
 ## Download from PyPI.org

app/.gitignore

@@ -2,3 +2,4 @@ test-output.xml
 LICENSE
 README.md
 README.md.txt
+sqlite.db

app/meticulous/__main__.py

@@ -5,12 +5,10 @@
 """
 from __future__ import absolute_import, division, print_function
 
-import sys
-
 import click
 
-from meticulous._github import check_forked, fork
-from meticulous._sources import obtain_sources
+from meticulous._github import is_archived
+from meticulous._process import run_invocation
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
@@ -19,35 +17,31 @@
 
 @click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True)
 @click.version_option(version=__version__)
+@click.option("--target", nargs=1)
 @click.pass_context
-def main(ctxt):
+def main(ctxt, target):
     """
     Main click group handler
     """
     if ctxt.invoked_subcommand is None:
-        run_invocation()
+        run_invocation(target)
 
 
 @main.command()
-def invoke():
+@click.option("--target", nargs=1)
+def invoke(target):
     """
     Primary command handler
     """
-    run_invocation()
+    run_invocation(target)
 
 
-def run_invocation():
+@main.command()
+def test():
     """
-    Execute the invocation
+    Test command handler
     """
-    for orgrepo in obtain_sources():
-        _, repo = orgrepo.split("/", 1)
-        print(f"Checking {orgrepo}")
-        if not check_forked(repo):
-            print(f"Have not forked {orgrepo}")
-            print(f"Forking {orgrepo}")
-            fork(orgrepo)
-            sys.exit(0)
+    print(is_archived("kennethreitz/clint"))
 
 
 if __name__ == "__main__":

app/meticulous/_github.py

@@ -2,9 +2,13 @@
 Handlers for checking existing forks and creating new ones.
 """
 
+import sys
+
 import github
+from plumbum import FG, local
 
 from meticulous._secrets import load_api_key
+from meticulous._storage import get_value, set_value
 
 
 def get_api():
@@ -15,6 +19,19 @@ def get_api():
 
 
 def check_forked(repository):
+    """
+    Check cache to check for an existing fork
+    """
+    key = f"forked|{repository}"
+    value = get_value(key)
+    if value is None:
+        result = _check_forked(repository)
+        value = "Y" if result else "N"
+        set_value(key, value)
+    return value == "Y"
+
+
+def _check_forked(repository):
     """
     Use the API to check for an existing fork
     """
@@ -27,13 +44,44 @@ def check_forked(repository):
         return False
 
 
+def is_archived(orgrepo):
+    """
+    Check if a repository is archived
+    """
+    api = get_api()
+    repo = api.get_repo(orgrepo)
+    return repo.archived
+
+
 def fork(orgrepo):
     """
     Use the API to fork a repository
     """
     api = get_api()
     repo = api.get_repo(orgrepo)
     repo.create_fork()
+    repository = orgrepo.split("/", 1)[-1]
+    key = f"forked|{repository}"
+    set_value(key, "Y")
+
+
+def checkout(repo, target):
+    """
+    Clone a repository to under the target path
+    if it does not already exist.
+    """
+    api = get_api()
+    user_org = api.get_user().login
+    clone_target = target / repo
+    if clone_target.exists():
+        print(f"{clone_target} already exists, clone aborted.", file=sys.stderr)
+        sys.exit(1)
+    git = local["/usr/bin/git"]
+    with local.cwd(str(target)):
+        _ = (
+            git["clone", f"ssh://git@github.com/{user_org}/{repo}", str(clone_target)]
+            & FG
+        )
 
 
 if __name__ == "__main__":

app/meticulous/_process.py

@@ -0,0 +1,169 @@
+"""
+Main processing for meticulous
+"""
+from __future__ import absolute_import, division, print_function
+
+import io
+import os
+import sys
+from pathlib import Path
+
+from plumbum import FG, local
+from PyInquirer import prompt
+from spelling.check import check  # noqa=I001
+
+from meticulous._github import check_forked, checkout, fork, is_archived
+from meticulous._sources import obtain_sources
+from meticulous._storage import get_json_value, prepare, set_json_value
+
+MAIN_MENU = [
+    {
+        "type": "list",
+        "name": "option",
+        "message": "What do you want to do?",
+        "choices": [
+            "add a new repository",
+            "manually add a new repository",
+            "examine a repository",
+            "remove a repository",
+            "- quit -",
+        ],
+    }
+]
+
+SELECT_REPO = {"type": "list", "name": "option", "message": "Which Repository?"}
+
+
+class NoRepoException(Exception):
+    """
+    Raised if no repositories are available/selected
+    """
+
+
+def run_invocation(target):
+    """
+    Execute the invocation
+    """
+    if target is None:
+        target = Path(os.environ["HOME"]) / "data"
+    else:
+        target = Path(target)
+    if not target.is_dir():
+        print(f"Target {target} is not a directory.", file=sys.stderr)
+        sys.exit(1)
+    prepare()
+    while True:
+        answers = prompt(MAIN_MENU)
+        option = answers.get("option", "- quit -")
+        try:
+            if option == "examine a repository":
+                examine_repo_selection()
+            elif option == "manually add a new repository":
+                manually_add_new_repo(target)
+            elif option == "remove a repository":
+                remove_repo_selection()
+            elif option == "add a new repository":
+                add_new_repo(target)
+            elif option == "- quit -":
+                print("Goodbye.")
+                return
+            else:
+                print(f"Unknown option {option}.", file=sys.stderr)
+                sys.exit(1)
+        except NoRepoException:
+            continue
+
+
+def remove_repo_selection():
+    """
+    Select an available repository to remove
+    """
+    repo, _ = pick_repo()
+    repository_map = get_json_value("repository_map", {})
+    del repository_map[repo]
+    set_json_value("repository_map", repository_map)
+
+
+def examine_repo_selection():
+    """
+    Select an available repository to examine
+    """
+    _, repodir = pick_repo()
+    examine_repo(repodir)
+
+
+def pick_repo():
+    """
+    Select an available repository
+    """
+    repository_map = get_json_value("repository_map", {})
+    if not repository_map:
+        print("No repositories available.", file=sys.stderr)
+        raise NoRepoException()
+    choice = dict(SELECT_REPO)
+    choices = list(repository_map.keys())
+    choices.append("- quit -")
+    choice["choices"] = choices
+    answers = prompt([choice])
+    option = answers.get("option", "- quit -")
+    if option == "- quit -":
+        raise NoRepoException()
+    repodir = repository_map[option]
+    return option, repodir
+
+
+def examine_repo(repodir):
+    """
+    Inspect an available repository
+    """
+    print("Opening editor")
+    editor = local["/usr/bin/vim"]
+    with local.cwd(repodir):
+        _ = editor["spelling.txt"] & FG
+
+
+def manually_add_new_repo(target):
+    """
+    Allow entry of a new repository manually
+    """
+    choice = dict(SELECT_REPO)
+    choices = sorted(os.listdir(target))
+    choices.append("- quit -")
+    choice["choices"] = choices
+    answers = prompt([choice])
+    option = answers.get("option", "- quit -")
+    if option == "- quit -":
+        raise NoRepoException()
+    repository_map = get_json_value("repository_map", {})
+    repository_map[option] = os.path.join(target, option)
+    set_json_value("repository_map", repository_map)
+
+
+def add_new_repo(target):
+    """
+    Locate a new repository and add it to the available set.
+    """
+    for orgrepo in obtain_sources():
+        _, repo = orgrepo.split("/", 1)
+        print(f"Checking {orgrepo}")
+        if not check_forked(repo):
+            print(f"Have not forked {orgrepo}")
+            print(f"Forking {orgrepo}")
+            fork(orgrepo)
+            if is_archived(orgrepo):
+                print(f"Skipping archived repo {orgrepo}")
+                continue
+            print(f"Checkout {repo}")
+            checkout(repo, target)
+            repodir = target / repo
+            print(f"Running spell check on {repodir}")
+            spellpath = repodir / "spelling.txt"
+            print(f"Spelling output {spellpath}")
+            with io.open(spellpath, "w", encoding="utf-8") as fobj:
+                os.chdir(repodir)
+                check(True, True, None, fobj)
+            repository_map = get_json_value("repository_map", {})
+            repository_map[repo] = str(repodir)
+            set_json_value("repository_map", repository_map)
+            return repo
+    return None

app/meticulous/_sources.py

@@ -9,6 +9,8 @@
 
 import urllib3
 
+from meticulous._storage import get_value, set_value
+
 SOURCE_MARKDOWN_URLS = [
     "https://raw.githubusercontent.com/vinta/awesome-python/master/README.md"
 ]
@@ -26,8 +28,12 @@ def check_url(url):
     """
     Download and process the
     """
-    for link in get_all_markdown_github_links(url):
-        yield link
+    key = "github_links|{url}"
+    results = get_value(key)
+    if results is None:
+        results = "\n".join(get_all_markdown_github_links(url))
+        set_value(key, results)
+    return results.splitlines()
 
 
 def get_all_markdown_github_links(url):