issue-comments command, closes #7

Also added --issue option to issues command, for fetching one specific issue.

README.md

@@ -32,6 +32,23 @@ You can point to a different location of `auth.json` using `-a`:
 
     $ github-to-sqlite issues github.db simonw/datasette -a /path/to/auth.json
 
+You can use the `--issue` option to only load just one specific issue:
+
+    $ github-to-sqlite issues github.db simonw/datasette --issue=1
+
+## Retrieving issue comments for a repository
+
+The `issue-comments` command retrieves all of the comments on all of the issues in a repository.
+
+It is recommended you run `issues` first, so that each imported comment can have a foreign key poining to its issue.
+
+    $ github-to-sqlite issues github.db simonw/datasette
+    $ github-to-sqlite issue-comments github.db simonw/datasette
+
+You can use the `--issue` option to only load comments for a specific issue within that repository, for example:
+
+    $ github-to-sqlite issue-comments github.db simonw/datasette --issue=1
+
 ## Fetching repos belonging to a user or organization
 
 The `repos` command fetches repos belonging to a user or organization.

github_to_sqlite/cli.py

@@ -40,6 +40,7 @@ def auth(auth):
     required=True,
 )
 @click.argument("repo", required=False)
+@click.option("--issue", help="Just pull this issue number")
 @click.option(
     "-a",
     "--auth",
@@ -52,17 +53,40 @@ def auth(auth):
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
     help="Load issues JSON from this file instead of the API",
 )
-def issues(db_path, repo, auth, load):
+def issues(db_path, repo, issue, auth, load):
     "Save issues for a specified repository, e.g. simonw/datasette"
     db = sqlite_utils.Database(db_path)
+    token = load_token(auth)
     if load:
         issues = json.load(open(load))
-    try:
-        token = json.load(open(auth))["github_personal_token"]
-    except (KeyError, FileNotFoundError):
-        token = None
+    else:
+        issues = utils.fetch_issues(repo, token, issue)
 
-    utils.save_issues(db, utils.fetch_all_issues(repo, token))
+    issues = list(issues)
+    utils.save_issues(db, issues)
+
+
+@cli.command(name="issue-comments")
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("repo")
+@click.option("--issue", help="Just pull comments for this issue")
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def issue_comments(db_path, repo, issue, auth):
+    "Retrieve issue comments for a specific repository"
+    db = sqlite_utils.Database(db_path)
+    token = load_token(auth)
+    for comment in utils.fetch_issue_comments(repo, token, issue):
+        utils.save_issue_comment(db, comment)
 
 
 @cli.command()
@@ -87,11 +111,7 @@ def issues(db_path, repo, auth, load):
 def starred(db_path, username, auth, load):
     "Save repos starred by the specified (or authenticated) username"
     db = sqlite_utils.Database(db_path)
-    try:
-        token = json.load(open(auth))["github_personal_token"]
-    except (KeyError, FileNotFoundError):
-        token = None
-
+    token = load_token(auth)
     if load:
         stars = json.load(open(load))
     else:
@@ -130,11 +150,7 @@ def starred(db_path, username, auth, load):
 def repos(db_path, username, auth, load):
     "Save repos owened by the specified (or authenticated) username or organization"
     db = sqlite_utils.Database(db_path)
-    try:
-        token = json.load(open(auth))["github_personal_token"]
-    except (KeyError, FileNotFoundError):
-        token = None
-
+    token = load_token(auth)
     if load:
         repos = json.load(open(load))
     else:
@@ -146,3 +162,11 @@ def repos(db_path, username, auth, load):
         utils.save_repo(db, repo)
     utils.ensure_repo_fts(db)
     utils.ensure_foreign_keys(db)
+
+
+def load_token(auth):
+    try:
+        token = json.load(open(auth))["github_personal_token"]
+    except (KeyError, FileNotFoundError):
+        token = None
+    return token

github_to_sqlite/utils.py

@@ -75,6 +75,34 @@ def save_milestone(db, milestone):
     )
 
 
+def save_issue_comment(db, comment):
+    comment = dict(comment)
+    comment["user"] = save_user(db, comment["user"])
+    # We set up a 'issue' foreign key, but only if issue is in the DB
+    comment["issue"] = None
+    issue_url = comment["issue_url"]
+    bits = issue_url.split("/")
+    user_slug, repo_slug, issue_number = bits[-4], bits[-3], bits[-1]
+    # Is the issue in the DB already?
+    issue_rows = list(
+        db["issues"].rows_where(
+            "number = :number and repo = :repo",
+            {"repo": "{}/{}".format(user_slug, repo_slug), "number": issue_number},
+        )
+    )
+    if len(issue_rows) == 1:
+        comment["issue"] = issue_rows[0]["id"]
+    comment.pop("url", None)
+    if "url" in comment.get("reactions", {}):
+        comment["reactions"].pop("url")
+    last_pk = (
+        db["issue_comments"]
+        .upsert(comment, pk="id", foreign_keys=("user", "issue"), alter=True)
+        .last_pk
+    )
+    return last_pk
+
+
 def fetch_repo(repo, token=None):
     headers = make_headers(token)
     owner, slug = repo.split("/")
@@ -116,11 +144,28 @@ def ensure_foreign_keys(db):
             db[expected_key[0]].add_foreign_key(*expected_key[1:])
 
 
-def fetch_all_issues(repo, token=None):
+def fetch_issues(repo, token=None, issue=None):
+    headers = make_headers(token)
+    if issue is not None:
+        url = "https://api.github.com/repos/{}/issues/{}".format(repo, issue)
+        yield from [requests.get(url).json()]
+    else:
+        url = "https://api.github.com/repos/{}/issues?state=all&filter=all".format(repo)
+        for issues in paginate(url, headers):
+            yield from issues
+
+
+def fetch_issue_comments(repo, token=None, issue=None):
+    assert "/" in repo
     headers = make_headers(token)
-    url = "https://api.github.com/repos/{}/issues?state=all&filter=all".format(repo)
-    for issues in paginate(url, headers):
-        yield from issues
+    # Get reactions:
+    headers["Accept"] = "application/vnd.github.squirrel-girl-preview"
+    path = "/repos/{}/issues/comments".format(repo)
+    if issue is not None:
+        path = "/repos/{}/issues/{}/comments".format(repo, issue)
+    url = "https://api.github.com{}".format(path)
+    for comments in paginate(url, headers):
+        yield from comments
 
 
 def fetch_all_starred(username=None, token=None):