wltool

#! /usr/bin/env python
# -*- mode: python ; coding: utf-8 -*-
# Copyright 2014-2022 Peter Williams <peter@newton.cx>
# Licensed under the GNU General Public License, version 3 or higher.

"""usage: wltool <command> [arguments...]

workog commands are:

  bootstrap-bibtex  Stub publication records from an ADS BibTeX file
  extract           Print out worklog records of a specific type
  github-repos      Print list of GitHub repositories contributed to
  html              Fill in an HTML-formatted template
  latex             Fill in a LaTeX-formatted template
  nsf-collabs       Print stub list of collaborators in past 48 months
  summarize         Summarize the records present in the worklog data files
  update-cites      Update ADS citation counts in the worklog data files
  update-github     Update statistics about contributions to GitHub repositories

Running 'wltool <command> --help' may give more information on the command
in question."""

import io
from itertools import chain
from os.path import join as pjoin
import sys

from worklog import *


def cli_bootstrap_bibtex(argv):
    """usage: wltool bootstrap-bibtex <bibtex-file> <your-surname> <output-dir>

    Create a new set of worklog files, seeding them with publication information
    gathered from from <bibtex-file>. Specify <your-surname> to assist in guessing
    of the "mypos" field (your position in the author list). The output directory
    <output-dir> must not exist when you run the program; this is to avoid
    accidental overwriting of existing files.

    Search the output files for "XXX" to find issues that need manual attention.

    WARNING: the current implementation assumes that the file is one from ADS.
    Parsing files from other origins will probably fail."""

    import os.path

    # note: we'll leave the output directory lying around if we die. I figure
    # that could be helpful for debugging.

    if len(argv) != 4 or "--help" in argv:
        print(cli_bootstrap_bibtex.__doc__)
        raise SystemExit(1)

    bibpath = argv[1]
    mysurname = argv[2]
    outdir = argv[3]

    try:
        os.mkdir(outdir)
    except Exception as e:
        die(
            'cannot create output directory "%s": %s (%s)',
            outdir,
            e,
            e.__class__.__name__,
        )

    with open(bibpath) as bibfile:
        bootstrap_bibtex(bibfile, outdir, mysurname)


def cli_extract(argv):
    """usage: wltool extract <section-name> [datadir]

    Print out all records with the given section name. If not specified, the data
    directory is assumed to be the current directory.

    See the README.md that came with this package for more detailed information."""

    from inifile import write
    import sys

    if len(argv) not in (2, 3) or "--help" in argv:
        print(cli_extract.__doc__)
        raise SystemExit(1)

    sectname = argv[1]

    if len(argv) < 3:
        datadir = "."
    else:
        datadir = argv[2]

    write(sys.stdout, (i for i in load(datadir) if i.section == sectname))


def cli_github_repos(argv):
    """
    usage: wltool github-repos [datadir]

    Scan GitHub APIs and event data to find repositories that you have contributed to,
    and report potentially new ones whose stats should be reported.

    This uses the Google BigQuery dataset set up by githubarchive.org, and requires
    that API access to Google BigQuery and GitHub have been set up.
    """

    if len(argv) not in (1, 2) or "--help" in argv:
        print(cli_github_repos.__doc__)
        raise SystemExit(1)

    if len(argv) < 2:
        datadir = "."
    else:
        datadir = argv[1]

    # Get the set of already-known repositories

    known_reponames = set()

    with open(pjoin(datadir, "repos.github_scan.dat"), "rt") as f:
        for line in f:
            line = line.split("#", 1)[0].strip()
            if not line:
                continue

            known_reponames.add(line.split(None, 1)[0])

    # Query GitHub repos where I have a merged pull request, or I have a direct
    # push and the repo is not a fork.
    #
    # Here and throughout, we ignore conda-forge feedstock repos, of
    # which I've contributed to literally hundreds.

    import wlgithub

    gh = wlgithub.get_github_service(datadir)
    login = gh.get_user().login
    jobs = wlgithub.get_bigquery_jobs_service(datadir, [])

    all_reponames = set(
        chain(
            wlgithub.get_repos_with_merged_prs_from_user(jobs, login),
            wlgithub.get_root_repos_with_pushes_from_user(gh, jobs, login),
        )
    )

    # Filter down

    new_reponames = set()
    n_feedstock = 0
    n_known = 0

    for reponame in all_reponames:
        if reponame.endswith("-feedstock"):
            n_feedstock += 1
        elif reponame in known_reponames:
            n_known += 1
        else:
            new_reponames.add(reponame)

    # Report

    for reponame in sorted(new_reponames):
        print(reponame)

    if n_feedstock or n_known:
        print()
        print(f"[... and {n_known} already-known repos]", file=sys.stderr)
        print(
            f"[... and {n_feedstock} conda-forge-related feedstock repos]",
            file=sys.stderr,
        )


def cli_nsf_collabs(argv):
    """usage: wltool nsf-collabs [datadir]

    Print out a list of collaborators in the past 48 months, a suitable basis for
    the generation of a list of collaborators for an NSF biographical sketch. You
    must manually check names, remove duplicates, split identical names, add
    organizational affiliations, and generally use your common sense. This tool
    uses authors on [pub] items; this does not necessarily map cleanly to the set
    of names that the NSF wants for such a list.

    The output will be formatted for inclusion in a LaTeX document."""

    if len(argv) not in (1, 2) or "--help" in argv:
        print(cli_nsf_collabs.__doc__)
        raise SystemExit(1)

    if len(argv) < 2:
        datadir = "."
    else:
        datadir = argv[1]

    # Set up for filtering by date

    import time

    cutoff_year, cutoff_month = time.localtime()[:2]
    cutoff_year -= 4

    # Load up those names

    names = set()

    for i in load(datadir):
        if i.section != "pub":
            continue

        dbits = i.pubdate.split("/")
        iyear, imonth = list(map(int, dbits[:2]))
        if iyear < cutoff_year or imonth < cutoff_month:
            continue

        for aubase in i.authors.split(";"):
            bits = aubase.strip().split()
            surname = bits[-1].replace("_", " ")
            rest = " ".join(bits[:-1])
            names.add((rest, surname))

    # Print, sorted by last name, in a format just ripe for filling in.

    for rest, surname in sorted(names, key=lambda t: t[1]):
        print("%s, %s" % (surname, rest))


def _cli_render(argv):
    """usage: wltool %s <template> [datadir=.]

    Process log files and use the information to fill in <template>.
    If not specified, the data directory is assumed to be the current
    directory.

    See the README.md that came with this package for more detailed information."""

    fmtname = argv[0]

    if len(argv) not in (2, 3) or "--help" in argv:
        print(_cli_render.__doc__ % fmtname)
        raise SystemExit(1)

    tmpl = argv[1]

    if len(argv) < 3:
        datadir = "."
    else:
        datadir = argv[2]

    if fmtname == "latex":
        render = render_latex
    elif fmtname == "html":
        render = render_html
    else:
        die('unknown output format "%s"', fmtname)

    context, commands = setup_processing(render, datadir)

    with io.open(tmpl, "rb") as f:
        for outline in process_template(f, commands, context):
            print(outline)


cli_latex = _cli_render
cli_html = _cli_render


def cli_summarize(argv):
    """usage: wltool summarize [datadir]

    Print out a summary of the different kinds of records in the log file. If not
    specified, the data directory is assumed to be the current directory.

    See the README.md that came with this package for more detailed information."""

    if len(argv) not in (1, 2) or "--help" in argv:
        print(cli_summarize.__doc__)
        raise SystemExit(1)

    if len(argv) < 2:
        datadir = "."
    else:
        datadir = argv[1]

    counts = {}
    maxsectlen = 0

    for i in load(datadir):
        counts[i.section] = counts.get(i.section, 0) + 1
        maxsectlen = max(maxsectlen, len(i.section))

    for section, count in sorted(counts.items()):
        print("% *s: %d" % (maxsectlen, section, count))


_update_minwait = 7 * 24 * 3600  # 1 week


def cli_update_cites(argv):
    """usage: wltool update-cites [datadir]

    Fetch citation counts from NASA ADS and insert them into the log files.
    Prints out a summary of new citations by bibcode.

    See the README.md that came with this package for more detailed information."""

    if len(argv) not in (1, 2) or "--help" in argv:
        print(cli_update_cites.__doc__)
        raise SystemExit(1)

    if len(argv) < 2:
        datadir = "."
    else:
        datadir = argv[1]

    import time
    from inifile import mutateInPlace

    now = int(time.time())
    nowstr = time.strftime("%Y/%m/%d ", time.gmtime(now))

    for path in list_data_files(datadir):
        for item in mutateInPlace(path):
            if not item.data.has("bibcode"):
                continue

            bibcode = item.data.bibcode
            firstauth = item.data.has("mypos") and int(item.data.mypos) == 1
            reffed = item.data.has("refereed") and item.data.refereed == "y"

            if not item.data.has("adscites"):
                lastupdate = curcites = 0
            else:
                try:
                    a = item.data.adscites.split()[:2]
                    y, m, d = [int(x) for x in a[0].split("/")]
                    lastupdate = time.mktime((y, m, d, 0, 0, 0, 0, 0, 0))
                    curcites = int(a[1])
                except Exception:
                    warn("cannot parse adscites entry: %s", item.data.adscites)
                    continue

            if lastupdate + _update_minwait > now:
                continue

            print(bibcode, " *"[firstauth] + " R"[reffed], "...", end=" ")
            try:
                newcites = get_ads_cite_count(bibcode)
                item.set("adscites", nowstr + str(newcites))
                print("%d (%+d)" % (newcites, newcites - curcites))
            except ADSCountError as e:
                print("error!: %s" % e)


def cli_update_github(argv):
    """usage: wltool update-github [datadir]

    Fetch contribution statistics regarding GitHub repositories that you've
    contributed to, and update the worklog records as appropriate. This requires
    that API access to GitHub has been set up."""

    if len(argv) not in (1, 2) or "--help" in argv:
        print(cli_update_github.__doc__)
        raise SystemExit(1)

    if len(argv) < 2:
        datadir = "."
    else:
        datadir = argv[1]

    import time, wlgithub
    from github import GithubException
    from inifile import mutateInPlace

    debug_requests = False
    if debug_requests:
        import logging

        logging.basicConfig()
        logging.getLogger("github.Requester").level = logging.DEBUG

    gh = wlgithub.get_github_service(datadir)

    now = int(time.time())
    nowstr = time.strftime("%Y/%m/%d", time.gmtime(now))

    for path in list_data_files(datadir):
        for item in mutateInPlace(path):
            if item.data.section != "repo" or item.data.service != "github":
                continue
            if item.data.has("skip") and item.data.skip == "y":
                continue

            if not item.data.has("updated"):
                lastupdate = 0
            else:
                try:
                    y, m, d = [int(x) for x in item.data.updated.split("/")]
                    lastupdate = time.mktime((y, m, d, 0, 0, 0, 0, 0, 0))
                except Exception:
                    warn('cannot parse "updated" entry: %s', item.data.updated)
                    continue

            if lastupdate + _update_minwait > now:
                continue

            name = item.data.name

            print(name, "...")
            try:
                # print gh.get_rate_limit().rate.remaining
                mycommits = wlgithub.get_repo_commit_stats(
                    gh, name, branch=item.data.get("branch")
                )
                item.set("usercommits", mycommits.commits)
                if mycommits.latest_date is not None:
                    item.set(
                        "lastusercommit",
                        "%04d/%02d/%02d"
                        % (
                            mycommits.latest_date.year,
                            mycommits.latest_date.month,
                            mycommits.latest_date.day,
                        ),
                    )

                impact = wlgithub.get_repo_impact_stats(gh, name)

                if item.data.get("desc") is None:
                    item.set("desc", impact.description)
                item.set("allcommits", impact.commits)
                item.set("forks", impact.forks)
                item.set("stars", impact.stars)
                item.set("contributors", impact.contributors)

                item.set("updated", nowstr)
            except GithubException as e:
                warn("GitHub exception: %s", e)
                continue
            except Exception as e:
                warn("exception: %s (%s)", e, e.__class__.__name__)
                continue


# The dispatcher

if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2 or sys.argv[1] == "--help":
        print(__doc__)
        raise SystemExit(0)

    cmdname = sys.argv[1]
    clicmd = globals().get("cli_" + cmdname.replace("-", "_"))

    if not callable(clicmd):
        die(
            'unknown subcommand "%s"; run this program without arguments '
            "for usage help",
            cmdname,
        )

    clicmd(sys.argv[1:])