fetch-github-events

#!/bin/bash
#
# fetch-github-events - store GitHub user events locally as JSONL
#
# Written in 2019 by Aron Griffis <aron@arongriffis.com>
#
# To the extent possible under law, the author(s) have dedicated all copyright
# and related and neighboring rights to this software to the public domain
# worldwide. This software is distributed without any warranty.
#
# CC0 Public Domain Dedication at
# http://creativecommons.org/publicdomain/zero/1.0/
#==============================================================================

main() {
    declare documents db login id tmp

    # Exit script if any part of a pipeline returns non-zero exit status.
    set -e -o pipefail

    # Find where documents are stored.
    documents=$(xdg-user-dir DOCUMENTS 2>/dev/null || true)
    documents=${documents:-~/Documents}
    assert documents

    # Store the downloaded events in a JSONL database, which works really well
    # for appending new data. For more information about JSONL see
    # http://jsonlines.org/
    db=$documents/github-events.jsonl

    # Who am I on GitHub?
    login=$(hub api user | jq -r .login)

    # Our database is in ascending order, with the most recent event last, so
    # that we can append new events to it. The API returns events with the most
    # recent event first, so we'll collect them in a temporary file that gets
    # cleaned up automatically when the script exits.
    tmp=$(mktemp)
    cleanup() { rm -f "$tmp"; }
    trap 'cleanup; exit' 0

    # If the database exists, get the last id received from the API.
    if [[ -s $db ]]; then
        id=$(tail -n1 "$db" | jq -r .id)
    else
        id=0
    fi
    assert id

    # The events API returns 30 records per page, with 10 pages available.
    # jq -c '.[]' converts the incoming array to JSONL. The select filters
    # records that aren't newer than our most recent.
    hub api --paginate "users/$login/events" | jq -c ".[] | select(.id | tonumber > $id)" >> "$tmp"

    # Avoid updating the timestamp on the DB if there were no new records
    # available.
    if [[ -s $tmp ]]; then
        info "Fetched $(wc -l < "$tmp") events"
        # New records will be in descending order. Flip them around and append
        # to the end of our DB.
        tac "$tmp" >> "$db"
    fi

    exit # always at bottom of main
}

info() {
    if [[ -t 1 ]]; then
        echo "$*"
    fi
}

assert() {
    declare v
    for v; do
        [[ -n "${!v}" ]] || die "\$$v is empty"
    done
}

die() {
    echo "fatal: $*" >&2
    exit 1
}

[[ "$0" != "$BASH_SOURCE" ]] || main "$@"