diff --git a/.npmignore b/.npmignore index 0e52827..0055389 100644 --- a/.npmignore +++ b/.npmignore @@ -1,3 +1,4 @@ +/.ghuser.io.json /.nyc_output/ /.travis.yml /coverage/ diff --git a/README.md b/README.md index 9883b0b..6cff73e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ List **all** GitHub repos a user has contributed to **since the beginning of tim $ github-contribs AurelienLourot ✔ Fetched first day at GitHub: 2015-04-04. ⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until -✔ Fetched all commits and PRs. +✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well. 35 repo(s) found: AurelienLourot/lsankidb reframejs/reframe @@ -65,9 +65,26 @@ Instead we noticed that the "Contribution Activity" on the * https://github.com/users/AurelienLourot/created_repositories?from=2018-05-17&to=2018-05-17 * https://github.com/users/AurelienLourot/created_pull_requests?from=2018-05-17&to=2018-05-17 * https://github.com/users/AurelienLourot/created_pull_request_reviews?from=2018-05-17&to=2018-05-17 +* https://github.com/users/AurelienLourot/created_issues?from=2018-07-10&to=2018-07-10 So we're doing the same :) +> **NOTE**: it seems like `created_issues` URLs don't deliver "hot issues" (issues which received +> more comments than others): +> +> ```bash +> $ curl -s "https://github.com/users/AurelienLourot/created_issues?from=2015-09-23&to=2015-09-23" +>
+>
+> ``` +> +> To get these, we also query the profile itself: +> +> ```bash +> $ curl -s "https://github.com/AurelienLourot?from=2015-09-23" | grep issues/ +> Publish properties aren't used by build-info-extractor-gradle? +> ``` + ### Why is it so slow? We hit a [rate limit](https://en.wikipedia.org/wiki/Rate_limiting). And since it's not an official @@ -75,11 +92,17 @@ API, we can't use a token to raise the limit. ### Isn't it likely to break? -Yes, it is since that interface isn't public. We're monitoring it and will react as fast as we can -when it breaks. +Yes, it is since that interface isn't public. We're monitoring it[1](#footnote1) and will +react as fast as we can when it breaks. + +1 [ghuser.io](https://github.com/AurelienLourot/ghuser.io) runs +this tool every day.
## Changelog +**2.2.0** (2018-08-09): + * [#1](https://github.com/AurelienLourot/github-contribs/issues/1) - Added `--issues` flag. + **2.1.0** (2018-06-25): * Exported helper function `prevDay()`. diff --git a/cli.js b/cli.js index a0d4dc4..a2d5b90 100755 --- a/cli.js +++ b/cli.js @@ -10,7 +10,7 @@ const cli = meow(` usage: - $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] USER + $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] [--issues] USER $ github-contribs --help $ github-contribs --version @@ -22,12 +22,14 @@ optional arguments: --verbose show debugging information --since YYYY-MM-DD limit the results (default: first day at GitHub) --until YYYY-MM-DD limit the results (default: today) + --issues fetches not only commits and PRs but also issues --version show program's version number and exit --help show this help message and exit `, { boolean: [ 'quiet', 'verbose', + 'issues', ], string: [ 'since', @@ -51,8 +53,10 @@ optional arguments: } const user = cli.input[0]; - const repos = await githubContribs.fetch(user, cli.flags.since, cli.flags.until, - !cli.flags.quiet && ora, cli.flags.verbose && console); + const repos = await githubContribs.fetch( + user, cli.flags.since, cli.flags.until, !cli.flags.quiet && ora, cli.flags.verbose && console, + cli.flags.issues + ); if (!cli.flags.quiet) { console.log(`${repos.size} repo(s) found:`); } diff --git a/docs/advanced.md b/docs/advanced.md index 9067f8a..55f41fc 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -14,7 +14,7 @@ $ github-contribs AurelienLourot GitHub API key found. ✔ Fetched first day at GitHub: 2015-04-04. ⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until -✔ Fetched all commits and PRs. +✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well. 35 repo(s) found: AurelienLourot/lsankidb reframejs/reframe diff --git a/index.js b/index.js index 7191cf3..4a357eb 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const htmlparser = require('htmlparser'); const moment = require('moment'); - const fetchContribs = async (user, since, until, ora, console) => { + const fetchContribs = async (user, since, until, ora, console, alsoIssues) => { ora = ora || (() => { return { start() { return this; }, @@ -21,7 +21,7 @@ }; const joinDate = await getFirstDayAtGithub(user, ora); - const result = await getContribs(user, joinDate, since, until, ora, console); + const result = await getContribs(user, joinDate, since, until, ora, console, alsoIssues); return result; }; @@ -82,7 +82,7 @@ return result; }; - const getContribs = async (user, joinDate, since, until, ora, console) => { + const getContribs = async (user, joinDate, since, until, ora, console, alsoIssues) => { const commitsHtmlToRepos = html => { const repos = new Set(); @@ -116,7 +116,7 @@ return repos; }; - const prsHtmlToRepos = html => { + const issuesHtmlToRepos = html => { const repos = new Set(); const handler = new htmlparser.DefaultHandler((error, dom) => {}); @@ -149,6 +149,39 @@ return repos; }; + const hotIssuesHtmlToRepos = html => { + const repos = new Set(); + + const regex = / { + let result = (isDone && 'Fetched') || 'Fetching'; + result += ' all commits'; + result += (alsoIssues && ', PRs and issues') || ' and PRs'; + + if (isDone) { + result += '.'; + } else if (numOfQueriedDays && numOfDaysToQuery) { + result += ` [${numOfQueriedDays}/${numOfDaysToQuery}]`; + } else { + result += '...'; + } + + if (!alsoIssues) { + result += ' Consider using --issues to fetch issues as well.'; + } + return result; + }; + let oldestDate = joinDate; if (since) { oldestDate = new Date(Math.max(oldestDate, stringToDate(since))); @@ -175,22 +208,50 @@ `https://github.com/users/${user}/created_commits?from=${currDateStr}&to=${currDateStr}` ); const userCommitsHtml = await userCommits.text(); + const commitsRepos = commitsHtmlToRepos(userCommitsHtml); + const userPRs = await fetchRetry( `https://github.com/users/${user}/created_pull_requests?from=${currDateStr}&to=${currDateStr}`, ); const userPRsHtml = await userPRs.text(); - const commitsRepos = commitsHtmlToRepos(userCommitsHtml); - const prsRepos = prsHtmlToRepos(userPRsHtml); + const prsRepos = issuesHtmlToRepos(userPRsHtml); + + let issuesRepos = []; + let hotIssuesRepos = []; + if (alsoIssues) { + const userIssues = await fetchRetry( + `https://github.com/users/${user}/created_issues?from=${currDateStr}&to=${currDateStr}`, + ); + const userIssuesHtml = await userIssues.text(); + issuesRepos = issuesHtmlToRepos(userIssuesHtml); + + const userHotIssues = await fetchRetry( + `https://github.com/${user}?from=${currDateStr}`, + ); + const userHotIssuesHtml = await userHotIssues.text(); + hotIssuesRepos = hotIssuesHtmlToRepos(userHotIssuesHtml); + } + progressSpinner.stop(); // temporary stop for logging for (const repo of commitsRepos) { - console.log(`${currDateStr}: (commits) ${repo}`); + console.log(`${currDateStr}: (commits) ${repo}`); result.add(repo); } for (const repo of prsRepos) { - console.log(`${currDateStr}: (PRs) ${repo}`); + console.log(`${currDateStr}: (PRs) ${repo}`); + result.add(repo); + } + for (const repo of issuesRepos) { + console.log(`${currDateStr}: (issues) ${repo}`); result.add(repo); } - progressSpinner.start(`Fetching all commits and PRs [${++numOfQueriedDays}/${numOfDaysToQuery}]`); + for (const repo of hotIssuesRepos) { + console.log(`${currDateStr}: (hot issues) ${repo}`); + result.add(repo); + } + progressSpinner.start( + progressMsg(false, alsoIssues, ++numOfQueriedDays, numOfDaysToQuery) + ); })(); }; })(); @@ -205,9 +266,9 @@ ora(warning).warn(); const result = new Set(); - const progressSpinner = ora('Fetching all commits and PRs...').start(); + const progressSpinner = ora(progressMsg(false, alsoIssues)).start(); await new PromisePool(getContribsOnOneDay, 5).start(); - progressSpinner.succeed('Fetched all commits and PRs.'); + progressSpinner.succeed(progressMsg(true, alsoIssues)); return result; }; diff --git a/package.json b/package.json index 7177c97..f541990 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ghuser/github-contribs", - "version": "2.1.0", + "version": "2.2.0", "description": "List all GitHub repos a user has contributed to since the beginning of time.", "license": "Unlicense", "repository": { @@ -72,6 +72,6 @@ ".js" ], "check-coverage": true, - "lines": 80 + "lines": 84 } } diff --git a/test.js b/test.js index 2f9f9d2..e029657 100644 --- a/test.js +++ b/test.js @@ -1,35 +1,55 @@ import test from 'ava'; import m from '.'; +const ora = string => { + if (string) { + console.log(string); + } + return { + start(string) { + if (string) { + console.log(string); + } + return this; + }, + stop() {}, + succeed(string) { + if (string) { + console.log(string); + } + }, + warn() {}, + }; +}; + test('fetches commits and PRs', async t => { /* AurelienLourot had the following activity: * 2017-08-26: nothing * 2017-08-27: 14 commits in AurelienLourot/mybeir.ut * 2017-08-28: 1 PR in tt-gf/ant-ivy */ - const ora = string => { - if (string) { - console.log(string); - } - return { - start(string) { - if (string) { - console.log(string); - } - return this; - }, - stop() {}, - succeed(string) { - if (string) { - console.log(string); - } - }, - warn() {}, - }; - }; - const result = await m.fetch('AurelienLourot', '2017-08-26', '2017-08-28', ora, console); t.is(result.size, 2); t.true(result.has('AurelienLourot/mybeir.ut')); t.true(result.has('tt-gf/ant-ivy')); }); + +test('fetches issues', async t => { + /* RichardLitt had the following activity: + * 2018-08-07: 1 issue in orbitdb/welcome */ + + const result = await m.fetch('RichardLitt', '2018-08-07', '2018-08-07', ora, console, true); + t.is(result.size, 1); + t.true(result.has('orbitdb/welcome')); +}); + +test('fetches hot issues', async t => { + /* AurelienLourot had the following activity: + * 2015-09-23: 1 commit in AurelienLourot/AurelienLourot.github.io + 1 hot issue in jfrog/build-info */ + + const result = await m.fetch('AurelienLourot', '2015-09-23', '2015-09-23', ora, console, true); + t.is(result.size, 2); + t.true(result.has('AurelienLourot/AurelienLourot.github.io')); + t.true(result.has('jfrog/build-info')); +});