diff --git a/.npmignore b/.npmignore
index 0e52827..0055389 100644
--- a/.npmignore
+++ b/.npmignore
@@ -1,3 +1,4 @@
+/.ghuser.io.json
/.nyc_output/
/.travis.yml
/coverage/
diff --git a/README.md b/README.md
index 9883b0b..6cff73e 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ List **all** GitHub repos a user has contributed to **since the beginning of tim
$ github-contribs AurelienLourot
✔ Fetched first day at GitHub: 2015-04-04.
⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until
-✔ Fetched all commits and PRs.
+✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well.
35 repo(s) found:
AurelienLourot/lsankidb
reframejs/reframe
@@ -65,9 +65,26 @@ Instead we noticed that the "Contribution Activity" on the
* https://github.com/users/AurelienLourot/created_repositories?from=2018-05-17&to=2018-05-17
* https://github.com/users/AurelienLourot/created_pull_requests?from=2018-05-17&to=2018-05-17
* https://github.com/users/AurelienLourot/created_pull_request_reviews?from=2018-05-17&to=2018-05-17
+* https://github.com/users/AurelienLourot/created_issues?from=2018-07-10&to=2018-07-10
So we're doing the same :)
+> **NOTE**: it seems like `created_issues` URLs don't deliver "hot issues" (issues which received
+> more comments than others):
+>
+> ```bash
+> $ curl -s "https://github.com/users/AurelienLourot/created_issues?from=2015-09-23&to=2015-09-23"
+>
+>
+> ```
+>
+> To get these, we also query the profile itself:
+>
+> ```bash
+> $ curl -s "https://github.com/AurelienLourot?from=2015-09-23" | grep issues/
+> Publish properties aren't used by build-info-extractor-gradle?
+> ```
+
### Why is it so slow?
We hit a [rate limit](https://en.wikipedia.org/wiki/Rate_limiting). And since it's not an official
@@ -75,11 +92,17 @@ API, we can't use a token to raise the limit.
### Isn't it likely to break?
-Yes, it is since that interface isn't public. We're monitoring it and will react as fast as we can
-when it breaks.
+Yes, it is since that interface isn't public. We're monitoring it[1](#footnote1) and will
+react as fast as we can when it breaks.
+
+1 [ghuser.io](https://github.com/AurelienLourot/ghuser.io) runs
+this tool every day.
## Changelog
+**2.2.0** (2018-08-09):
+ * [#1](https://github.com/AurelienLourot/github-contribs/issues/1) - Added `--issues` flag.
+
**2.1.0** (2018-06-25):
* Exported helper function `prevDay()`.
diff --git a/cli.js b/cli.js
index a0d4dc4..a2d5b90 100755
--- a/cli.js
+++ b/cli.js
@@ -10,7 +10,7 @@
const cli = meow(`
usage:
- $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] USER
+ $ github-contribs [--quiet|--verbose] [--since YYYY-MM-DD] [--until YYYY-MM-DD] [--issues] USER
$ github-contribs --help
$ github-contribs --version
@@ -22,12 +22,14 @@ optional arguments:
--verbose show debugging information
--since YYYY-MM-DD limit the results (default: first day at GitHub)
--until YYYY-MM-DD limit the results (default: today)
+ --issues fetches not only commits and PRs but also issues
--version show program's version number and exit
--help show this help message and exit
`, {
boolean: [
'quiet',
'verbose',
+ 'issues',
],
string: [
'since',
@@ -51,8 +53,10 @@ optional arguments:
}
const user = cli.input[0];
- const repos = await githubContribs.fetch(user, cli.flags.since, cli.flags.until,
- !cli.flags.quiet && ora, cli.flags.verbose && console);
+ const repos = await githubContribs.fetch(
+ user, cli.flags.since, cli.flags.until, !cli.flags.quiet && ora, cli.flags.verbose && console,
+ cli.flags.issues
+ );
if (!cli.flags.quiet) {
console.log(`${repos.size} repo(s) found:`);
}
diff --git a/docs/advanced.md b/docs/advanced.md
index 9067f8a..55f41fc 100644
--- a/docs/advanced.md
+++ b/docs/advanced.md
@@ -14,7 +14,7 @@ $ github-contribs AurelienLourot
GitHub API key found.
✔ Fetched first day at GitHub: 2015-04-04.
⚠ Be patient. The whole process might take up to an hour... Consider using --since and/or --until
-✔ Fetched all commits and PRs.
+✔ Fetched all commits and PRs. Consider using --issues to fetch issues as well.
35 repo(s) found:
AurelienLourot/lsankidb
reframejs/reframe
diff --git a/index.js b/index.js
index 7191cf3..4a357eb 100644
--- a/index.js
+++ b/index.js
@@ -7,7 +7,7 @@
const htmlparser = require('htmlparser');
const moment = require('moment');
- const fetchContribs = async (user, since, until, ora, console) => {
+ const fetchContribs = async (user, since, until, ora, console, alsoIssues) => {
ora = ora || (() => {
return {
start() { return this; },
@@ -21,7 +21,7 @@
};
const joinDate = await getFirstDayAtGithub(user, ora);
- const result = await getContribs(user, joinDate, since, until, ora, console);
+ const result = await getContribs(user, joinDate, since, until, ora, console, alsoIssues);
return result;
};
@@ -82,7 +82,7 @@
return result;
};
- const getContribs = async (user, joinDate, since, until, ora, console) => {
+ const getContribs = async (user, joinDate, since, until, ora, console, alsoIssues) => {
const commitsHtmlToRepos = html => {
const repos = new Set();
@@ -116,7 +116,7 @@
return repos;
};
- const prsHtmlToRepos = html => {
+ const issuesHtmlToRepos = html => {
const repos = new Set();
const handler = new htmlparser.DefaultHandler((error, dom) => {});
@@ -149,6 +149,39 @@
return repos;
};
+ const hotIssuesHtmlToRepos = html => {
+ const repos = new Set();
+
+ const regex = / {
+ let result = (isDone && 'Fetched') || 'Fetching';
+ result += ' all commits';
+ result += (alsoIssues && ', PRs and issues') || ' and PRs';
+
+ if (isDone) {
+ result += '.';
+ } else if (numOfQueriedDays && numOfDaysToQuery) {
+ result += ` [${numOfQueriedDays}/${numOfDaysToQuery}]`;
+ } else {
+ result += '...';
+ }
+
+ if (!alsoIssues) {
+ result += ' Consider using --issues to fetch issues as well.';
+ }
+ return result;
+ };
+
let oldestDate = joinDate;
if (since) {
oldestDate = new Date(Math.max(oldestDate, stringToDate(since)));
@@ -175,22 +208,50 @@
`https://github.com/users/${user}/created_commits?from=${currDateStr}&to=${currDateStr}`
);
const userCommitsHtml = await userCommits.text();
+ const commitsRepos = commitsHtmlToRepos(userCommitsHtml);
+
const userPRs = await fetchRetry(
`https://github.com/users/${user}/created_pull_requests?from=${currDateStr}&to=${currDateStr}`,
);
const userPRsHtml = await userPRs.text();
- const commitsRepos = commitsHtmlToRepos(userCommitsHtml);
- const prsRepos = prsHtmlToRepos(userPRsHtml);
+ const prsRepos = issuesHtmlToRepos(userPRsHtml);
+
+ let issuesRepos = [];
+ let hotIssuesRepos = [];
+ if (alsoIssues) {
+ const userIssues = await fetchRetry(
+ `https://github.com/users/${user}/created_issues?from=${currDateStr}&to=${currDateStr}`,
+ );
+ const userIssuesHtml = await userIssues.text();
+ issuesRepos = issuesHtmlToRepos(userIssuesHtml);
+
+ const userHotIssues = await fetchRetry(
+ `https://github.com/${user}?from=${currDateStr}`,
+ );
+ const userHotIssuesHtml = await userHotIssues.text();
+ hotIssuesRepos = hotIssuesHtmlToRepos(userHotIssuesHtml);
+ }
+
progressSpinner.stop(); // temporary stop for logging
for (const repo of commitsRepos) {
- console.log(`${currDateStr}: (commits) ${repo}`);
+ console.log(`${currDateStr}: (commits) ${repo}`);
result.add(repo);
}
for (const repo of prsRepos) {
- console.log(`${currDateStr}: (PRs) ${repo}`);
+ console.log(`${currDateStr}: (PRs) ${repo}`);
+ result.add(repo);
+ }
+ for (const repo of issuesRepos) {
+ console.log(`${currDateStr}: (issues) ${repo}`);
result.add(repo);
}
- progressSpinner.start(`Fetching all commits and PRs [${++numOfQueriedDays}/${numOfDaysToQuery}]`);
+ for (const repo of hotIssuesRepos) {
+ console.log(`${currDateStr}: (hot issues) ${repo}`);
+ result.add(repo);
+ }
+ progressSpinner.start(
+ progressMsg(false, alsoIssues, ++numOfQueriedDays, numOfDaysToQuery)
+ );
})();
};
})();
@@ -205,9 +266,9 @@
ora(warning).warn();
const result = new Set();
- const progressSpinner = ora('Fetching all commits and PRs...').start();
+ const progressSpinner = ora(progressMsg(false, alsoIssues)).start();
await new PromisePool(getContribsOnOneDay, 5).start();
- progressSpinner.succeed('Fetched all commits and PRs.');
+ progressSpinner.succeed(progressMsg(true, alsoIssues));
return result;
};
diff --git a/package.json b/package.json
index 7177c97..f541990 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "@ghuser/github-contribs",
- "version": "2.1.0",
+ "version": "2.2.0",
"description": "List all GitHub repos a user has contributed to since the beginning of time.",
"license": "Unlicense",
"repository": {
@@ -72,6 +72,6 @@
".js"
],
"check-coverage": true,
- "lines": 80
+ "lines": 84
}
}
diff --git a/test.js b/test.js
index 2f9f9d2..e029657 100644
--- a/test.js
+++ b/test.js
@@ -1,35 +1,55 @@
import test from 'ava';
import m from '.';
+const ora = string => {
+ if (string) {
+ console.log(string);
+ }
+ return {
+ start(string) {
+ if (string) {
+ console.log(string);
+ }
+ return this;
+ },
+ stop() {},
+ succeed(string) {
+ if (string) {
+ console.log(string);
+ }
+ },
+ warn() {},
+ };
+};
+
test('fetches commits and PRs', async t => {
/* AurelienLourot had the following activity:
* 2017-08-26: nothing
* 2017-08-27: 14 commits in AurelienLourot/mybeir.ut
* 2017-08-28: 1 PR in tt-gf/ant-ivy */
- const ora = string => {
- if (string) {
- console.log(string);
- }
- return {
- start(string) {
- if (string) {
- console.log(string);
- }
- return this;
- },
- stop() {},
- succeed(string) {
- if (string) {
- console.log(string);
- }
- },
- warn() {},
- };
- };
-
const result = await m.fetch('AurelienLourot', '2017-08-26', '2017-08-28', ora, console);
t.is(result.size, 2);
t.true(result.has('AurelienLourot/mybeir.ut'));
t.true(result.has('tt-gf/ant-ivy'));
});
+
+test('fetches issues', async t => {
+ /* RichardLitt had the following activity:
+ * 2018-08-07: 1 issue in orbitdb/welcome */
+
+ const result = await m.fetch('RichardLitt', '2018-08-07', '2018-08-07', ora, console, true);
+ t.is(result.size, 1);
+ t.true(result.has('orbitdb/welcome'));
+});
+
+test('fetches hot issues', async t => {
+ /* AurelienLourot had the following activity:
+ * 2015-09-23: 1 commit in AurelienLourot/AurelienLourot.github.io
+ 1 hot issue in jfrog/build-info */
+
+ const result = await m.fetch('AurelienLourot', '2015-09-23', '2015-09-23', ora, console, true);
+ t.is(result.size, 2);
+ t.true(result.has('AurelienLourot/AurelienLourot.github.io'));
+ t.true(result.has('jfrog/build-info'));
+});