Skip to content
Permalink
Browse files
wip
  • Loading branch information
robertkowalski committed Nov 21, 2014
1 parent e5a2458 commit 0ec747a14ef0b4b23992379819b20ebd92bb6c3e
Showing 6 changed files with 225 additions and 72 deletions.
@@ -1 +1,2 @@
node_modules
npm-debug.log
@@ -11,41 +11,46 @@
// License for the specific language governing permissions and limitations under
// the License.

const crawler = require('./lib/index.js');
const crawler = require('./lib/index.js'),
argument = require('./lib/argument.js');

const arg = process.argv[2];
const template = '{{count}} {{messages}} since {{month}} ' +
'({{diff}} change)';

if (arg === '-h' || arg === '--help') {
return printUsage();
}

if (!validArg(arg)) {
console.log('Error: format is YYYYMM-YYYYMM - see ' +
if (!argument.validArg(arg)) {
console.error('Error: format is YYYYMM - see ' +
"'generate-report --help'");
process.exit(1);
return;
}

const template = '{{count}} {{messages}} since {{month}} ' +
'(DIFF change)';

crawler(arg, function (err, data) {
var monthsForDiff = 3
const queryParams = argument.prepareQueryParams(arg);
crawler(queryParams, monthsForDiff, function (err, data) {
if (err) {
logLine('Error:');
console.log(err);
console.error(err);
process.exit(1);
return;
}

logLine('Your message counts:');
data.forEach(function (el) {
const month = getStartMonthAsWord(arg),
messageCount = +el[1].replace(',', ''),
Object.keys(data).forEach(function (el) {
const month = argument.getMonthAsWordFromNow(arg, monthsForDiff),
messageCount = data[el].curr,
message = messageCount > 1 ? 'messages' : 'message',
wikitext = template
.replace('{{count}}', el[1])
.replace('{{count}}', messageCount)
.replace('{{month}}', month)
.replace('{{messages}}', message);
.replace('{{messages}}', message)
.replace('{{diff}}', data[el].diff);

console.log(el[0] + ':');
console.log(el + ':');
logLine(wikitext);
});
console.log('Happy reporting! :)');
@@ -55,54 +60,10 @@ function logLine (line) {
console.log(line + '\n');
}

function getStartMonthAsWord (date) {
const i = parseInt(date.substr(4, 2), 10);
return getMonthAsWord(i);
}

function getMonthAsWord (i) {
return [
'January',
'February',
'March',
'April',
'May',
'June',
'July',
'August',
'September',
'October',
'November',
'December'
][i - 1];
}

function validArg (arg) {
if (!arg) {
return false;
}

const dates = arg.split('-');

if (dates.length !== 2) {
return false;
}

if (dates[0].length !== 6 || dates[1].length !== 6) {
return false;
}

if (Number.isNaN(+dates[0]) || Number.isNaN(+dates[1])) {
return false;
}

return true;
}

function printUsage () {
logLine('usage: generate-report <daterange> | --help');
logLine('usage: generate-report <date> | --help');
console.log('If you are reporting for February 2014, for instance,');
logLine('set the date range to: 201311-201402');
logLine('set the date to: 201402');
console.log('guide:');
console.log('https://cwiki.apache.org/confluence/display' +
'/COUCHDB/Guide');
@@ -0,0 +1,67 @@
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

const moment = require('moment');

exports.prepareQueryParams = prepareQueryParams;
function prepareQueryParams (arg) {
const startEnd = getStartEndDates(arg, 3);
startAsString = startEnd[0].format('YYYYMM'),
diffStartEnd = getStartEndDates(startAsString, 3);

return {
queryCurr: formatQuery(startEnd),
queryDiff: formatQuery(diffStartEnd)
};
}

exports.validArg = validArg;
function validArg (arg) {
if (!arg) {
return false;
}

if (arg.length !== 6) {
return false;
}

if (Number.isNaN(+arg)) {
return false;
}

return moment(arg, 'YYYYMM').isValid();
}

exports.getMonthAsWordFromNow = getMonthAsWordFromNow;
function getMonthAsWordFromNow (reportEnd, time) {
const inter = moment(reportEnd, 'YYYYMM')
.subtract(time, 'months')
.format('YYYYMM');

return moment(inter, 'YYYYMM').format('MMMM');
}

function getMonthAsWord (reportEnd) {
return moment(reportEnd, 'YYYYMM').format('MMMM');
}

function formatQuery (startEnd) {
return startEnd[0].format('YYYYMM') + '-' +
startEnd[1].format('YYYYMM');
}

function getStartEndDates (arg, time) {
const end = moment(arg, 'YYYYMM'),
start = moment(arg, 'YYYYMM').subtract(time, 'months');

return [start, end];
}
@@ -12,7 +12,9 @@

const async = require('async'),
request = require('request'),
cheerio = require('cheerio');
cheerio = require('cheerio'),
assert = require('assert'),
argument = require('./argument.js');

const urlTemplate = 'http://markmail.org/search/?q=list%3A' +
'org.apache.{{listname}}%20date%3A{{date}}';
@@ -28,24 +30,100 @@ const lists = [
'couchdb-marketing'
];

function getMessageCounts (date, cb) {
const listUrls = lists.map(function (list) {
function api (queryParams, timeframe, cb) {
assert.ok(queryParams.queryCurr, 'queryParams must be defined');
assert.ok(queryParams.queryDiff, 'queryParams must be defined');
assert.equal(typeof timeframe, 'number',
'timeframe must be a number');
assert.equal(typeof cb, 'function', 'callback must a a function');

const listUrlsCurr = getUrls(queryParams.queryCurr),
listUrlsDiff = getUrls(queryParams.queryDiff);

console.log(listUrlsCurr);
console.log(listUrlsDiff);

async.parallel({
current: function (cb) {
getMessageCounts(listUrlsCurr, cb);
},
diff: function (cb) {
getMessageCounts(listUrlsDiff, cb);
}
},
function (err, res) {
const data = joinDiffWithCurrent(res);
cb(null, data);
});
}

function joinDiffWithCurrent (structure) {
const curr = structure.current,
diff = structure.diff;

return curr.reduce(function (acc, el) {
const name = el[0],
count = el[1],
countOld = pick(name, diff);

acc[name] = {
curr: normalize(count),
old: normalize(countOld),
diff: getDiffString(count, countOld)
};

return acc;
}, {});
}

function pick (element, structure) {
return structure.reduce(function (acc, row) {
if (row[0] === element) {
acc = acc + row[1];
}
return acc;
}, 0);
}

function getDiffString (count, countOld) {
const result = normalize(countOld) - normalize(count);

if (result >= 0) {
return '+' + result;
}
return '' + result;
}

function normalize (string) {
return parseInt(string.replace(',', ''), 10);
}

function getUrls (date) {
return lists.map(function (list) {
return urlTemplate
.replace('{{date}}', date)
.replace('{{listname}}', list);
});
}

function requestWithOptions (url, cb) {
request({
uri: url,
pool: {
maxSockets: Infinity
}
}, function (err, res, body) {
cb(err, body);
})
}

async.map(listUrls, request, function (err, results) {
function getMessageCounts (urlList, cb) {
async.map(urlList, requestWithOptions, function (err, results) {
if (err) {
return cb(err);
}

const bodies = results.reduce(function (acc, cur) {
acc.push(cur.request.req.res.body);
return acc;
}, []);

const res = bodies.map(function (markup) {
const res = results.map(function (markup) {
const $ = cheerio.load(markup),
count = $('#lists .count').text(),
list = $('#lists a').text().replace('org.apache.couchdb.', '');
@@ -57,4 +135,4 @@ function getMessageCounts (date, cb) {
});
}

module.exports = getMessageCounts;
module.exports = api;
@@ -5,13 +5,17 @@
"description": "I'm helping to prepare board reports",
"main": "lib/index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "mocha -R spec"
},
"author": "Robert Kowalski <rok@kowalski.gd>",
"license": "Apache License, Version 2.0",
"dependencies": {
"async": "0.9.0",
"cheerio": "0.18.0",
"moment": "2.8.3",
"request": "2.48.0"
},
"devDependencies": {
"mocha": "~2.0.1"
}
}
@@ -0,0 +1,42 @@
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

const assert = require('assert'),
argument = require('../lib/argument.js');

describe('arguments', function () {

it('validates arguments', function () {
assert.ok(argument.validArg('201401'));
assert.equal(argument.validArg('2ente'), false);
assert.equal(argument.validArg('2014123'), false);
});

it('has a method for getting names of months', function () {
assert.equal(argument.getMonthAsWord('201401'), 'January');
assert.equal(argument.getMonthAsWord('201412'), 'December');
});

it('prepares query parameters', function () {
const queryParams = argument.prepareQueryParams('201411');

assert.equal(queryParams.queryCurr, '201408-201411');
assert.equal(queryParams.queryDiff, '201405-201408');
});

it('prepares urls for the current and the diff', function () {
const queryParams = argument.prepareQueryParams('201411');

assert.equal(queryParams.queryCurr, '201408-201411');
assert.equal(queryParams.queryDiff, '201405-201408');
});
});

0 comments on commit 0ec747a

Please sign in to comment.