Permalink
Browse files

Merge pull request #17 from dsquier/master

Add util functions, callback response object
  • Loading branch information...
2 parents 4fd5ba2 + f3b9464 commit 42148a895ac248233d2098c13c87b573d85770fc @cliffano committed Jun 8, 2017
Showing with 149 additions and 71 deletions.
  1. +1 −1 .travis.yml
  2. +8 −1 CHANGELOG.md
  3. +25 −9 README.md
  4. +8 −1 examples/couchtato.js
  5. +4 −6 lib/cli.js
  6. +20 −18 lib/couchtato.js
  7. +12 −22 lib/db.js
  8. +35 −4 lib/util.js
  9. +11 −8 package.json
  10. +12 −0 test/buster.js
  11. +13 −1 test/util.js
View
@@ -1,8 +1,8 @@
sudo: false
language: node_js
node_js:
- - "4"
- "6"
+ - "8"
env:
- CXX=g++-4.8
addons:
View
@@ -1,3 +1,10 @@
+### 0.3.0
+* Implement ES6 features and require Node 6.10+ [David Squier](https://github.com/dsquier)
+* Add response object to report callback [David Squier](https://github.com/dsquier)
+* Add util.audit() adds an object to an array that is returned upon [David Squier](https://github.com/dsquier)
+completion
+* Add util.hash() which returns a SHA256 hash using node-object-hash [David Squier](https://github.com/dsquier)
+
### 0.2.2
*
@@ -37,7 +44,7 @@
* Replace 'c' variable in couchtato.js tasks module with a more descriptive 'util'
* Replace cradle with nano, replace nomnom and Config with bagofholding
* Remove -d option, nano as a couchdb driver is fine
-* Set min node engine to >= 0.6.0, max node engine to < 0.9.0
+* Set min node engine to >= 0.6.0, max node engine to < 0.9.0
* Replace init command with config
### 0.0.5
View
@@ -4,10 +4,8 @@
[![Dependencies Status](https://img.shields.io/david/cliffano/couchtato.svg)](http://david-dm.org/cliffano/couchtato)
[![Coverage Status](https://img.shields.io/coveralls/cliffano/couchtato.svg)](https://coveralls.io/r/cliffano/couchtato?branch=master)
[![Published Version](https://img.shields.io/npm/v/couchtato.svg)](http://www.npmjs.com/package/couchtato)
-<br/>
[![npm Badge](https://nodei.co/npm/couchtato.png)](http://npmjs.org/package/couchtato)
-
Couchtato
---------
@@ -21,14 +19,14 @@ Installation
------------
npm install -g couchtato
-
+
Usage
-----
Create sample couchtato.js configuration file:
couchtato config
-
+
Iterate through all documents in a CouchDB database:
couchtato iterate -u http://user:pass@host:port/db
@@ -48,7 +46,7 @@ Iterate through documents within a range of IDs:
Only iterate the first 5 pages where each page contains 1000 documents:
couchtato iterate -u http://user:pass@host:port/db -n 5 -p 1000
-
+
Save/remove docs in bulk of 20000 documents at a time:
couchtato iterate -u http://user:pass@host:port/db -b 20000
@@ -90,6 +88,13 @@ Specify the task functions in config file. Each function in exports.conf.tasks w
"count-by-field": function (util, doc) {
util.count(doc.status);
},
+ "hash-doc": function (util, doc) {
+ const hash = util.hash(doc);
+ util.log('hash:' + hash);
+ },
+ "audit-object": function (util, doc) {
+ util.audit(doc);
+ },
"whatever": function (util, doc) {
// you need to implement whatever function
whatever(doc);
@@ -116,19 +121,26 @@ That 'util' in function (util, doc) is a utility variable, it provides you with
# save the document back to the database
util.save(doc)
-
+
# remove the document from the database
util.remove(doc)
-
+
# increment a counter associated with a particular key
# all counters will be displayed in the summary report
util.count('somekey')
-
+
# log a message to both the console and to couchtato.log file
# if you only want to display a message on the console,
# simply use good old console.log(message)
util.log(message)
+ # generate a SHA256 hash for a given document, object, or string
+ util.hash(doc)
+
+ # add an object to the audit array, which is returned in the
+ # callback and can be used for downstream processing
+ util.audit(doc)
+
Report
------
@@ -148,7 +160,11 @@ FAQ
Q: Why am I getting 'exports' is undefined Microsoft JScript runtime error on Windows?
-A: Since Couchtato's default config file is called couchtato.js, Windows tried to execute couchtato.js instead of couchtato command, which then resulted in the above error. A workaround to this problem is to rename couchtato.js to config.js, and then use -c/--config-file flag, e.g. `couchtato --config-file config.js iterate --url http://user:pass@host:port/db` .
+A: Since Couchtato's default config file is called couchtato.js, Windows tried to execute couchtato.js instead of couchtato command, which then resulted in the above error. A workaround to this problem is to rename couchtato.js to config.js, and then use -c/--config-file flag, e.g. `couchtato --config-file config.js iterate --url http://user:pass@host:port/db`.
+
+Q: What is the purpose of `util.audit` and/or the audit array?
+
+A: The audit array is a convenient way to store data while iterating through documents. All objects added via `util.audit()` will be returned in the callback response upon completion. This is a powerful way to chain processing steps via messaging queues, lambda functions, or monitoring tools.
Colophon
--------
View
@@ -2,7 +2,7 @@ exports.conf = {
"tasks": {
"all_docs": function (util, doc) {
util.log(doc);
-
+
// That 'util' in function (util, doc) is a utility variable,
// it provides you with the following convenient functions:
@@ -16,6 +16,13 @@ exports.conf = {
// all counters will be displayed in the summary report
// util.count('somekey')
+ // generate an SHA-256 hash from any object or document
+ // const hash = util.hash(doc)
+
+ // add an object to an audit array which can be used for
+ // downstream processing after completion of run.
+ // util.audit(doc)
+
// log a message to both the console and to couchtato.log file
// if you only want to display a message on the console,
// simply use good old console.log(message)
View
@@ -1,4 +1,4 @@
-var cli = require('bagofcli'),
+const cli = require('bagofcli'),
Couchtato = require('./couchtato'),
p = require('path');
@@ -8,8 +8,7 @@ function _config() {
}
function _iterate(args) {
-
- var config = require(p.join(process.cwd(), args.configFile || 'couchtato')),
+ const config = require(p.join(process.cwd(), args.configFile || 'couchtato')),
opts = {
batchSize: (args.batchSize) ? parseInt(args.batchSize, 10) : undefined,
pageSize: (args.pageSize) ? parseInt(args.pageSize, 10) : undefined,
@@ -26,8 +25,7 @@ function _iterate(args) {
* Execute Couchtato CLI.
*/
function exec() {
-
- var actions = {
+ const actions = {
commands: {
config: { action: _config },
iterate: { action: _iterate }
@@ -37,4 +35,4 @@ function exec() {
cli.command(__dirname, actions);
}
-exports.exec = exec;
+exports.exec = exec;
View
@@ -1,4 +1,4 @@
-var _ = require('lodash'),
+const _ = require('lodash'),
Db = require('./db'),
fsx = require('fs.extra'),
p = require('path'),
@@ -28,15 +28,14 @@ Couchtato.prototype.config = function (cb) {
* @param {Function} cb: standard cb(err, result) callback
*/
Couchtato.prototype.iterate = function (tasks, url, opts, cb) {
-
opts.batchSize = opts.batchSize || 1000;
opts.pageSize = opts.pageSize || 1000;
opts.numPages = opts.numPages || undefined;
opts.startKey = opts.startKey || null;
opts.interval = opts.interval || 1000;
opts.quiet = opts.quiet || false;
- var _db = new Db(url),
+ const _db = new Db(url),
_util = new Util({
_couchtato_docs: 0,
_couchtato_pages: 0,
@@ -45,12 +44,11 @@ Couchtato.prototype.iterate = function (tasks, url, opts, cb) {
}, [], _db.db);
function _pageCb(rows) {
-
- var docsCount = (rows.length === opts.pageSize + 1) ? opts.pageSize : rows.length;
+ const docsCount = (rows.length === opts.pageSize + 1) ? opts.pageSize : rows.length;
if (opts.quiet === false) {
console.log('retrieved %d doc%s - %s', docsCount, docsCount > 1 ? 's' : '', rows[0].doc._id);
}
-
+
_util.increment('_couchtato_docs', docsCount);
_util.increment('_couchtato_pages', 1);
@@ -62,7 +60,8 @@ Couchtato.prototype.iterate = function (tasks, url, opts, cb) {
});
// bulk update queued documents
- var queuedDocs = _util.getQueue().slice(0); // a copy of the queue
+ const queuedDocs = _util.getQueue().slice(0); // a copy of the queue
+
if (queuedDocs.length >= opts.batchSize) {
console.log('updating %d doc%s - %s', queuedDocs.length, queuedDocs.length > 1 ? 's' : '', queuedDocs[0]._id);
_db.update(queuedDocs, function (err, result) {
@@ -76,32 +75,35 @@ Couchtato.prototype.iterate = function (tasks, url, opts, cb) {
}
}
- function _endCb(err) {
+ function _endCb(err, result) {
+ function _report(err, result) {
+ const audit = _util.getAudit();
+ const stat = _util.getStat();
- function _report(err) {
if (opts.quiet === false) {
- var stat = _util.getStat(),
- report = '\n------------------------\n' +
- 'Retrieved ' + stat._couchtato_docs + ' documents in ' + stat._couchtato_pages + ' pages\n' +
- 'Processed ' + stat._couchtato_save + ' saves and ' + stat._couchtato_remove + ' removes\n';
+ let report = '\n------------------------\n' +
+ 'Retrieved ' + stat._couchtato_docs + ' documents in ' + stat._couchtato_pages + ' pages\n' +
+ 'Processed ' + stat._couchtato_save + ' saves and ' + stat._couchtato_remove + ' removes\n';
_.keys(stat).forEach(function (prop) {
if (!prop.match(/^_couchtato_.+/)) {
report += '- ' + prop + ': ' + stat[prop] + '\n';
}
});
_util.log(report);
}
- cb(err);
+
+ // return a callback response object containing the stat object and audit array
+ cb(err, { stat, audit });
}
- var queue = _util.getQueue();
- if (!_.isEmpty(queue)) {
+ const queue = _util.getQueue();
+ if (!_.isEmpty(queue)) {
// update the remaining queued documents
_db.update(queue, function (err, result) {
function _wait() {
if (_db.done()) {
- _report(err);
+ _report(err, result);
} else {
setImmediate(_wait);
}
@@ -110,7 +112,7 @@ Couchtato.prototype.iterate = function (tasks, url, opts, cb) {
});
// if queue is empty, then just report regardless there's an error or not
} else {
- _report(err);
+ _report(err, result);
}
}
View
@@ -1,5 +1,4 @@
-var _ = require('lodash'),
- async = require('async'),
+const async = require('async'),
nano = require('nano'),
url = require('url');
@@ -11,10 +10,8 @@ var _ = require('lodash'),
*/
function Db(_url, opts) {
opts = opts || {};
-
- var dbPath = url.parse(_url.replace(/\/$/, '')).pathname.replace(/^\//, ''),
- dbPathElems = dbPath.split('/'),
- dbUrl;
+ const dbPath = url.parse(_url.replace(/\/$/, '')).pathname.replace(/^\//, '');
+ const dbPathElems = dbPath.split('/');
if (dbPathElems.length === 1) {
this.dbName = dbPathElems[0];
@@ -24,7 +21,7 @@ function Db(_url, opts) {
this.dbView = dbPathElems[2];
}
- dbUrl = _url.replace(new RegExp(dbPath + '$', 'g'), '');
+ const dbUrl = _url.replace(new RegExp(dbPath + '$', 'g'), '');
this.db = opts.db || nano(dbUrl).use(this.dbName);
this.inProgress = 0;
}
@@ -42,17 +39,15 @@ function Db(_url, opts) {
* @param {Function} cb: standard cb(err, result) callback
*/
Db.prototype.paginate = function (interval, startKey, endKey, pageSize, numPages, pageCb, cb) {
-
- var done = false,
- pageCount = 0,
- key = startKey,
- key_docid = startKey,
- self = this;
+ const self = this;
+ let done = false;
+ let pageCount = 0;
+ let key = startKey;
+ let key_docid = startKey;
function _action(cb) {
-
// both startkey and startkey_docid are specified to handle a view index with duplicated keys
- var opts = {
+ const opts = {
startkey: key,
startkey_docid: key_docid,
limit: pageSize + 1,
@@ -67,13 +62,10 @@ Db.prototype.paginate = function (interval, startKey, endKey, pageSize, numPages
}
function _actionCb(err, result) {
-
if (err) {
cb(err);
-
} else {
pageCb(result.rows);
-
// when the number of retrieved documents is less than the limit,
// or the page count is the number of pages to retrieve,
// that means it's the last page
@@ -115,17 +107,15 @@ Db.prototype.paginate = function (interval, startKey, endKey, pageSize, numPages
* @param {Function} cb: standard cb(err, result) callback
*/
Db.prototype.update = function (docs, cb) {
-
this.inProgress += 1;
-
- var self = this;
+ const self = this;
// NOTE: when CouchDB instance requires authentication and the URL does not supply a username and password,
// nano bulk callback is called with null error and unmodified documents in CouchDB as the result, ignoring the modified documents.
// Ideally nano should pass an error.
this.db.bulk({ docs: docs }, function (err, result) {
self.inProgress -= 1;
cb(err, result);
- });
+ });
};
/**
Oops, something went wrong.

0 comments on commit 42148a8

Please sign in to comment.