This repository has been archived by the owner on Nov 16, 2019. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
MULTISKIM all squashed and cleaned up.
* Replaced cuttlefish with multi-fs. * Removed the dependency on mcouch; restructured signficantly. * Added bunyan logging output for more diagnosible logs. * Added 'log' events to emit debugging information. * Switched from tap testing to lab testing. * Expanded the tests somewhat & based them on a couchdb snapshot starting point. * Tweaked command-line options slightly. * Added semicolons like a stodgy person. * Added a functional test in `npm run dev` that skims from fullfatdb into a local couchdb.
- Loading branch information
C J Silverio
committed
May 7, 2014
1 parent
841ad86
commit 96609ed
Showing
21 changed files
with
1,306 additions
and
547 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| node_modules | ||
| .sequence | ||
| .DS_Store |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| test/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| box: wercker/nodejs | ||
| services: | ||
| - mies/couchdb | ||
|
|
||
| build: | ||
| steps: | ||
| - npm-install | ||
| - npm-test | ||
|
|
||
| - script: | ||
| name: echo nodejs information | ||
| code: | | ||
| echo "node version $(node -v) running" | ||
| echo "npm version $(npm -v) running" | ||
| - script: | ||
| name: curl | ||
| code: | | ||
| echo $WERCKER_COUCHDB_HOST | ||
| curl -X PUT "$WERCKER_COUCHDB_HOST:5984/registry" | ||
| curl -X GET "$WERCKER_COUCHDB_HOST:5984/_all_dbs" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,92 +1,116 @@ | ||
| #!/usr/bin/env node | ||
| var createClient = require('manta-client'); | ||
| var manta = require('manta'); | ||
| var Skim = require('../'); | ||
| var dashdash = require('dashdash'); | ||
|
|
||
| var | ||
| Skimmer = require('../skim'), | ||
| bunyan = require('bunyan'), | ||
| dashdash = require('dashdash'), | ||
| MultiFS = require('multi-fs'), | ||
| path = require('path'), | ||
| util = require('util') | ||
| ; | ||
|
|
||
| var parser = dashdash.createParser({ | ||
| options: [ | ||
| { names: [ 'config', 'f' ], | ||
| type: 'string', | ||
| help: 'config file for multifs targets; required', | ||
| helpArg: 'FILE' }, | ||
| { names: [ 'seq-file', 'Q' ], | ||
| type: 'string', | ||
| help: 'File to store the sequence in', | ||
| help: 'File to store the sequence in, required', | ||
| helpArg: 'FILE' }, | ||
| { names: [ 'seq', 'q' ], | ||
| type: 'number', | ||
| help: 'Sequence ID to start at', | ||
| help: 'Sequence ID to start at; overrides sequence in file', | ||
| helpArg: 'NUMBER' }, | ||
| { names: [ 'registry', 'r' ], | ||
| type: 'string', | ||
| help: 'The registry where attachments can be found. Optional.', | ||
| help: 'The registry where attachments can be found; optional', | ||
| helpArg: 'URL' }, | ||
| { names: [ 'inactivity-ms' ], | ||
| type: 'number', | ||
| help: 'Max ms to wait before assuming disconnection.', | ||
| helpArg: 'MS' }, | ||
| { names: [ 'delete', 'd' ], | ||
| type: 'bool', | ||
| help: 'Delete removed attachments and docs from manta' }, | ||
| { names: [ 'skim', 's'] , | ||
| help: 'Delete removed attachments and docs from targets' }, | ||
| { names: [ 'skimdb', 's'] , | ||
| type: 'string', | ||
| helpArg: 'URL', | ||
| help: 'Target to write attachment free docs. ' + | ||
| 'Defaults to put back into COUCHDB arg.' } | ||
|
|
||
| ].concat(manta.DEFAULT_CLI_OPTIONS) | ||
| help: 'Target to write attachment-free docs. ' + | ||
| 'Defaults to put back into COUCHDB arg.' }, | ||
| { names: ['help', 'h'], | ||
| type: 'bool', | ||
| help: 'Print this help and exit' }, | ||
| ] | ||
| }); | ||
|
|
||
| var opts = parser.parse(process.argv, process.env); | ||
| var args = opts._args; | ||
|
|
||
| if (opts.help || args.length !== 4) | ||
| return usage(); | ||
|
|
||
| var client = createClient(process.argv, process.env); | ||
| var db = args[2]; | ||
| var path = args[3]; | ||
| var seqFile = opts.seq_file; | ||
| var seq = opts.seq; | ||
| var inactivity_ms = opts.inactivity_ms; | ||
| var del = opts.delete; | ||
| var skim = opts.skim || opts.db; | ||
| var registry = opts.registry || null; | ||
| if (opts.help || args.length !== 3) | ||
| { | ||
| usage(); | ||
| process.exit(); | ||
| } | ||
|
|
||
| if (args.length !== 3) | ||
| { | ||
| usage(); | ||
| process.exit(1); | ||
| } | ||
|
|
||
| if (!db || !path) { | ||
| usage(); | ||
| process.exit(1); | ||
| function usage() | ||
| { | ||
| console.log( | ||
| 'npm-skim-registry - Skim the fat out of your registry couchdb\n' + | ||
| 'Usage: npm-skim-registry [args] COUCHDB\n' + | ||
| '\n' + | ||
| ' COUCHDB Full url to your couch, e.g.,\n'+ | ||
| ' http://localhost:5984/database'); | ||
| console.log(parser.help()); | ||
| } | ||
|
|
||
| function usage() { | ||
| console.log(usage.toString().split(/\n/).slice(4, -2).join('\n')); | ||
| console.log(parser.help()); | ||
| /* | ||
| npm-skim-registry - Skim the fat out of your registry couchdb | ||
| Usage: npm-skim-registry [args] COUCHDB MANTAPATH | ||
| var logopts = | ||
| { | ||
| name: 'npm-skim-registry', | ||
| serializers: bunyan.stdSerializers, | ||
| streams: [ ] | ||
| }; | ||
|
|
||
| COUCHDB Full url to your couch, like | ||
| http://localhost:5984/database | ||
| MANTAPATH Remote path in Manta, like | ||
| ~~/stor/database | ||
| */ | ||
| } | ||
| if (process.env.NODE_ENV === 'dev') | ||
| logopts.streams.push({level: 'debug', stream: process.stdout}); | ||
| else | ||
| logopts.streams.push({level: 'info', stream: process.stdout}); | ||
|
|
||
| var logger = bunyan.createLogger(logopts); | ||
|
|
||
| Skim({ | ||
| client: client, | ||
| db: db, | ||
| path: path, | ||
| seqFile: seqFile, | ||
| inactivity_ms: inactivity_ms, | ||
| seq: seq, | ||
| delete: del, | ||
| skim: skim, | ||
| registry: registry | ||
|
|
||
| var targets = require(path.resolve(opts.config)); | ||
| var client = new MultiFS(targets); | ||
|
|
||
| var skimmer = new Skimmer({ | ||
| client: client, | ||
| sequence: opts.seq, | ||
| sequenceFile: opts.seq_file, | ||
| inactivity_ms: opts.inactivity_ms, | ||
| delete: opts.delete, | ||
| source: args[2], | ||
| skimdb: opts.skimdb, | ||
| registry: opts.registry || null, | ||
| }).on('put', function(change) { | ||
| console.log('PUT %s', change.id); | ||
| logger.info('PUT ' + change.id); | ||
| }).on('rm', function(change) { | ||
| console.log('RM %s', change.id); | ||
| logger.info('RM ' + change.id); | ||
| }).on('send', function(change, file) { | ||
| console.log('-> sent %s/%s', change.id, file.name); | ||
| logger.info(util.format('-> sent %s', file)); | ||
| }).on('delete', function(change, remote) { | ||
| console.log('-> deleted %s/%s', change.id, remote); | ||
| logger.info(util.format('-> deleted %s/%s', change.id, remote)); | ||
| }).on('putBack', function(change) { | ||
| console.error('-> putback %s', change.id); | ||
| logger.warning(util.format('-> putback %s', change.id)); | ||
| }).on('log', function(msg) { | ||
| // logger.debug('LOG: ' + msg); | ||
| }); | ||
|
|
||
| skimmer.start(); | ||
| logger.info('now skimming ' + skimmer.source + ' into ' + skimmer.skimdb); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.