Permalink
Browse files

Initial commit.

  • Loading branch information...
0 parents commit c1ed28a4237d0a7132c7d31d4c699b765f226802 @cliffano committed Apr 27, 2012
Showing with 274 additions and 0 deletions.
  1. +4 −0 .gitignore
  2. +2 −0 CHANGELOG.md
  3. +20 −0 LICENSE
  4. +20 −0 README.md
  5. +5 −0 bin/datagen
  6. +1 −0 examples/footer
  7. +2 −0 examples/header
  8. +3 −0 examples/segment
  9. +47 −0 lib/cli.js
  10. +41 −0 lib/datagen.js
  11. +29 −0 lib/reader.js
  12. +15 −0 lib/worker.js
  13. +39 −0 lib/writer.js
  14. +46 −0 package.json
@@ -0,0 +1,4 @@
+.DS_Store
+build
+node_modules
+npm-debug.log
@@ -0,0 +1,2 @@
+### 0.0.1
+* Initial version
20 LICENSE
@@ -0,0 +1,20 @@
+Copyright (C) 2012 by Cliffano Subagio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
@@ -0,0 +1,20 @@
+datagen [![http://travis-ci.org/cliffano/datagen](https://secure.travis-ci.org/cliffano/datagen.png?branch=master)](http://travis-ci.org/cliffano/datagen)
+-----------
+
+TODO
+
+Installation
+------------
+
+ npm install -g datagen
+
+Usage
+-----
+
+TODO
+
+Colophon
+--------
+
+Follow [@cliffano](http://twitter.com/cliffano) on Twitter.
+
@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+var cli = require('../lib/cli');
+cli.exec();
+
@@ -0,0 +1 @@
+</data>
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<data>
@@ -0,0 +1,3 @@
+<segment>
+ <id>{run_id}-{worker_id}-{segment_id}</id>
+</segment>
@@ -0,0 +1,47 @@
+var cly = require('cly'),
+ DataGen = require('./datagen').DataGen,
+ p = require('path');
+
+function exec() {
+
+ var dataGenDir = __dirname,
+ commands = {
+ init: {
+ callback: function (args) {
+ console.log('Creating DataGen configuration files');
+ cly.copyDir(p.join(dataGenDir, '../examples'), '.', cly.exit);
+ }
+ },
+ run: {
+ options: {
+ runId: {
+ string: '-r run_id',
+ help: 'An ID unique the execution of this command. Default: master process PID'
+ },
+ numSegments: {
+ string: '-s num_segments',
+ help: 'How many segments in a data file. Default: 1'
+ },
+ numWorkers: {
+ string: '-w num_workers',
+ help: 'How many worker processes, each worker creates a data file. Default: 1'
+ },
+ outFile: {
+ string: '-o output_file',
+ help: 'Output file name, which will be appended with worker ID. Default: \'out\''
+ }
+ },
+ callback: function (args) {
+ new DataGen(
+ args.numSegments,
+ args.numWorkers,
+ args.outFile
+ ).run(args.runId, cly.exit);
+ }
+ }
+ };
+
+ cly.parse(dataGenDir, 'datagen', commands);
+}
+
+exports.exec = exec;
@@ -0,0 +1,41 @@
+var childProcess = require('child_process'),
+ Reader = require('./reader').Reader,
+ worker = require('./worker'),
+ p = require('path');
+
+function DataGen(numSegments, numWorkers, outFile) {
+
+ var templates = ['header', 'segment', 'footer'],
+ numSegments = numSegments || 1,
+ numWorkers = numWorkers || 1,
+ outFile = outFile || 'out';
+
+ function run(runId, cb) {
+
+ var runId = runId || process.pid,
+ data = new Reader(templates).data(),
+ workers = [];
+
+ // prepare all workers
+ for (var i = 0; i < numWorkers; i += 1) {
+ workers.push(childProcess.fork(p.join(__dirname, 'worker.js')));
+ }
+
+ // run all workers
+ for (var i = 0, ln = workers.length; i < ln; i += 1) {
+ workers[i].send({
+ workerId: i,
+ data: data,
+ numSegments: numSegments,
+ params: { run_id: runId, worker_id: i },
+ outFile: outFile + i
+ });
+ }
+ }
+
+ return {
+ run: run
+ };
+}
+
+exports.DataGen = DataGen;
@@ -0,0 +1,29 @@
+var fs = require('fs');
+
+function Reader(templates) {
+
+ function _read(file) {
+ var data;
+ try {
+ fs.stat(file);
+ data = fs.readFileSync(file).toString();
+ } catch (err) {
+ console.warn('Ignoring ' + file + ' due to ' + err.message);
+ }
+ return data;
+ }
+
+ function data() {
+ var data = {};
+ templates.forEach(function (template) {
+ data[template] = _read(template);
+ });
+ return data;
+ }
+
+ return {
+ data: data
+ };
+}
+
+exports.Reader = Reader;
@@ -0,0 +1,15 @@
+var Writer = require('./writer').Writer;
+
+process.on('message', function (message) {
+ console.log('Start worker ' + message.workerId);
+ new Writer().gen(
+ message.data,
+ message.numSegments,
+ message.params,
+ message.outFile,
+ function () {
+ console.log('Finish worker ' + message.workerId);
+ process.exit(1);
+ }
+ );
+});
@@ -0,0 +1,39 @@
+var _ = require('underscore'),
+ fs = require('fs');
+
+function Writer() {
+
+ function gen(data, numSegments, params, outFile, cb) {
+ var stream = fs.createWriteStream(outFile,
+ { flags: 'w', encoding: 'utf-8' }),
+ segmentId = 0,
+ segment;
+
+ stream.on('error', function (err) {
+ console.error('Error: ' + err);
+ });
+
+ stream.on('close', function () {
+ cb();
+ });
+
+ stream.write(data.header);
+
+ while (segmentId++ < numSegments) {
+ segment = data.segment;
+ params.segment_id = segmentId;
+ _.keys(params).forEach(function (param) {
+ segment = segment.replace(new RegExp('{' + param + '}', 'g'), params[param]);
+ });
+ stream.write(segment);
+ }
+
+ stream.end(data.footer);
+ }
+
+ return {
+ gen: gen
+ };
+}
+
+exports.Writer = Writer;
@@ -0,0 +1,46 @@
+{
+ "name": "datagen",
+ "description": "Multi-process data file generator",
+ "keywords": [
+ "multi",
+ "process",
+ "file",
+ "generator"
+ ],
+ "version": "0.0.1",
+ "homepage": "http://github.com/cliffano/datagen",
+ "author": "Cliffano Subagio <blah@cliffano.com> (http://blog.cliffano.com)",
+ "bin": {
+ "datagen": "./bin/datagen"
+ },
+ "repository": {
+ "type": "git",
+ "url": "http://github.com/cliffano/datagen.git"
+ },
+ "bugs": {
+ "url": "http://github.com/cliffano/datagen/issues"
+ },
+ "directories": {
+ "bin": "./bin",
+ "lib": "./lib",
+ "test": "./test"
+ },
+ "dependencies": {
+ "cly": "0.0.4",
+ "underscore": "1.3.3"
+ },
+ "devDependencies": {
+ "sandboxed-module": "0.1.3",
+ "vows": "0.6.1"
+ },
+ "engines": {
+ "node": ">= 0.6.0"
+ },
+ "licenses": [
+ {
+ "type": "MIT",
+ "url": "http://github.com/cliffano/datagen/raw/master/LICENSE"
+ }
+ ]
+}
+

0 comments on commit c1ed28a

Please sign in to comment.