Permalink
Browse files

Merge pull request #14 from esatterwhite/readstream_update

Adding a way to manage the read stream

For very large file ( 1GB+ ) the read stream will pull in data
4-5x faster then node can write it out which causes excessive
memory usage. By providing a way to manage the read stream via
pause / resume methods, one is able to parse & manage very large files
  • Loading branch information...
2 parents 9cd26e4 + e9260db commit 6e9796ec3a55dc3b18fcf3e30ec21b85a2fcb1e0 @koles committed Feb 26, 2012
Showing with 65 additions and 2 deletions.
  1. +65 −2 lib/ya-csv.js
View
67 lib/ya-csv.js
@@ -4,6 +4,24 @@ var sys = require('sys'),
var csv = exports;
+/**
+ * Provides Base CSV Reading capabilities
+ * @class CsvReader
+ * @extends EventEmitter
+ */
+
+/**
+ * The constructor
+ * @constructor
+ * @param readStream {ReadStread} An instance of the ReadStream Cl
+ * @param options {Object} optional paramaters for the reader <br/>
+ * - rows {Number}
+ * - openRecord {Array}
+ * - openField {String}
+ * - lastChar {String}
+ * - quetedField {Boolean}
+ * - commentedLine{Boolean}
+ */
var CsvReader = csv.CsvReader = function(readStream, options) {
var self = this;
_setOptions(self, options);
@@ -21,10 +39,55 @@ var CsvReader = csv.CsvReader = function(readStream, options) {
readStream.addListener('data', this.parse.bind(this));
readStream.addListener('error', this.emit.bind(this, 'error'));
readStream.addListener('end', this.end.bind(this));
+
+
+ /**
+ * Pauses the readStream
+ * @method pause
+ * @return {ReadStream} the readstream instance
+ */
+ self.pause = function(){
+ readStream.pause();
+ return self;
+ }
+
+ /**
+ * Resumes the readStream
+ * @method resume
+ * @return {ReadStream} the readstream instance
+ */
+ self.resume = function(){
+ readStream.resume();
+ return self;
+ }
+ /**
+ * Closes the readStream
+ * @method destroy
+ * @return {ReadStream} the readstream instance
+ */
+ self.destoy = function(){
+ readStream.destroy();
+ return self;
+ }
+ /**
+ * Closes the readStream when its file stream has been drained
+ * @method destroySoon
+ * @return {ReadStream} the readstream instance
+ */
+ self.destroySoon = function(){
+ readstream.destroy();
+ return self;
+ }
}
+
};
sys.inherits(CsvReader, events.EventEmitter);
+/**
+ * Parses incoming data as a readable CSV file
+ * @method parse
+ * @param data {Array} Array of values to parse from the incommin file
+ */
CsvReader.prototype.parse = function(data) {
var ps = this.parsingStatus;
if (ps.openRecord.length == 0) {
@@ -104,6 +167,7 @@ CsvReader.prototype.parse = function(data) {
}
};
+
CsvReader.prototype.end = function() {
var ps = this.parsingStatus;
if (ps.quotedField) {
@@ -119,7 +183,6 @@ CsvReader.prototype.end = function() {
this.emit('end');
}
}
-
CsvReader.prototype._isEscapable = function(c) {
if ((c === this.escapechar) || (c === this.quotechar)) {
return true;
@@ -224,7 +287,7 @@ function _appendField(outArr, writer, field) {
if(typeof(field) !== 'undefined' && field !== null) {
field = String(field);
} else {
- outArr.push('');
+ outArr.push('');
return;
}
}

0 comments on commit 6e9796e

Please sign in to comment.