Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Add columns support

  • Loading branch information...
commit d41f96dd01f616f75bae48ac32eefcba68af788d 1 parent 9e0965a
@wdavidw authored
View
32 lib/csv.js
@@ -33,6 +33,7 @@ module.exports = function(){
delimiter: ',',
quote: '"',
escape: '"',
+ columns: null,
flags: 'r',
encoding: 'utf8',
bufferSize: 8 * 1024 * 1024
@@ -201,12 +202,41 @@ module.exports = function(){
}
function flush(){
+ if(csv.readOptions.columns){
+ if(state.count===0&&csv.readOptions.columns===true){
+ csv.readOptions.columns = state.line;
+ state.line = [];
+ state.lastC = '';
+ return;
+ }
+ var line = {};
+ csv.readOptions.columns.forEach(function(column,i){
+ line[column] = state.line[i]||null;
+ })
+ state.line = line;
+ line = null;
+ }
var line = csv.transformer?csv.transformer(state.line,state.count):state.line;
if(line !== null){
csv.emit('data',line,state.count);
}
if(line !== null){
if(typeof line === 'object'){
+ if(!(line instanceof Array)){
+ var columns = csv.writeOptions.columns||csv.readOptions.columns;
+ var _line = [];
+ if(columns){
+ columns.forEach(function(column,i){
+ _line[i] = line[column]||'';
+ })
+ }else{
+ for(var column in line){
+ _line.push(line[column]);
+ }
+ }
+ line = _line;
+ _line = null;
+ }
if(line instanceof Array){
var newLine = state.count?csv.writeOptions.lineBreaks:'';
line.forEach(function(field,i){
@@ -224,8 +254,6 @@ module.exports = function(){
}
});
line = newLine;
- }else{
-
}
}
if(state.buffer){
View
71 readme.md
@@ -27,7 +27,6 @@ Using the library is a 4 steps process:
3. Transform the data (optional)
4. Listen to events (optional)
-<pre class="javascript">
var csv = require('csv');
csv()
.fromPath(__dirname+'/sample.in')
@@ -45,16 +44,19 @@ Using the library is a 4 steps process:
.on('error',function(error){
console.log(error.message);
});
-</pre>
Installing
----------
-Manually
-Simply copy or link the lib/csv.js file into your $HOME/.node_libraries folder or inside declared path folder.
+Via git (or downloaded tarball):
+
+ $ git clone git://github.com/ajaxorg/cloud9.git
+
+Then, simply copy or link the lib/csv.js file into your $HOME/.node_libraries folder or inside a declared path folder.
+
+Via [npm](http://github.com/isaacs/npm):
-NPM
-Simply install the project with `npm install node-csv` and you'll be ready to go.
+ $ npm install csv
Creating a source
-----------------
@@ -69,6 +71,9 @@ Options are:
- *escape*
Set the field delimiter, one character only, default to double quotes.
+
+- *columns*
+ List of fields or true if autodiscovered in the first CSV lien, impact the `transform` argument and the `data` event by providing an object instead of an array, order matters, see the transform and the columns section below.
The following method are available:
@@ -87,16 +92,19 @@ Creating a destination
Options are:
- *delimiter*
- Default to the delimiter read option
+ Default to the delimiter read option.
- *quote*
- Default to the quote read option
+ Default to the quote read option.
- *escape*
- Default to the escape read option
+ Default to the escape read option.
+
+- *columns*
+ List of fields, apply when `transform` return an object, order matters, see the transform and the columns sections below.
- *encoding*
- Default to 'utf8'
+ Default to 'utf8', apply when a writable stream is created.
- *lineBreaks*
String used to delimite record rows or a special value; special values are 'auto', 'unix', 'mac', 'windows', 'unicode'; default to 'auto' (discovered in source).
@@ -120,6 +128,22 @@ Transforming data
You may provide a callback to the `transform` method. The contract is quite simple, you recieve an array of fields for each record and return the transformed record. The return value may be an array, an associative array, a string or null. If null, the record will simply be skipped.
+Unless you specify the `columns` read option, `data` are provided as arrays, otherwise they are objects with keys matching columns names.
+
+When the returned value is an array, the fields are merge in order. When the returned value is an object, it will search for the `columns` property in the write or in the read options and smartly order the values. If no `columns` options are found, it will merge the values in their order of appearance. When the returned value is a string, it directly sent to the destination source and it is your responsibility to delimit, quote, escape or define line breaks.
+
+Exemple returning a string
+
+ csv()
+ .fromPath('user.csv')
+ .toStream(process.stdout)
+ .transform(function(data,index){
+ return (index>0 ? ',' : '') + data[4] + ":" + data[3];
+ });
+
+ // Print sth like:
+ // david:32,ewa:30
+
Events
------
@@ -137,6 +161,33 @@ By extending the Node `EventEmitter` class, the library provide a few usefull ev
- *error*
Thrown whenever an error is captured.
+Columns
+-------
+
+Columns names may be provided or discovered in the first line with the read options `columns`. If defined as an array, the order must match the input source. If set to `true`, the fields are expected to be present in the first line of the input source.
+
+You can define a different order and even different columns in the read options and in the write options. If the `columns` is not defined in the write options, it will default to the one present in the read options.
+
+When working with fields, the `transform` method and the `data` events recieve their `data` parameter as an object instead of an array where the keys are the field names.
+
+ var csv = require('csv');
+ csv()
+ .fromPath(__dirname+'/sample.in')
+ .toPath(__dirname+'/sample.out')
+ .transform(function(data){
+ data.
+ return data;
+ })
+ .on('data',function(data,index){
+ console.log('#'+index+' '+JSON.stringify(data));
+ })
+ .on('end',function(count){
+ console.log('Number of lines '+count);
+ })
+ .on('error',function(error){
+ console.log(error.message);
+ });
+
Running the tests
-----------------
View
1  test/buffer.js
@@ -4,7 +4,6 @@
var fs = require('fs'),
csv = require('csv');
-
module.exports = {
'Buffer smaller than in': function(assert){
csv()
View
88 test/columns.js
@@ -0,0 +1,88 @@
+
+// Test CSV - Copyright David Worms <open@adaltas.com> (MIT Licensed)
+
+var fs = require('fs'),
+ csv = require('csv');
+
+module.exports = {
+ 'Test columns in true': function(assert){
+ // Note: if true, columns are expected to be in first line
+ csv()
+ .fromPath(__dirname+'/columns/in_true.in',{
+ columns: true
+ })
+ .toPath(__dirname+'/columns/in_true.tmp')
+ .transform(function(data,index){
+ assert.equal(true,data instanceof Object);
+ assert.equal(false,data instanceof Array);
+ if(index===0){
+ assert.strictEqual('20322051544',data.FIELD_1);
+ }else if(index===1){
+ assert.strictEqual('DEF',data.FIELD_4);
+ }
+ return data;
+ })
+ .on('end',function(count){
+ assert.strictEqual(2,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/columns/in_true.out').toString(),
+ fs.readFileSync(__dirname+'/columns/in_true.tmp').toString()
+ );
+ fs.unlink(__dirname+'/columns/in_true.tmp');
+ });
+ },
+ 'Test columns in named': function(assert){
+ // Note: if true, columns are expected to be in first line
+ csv()
+ .fromPath(__dirname+'/columns/in_named.in',{
+ columns: ["FIELD_1","FIELD_2","FIELD_3","FIELD_4","FIELD_5","FIELD_6"]
+ })
+ .toPath(__dirname+'/columns/in_named.tmp')
+ .transform(function(data,index){
+ assert.equal(true,data instanceof Object);
+ assert.equal(false,data instanceof Array);
+ if(index===0){
+ assert.strictEqual('20322051544',data.FIELD_1);
+ }else if(index===1){
+ assert.strictEqual('DEF',data.FIELD_4);
+ }
+ return data;
+ })
+ .on('data',function(data,index){
+ assert.equal(true,data instanceof Object);
+ assert.equal(false,data instanceof Array);
+ })
+ .on('end',function(count){
+ assert.strictEqual(2,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/columns/in_named.out').toString(),
+ fs.readFileSync(__dirname+'/columns/in_named.tmp').toString()
+ );
+ fs.unlink(__dirname+'/columns/in_named.tmp');
+ });
+ },
+ 'Test columns out named': function(assert){
+ // Note: if true, columns are expected to be in first line
+ csv()
+ .fromPath(__dirname+'/columns/out_named.in')
+ .toPath(__dirname+'/columns/out_named.tmp',{
+ columns: ["FIELD_1","FIELD_2"]
+ })
+ .transform(function(data,index){
+ assert.equal(true,data instanceof Array);
+ return {FIELD_2:data[3],FIELD_1:data[4]};
+ })
+ .on('data',function(data,index){
+ assert.equal(true,data instanceof Object);
+ assert.equal(false,data instanceof Array);
+ })
+ .on('end',function(count){
+ assert.strictEqual(2,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/columns/out_named.out').toString(),
+ fs.readFileSync(__dirname+'/columns/out_named.tmp').toString()
+ );
+ fs.unlink(__dirname+'/columns/out_named.tmp');
+ });
+ },
+}
View
2  test/columns/in_named.in
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
2  test/columns/in_named.out
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
3  test/columns/in_true.in
@@ -0,0 +1,3 @@
+FIELD_1,FIELD_2,FIELD_3,FIELD_4,FIELD_5,FIELD_6
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
2  test/columns/in_true.out
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
2  test/columns/out_named.in
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
2  test/columns/out_named.out
@@ -0,0 +1,2 @@
+45,ABC
+23,DEF
View
1  test/escape.js
@@ -4,7 +4,6 @@
var fs = require('fs'),
csv = require('csv');
-
module.exports = {
// Note: we only escape quote and escape character
'Test default': function(assert){
View
1  test/fromto.js
@@ -4,7 +4,6 @@
var fs = require('fs'),
csv = require('csv');
-
module.exports = {
'Test fs stream': function(assert){
csv()
View
1  test/quotes.js
@@ -4,7 +4,6 @@
var fs = require('fs'),
csv = require('csv');
-
module.exports = {
'Test regular quotes': function(assert){
csv()
View
36 test/transform.js
@@ -4,7 +4,6 @@
var fs = require('fs'),
csv = require('csv');
-
module.exports = {
'Test reorder fields': function(assert){
var count = 0;
@@ -44,5 +43,40 @@ module.exports = {
);
fs.unlink(__dirname+'/transform/empty.tmp');
});
+ },
+ 'Test return object': function(assert){
+ // we don't define columns
+ // recieve and array and return an object
+ // also see the columns test
+ csv()
+ .fromPath(__dirname+'/transform/object.in')
+ .toPath(__dirname+'/transform/object.tmp')
+ .transform(function(data,index){
+ return {field_1:data[4],field_2:data[3]};
+ })
+ .on('end',function(count){
+ assert.strictEqual(2,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/transform/object.out').toString(),
+ fs.readFileSync(__dirname+'/transform/object.tmp').toString()
+ );
+ fs.unlink(__dirname+'/transform/object.tmp');
+ });
+ },
+ 'Test return string': function(assert){
+ csv()
+ .fromPath(__dirname+'/transform/string.in')
+ .toPath(__dirname+'/transform/string.tmp')
+ .transform(function(data,index){
+ return (index>0 ? ',' : '') + data[4] + ":" + data[3];
+ })
+ .on('end',function(count){
+ assert.strictEqual(2,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/transform/string.out').toString(),
+ fs.readFileSync(__dirname+'/transform/string.tmp').toString()
+ );
+ fs.unlink(__dirname+'/transform/string.tmp');
+ });
}
}
View
2  test/transform/object.in
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
2  test/transform/object.out
@@ -0,0 +1,2 @@
+45,ABC
+23,DEF
View
2  test/transform/string.in
@@ -0,0 +1,2 @@
+20322051544,1979,8.8017226E7,ABC,45,2000-01-01
+28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
1  test/transform/string.out
@@ -0,0 +1 @@
+45:ABC,23:DEF
Please sign in to comment.
Something went wrong with that request. Please try again.