Permalink
Browse files

Isolate trim tests; Reimplement trim to preserve csv parsing; Rename …

…trim options
  • Loading branch information...
1 parent 9bccb5b commit 92a9687333058505e1199e9e1c024a02a4839b1f @wdavidw committed Apr 10, 2011
View
@@ -34,11 +34,12 @@ module.exports = function(){
quote: '"',
escape: '"',
columns: null,
- ltrimFields: false,
- rtrimFields: false,
flags: 'r',
encoding: 'utf8',
- bufferSize: 8 * 1024 * 1024
+ bufferSize: 8 * 1024 * 1024,
+ trim: false,
+ ltrim: false,
+ rtrim: false
};
this.writeOptions = {
delimiter: null,
@@ -60,7 +61,6 @@ module.exports = function(){
var self = this;
process.nextTick(function(){
if(data instanceof Array){
- // todo, check if we can now remove those
if( csv.writeOptions.lineBreaks === null ){
csv.writeOptions.lineBreaks = "\r\n";
}
@@ -119,7 +119,11 @@ module.exports = function(){
} else {
// dump open record
if (state.field) {
- flushField();
+ if(csv.readOptions.trim || csv.readOptions.rtrim){
+ state.field = state.field.trimRight();
+ }
+ state.line.push(state.field);
+ state.field = '';
}
if (state.line.length > 0) {
flush();
@@ -300,7 +304,11 @@ module.exports = function(){
if( state.quoted ) {
state.field += c;
}else{
- flushField();
+ if(csv.readOptions.trim || csv.readOptions.rtrim){
+ state.field = state.field.trimRight();
+ }
+ state.line.push(state.field);
+ state.field = '';
}
break;
case '\n':
@@ -319,10 +327,20 @@ module.exports = function(){
if( csv.writeOptions.lineBreaks === null ){
csv.writeOptions.lineBreaks = c + ( c === '\r' && chars.charAt(i+1) === '\n' ? '\n' : '' );
}
-
- flushField();
+ if(csv.readOptions.trim || csv.readOptions.rtrim){
+ state.field = state.field.trimRight();
+ }
+ state.line.push(state.field);
+ state.field = '';
flush();
break;
+ case ' ':
+ case '\t':
+ if(state.quoted || (!csv.readOptions.trim && !csv.readOptions.ltrim ) || state.field) {
+ state.field += c;
+ break;
+ }
+ break;
default:
if(state.commented) break;
state.field += c;
@@ -331,19 +349,6 @@ module.exports = function(){
}
}
- function flushField() {
- var newField = state.field;
- if ( csv.readOptions.ltrimFields ) {
- newField = newField.replace(/^\s+/, '');
- }
- if ( csv.readOptions.rtrimFields ) {
- newField = newField.replace(/\s+$/, '');
- }
-
- state.line.push(newField);
- state.field = '';
- }
-
function flush(){
if(csv.readOptions.columns){
if(state.count === 0 && csv.readOptions.columns === true){
View
@@ -94,11 +94,14 @@ Options are:
- *columns*
List of fields or true if autodiscovered in the first CSV line, impact the `transform` argument and the `data` event by providing an object instead of an array, order matters, see the transform and the columns section below.
-
-- *ltrimFields*
+
+- *trim*
+ If true, ignore whitespace immediately around the delimiter, default to false.
+
+- *ltrim*
If true, ignore whitespace immediately following the delimiter (i.e. left-trim all fields), default to false.
-
-- *rtrimFields*
+
+- *rtrim*
If true, ignore whitespace immediately preceding the delimiter (i.e. right-trim all fields), default to false.
Writing API
View
@@ -30,53 +30,5 @@ module.exports = {
);
fs.unlink(__dirname+'/delimiter/empty_value.tmp');
});
- },
- 'Test ignoring whitespace immediately following the delimiter': function(){
- csv()
- .fromPath(__dirname+'/delimiter/init_whitespace.in', {ltrimFields: true})
- .toPath(__dirname+'/delimiter/init_whitespace.tmp')
- .transform(function(data,index){
- return data;
- })
- .on('end',function(count){
- assert.strictEqual(3,count);
- assert.equal(
- fs.readFileSync(__dirname+'/delimiter/init_whitespace.out').toString(),
- fs.readFileSync(__dirname+'/delimiter/init_whitespace.tmp').toString()
- );
- fs.unlink(__dirname+'/delimiter/init_whitespace.tmp');
- });
- },
- 'Test ignoring whitespace immediately preceding the delimiter': function(){
- csv()
- .fromPath(__dirname+'/delimiter/trail_whitespace.in', {rtrimFields: true})
- .toPath(__dirname+'/delimiter/trail_whitespace.tmp')
- .transform(function(data,index){
- return data;
- })
- .on('end',function(count){
- assert.strictEqual(3,count);
- assert.equal(
- fs.readFileSync(__dirname+'/delimiter/trail_whitespace.out').toString(),
- fs.readFileSync(__dirname+'/delimiter/trail_whitespace.tmp').toString()
- );
- fs.unlink(__dirname+'/delimiter/trail_whitespace.tmp');
- });
- },
- 'Test ignoring whitespace both immediately preceding and following the delimiter': function(){
- csv()
- .fromPath(__dirname+'/delimiter/init_and_trail_whitespace.in', {ltrimFields: true, rtrimFields: true})
- .toPath(__dirname+'/delimiter/init_and_trail_whitespace.tmp')
- .transform(function(data,index){
- return data;
- })
- .on('end',function(count){
- assert.strictEqual(3,count);
- assert.equal(
- fs.readFileSync(__dirname+'/delimiter/init_and_trail_whitespace.out').toString(),
- fs.readFileSync(__dirname+'/delimiter/init_and_trail_whitespace.tmp').toString()
- );
- fs.unlink(__dirname+'/delimiter/init_and_trail_whitespace.tmp');
- });
}
}
@@ -1,3 +0,0 @@
-FIELD_1, FIELD_2, FIELD_3, FIELD_4, FIELD_5, FIELD_6
-20322051544, 1979,8.8017226E7, ABC,45,2000-01-01
-28392898392, 1974, 8.8392926E7,DEF, 23, 2050-11-27
@@ -1,3 +0,0 @@
-FIELD_1,FIELD_2,FIELD_3,FIELD_4,FIELD_5,FIELD_6
-20322051544,1979,8.8017226E7,ABC,45,2000-01-01
-28392898392,1974,8.8392926E7,DEF,23,2050-11-27
View
@@ -0,0 +1,77 @@
+
+// Test CSV - Copyright David Worms <open@adaltas.com> (MIT Licensed)
+
+var fs = require('fs'),
+ assert = require('assert'),
+ csv = require('csv');
+
+module.exports = {
+ 'Test ignoring whitespace immediately following the delimiter': function(){
+ csv()
+ .fromPath(__dirname+'/trim/ltrim.in', {ltrim: true})
+ .toPath(__dirname+'/trim/ltrim.tmp')
+ .transform(function(data,index){
+ return data;
+ })
+ .on('end',function(count){
+ assert.strictEqual(3,count);
+ // console.log('');
+ // console.log(fs.readFileSync(__dirname+'/trim/ltrim.tmp').toString());
+ // console.log('');
+ // console.log(fs.readFileSync(__dirname+'/trim/ltrim.out').toString());
+ assert.equal(
+ fs.readFileSync(__dirname+'/trim/ltrim.out').toString(),
+ fs.readFileSync(__dirname+'/trim/ltrim.tmp').toString()
+ );
+ fs.unlink(__dirname+'/trim/ltrim.tmp');
+ });
+ },
+ 'Test rtrim - ignoring whitespace immediately preceding the delimiter': function(){
+ csv()
+ .fromPath(__dirname+'/trim/rtrim.in', {rtrim: true})
+ .toPath(__dirname+'/trim/rtrim.tmp')
+ .transform(function(data,index){
+ return data;
+ })
+ .on('end',function(count){
+ assert.strictEqual(3,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/trim/rtrim.out').toString(),
+ fs.readFileSync(__dirname+'/trim/rtrim.tmp').toString()
+ );
+ fs.unlink(__dirname+'/trim/rtrim.tmp');
+ });
+ },
+ 'Test trim - ignoring whitespace both immediately preceding and following the delimiter': function(){
+ csv()
+ .fromPath(__dirname+'/trim/trim.in', {trim: true})
+ .toPath(__dirname+'/trim/trim.tmp')
+ .transform(function(data,index){
+ return data;
+ })
+ .on('end',function(count){
+ assert.strictEqual(3,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/trim/trim.out').toString(),
+ fs.readFileSync(__dirname+'/trim/trim.tmp').toString()
+ );
+ fs.unlink(__dirname+'/trim/trim.tmp');
+ });
+ },
+ 'Test no trim': function(){
+ csv()
+ .fromPath(__dirname+'/trim/notrim.in')
+ .toPath(__dirname+'/trim/notrim.tmp')
+ .transform(function(data,index){
+ return data;
+ })
+ .on('end',function(count){
+ assert.strictEqual(3,count);
+ assert.equal(
+ fs.readFileSync(__dirname+'/trim/notrim.out').toString(),
+ fs.readFileSync(__dirname+'/trim/notrim.tmp').toString()
+ );
+ fs.unlink(__dirname+'/trim/notrim.tmp');
+ });
+ }
+}
View
@@ -0,0 +1,3 @@
+FIELD_1, FIELD_2, FIELD_3, FIELD_4, FIELD_5, FIELD_6
+20322051544," 1979",8.8017226E7, ABC,45,2000-01-01
+28392898392, " 1974", 8.8392926E7,DEF, 23, 2050-11-27
View
@@ -0,0 +1,3 @@
+FIELD_1,FIELD_2,FIELD_3,FIELD_4,FIELD_5,FIELD_6
+20322051544, 1979,8.8017226E7,ABC,45,2000-01-01
+28392898392, 1974,8.8392926E7,DEF,23,2050-11-27
File renamed without changes.
View
@@ -0,0 +1,3 @@
+ FIELD_1 , FIELD_2 , FIELD_3,FIELD_4 , FIELD_5,FIELD_6
+20322051544,1979 ,8.8017226E7,ABC , 45 , 2000-01-01
+ 28392898392, 1974,8.8392926E7,DEF , 23 , 2050-11-27
File renamed without changes.
File renamed without changes.
View
@@ -0,0 +1,3 @@
+ FIELD_1 , FIELD_2 , FIELD_3,FIELD_4 , FIELD_5,FIELD_6
+20322051544,1979 ,8.8017226E7,ABC , 45 , 2000-01-01
+ 28392898392, 1974,8.8392926E7,DEF , 23 , 2050-11-27
File renamed without changes.

0 comments on commit 92a9687

Please sign in to comment.