Fix bug #1 where empty field were not correctly handled, not perfect …

…yet but now tests are all passing
adaltas · Sep 28, 2010 · a969685 · a969685
1 parent e9767af
commit a969685
Show file tree

Hide file tree

Showing 12 changed files with 106 additions and 10 deletions.
diff --git a/lib/csv.js b/lib/csv.js
@@ -140,14 +140,22 @@ module.exports = function(){
 		chars = ''+chars;
 		for (var i = 0, l = chars.length; i < l; i++) {
 			var c = chars.charAt(i);
+//			console.log(c);
 			switch (c) {
 				case csv.readOptions.escape:
 				case csv.readOptions.quote:
 					if( state.commented ) break;
 					var isEscape = false;
 					if (c === csv.readOptions.escape) {
+						// Make sure the escape is really here for escaping:
+						// if escape is same as quote, and escape is first char of a field and it's not quoted, then it is a quote
+						// next char should be an escape or a quote
 						var nextChar = chars.charAt(i + 1);
-						if (nextChar === csv.readOptions.escape || nextChar === csv.readOptions.quote) {
+//						console.log('-------------- '+c+' '+nextChar+' |'+''+'|');
+//						console.log('-------------- '+(!( csv.readOptions.escape === csv.readOptions.quote && !state.field && !state.quoted )));
+
+						if ( !( csv.readOptions.escape === csv.readOptions.quote && !state.field && !state.quoted )
+						&&   ( nextChar === csv.readOptions.escape || nextChar === csv.readOptions.quote ) ) {
 							i++;
 							isEscape = true;
 							c = chars.charAt(i);
@@ -246,8 +254,13 @@ module.exports = function(){
 						}
 						var containsdelimiter = field.indexOf(csv.writeOptions.delimiter||csv.readOptions.delimiter)>=0;
 						var containsQuote = field.indexOf(csv.writeOptions.quote||csv.readOptions.quote)>=0;
+						// http://simonwillison.net/2006/Jan/20/escape/
+						RegExp.escape = function(text) {
+						    return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+						}
+						var quote = new RegExp(csv.writeOptions.quote||csv.readOptions.quote,'g');
 						if(containsQuote){
-							field = field.replace(csv.writeOptions.quote||csv.readOptions.quote,(csv.writeOptions.escape||csv.readOptions.escape)+(csv.writeOptions.quote||csv.readOptions.quote));
+							field = field.replace(quote,(csv.writeOptions.escape||csv.readOptions.escape)+(csv.writeOptions.quote||csv.readOptions.quote));
 						}
 						if(containsQuote||containsdelimiter){
 							field = (csv.writeOptions.quote||csv.readOptions.quote)+field+(csv.writeOptions.quote||csv.readOptions.quote);

diff --git a/readme.md b/readme.md
@@ -1,11 +1,12 @@
 
 <pre>
-             _   _           _         _____  _______      __
-            | \ | |         | |       / ____|/ ____\ \    / /
-            |  \| | ___   __| | ___  | |    | (___  \ \  / / 
-            | . ` |/ _ \ / _` |/ _ \ | |     \___ \  \ \/ /  
-            | |\  | (_) | (_| |  __/ | |____ ____) |  \  /   
-            |_| \_|\___/ \__,_|\___|  \_____|_____/    \/   
+ _   _           _        _____  _______      __ _____  
+| \ | |         | |      / ____|/ ____\ \    / /|  __ \  
+|  \| | ___   __| | ___ | |    | (___  \ \  / / | |__) |_ _ _ __ ___  ___ _ __ 
+| . ` |/ _ \ / _` |/ _ \| |     \___ \  \ \/ /  |  ___/ _` | '__/ __|/ _ \ '__|
+| |\  | (_) | (_| |  __/| |____ ____) |  \  /   | |  | (_| | |  \__ \  __/ |  
+|_| \_|\___/ \__,_|\___| \_____|_____/    \/    |_|   \__,_|_|  |___/\___|_|  
+
 </pre>
 
 This project provide CSV parsing and has been tested and used on large source file (over 2Gb).

diff --git a/test/delimiter.js b/test/delimiter.js
@@ -0,0 +1,33 @@
+
+// Test CSV - Copyright David Worms <open@adaltas.com> (MIT Licensed)
+
+var fs = require('fs'),
+	csv = require('csv');
+
+module.exports = {
+	'Test empty value': function(assert){
+		csv()
+		.fromPath(__dirname+'/delimiter/empty_value.in')
+		.toPath(__dirname+'/delimiter/empty_value.tmp')
+		.transform(function(data,index){
+			assert.strictEqual(5,data.length);
+			if(index===0){
+				assert.strictEqual('',data[1]);
+				assert.strictEqual('',data[4]);
+			}else if(index===1){
+				assert.strictEqual('',data[0]);
+				assert.strictEqual('',data[3]);
+				assert.strictEqual('',data[4]);
+			}
+			return data;
+		})
+		.on('end',function(count){
+			assert.strictEqual(2,count);
+			assert.equal(
+				fs.readFileSync(__dirname+'/delimiter/empty_value.out').toString(),
+				fs.readFileSync(__dirname+'/delimiter/empty_value.tmp').toString()
+			);
+			fs.unlink(__dirname+'/delimiter/empty_value.tmp');
+		});
+	}
+}
diff --git a/test/delimiter/empty_value.in b/test/delimiter/empty_value.in
@@ -0,0 +1,2 @@
+20322051544,,8.8017226E7,45,
+,1974,8.8392926E7,,
diff --git a/test/delimiter/empty_value.out b/test/delimiter/empty_value.out
@@ -0,0 +1,2 @@
+20322051544,,8.8017226E7,45,
+,1974,8.8392926E7,,
diff --git a/test/escape/default.in b/test/escape/default.in
@@ -1,2 +1,2 @@
-20322051544,"19""79.0",8.8017226E7,ABC,45,2000-01-01
+20322051544,"19""79.0",8.8017226E7,"A""B""C",45,2000-01-01
 28392898392,1974.0,8.8392926E7,DEF,23,2050-11-27
diff --git a/test/escape/default.out b/test/escape/default.out
@@ -1,2 +1,2 @@
-20322051544,"19""79.0",8.8017226E7,ABC,45,2000-01-01
+20322051544,"19""79.0",8.8017226E7,"A""B""C",45,2000-01-01
 28392898392,1974.0,8.8392926E7,DEF,23,2050-11-27
diff --git a/test/quotes.js b/test/quotes.js
@@ -46,5 +46,42 @@ module.exports = {
 			);
 			fs.unlink(__dirname+'/quotes/in_field.tmp');
 		});
+	},
+	'Test empty value': function(assert){
+		csv()
+		.fromPath(__dirname+'/quotes/empty_value.in',{
+			quote: '"',
+			escape: '"',
+		})
+		.toPath(__dirname+'/quotes/empty_value.tmp')
+		.on('end',function(){
+			assert.equal(
+				fs.readFileSync(__dirname+'/quotes/empty_value.out').toString(),
+				fs.readFileSync(__dirname+'/quotes/empty_value.tmp').toString()
+			);
+			fs.unlink(__dirname+'/quotes/empty_value.tmp');
+		});
+	},
+	'Test quoted quote': function(assert){
+		csv()
+		.fromPath(__dirname+'/quotes/quoted.in',{
+			quote: '"',
+			escape: '"',
+		})
+		.toPath(__dirname+'/quotes/quoted.tmp')
+		.on('data',function(data,index){
+			assert.strictEqual(5,data.length);
+			if(index===0){
+				assert.strictEqual('"',data[1]);
+				assert.strictEqual('"ok"',data[4]);
+			}
+		})
+		.on('end',function(){
+			assert.equal(
+				fs.readFileSync(__dirname+'/quotes/quoted.out').toString(),
+				fs.readFileSync(__dirname+'/quotes/quoted.tmp').toString()
+			);
+			fs.unlink(__dirname+'/quotes/quoted.tmp');
+		});
 	}
 }
diff --git a/test/quotes/empty_value.in b/test/quotes/empty_value.in
@@ -0,0 +1,2 @@
+20322051544,"",8.8017226E7,45,""
+"",1974,8.8392926E7,"",""
diff --git a/test/quotes/empty_value.out b/test/quotes/empty_value.out
@@ -0,0 +1,2 @@
+20322051544,,8.8017226E7,45,
+,1974,8.8392926E7,,
diff --git a/test/quotes/quoted.in b/test/quotes/quoted.in
@@ -0,0 +1,2 @@
+20322051544,"""",8.8017226E7,45,"""ok"""
+"",1974,8.8392926E7,"",""
diff --git a/test/quotes/quoted.out b/test/quotes/quoted.out
@@ -0,0 +1,2 @@
+20322051544,"""",8.8017226E7,45,"""ok"""
+,1974,8.8392926E7,,