Skip to content

Commit

Permalink
Fix for issue #37.
Browse files Browse the repository at this point in the history
  • Loading branch information
@martindrapeau
martindrapeau committed Apr 1, 2018
1 parent 37e71c9 commit 632b070
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 143 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ if (defined('CURL_SSLVERSION_TLSv1')) {
define('AWS_S3_SECRET', 'XXXX');
define('AWS_S3_REGION', 'us-east-2');
define('AWS_S3_BUCKET', 'csvjson');
define('AWS_S3_URL', 'http://s3.'.AWS_S3_REGION.'.amazonaws.com/'.AWS_S3_BUCKET.'/');
define('AWS_S3_URL', 'https://s3.'.AWS_S3_REGION.'.amazonaws.com/'.AWS_S3_BUCKET.'/');
}
$config['aws_s3'] = array('supported' => defined('AWS_S3_URL'));
```
Expand Down
11 changes: 5 additions & 6 deletions js/csvjson.min.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

226 changes: 91 additions & 135 deletions js/csvjson/csv2json.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,134 +19,90 @@
* Copyright (c) 2014 Martin Drapeau
*
*/

var errorDetectingSeparator = "We could not detect the separator.",
errorEmpty = "Please upload a file or type in something.",
errorEmptyHeader = "Could not detect header. Ensure first row cotains your column headers.",
separators = [",", ";", "\t"];

function detectSeparator(csv) {
var counts = {},
sepMax;
_.each(separators, function(sep, i) {
var re = new RegExp(sep, 'g');
counts[sep] = (csv.match(re) || []).length;
sepMax = !sepMax || counts[sep] > counts[sepMax] ? sep : sepMax;
});
return sepMax ? sepMax : undefined;
}

// source: http://stackoverflow.com/a/1293163/2343
function CSVtoArray( strData, strDelimiter ){
// Check to see if the delimiter is defined. If not,
// then default to comma.
strDelimiter = (strDelimiter || ",");

// Create a regular expression to parse the CSV values.
var objPattern = new RegExp(
(
// Delimiters.
"(\\" + strDelimiter + "|\\r?\\n|\\r|^)" +

// Quoted fields.
"(?:\"([^\"]*(?:\"\"[^\"]*)*)\"|" +

// Standard fields.
"([^\\" + strDelimiter + "\\r\\n]*))"
),
"gi"
);


// Create an array to hold our data. Give the array
// a default empty first row.
var arrData = [[]];

// Create an array to hold our individual pattern
// matching groups.
var arrMatches = null;


// Keep looping over the regular expression matches
// until we can no longer find a match.
while (arrMatches = objPattern.exec( strData )){

// Get the delimiter that was found.
var strMatchedDelimiter = arrMatches[ 1 ];

// Check to see if the given delimiter has a length
// (is not the start of string) and if it matches
// field delimiter. If id does not, then we know
// that this delimiter is a row delimiter.
if (
strMatchedDelimiter.length &&
strMatchedDelimiter !== strDelimiter
){

// Since we have reached a new row of data,
// add an empty row to our data array.
arrData.push( [] );

}

var strMatchedValue;

// Now that we have our delimiter out of the way,
// let's check to see which kind of value we
// captured (quoted or unquoted).
if (arrMatches[ 2 ]){

// We found a quoted value. When we capture
// this value, unescape any double quotes.
strMatchedValue = arrMatches[ 2 ].replace(
new RegExp( "\"\"", "g" ),
"\""
);

} else {

// We found a non-quoted value.
strMatchedValue = arrMatches[ 3 ];

}


// Now that we have our value string, let's add
// it to the data array.
arrData[ arrData.length - 1 ].push( strMatchedValue );
}
var errorDetectingSeparator = "We could not detect the separator.",
errorNotWellFormed = "CSV is not well formed",
errorEmpty = "Please upload a file or type in something.",
errorEmptyHeader = "Could not detect header. Ensure first row cotains your column headers.",
separators = [",", ";", "\t"];

function detectSeparator(csv) {
var counts = {},
sepMax;
_.each(separators, function(sep, i) {
var re = new RegExp(sep, 'g');
counts[sep] = (csv.match(re) || []).length;
sepMax = !sepMax || counts[sep] > counts[sepMax] ? sep : sepMax;
});
return sepMax ? sepMax : undefined;
}

// source: https://stackoverflow.com/questions/8493195/how-can-i-parse-a-csv-string-with-javascript-which-contains-comma-in-data
function CSVtoArray(text, delimiter) {
delimiter || (delimiter = ',');
// convert two doublw quotes "" by escaped double quotes \".
text = text.replace(/(,|{\s)""(.*)""(\s*:)/g, '$1\\"$2\\"$3').replace(/(?<!,\s*)""(?!\s*,)/g, '\\"');

var re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/;
var re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;
if (delimiter == ';') {
re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^;'"\s\\]*(?:\s+[^;'"\s\\]+)*)\s*(?:;\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^;'"\s\\]*(?:\s+[^;'"\s\\]+)*)\s*)*$/;
re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^;'"\s\\]*(?:\s+[^;'"\s\\]+)*))\s*(?:;|$)/g;
}
else if (delimiter == '\t') {
re_valid = /^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^\t'"\s\\]*(?:\s+[^\t'"\s\\]+)*)\s*(?:\t\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^\t'"\s\\]*(?:\s+[^\t'"\s\\]+)*)\s*)*$/;
re_value = /(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^\t'"\s\\]*(?:\s+[^\t'"\s\\]+)*))\s*(?:\t|$)/g;
}

// Return the parsed data.
return( arrData );
if (!re_valid.test(text)) return null;
var a = []; // Initialize array to receive values.
text.replace(re_value, // "Walk" the string using replace with callback.
function(m0, m1, m2, m3) {
// Remove backslash from \' in single quoted values.
if (m1 !== undefined) a.push(m1.replace(/\\'/g, "'"));
// Remove backslash from \" in double quoted values.
else if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
else if (m3 !== undefined) a.push(m3);
return ''; // Return empty string.
});
// Handle special case of empty last value.
if (/,\s*$/.test(text)) a.push('');
return a;
};

function convert(csv, options) {
options || (options = {});
if (csv.length == 0) throw errorEmpty;

var separator = options.separator || detectSeparator(csv);
if (!separator) throw errorDetectingSeparator;

var lines = csv.split(/\r?\n/),
a = [];
for (var l = 0; l < lines.length; l++) {
var line = CSVtoArray(lines[l], options.separator);
if (line === null) throw errorNotWellFormed+'\n'+lines[l];
a.push(line)
}

function convert(csv, options) {
options || (options = {});
if (csv.length == 0) throw errorEmpty;

var separator = options.separator || detectSeparator(csv);
if (!separator) throw errorDetectingSeparator;

var a = CSVtoArray(csv, separator);
if (!a) throw errorEmpty;

if (options.transpose) a = _.zip.apply(_, a);
if (options.transpose) a = _.zip.apply(_, a);

var keys = a.shift();
if (keys.length == 0) throw errorEmptyHeader;
keys = _.map(keys, function(key) {
return _(key).chain().trim().trim('"').value();
});

var keys = a.shift();
if (keys.length == 0) throw errorEmptyHeader;
keys = _.map(keys, function(key) {
return _(key).chain().trim().trim('"').value();
});

var json = options.hash ? {} : [];
for (var l = 0; l < a.length; l++) {
var row = {},
hashKey;
for (var i = 0; i < keys.length; i++) {
var value = _(a[l][i]).chain().trim().trim('"').value(),
number = value === "" ? NaN : value - 0;
if (options.hash && i == 0) {
hashKey = value;
var json = options.hash ? {} : [];
for (var l = 0; l < a.length; l++) {
var row = {},
hashKey;
for (var i = 0; i < keys.length; i++) {
var value = _(a[l][i]).chain().trim().trim('"').value(),
number = value === "" ? NaN : value - 0;
if (options.hash && i == 0) {
hashKey = value;
}
else {
if (options.parseJSON || options.parseNumbers && !isNaN(number)) {
Expand All @@ -160,17 +116,17 @@
row[keys[i]] = value;
}
}
}
if (options.hash)
json[hashKey] = row;
else
json.push(row);
}
return json;
};
this.CSVJSON || (this.CSVJSON = {});
this.CSVJSON.csv2json = convert;
}
if (options.hash)
json[hashKey] = row;
else
json.push(row);
}

return json;
};

this.CSVJSON || (this.CSVJSON = {});
this.CSVJSON.csv2json = convert;

}).call(this);
2 changes: 1 addition & 1 deletion js/src/version
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.184
0.186

0 comments on commit 632b070

Please sign in to comment.