Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactored libs/core and tests #150

Merged
merged 2 commits into from
Mar 27, 2017
Merged

Refactored libs/core and tests #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e834303
Refactored libs/core.
kakts Mar 16, 2017
64b7291
Refactored:
kakts Mar 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 35 additions & 41 deletions libs/core/Converter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,26 +176,26 @@ Converter.prototype.workerProcess = function (fileLine, cb) {
var eol = this.getEol();
this.setPartialData(line.partial);
this.workerMgr.sendWorker(line.lines.join(eol) + eol, this.lastIndex, cb, function (results, lastIndex) {
var buf;
var cur = self.sequenceBuffer[0];
if (cur.idx === lastIndex) {
cur.result = results;
var records = [];
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result) {
var buf = self.sequenceBuffer.shift();
buf = self.sequenceBuffer.shift();
records = records.concat(buf.result);
}
self.processResult(records);
self.recordNum += records.length;
} else {
for (var i = 0; i < self.sequenceBuffer.length; i++) {
var buf = self.sequenceBuffer[i];
for (var i = 0, len = self.sequenceBuffer.length; i < len; i++) {
buf = self.sequenceBuffer[i];
if (buf.idx === lastIndex) {
buf.result = results;
break;
}
}
}
// self.processResult(JSON.parse(results),function(){},true);
});
this.sequenceBuffer.push({
idx: this.lastIndex,
Expand All @@ -206,38 +206,38 @@ Converter.prototype.workerProcess = function (fileLine, cb) {

Converter.prototype.processHead = function (fileLine, cb) {
var params = this.param;
if (!params._headers) { //header is not inited. init header
var lines = fileLineToCSVLine(fileLine, params);
this.setPartialData(lines.partial);
if (params.noheader) {
if (params.headers) {
params._headers = params.headers;
} else {
params._headers = [];
}
if (params._headers) {
return cb();
}

// if header is not inited. init header
var lines = fileLineToCSVLine(fileLine, params);
this.setPartialData(lines.partial);
if (params.noheader) {
if (params.headers) {
params._headers = params.headers;
} else {
var headerRow = lines.lines.shift();
if (params.headers) {
params._headers = params.headers;
} else {
params._headers = headerRow;
}
}
if (this.param.workerNum > 1) {
this.workerMgr.setParams(params);
params._headers = [];
}
var res = linesToJson(lines.lines, params, 0);
this.processResult(res);
this.lastIndex += res.length;
this.recordNum += res.length;
cb();
} else {
cb();
var headerRow = lines.lines.shift();
if (params.headers) {
params._headers = params.headers;
} else {
params._headers = headerRow;
}
}
if (this.param.workerNum > 1) {
this.workerMgr.setParams(params);
}
var res = linesToJson(lines.lines, params, 0);
this.processResult(res);
this.lastIndex += res.length;
this.recordNum += res.length;
cb();
};

Converter.prototype.processResult = function (result) {

for (var i = 0, len = result.length; i < len; i++) {
var r = result[i];
if (r.err) {
Expand All @@ -246,8 +246,6 @@ Converter.prototype.processResult = function (result) {
this.emitResult(r);
}
}
// this.lastIndex+=result.length;
// cb();
};

Converter.prototype.emitResult = function (r) {
Expand Down Expand Up @@ -305,6 +303,7 @@ Converter.prototype.preProcessRaw = function (data, cb) {
cb(data);
};

// FIXME: lineNumber is not used.
Converter.prototype.preProcessLine = function (line, lineNumber) {
return line;
};
Expand All @@ -322,8 +321,9 @@ Converter.prototype._flush = function (cb) {
}
};
if (this._csvLineBuffer.length > 0) {
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] != this.getEol()) {
this._csvLineBuffer += this.getEol();
var eol = this.getEol();
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] !== eol) {
this._csvLineBuffer += eol;
}
this.processData(this._csvLineBuffer, function () {
this.checkAndFlush();
Expand All @@ -333,13 +333,7 @@ Converter.prototype._flush = function (cb) {
}
return;
};
// Converter.prototype._transformFork = function(data, encoding, cb) {
// this.child.stdin.write(data, encoding, cb);
// }
// Converter.prototype._flushFork = function(cb) {
// this.child.stdin.end();
// this.child.on("exit", cb);
// }

Converter.prototype.checkAndFlush = function () {
if (this._csvLineBuffer.length !== 0) {
this.emit("error", CSVError.unclosed_quote(this.recordNum, this._csvLineBuffer), this._csvLineBuffer);
Expand Down Expand Up @@ -410,7 +404,7 @@ Converter.prototype.transf = function (func) {
};

Converter.prototype.fromString = function (csvString, cb) {
if (typeof csvString != "string") {
if (typeof csvString !== "string") {
return cb(new Error("Passed CSV Data is not a string."));
}
if (cb && typeof cb === "function") {
Expand Down
8 changes: 4 additions & 4 deletions libs/core/defaultParsers/parser_flat.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
module.exports = {
"name": "flat",
"processSafe":true,
"processSafe": true,
"regExp": /^\*flat\*/,
"parserFunc": function parser_flat (params) {
var key=this.getHeadStr();
var val=params.item;
params.resultRow[key]=val;
var key = this.getHeadStr();
var val = params.item;
params.resultRow[key] = val;
}
};
4 changes: 2 additions & 2 deletions libs/core/defaultParsers/parser_json.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ module.exports = {
"parserFunc": function parser_json(params) {
var fieldStr = this.getHeadStr();
var headArr = (params.config && params.config.flatKeys) ? [fieldStr] : fieldStr.split('.');
var match, index, key, pointer;
var match, index, key;
//now the pointer is pointing the position to add a key/value pair.
var pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
key = headArr.shift();
Expand All @@ -60,7 +60,7 @@ module.exports = {
params.resultRow[fieldStr] = params.item;
}
} else {
if (typeof pointer=== "string"){
if (typeof pointer === "string"){
params.resultRow[fieldStr] = params.item;
}else{
pointer[key] = params.item;
Expand Down
10 changes: 1 addition & 9 deletions libs/core/fileline.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,5 @@ module.exports = function(data, param) {
var eol = getEol(data,param);
var lines = data.split(eol);
var partial = lines.pop();
// if (param.ignoreEmpty){
// var trimmedLines=[];
// for (var i=0;i<lines.length;i++){
// trimmedLines.push(lines[i].trim())
// }
// return {lines:trimmedLines,partial:partial};
// }else{
return {lines: lines, partial: partial};
// }
return {lines: lines, partial: partial};
};
88 changes: 18 additions & 70 deletions libs/core/linesToJson.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
var parserMgr = require("./parserMgr.js");
var Parser = require("./parser");
var CSVError = require("./CSVError");
var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;
/**
Expand Down Expand Up @@ -48,18 +47,9 @@ function processRows(csvRows, params, startIndex) {
return res;
}

function getConstParser(number, param) {
var inst = new Parser("field" + number, /.*/, function (params) {
var name = this.getName();
params.resultRow[name] = params.item;
}, true);
inst.setParam(param);
return inst;
}

function processRow(row, param, index) {
var parseRules = param.parseRules;
if (param.checkColumn && row.length != parseRules.length) {
if (param.checkColumn && row.length !== parseRules.length) {
return {
err: CSVError.column_mismatched(index)
};
Expand Down Expand Up @@ -91,14 +81,9 @@ function convertRowToJson(row, headRow, param) {
}
hasValue = true;

// parser = parseRules[i];
// if (!parser) {
// parser = parseRules[i] = getConstParser(i + 1, param);
// }
head = headRow[i];
if (!head || head === "") {
head = headRow[i] = "field" + (i + 1);
// parser.initHead(head);
}
var flag = getFlag(head, i, param);
if (flag === 'omit') {
Expand All @@ -114,16 +99,6 @@ function convertRowToJson(row, headRow, param) {
} else {
setPath(resultRow, title, item);
}
// _.set(resultRow,head,item)
// parser.parse({
// head: head,
// item: item,
// itemIndex: i,
// rawRow: row,
// resultRow: resultRow,
// rowIndex: index,
// config: param || {}
// });
}
if (hasValue) {
return resultRow;
Expand All @@ -145,41 +120,37 @@ function setPath(json, path, value) {
function getFlag(head, i, param) {
if (typeof param._headerFlag[i] === "string") {
return param._headerFlag[i];
} else if (head.indexOf('*omit*') > -1) {
return param._headerFlag[i] = 'omit';
} else if (head.indexOf('*flat*') > -1) {
return param._headerFlag[i] = 'flat';
} else {
if (head.indexOf('*omit*') > -1) {
return param._headerFlag[i] = 'omit';
} else if (head.indexOf('*flat*') > -1) {
return param._headerFlag[i] = 'flat';
} else {
return param._headerFlag[i] = '';
}
return param._headerFlag[i] = '';
}
}

function getTitle(head, i, param) {
if (param._headerTitle[i]) {
return param._headerTitle[i];
} else {
var flag = getFlag(head, i, param);
var str = head.replace(flag, '');
str = str.replace('string#!', '').replace('number#!', '');
return param._headerTitle[i] = str;
}

var flag = getFlag(head, i, param);
var str = head.replace(flag, '');
str = str.replace('string#!', '').replace('number#!', '');
return param._headerTitle[i] = str;
}

function checkType(item, head, headIdx, param) {
if (param._headerType[headIdx]) {
return param._headerType[headIdx];
} else if (head.indexOf('number#!') > -1) {
return param._headerType[headIdx] = numberType;
} else if (head.indexOf('string#!') > -1) {
return param._headerType[headIdx] = stringType;
} else if (param.checkType) {
return param._headerType[headIdx] = dynamicType;
} else {
if (head.indexOf('number#!') > -1) {
return param._headerType[headIdx] = numberType;
} else if (head.indexOf('string#!') > -1) {
return param._headerType[headIdx] = stringType;
} else if (param.checkType) {
return param._headerType[headIdx] = dynamicType;
} else {
return param._headerType[headIdx] = stringType;
}
return param._headerType[headIdx] = stringType;
}
}

Expand All @@ -206,7 +177,6 @@ function dynamicType(item) {
return booleanType(item);
} else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
return jsonType(item);

} else {
return stringType(item);
}
Expand All @@ -228,25 +198,3 @@ function jsonType(item) {
return item;
}
}
// function dynamicType(item) {
// var trimed = item.trim();
// if (trimed === "") {
// return trimed;
// }
// if (!isNaN(trimed)) {
// return parseFloat(trimed);
// } else if (trimed.length === 5 && trimed.toLowerCase() === "false") {
// return false;
// } else if (trimed.length === 4 && trimed.toLowerCase() === "true") {
// return true;
// } else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
// try {
// return JSON.parse(trimed);
// } catch (e) {
// return item;
// }
// } else {
// return item;

// }
// }
1 change: 0 additions & 1 deletion libs/core/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ Parser.prototype.clone = function() {
newParser[key] = obj[key];
}
return newParser;
//return new Parser(this.name, this.regExp, this.parse, this.processSafe);
};

Parser.prototype.getName = function() {
Expand Down
Loading