Skip to content

Add rowIndex to convertRowToJson #374

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Add rowIndex to convertRowToJson #374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 16 additions & 10 deletions src/Parameters.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export interface CSVParseParam {
*/
trim: boolean;
/**
* This parameter turns on and off whether check field type. Default is false.
* This parameter turns on and off whether check field type. Default is false.
*/
checkType: boolean;
/**
Expand Down Expand Up @@ -59,7 +59,7 @@ export interface CSVParseParam {
* Allows override parsing logic for a specific column. It accepts a JSON object with fields like: headName: <String | Function> . e.g. {field1:'number'} will use built-in number parser to convert value of the field1 column to number. Another example {"name":nameProcessFunc} will use specified function to parse the value.
*/
colParser: {
[key: string]: string | CellParser | ColumnParam
[key: string]: string | CellParser | ColumnParam;
};
/**
* End of line character. If omitted, parser will attempt to retrieve it from the first chunks of CSV data
Expand All @@ -70,14 +70,14 @@ export interface CSVParseParam {
*/
alwaysSplitAtEOL: boolean;
/**
* The format to be converted to. "json" (default) -- convert csv to json. "csv" -- convert csv to csv row array. "line" -- convert csv to csv line string
* The format to be converted to. "json" (default) -- convert csv to json. "csv" -- convert csv to csv row array. "line" -- convert csv to csv line string
*/
output: "json" | "csv" | "line";

/**
* Convert string "null" to null object in JSON outputs. Default is false.
*/
nullObject:boolean;
nullObject: boolean;
/**
* Define the format required by downstream (this parameter does not work if objectMode is on). `line` -- json is emitted in a single line separated by a line breake like "json1\njson2" . `array` -- downstream requires array format like "[json1,json2]". Default is line.
*/
Expand All @@ -88,7 +88,14 @@ export interface CSVParseParam {
needEmitAll: boolean;
}

export type CellParser = (item: string, head: string, resultRow: any, row: string[], columnIndex: number) => any;
export type CellParser = (
item: string,
head: string,
resultRow: any,
row: string[],
columnIndex: number,
rowIndex: number
) => any;

export interface ColumnParam {
flat?: boolean;
Expand All @@ -97,7 +104,7 @@ export interface ColumnParam {

export function mergeParams(params?: Partial<CSVParseParam>): CSVParseParam {
const defaultParam: CSVParseParam = {
delimiter: ',',
delimiter: ",",
ignoreColumns: undefined,
includeColumns: undefined,
quote: '"',
Expand All @@ -116,9 +123,9 @@ export function mergeParams(params?: Partial<CSVParseParam>): CSVParseParam {
alwaysSplitAtEOL: false,
output: "json",
nullObject: false,
downstreamFormat:"line",
needEmitAll:true
}
downstreamFormat: "line",
needEmitAll: true
};
if (!params) {
params = {};
}
Expand All @@ -133,4 +140,3 @@ export function mergeParams(params?: Partial<CSVParseParam>): CSVParseParam {
}
return defaultParam;
}

124 changes: 78 additions & 46 deletions src/lineToJson.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { ParseRuntime } from "./ParseRuntime";

var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;

export default function (csvRows: string[][], conv: Converter): JSONResult[] {
export default function(csvRows: string[][], conv: Converter): JSONResult[] {
const res: JSONResult[] = [];
for (let i = 0, len = csvRows.length; i < len; i++) {
const r = processRow(csvRows[i], conv, i);
Expand All @@ -15,34 +15,44 @@ export default function (csvRows: string[][], conv: Converter): JSONResult[] {
}
}
return res;
};
export type JSONResult = {
[key: string]: any
}
export type JSONResult = {
[key: string]: any;
};

function processRow(row: string[], conv: Converter, index): JSONResult | null {

if (conv.parseParam.checkColumn && conv.parseRuntime.headers && row.length !== conv.parseRuntime.headers.length) {
throw (CSVError.column_mismatched(conv.parseRuntime.parsedLineNumber + index))
if (
conv.parseParam.checkColumn &&
conv.parseRuntime.headers &&
row.length !== conv.parseRuntime.headers.length
) {
throw CSVError.column_mismatched(
conv.parseRuntime.parsedLineNumber + index
);
}

const headRow = conv.parseRuntime.headers || [];
const resultRow = convertRowToJson(row, headRow, conv);
const resultRow = convertRowToJson(row, headRow, conv, index);
if (resultRow) {
return resultRow;
} else {
return null;
}
}

function convertRowToJson(row: string[], headRow: string[], conv: Converter): { [key: string]: any } | null {
function convertRowToJson(
row: string[],
headRow: string[],
conv: Converter,
rowIndex: number
): { [key: string]: any } | null {
let hasValue = false;
const resultRow = {};

for (let i = 0, len = row.length; i < len; i++) {
let item = row[i];

if (conv.parseParam.ignoreEmpty && item === '') {
if (conv.parseParam.ignoreEmpty && item === "") {
continue;
}
hasValue = true;
Expand All @@ -53,9 +63,9 @@ function convertRowToJson(row: string[], headRow: string[], conv: Converter): {
}
const convFunc = getConvFunc(head, i, conv);
if (convFunc) {
const convRes = convFunc(item, head, resultRow, row, i);
const convRes = convFunc(item, head, resultRow, row, i, rowIndex);
if (convRes !== undefined) {
setPath(resultRow, head, convRes, conv,i);
setPath(resultRow, head, convRes, conv, i);
}
} else {
// var flag = getFlag(head, i, param);
Expand All @@ -67,7 +77,7 @@ function convertRowToJson(row: string[], headRow: string[], conv: Converter): {
item = convertFunc(item);
}
if (item !== undefined) {
setPath(resultRow, head, item, conv,i);
setPath(resultRow, head, item, conv, i);
}
}
}
Expand All @@ -79,17 +89,21 @@ function convertRowToJson(row: string[], headRow: string[], conv: Converter): {
}

const builtInConv: { [key: string]: CellParser } = {
"string": stringType,
"number": numberType,
"omit": function () { }
}
function getConvFunc(head: string, i: number, conv: Converter): CellParser | null {
string: stringType,
number: numberType,
omit: function() {}
};
function getConvFunc(
head: string,
i: number,
conv: Converter
): CellParser | null {
if (conv.parseRuntime.columnConv[i] !== undefined) {
return conv.parseRuntime.columnConv[i];
} else {
let flag = conv.parseParam.colParser[head];
if (flag === undefined) {
return conv.parseRuntime.columnConv[i] = null;
return (conv.parseRuntime.columnConv[i] = null);
}
if (typeof flag === "object") {
flag = (flag as ColumnParam).cellParser || "string";
Expand All @@ -98,34 +112,43 @@ function getConvFunc(head: string, i: number, conv: Converter): CellParser | nul
flag = flag.trim().toLowerCase();
const builtInFunc = builtInConv[flag];
if (builtInFunc) {
return conv.parseRuntime.columnConv[i] = builtInFunc;
return (conv.parseRuntime.columnConv[i] = builtInFunc);
} else {
return conv.parseRuntime.columnConv[i] = null;
return (conv.parseRuntime.columnConv[i] = null);
}
} else if (typeof flag === "function") {
return conv.parseRuntime.columnConv[i] = flag;
return (conv.parseRuntime.columnConv[i] = flag);
} else {
return conv.parseRuntime.columnConv[i] = null;
return (conv.parseRuntime.columnConv[i] = null);
}
}
}
function setPath(resultJson: any, head: string, value: any, conv: Converter,headIdx:number) {
function setPath(
resultJson: any,
head: string,
value: any,
conv: Converter,
headIdx: number
) {
if (!conv.parseRuntime.columnValueSetter[headIdx]) {
if (conv.parseParam.flatKeys) {
conv.parseRuntime.columnValueSetter[headIdx] = flatSetter;
} else {

if (head.indexOf(".") > -1) {
const headArr=head.split(".");
let jsonHead=true;
while(headArr.length>0){
const headCom=headArr.shift();
if (headCom!.length===0){
jsonHead=false;
const headArr = head.split(".");
let jsonHead = true;
while (headArr.length > 0) {
const headCom = headArr.shift();
if (headCom!.length === 0) {
jsonHead = false;
break;
}
}
if (!jsonHead || conv.parseParam.colParser[head] && (conv.parseParam.colParser[head] as ColumnParam).flat) {
if (
!jsonHead ||
(conv.parseParam.colParser[head] &&
(conv.parseParam.colParser[head] as ColumnParam).flat)
) {
conv.parseRuntime.columnValueSetter[headIdx] = flatSetter;
} else {
conv.parseRuntime.columnValueSetter[headIdx] = jsonSetter;
Expand All @@ -135,12 +158,11 @@ function setPath(resultJson: any, head: string, value: any, conv: Converter,head
}
}
}
if (conv.parseParam.nullObject ===true && value ==="null"){
value=null;
if (conv.parseParam.nullObject === true && value === "null") {
value = null;
}
conv.parseRuntime.columnValueSetter[headIdx](resultJson, head, value);
// flatSetter(resultJson, head, value);

}
function flatSetter(resultJson: any, head: string, value: any) {
resultJson[head] = value;
Expand All @@ -149,18 +171,22 @@ function jsonSetter(resultJson: any, head: string, value: any) {
set(resultJson, head, value);
}


function checkType(item: string, head: string, headIdx: number, conv: Converter): Function {
function checkType(
item: string,
head: string,
headIdx: number,
conv: Converter
): Function {
if (conv.parseRuntime.headerType[headIdx]) {
return conv.parseRuntime.headerType[headIdx];
} else if (head.indexOf('number#!') > -1) {
return conv.parseRuntime.headerType[headIdx] = numberType;
} else if (head.indexOf('string#!') > -1) {
return conv.parseRuntime.headerType[headIdx] = stringType;
} else if (head.indexOf("number#!") > -1) {
return (conv.parseRuntime.headerType[headIdx] = numberType);
} else if (head.indexOf("string#!") > -1) {
return (conv.parseRuntime.headerType[headIdx] = stringType);
} else if (conv.parseParam.checkType) {
return conv.parseRuntime.headerType[headIdx] = dynamicType;
return (conv.parseRuntime.headerType[headIdx] = dynamicType);
} else {
return conv.parseRuntime.headerType[headIdx] = stringType;
return (conv.parseRuntime.headerType[headIdx] = stringType);
}
}

Expand All @@ -183,9 +209,15 @@ function dynamicType(item) {
}
if (numReg.test(trimed)) {
return numberType(item);
} else if (trimed.length === 5 && trimed.toLowerCase() === "false" || trimed.length === 4 && trimed.toLowerCase() === "true") {
} else if (
(trimed.length === 5 && trimed.toLowerCase() === "false") ||
(trimed.length === 4 && trimed.toLowerCase() === "true")
) {
return booleanType(item);
} else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
} else if (
(trimed[0] === "{" && trimed[trimed.length - 1] === "}") ||
(trimed[0] === "[" && trimed[trimed.length - 1] === "]")
) {
return jsonType(item);
} else {
return stringType(item);
Expand Down