Skip to content

Commit

Permalink
Reemplazo quote funcs obsoletos por más simples
Browse files Browse the repository at this point in the history
  • Loading branch information
@diegoefe
diegoefe committed Oct 12, 2016
1 parent 054d523 commit edff503
Show file tree
Hide file tree
Showing 3 changed files with 175 additions and 59 deletions.
76 changes: 73 additions & 3 deletions bin/txt-to-sql-run.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,60 @@ function fastProcessEncodingOptions(info) {
});
}

function fastProcessLine(info, line) {
if(line && info.lines && info.lines.length<info.fastMaxLines) {
info.lines.push(line+'\n');
}
}

function fastAnalyzeLines(info) {
txtToSql.separateRows(info);
txtToSql.verifyColumnCount(info);
txtToSql.transformNames(info);
txtToSql.verifyColumnNames(info);
txtToSql.determineColumnTypes(info);
txtToSql.determineColumnValuesInfo(info);
txtToSql.quoteNames(info);
txtToSql.generateDropTable(info);
txtToSql.generateCreateScript(info);
}

function fastInsert(info, row) {
rows = txtToSql.createAdaptedRows(info, row);
var insertInto = txtToSql.createInsertInto(info);
txtToSql.createInsertValues(rows, info.columnsInfo).map(function(c) { return insertInto + c + ";"; }).join('\n')
}

function fastFinalize(info, outStream) {
//txtToSql.removeIgnoredLines(info);
txtToSql.generateInsertScript(info);
console.log("info", info.scripts)
info.scripts.forEach(function(script) {
outStream.write(script.sql+'\n');
});
}

/*
++ .then(verifyInputParams)
-- .then(processEncodingOptions)
-- .then(separateLines)
++ .then(determineSeparator)
++ .then(separateColumns)

++.then(separateRows)
++ .then(verifyColumnCount)
++.then(transformNames)
++.then(verifyColumnNames)
++.then(determineColumnTypes)
++.then(determineColumnValuesInfo)
-- .then(determinePrimaryKey);
++.then(quoteNames)
++.then(generateDropTable)
++.then(generateCreateScript)
++ -- .then(removeIgnoredLines)
++.then(generateInsertScript)
-- .then(processOutputBuffer)
*/
function doFast(params, inputBase) {
var inStream, outStream;
var rl;
Expand All @@ -141,23 +195,39 @@ function doFast(params, inputBase) {
//console.log("info", info);
inStream = fsSync.createReadStream(inputBase+'.txt', {encoding:'utf8'});
outStream = fsSync.createWriteStream(inputBase+'.sql', {encoding:'utf8'});
info.lines = [];
// maximo de lineas para procemiento viejo
info.fastMaxLines = 100;
rl = readline.createInterface({
input: inStream,
terminal: false
});
rl.on('line', function(line) {
console.log("line", line);
//console.log("line", line);
if(! info.headers) {
info.headers = line;
txtToSql.determineSeparator(info);
txtToSql.separateColumns(info);
} else {

fastProcessLine(info, line);
if(info.lines) {
if(info.lines.length===info.fastMaxLines) {
fastAnalyzeLines(info);
fastFinalize(info, outStream);
delete info.lines;
}
} else { // more than info.fastMaxLines
}
}
//outStream.write(line+'\n')
});
rl.on('close', function() {
console.log("info", info);
if(info.lines && info.lines.length<info.fastMaxLines) {
fastProcessLine(info);
fastAnalyzeLines(info);
fastFinalize(info, outStream);
}
//console.log("info", info);
});
});
}
Expand Down
79 changes: 51 additions & 28 deletions lib/txt-to-sql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ function adaptPlain(x){

function adaptText(x){
if(x===''){ return 'null'; }
return "'"+x.replace(/'/g,"''").replace(/\r/g,"' || LQ(10) || '").replace(/\n/g,"' || LQ(13) || '")+"'";
return "'"+x.replace(/'/g,"''").replace(/\r/g,"\\r").replace(/\n/g,"\\n")+"'";
//return "'"+x.replace(/'/g,"''")+"'";
}

function filling(columnLength, val) { return val.length>=columnLength?'':new Array(columnLength - val.length + 1).join(' '); }
Expand All @@ -61,10 +62,10 @@ function mapTypes(typeNames) {
return typeNames.map(function(type, index) { return Object.assign({typeName:type}, types[index]); });
}

var quoteBackTick = { LQ:'`', RQ:'`', fun:function(objectName) { return '`'+objectName.replace(/`/g,'``')+'`'; } };
function quoteBackTick(objectName) { return '`'+objectName.replace(/`/g,'``')+'`'; };
// Solo hay que escapar ']' de acuerdo con: https://technet.microsoft.com/en-us/library/ms176027(v=sql.105).aspx
var quoteBracket = { LQ:'[', RQ:']', fun:function(objectName) { return '['+objectName.replace(/]/g,']]')+']'; } };
var quoteDouble = { LQ:'"', RQ:'"', fun:function(objectName) { return '"'+objectName.replace(/"/g,'""')+'"'; } };
function quoteBracket(objectName) { return '['+objectName.replace(/]/g,']]')+']'; };
function quoteDouble(objectName) { return '"'+objectName.replace(/"/g,'""')+'"'; };

function dropTableIfExists(tableName) { return "drop table if exists "+tableName; }
function dropTable(tableName) { return "drop table "+tableName; }
Expand Down Expand Up @@ -164,7 +165,7 @@ function verifyInputParams(info){
checkEncodingParam(info.opts.outputEncoding, 'output', errors);
throwIfErrors(errors);
info.outputEngine=engines[info.opts.outputEngine];
info.quote = function(objectName) { return info.outputEngine.quote.fun(objectName); };
info.quote = info.outputEngine.quote;
info.transform = function(objectName) { return formatFunctions[info.opts.columnNamesFormat](objectName); };
info.nameColumn = function(columnInfo) {
var name = columnInfo.name+" "+columnInfo.typeInfo.typeName;
Expand Down Expand Up @@ -469,6 +470,35 @@ function generateCreateScript(info){
return info;
}

function removeIgnoredLines(info) {
if(info.opts.ignoreNullLines) {
info.rows = info.rows.filter(function(row) {
return row.filter(function(column) { return column !==''; }).length!==0;
});
}
return info;
}

function createAdaptedRows(info, rows) {
return rows.map(function(row, rowIndex) {
return info.columnsInfo.map(function(column, columnIndex) {
var adaptedValue = column.typeInfo.adapt(row[columnIndex]);
if(info.opts.columnAlignedCommas) {
if(adaptedValue.length>column.columnLength) {
column.columnLength = adaptedValue.length;
}
}
return adaptedValue;
});
});
}

function createInsertInto(info) {
return "insert into "+info.formatedTableName+" ("+info.columnsInfo.map(function(columnInfo){
return columnInfo.name;
}).join(', ')+") values";
}

function createInsertValues(rows, columnsInfo) {
return rows.map(function(row){
var owedLength = 0;
Expand All @@ -491,34 +521,13 @@ function createInsertValues(rows, columnsInfo) {
});
}

function removeIgnoredLines(info) {
if(info.opts.ignoreNullLines) {
info.rows = info.rows.filter(function(row) {
return row.filter(function(column) { return column !==''; }).length!==0;
});
}
return info;
}

function generateInsertScript(info){
var adaptedRows = info.rows.map(function(row, rowIndex) {
return info.columnsInfo.map(function(column, columnIndex) {
var adaptedValue = column.typeInfo.adapt(row[columnIndex]);
if(info.opts.columnAlignedCommas) {
if(adaptedValue.length>column.columnLength) {
column.columnLength = adaptedValue.length;
}
}
return adaptedValue;
});
});
var adaptedRows = createAdaptedRows(info, info.rows);
info.columnsInfo.forEach(function(column){
column.columnLength = info.opts.columnAlignedCommas?
Math.min(column.columnLength, info.opts.columnAlignedMaxWidth):0;
});
var insertInto = "insert into "+info.formatedTableName+" ("+info.columnsInfo.map(function(columnInfo){
return columnInfo.name;
}).join(', ')+") values";
var insertInto = createInsertInto(info);
var insertValues = createInsertValues(adaptedRows, info.columnsInfo);
info.scripts.push({
type:'insert',
Expand Down Expand Up @@ -597,5 +606,19 @@ txtToSql.verifyInputParams = verifyInputParams;
txtToSql.getEncoding = getEncoding;
txtToSql.determineSeparator = determineSeparator;
txtToSql.separateColumns = separateColumns;
txtToSql.transformNames = transformNames;
txtToSql.verifyColumnNames = verifyColumnNames;
txtToSql.separateRows = separateRows;
txtToSql.verifyColumnCount = verifyColumnCount;
txtToSql.determineColumnTypes = determineColumnTypes;
txtToSql.determineColumnValuesInfo = determineColumnValuesInfo;
txtToSql.quoteNames = quoteNames;
txtToSql.generateDropTable = generateDropTable;
txtToSql.generateCreateScript = generateCreateScript;
txtToSql.removeIgnoredLines = removeIgnoredLines;
txtToSql.generateInsertScript = generateInsertScript;
txtToSql.createAdaptedRows = createAdaptedRows;
txtToSql.createInsertInto = createInsertInto;
txtToSql.createInsertValues = createInsertValues;

module.exports = txtToSql;
79 changes: 51 additions & 28 deletions web/txt-to-sql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ function adaptPlain(x){

function adaptText(x){
if(x===''){ return 'null'; }
return "'"+x.replace(/'/g,"''").replace(/\r/g,"' || LQ(10) || '").replace(/\n/g,"' || LQ(13) || '")+"'";
return "'"+x.replace(/'/g,"''").replace(/\r/g,"\\r").replace(/\n/g,"\\n")+"'";
//return "'"+x.replace(/'/g,"''")+"'";
}

function filling(columnLength, val) { return val.length>=columnLength?'':new Array(columnLength - val.length + 1).join(' '); }
Expand All @@ -61,10 +62,10 @@ function mapTypes(typeNames) {
return typeNames.map(function(type, index) { return Object.assign({typeName:type}, types[index]); });
}

var quoteBackTick = { LQ:'`', RQ:'`', fun:function(objectName) { return '`'+objectName.replace(/`/g,'``')+'`'; } };
function quoteBackTick(objectName) { return '`'+objectName.replace(/`/g,'``')+'`'; };
// Solo hay que escapar ']' de acuerdo con: https://technet.microsoft.com/en-us/library/ms176027(v=sql.105).aspx
var quoteBracket = { LQ:'[', RQ:']', fun:function(objectName) { return '['+objectName.replace(/]/g,']]')+']'; } };
var quoteDouble = { LQ:'"', RQ:'"', fun:function(objectName) { return '"'+objectName.replace(/"/g,'""')+'"'; } };
function quoteBracket(objectName) { return '['+objectName.replace(/]/g,']]')+']'; };
function quoteDouble(objectName) { return '"'+objectName.replace(/"/g,'""')+'"'; };

function dropTableIfExists(tableName) { return "drop table if exists "+tableName; }
function dropTable(tableName) { return "drop table "+tableName; }
Expand Down Expand Up @@ -164,7 +165,7 @@ function verifyInputParams(info){
checkEncodingParam(info.opts.outputEncoding, 'output', errors);
throwIfErrors(errors);
info.outputEngine=engines[info.opts.outputEngine];
info.quote = function(objectName) { return info.outputEngine.quote.fun(objectName); };
info.quote = info.outputEngine.quote;
info.transform = function(objectName) { return formatFunctions[info.opts.columnNamesFormat](objectName); };
info.nameColumn = function(columnInfo) {
var name = columnInfo.name+" "+columnInfo.typeInfo.typeName;
Expand Down Expand Up @@ -469,6 +470,35 @@ function generateCreateScript(info){
return info;
}

function removeIgnoredLines(info) {
if(info.opts.ignoreNullLines) {
info.rows = info.rows.filter(function(row) {
return row.filter(function(column) { return column !==''; }).length!==0;
});
}
return info;
}

function createAdaptedRows(info, rows) {
return rows.map(function(row, rowIndex) {
return info.columnsInfo.map(function(column, columnIndex) {
var adaptedValue = column.typeInfo.adapt(row[columnIndex]);
if(info.opts.columnAlignedCommas) {
if(adaptedValue.length>column.columnLength) {
column.columnLength = adaptedValue.length;
}
}
return adaptedValue;
});
});
}

function createInsertInto(info) {
return "insert into "+info.formatedTableName+" ("+info.columnsInfo.map(function(columnInfo){
return columnInfo.name;
}).join(', ')+") values";
}

function createInsertValues(rows, columnsInfo) {
return rows.map(function(row){
var owedLength = 0;
Expand All @@ -491,34 +521,13 @@ function createInsertValues(rows, columnsInfo) {
});
}

function removeIgnoredLines(info) {
if(info.opts.ignoreNullLines) {
info.rows = info.rows.filter(function(row) {
return row.filter(function(column) { return column !==''; }).length!==0;
});
}
return info;
}

function generateInsertScript(info){
var adaptedRows = info.rows.map(function(row, rowIndex) {
return info.columnsInfo.map(function(column, columnIndex) {
var adaptedValue = column.typeInfo.adapt(row[columnIndex]);
if(info.opts.columnAlignedCommas) {
if(adaptedValue.length>column.columnLength) {
column.columnLength = adaptedValue.length;
}
}
return adaptedValue;
});
});
var adaptedRows = createAdaptedRows(info, info.rows);
info.columnsInfo.forEach(function(column){
column.columnLength = info.opts.columnAlignedCommas?
Math.min(column.columnLength, info.opts.columnAlignedMaxWidth):0;
});
var insertInto = "insert into "+info.formatedTableName+" ("+info.columnsInfo.map(function(columnInfo){
return columnInfo.name;
}).join(', ')+") values";
var insertInto = createInsertInto(info);
var insertValues = createInsertValues(adaptedRows, info.columnsInfo);
info.scripts.push({
type:'insert',
Expand Down Expand Up @@ -597,5 +606,19 @@ txtToSql.verifyInputParams = verifyInputParams;
txtToSql.getEncoding = getEncoding;
txtToSql.determineSeparator = determineSeparator;
txtToSql.separateColumns = separateColumns;
txtToSql.transformNames = transformNames;
txtToSql.verifyColumnNames = verifyColumnNames;
txtToSql.separateRows = separateRows;
txtToSql.verifyColumnCount = verifyColumnCount;
txtToSql.determineColumnTypes = determineColumnTypes;
txtToSql.determineColumnValuesInfo = determineColumnValuesInfo;
txtToSql.quoteNames = quoteNames;
txtToSql.generateDropTable = generateDropTable;
txtToSql.generateCreateScript = generateCreateScript;
txtToSql.removeIgnoredLines = removeIgnoredLines;
txtToSql.generateInsertScript = generateInsertScript;
txtToSql.createAdaptedRows = createAdaptedRows;
txtToSql.createInsertInto = createInsertInto;
txtToSql.createInsertValues = createInsertValues;

module.exports = txtToSql;

0 comments on commit edff503

Please sign in to comment.