Skip to content

Commit

Permalink
Preparando para #20
Browse files Browse the repository at this point in the history
  • Loading branch information
@diegoefe
diegoefe committed Oct 12, 2016
1 parent 48754d7 commit 054d523
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 64 deletions.
62 changes: 58 additions & 4 deletions bin/txt-to-sql-run.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ var program = require('commander');
var txtToSql = require('../lib/txt-to-sql.js');
var Promises = require('best-promise');
var fs = require('fs-promise');
var fsSync = require('fs');
var Path = require('path');
var miniTools = require('mini-tools');
var jsYaml = require('js-yaml');
var changing = require('best-globals').changing;
var changing = require('best-globals').changing;
var readline = require('readline');

function getOutputDir(inFile) {
return Promises.start(function() {
Expand Down Expand Up @@ -102,9 +104,61 @@ function doGenerate(params, inputYaml, create, inputName) {
});
}

function doFast(params, inputName) {
function fastProcessEncodingOptions(info) {
return txtToSql.getEncoding(info.rawTable).then(function(encoding) {
info.inputEncodingDetected = encoding;
if(! info.opts.inputEncoding) { info.opts.inputEncoding = info.inputEncodingDetected; }
if(! info.opts.outputEncoding) { info.opts.outputEncoding = info.inputEncodingDetected; }
//var inFromToString = info.rawTable.toString("utf8");
if(info.opts.inputEncoding==='ANSI') {
// if(inFromToString.substr(1).indexOf('\uFFFD')<0) {
// throw new Error('ansi -> utf8: replacement character not found');
// }
//info.decodedBuffer = iconv.decode(info.rawTable, "win1252");
// if(txtToSql.compareBuffers(info.decodedBuffer, info.rawTable) === -1) {
// throw new Error('ansi -> utf8: no conversion performed');
// }
} else if(info.opts.inputEncoding==='UTF8') {
//info.decodedBuffer = inFromToString;
// var result = txtToSql.compareBuffers(info.rawTable, new Buffer(info.decodedBuffer, 'utf8'));
// if(result !== -1) {
// throw new Error('utf8 check failed in position: '+result);
// }
} else {
//info.decodedBuffer = inFromToString;
}
return info;
});
}

function doFast(params, inputBase) {
var inStream, outStream;
var rl;
return Promise.resolve().then(function() {
return "not yet";
return txtToSql.verifyInputParams(params);
}).then(fastProcessEncodingOptions)
.then(function(info) {
//console.log("info", info);
inStream = fsSync.createReadStream(inputBase+'.txt', {encoding:'utf8'});
outStream = fsSync.createWriteStream(inputBase+'.sql', {encoding:'utf8'});
rl = readline.createInterface({
input: inStream,
terminal: false
});
rl.on('line', function(line) {
console.log("line", line);
if(! info.headers) {
info.headers = line;
txtToSql.determineSeparator(info);
txtToSql.separateColumns(info);
} else {

}
//outStream.write(line+'\n')
});
rl.on('close', function() {
console.log("info", info);
});
});
}

Expand All @@ -130,7 +184,7 @@ getOutputDir(cmdParams.input).then(function(dir) {
}).then(function(rawInput) {
params.rawTable = rawInput;
if(cmdParams.fast) {
return doFast(params, inputName);
return doFast(params, inputBase);
} else if (cmdParams.prepare) {
return doPrepare(params, inputYaml, createInputYaml);
} else {
Expand Down
71 changes: 41 additions & 30 deletions lib/txt-to-sql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,36 +107,6 @@ function throwIfErrors(errors) {
}
}

function getEncodingSinc(buf) {
// si es un continuador
function cont(code) { return code>=128 && code<192; }
function case1(code) { return code>=192 && code<224; }
function case2(code) { return code>=224 && code<240; }
function case3(code) { return code>=240 && code<248; }
var type = 'ASCII7';
var i=0;
var code;
while(i<buf.length) {
if(buf[i]>127) {
type = 'UTF8';
if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; }
if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; }
if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; }
type = 'ANSI';
break;
} else {
i+=1;
}
}
return type;
}

function getEncoding(buf) {
return Promise.resolve().then(function() {
return getEncodingSinc(buf);
});
}

txtToSql.defaultOpts = {
columnNamesFormat: 'lowercased_names',
separator: false,
Expand Down Expand Up @@ -207,6 +177,36 @@ function verifyInputParams(info){
return info;
}

function getEncodingSinc(buf) {
// si es un continuador
function cont(code) { return code>=128 && code<192; }
function case1(code) { return code>=192 && code<224; }
function case2(code) { return code>=224 && code<240; }
function case3(code) { return code>=240 && code<248; }
var type = 'ASCII7';
var i=0;
var code;
while(i<buf.length) {
if(buf[i]>127) {
type = 'UTF8';
if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; }
if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; }
if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; }
type = 'ANSI';
break;
} else {
i+=1;
}
}
return type;
}

function getEncoding(buf) {
return Promise.resolve().then(function() {
return getEncodingSinc(buf);
});
}

function compareBuffers(one, two) {
var max = Math.max(one.length, two.length);
for(var i=0; i<max; i++){ if(one[i]!==two[i]) { return i; } }
Expand Down Expand Up @@ -276,6 +276,10 @@ function separateColumns(info){
column.name = info.opts.columns[index].name;
});
}
return info;
}

function separateRows(info) {
info.rows = info.lines.filter(function(line){ return line.trim()!==""; })
.map(function(line){ return line.split(info.opts.separator);});
return info;
Expand Down Expand Up @@ -539,6 +543,7 @@ function setup(info) {
.then(separateLines)
.then(determineSeparator)
.then(separateColumns)
.then(separateRows)
.then(verifyColumnCount)
.then(transformNames)
.then(verifyColumnNames)
Expand Down Expand Up @@ -587,4 +592,10 @@ txtToSql.getEncodingSinc = getEncodingSinc;
txtToSql.getEncoding = getEncoding;
txtToSql.compareBuffers = compareBuffers;

// for --fast
txtToSql.verifyInputParams = verifyInputParams;
txtToSql.getEncoding = getEncoding;
txtToSql.determineSeparator = determineSeparator;
txtToSql.separateColumns = separateColumns;

module.exports = txtToSql;
71 changes: 41 additions & 30 deletions web/txt-to-sql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,36 +107,6 @@ function throwIfErrors(errors) {
}
}

function getEncodingSinc(buf) {
// si es un continuador
function cont(code) { return code>=128 && code<192; }
function case1(code) { return code>=192 && code<224; }
function case2(code) { return code>=224 && code<240; }
function case3(code) { return code>=240 && code<248; }
var type = 'ASCII7';
var i=0;
var code;
while(i<buf.length) {
if(buf[i]>127) {
type = 'UTF8';
if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; }
if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; }
if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; }
type = 'ANSI';
break;
} else {
i+=1;
}
}
return type;
}

function getEncoding(buf) {
return Promise.resolve().then(function() {
return getEncodingSinc(buf);
});
}

txtToSql.defaultOpts = {
columnNamesFormat: 'lowercased_names',
separator: false,
Expand Down Expand Up @@ -207,6 +177,36 @@ function verifyInputParams(info){
return info;
}

function getEncodingSinc(buf) {
// si es un continuador
function cont(code) { return code>=128 && code<192; }
function case1(code) { return code>=192 && code<224; }
function case2(code) { return code>=224 && code<240; }
function case3(code) { return code>=240 && code<248; }
var type = 'ASCII7';
var i=0;
var code;
while(i<buf.length) {
if(buf[i]>127) {
type = 'UTF8';
if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; }
if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; }
if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; }
type = 'ANSI';
break;
} else {
i+=1;
}
}
return type;
}

function getEncoding(buf) {
return Promise.resolve().then(function() {
return getEncodingSinc(buf);
});
}

function compareBuffers(one, two) {
var max = Math.max(one.length, two.length);
for(var i=0; i<max; i++){ if(one[i]!==two[i]) { return i; } }
Expand Down Expand Up @@ -276,6 +276,10 @@ function separateColumns(info){
column.name = info.opts.columns[index].name;
});
}
return info;
}

function separateRows(info) {
info.rows = info.lines.filter(function(line){ return line.trim()!==""; })
.map(function(line){ return line.split(info.opts.separator);});
return info;
Expand Down Expand Up @@ -539,6 +543,7 @@ function setup(info) {
.then(separateLines)
.then(determineSeparator)
.then(separateColumns)
.then(separateRows)
.then(verifyColumnCount)
.then(transformNames)
.then(verifyColumnNames)
Expand Down Expand Up @@ -587,4 +592,10 @@ txtToSql.getEncodingSinc = getEncodingSinc;
txtToSql.getEncoding = getEncoding;
txtToSql.compareBuffers = compareBuffers;

// for --fast
txtToSql.verifyInputParams = verifyInputParams;
txtToSql.getEncoding = getEncoding;
txtToSql.determineSeparator = determineSeparator;
txtToSql.separateColumns = separateColumns;

module.exports = txtToSql;

0 comments on commit 054d523

Please sign in to comment.