diff --git a/bin/txt-to-sql-run.js b/bin/txt-to-sql-run.js index 8cd4d91..aea9378 100644 --- a/bin/txt-to-sql-run.js +++ b/bin/txt-to-sql-run.js @@ -6,10 +6,12 @@ var program = require('commander'); var txtToSql = require('../lib/txt-to-sql.js'); var Promises = require('best-promise'); var fs = require('fs-promise'); +var fsSync = require('fs'); var Path = require('path'); var miniTools = require('mini-tools'); var jsYaml = require('js-yaml'); -var changing = require('best-globals').changing; +var changing = require('best-globals').changing; +var readline = require('readline'); function getOutputDir(inFile) { return Promises.start(function() { @@ -102,9 +104,61 @@ function doGenerate(params, inputYaml, create, inputName) { }); } -function doFast(params, inputName) { +function fastProcessEncodingOptions(info) { + return txtToSql.getEncoding(info.rawTable).then(function(encoding) { + info.inputEncodingDetected = encoding; + if(! info.opts.inputEncoding) { info.opts.inputEncoding = info.inputEncodingDetected; } + if(! info.opts.outputEncoding) { info.opts.outputEncoding = info.inputEncodingDetected; } + //var inFromToString = info.rawTable.toString("utf8"); + if(info.opts.inputEncoding==='ANSI') { + // if(inFromToString.substr(1).indexOf('\uFFFD')<0) { + // throw new Error('ansi -> utf8: replacement character not found'); + // } + //info.decodedBuffer = iconv.decode(info.rawTable, "win1252"); + // if(txtToSql.compareBuffers(info.decodedBuffer, info.rawTable) === -1) { + // throw new Error('ansi -> utf8: no conversion performed'); + // } + } else if(info.opts.inputEncoding==='UTF8') { + //info.decodedBuffer = inFromToString; + // var result = txtToSql.compareBuffers(info.rawTable, new Buffer(info.decodedBuffer, 'utf8')); + // if(result !== -1) { + // throw new Error('utf8 check failed in position: '+result); + // } + } else { + //info.decodedBuffer = inFromToString; + } + return info; + }); +} + +function doFast(params, inputBase) { + var inStream, outStream; + var rl; return Promise.resolve().then(function() { - return "not yet"; + return txtToSql.verifyInputParams(params); + }).then(fastProcessEncodingOptions) + .then(function(info) { + //console.log("info", info); + inStream = fsSync.createReadStream(inputBase+'.txt', {encoding:'utf8'}); + outStream = fsSync.createWriteStream(inputBase+'.sql', {encoding:'utf8'}); + rl = readline.createInterface({ + input: inStream, + terminal: false + }); + rl.on('line', function(line) { + console.log("line", line); + if(! info.headers) { + info.headers = line; + txtToSql.determineSeparator(info); + txtToSql.separateColumns(info); + } else { + + } + //outStream.write(line+'\n') + }); + rl.on('close', function() { + console.log("info", info); + }); }); } @@ -130,7 +184,7 @@ getOutputDir(cmdParams.input).then(function(dir) { }).then(function(rawInput) { params.rawTable = rawInput; if(cmdParams.fast) { - return doFast(params, inputName); + return doFast(params, inputBase); } else if (cmdParams.prepare) { return doPrepare(params, inputYaml, createInputYaml); } else { diff --git a/lib/txt-to-sql.js b/lib/txt-to-sql.js index 59e758c..789f296 100644 --- a/lib/txt-to-sql.js +++ b/lib/txt-to-sql.js @@ -107,36 +107,6 @@ function throwIfErrors(errors) { } } -function getEncodingSinc(buf) { - // si es un continuador - function cont(code) { return code>=128 && code<192; } - function case1(code) { return code>=192 && code<224; } - function case2(code) { return code>=224 && code<240; } - function case3(code) { return code>=240 && code<248; } - var type = 'ASCII7'; - var i=0; - var code; - while(i127) { - type = 'UTF8'; - if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; } - if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; } - if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; } - type = 'ANSI'; - break; - } else { - i+=1; - } - } - return type; -} - -function getEncoding(buf) { - return Promise.resolve().then(function() { - return getEncodingSinc(buf); - }); -} - txtToSql.defaultOpts = { columnNamesFormat: 'lowercased_names', separator: false, @@ -207,6 +177,36 @@ function verifyInputParams(info){ return info; } +function getEncodingSinc(buf) { + // si es un continuador + function cont(code) { return code>=128 && code<192; } + function case1(code) { return code>=192 && code<224; } + function case2(code) { return code>=224 && code<240; } + function case3(code) { return code>=240 && code<248; } + var type = 'ASCII7'; + var i=0; + var code; + while(i127) { + type = 'UTF8'; + if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; } + if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; } + if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; } + type = 'ANSI'; + break; + } else { + i+=1; + } + } + return type; +} + +function getEncoding(buf) { + return Promise.resolve().then(function() { + return getEncodingSinc(buf); + }); +} + function compareBuffers(one, two) { var max = Math.max(one.length, two.length); for(var i=0; i=128 && code<192; } - function case1(code) { return code>=192 && code<224; } - function case2(code) { return code>=224 && code<240; } - function case3(code) { return code>=240 && code<248; } - var type = 'ASCII7'; - var i=0; - var code; - while(i127) { - type = 'UTF8'; - if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; } - if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; } - if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; } - type = 'ANSI'; - break; - } else { - i+=1; - } - } - return type; -} - -function getEncoding(buf) { - return Promise.resolve().then(function() { - return getEncodingSinc(buf); - }); -} - txtToSql.defaultOpts = { columnNamesFormat: 'lowercased_names', separator: false, @@ -207,6 +177,36 @@ function verifyInputParams(info){ return info; } +function getEncodingSinc(buf) { + // si es un continuador + function cont(code) { return code>=128 && code<192; } + function case1(code) { return code>=192 && code<224; } + function case2(code) { return code>=224 && code<240; } + function case3(code) { return code>=240 && code<248; } + var type = 'ASCII7'; + var i=0; + var code; + while(i127) { + type = 'UTF8'; + if(case1(buf[i]) && cont(buf[i+1])) { i+=2; continue; } + if(case2(buf[i]) && cont(buf[i+1]) && cont(buf[i+2])) { i+=3; continue; } + if(case3(buf[i]) && cont(buf[i+1]) && cont(buf[i+2]) && cont(buf[i+3])) { i+=4; continue; } + type = 'ANSI'; + break; + } else { + i+=1; + } + } + return type; +} + +function getEncoding(buf) { + return Promise.resolve().then(function() { + return getEncodingSinc(buf); + }); +} + function compareBuffers(one, two) { var max = Math.max(one.length, two.length); for(var i=0; i