diff --git a/benchmarks/benchs/bench_promise_insert_batch.js b/benchmarks/benchs/bench_promise_insert_batch.js new file mode 100644 index 00000000..26015e24 --- /dev/null +++ b/benchmarks/benchs/bench_promise_insert_batch.js @@ -0,0 +1,86 @@ +const assert = require("assert"); + +const basechars = "123456789abcdefghijklmnop\\Z"; +const chars = basechars.split(""); +chars.push("😎"); +chars.push("🌶"); +chars.push("🎤"); +chars.push("🥂"); + +function randomString(length) { + let result = ""; + for (let i = length; i > 0; --i) result += chars[Math.round(Math.random() * (chars.length - 1))]; + return result; +} + +let sqlTable = + "CREATE TABLE testn.perfTestTextPipe (id MEDIUMINT NOT NULL AUTO_INCREMENT,t0 text" + + ", PRIMARY KEY (id))"; +sqlInsert = "INSERT INTO testn.perfTestTextPipe(t0) VALUES (?)"; + +module.exports.title = + "100 * insert 100 characters using promise and batch method (for mariadb only, since doesn't exist for others)"; +module.exports.displaySql = "INSERT INTO testn.perfTestTextPipe VALUES (?) (into BLACKHOLE ENGINE)"; +const iterations = 100; +module.exports.promise = true; +module.exports.benchFct = function(conn, deferred, connType) { + const params = [randomString(100)]; + // console.log(connType.desc); + if (!connType.desc.includes("mariadb")) { + //other driver doesn't have bulk method + let ended = 0; + for (let i = 0; i < iterations; i++) { + conn + .query(sqlInsert, params) + .then(rows => { + // let val = Array.isArray(rows) ? rows[0] : rows; + // assert.equal(1, val.info ? val.info.affectedRows : val.affectedRows); + if (++ended === iterations) { + deferred.resolve(); + } + }) + .catch(err => { + throw err; + }); + } + } else { + //use batch capability + const totalParams = new Array(iterations); + for (let i = 0; i < iterations; i++) { + totalParams[i] = params; + } + conn + .batch(sqlInsert, totalParams) + .then(rows => { + deferred.resolve(); + }) + .catch(err => { + throw err; + }); + } +}; + +module.exports.initFct = function(conn) { + return Promise.all([ + conn.query("DROP TABLE IF EXISTS testn.perfTestTextPipe"), + conn.query("INSTALL SONAME 'ha_blackhole'"), + conn.query(sqlTable + " ENGINE = BLACKHOLE COLLATE='utf8mb4_unicode_ci'") + ]) + .catch(err => { + return Promise.all([ + conn.query("DROP TABLE IF EXISTS testn.perfTestTextPipe"), + conn.query(sqlTable + " COLLATE='utf8mb4_unicode_ci'") + ]); + }) + .catch(e => { + console.log(e); + throw e; + }); +}; + +module.exports.onComplete = function(conn) { + conn.query("TRUNCATE TABLE testn.perfTestTextPipe").catch(e => { + console.log(e); + throw e; + }); +}; diff --git a/benchmarks/benchs/bench_promise_insert_pipelining.js b/benchmarks/benchs/bench_promise_insert_pipelining.js index ce29c713..af03ecf7 100644 --- a/benchmarks/benchs/bench_promise_insert_pipelining.js +++ b/benchmarks/benchs/bench_promise_insert_pipelining.js @@ -20,7 +20,7 @@ sqlInsert = "INSERT INTO testn.perfTestTextPipe(t0) VALUES (?)"; module.exports.title = "100 * insert 100 characters using promise"; module.exports.displaySql = "INSERT INTO testn.perfTestTextPipe VALUES (?) (into BLACKHOLE ENGINE)"; -const iterations = 10; +const iterations = 100; module.exports.promise = true; module.exports.benchFct = function(conn, deferred) { const params = [randomString(100)]; diff --git a/benchmarks/common_benchmarks.js b/benchmarks/common_benchmarks.js index dd1ea7df..7297735e 100644 --- a/benchmarks/common_benchmarks.js +++ b/benchmarks/common_benchmarks.js @@ -394,7 +394,7 @@ const getAddTest = function(self, suite, fct, minSamples, title, displaySql, onC suite.add({ name: title + " - " + name, fn: function(deferred) { - fct.call(self, usePool ? conn.pool : conn.drv, deferred); + fct.call(self, usePool ? conn.pool : conn.drv, deferred, conn); }, onComplete: () => { if (onComplete) onComplete.call(self, usePool ? conn.pool : conn.drv); diff --git a/lib/cmd/batch.js b/lib/cmd/batch.js index a293e7af..b92c7788 100644 --- a/lib/cmd/batch.js +++ b/lib/cmd/batch.js @@ -39,24 +39,16 @@ class Batch extends CommonText { info ); } + this.initialValues = Array.isArray(this.initialValues) + ? this.initialValues + : [this.initialValues]; if (this.opts.namedPlaceholders) { - try { - const parsed = Parse.splitQueryPlaceholder( - this.sql, - info, - this.initialValues, - this.displaySql.bind(this) - ); - this.queryParts = parsed.parts; - this.values = parsed.values; - } catch (err) { - this.emit("send_end"); - return this.throwError(err, info); - } + this.parseResults = Parse.splitRewritableNamedParameterQuery(this.sql, this.initialValues); + this.values = this.parseResults.values; } else { this.parseResults = Parse.splitRewritableQuery(this.sql); - this.values = Array.isArray(this.initialValues) ? this.initialValues : [this.initialValues]; + this.values = this.initialValues; if (!this.validateParameters(info)) return; } @@ -109,7 +101,7 @@ class Batch extends CommonText { function() { this.packet.writeInt8(QUOTE); //' this.currentParam++; - process.nextTick(this.paramWritten.bind(this)); + this.paramWritten(); }.bind(this) ); @@ -126,39 +118,105 @@ class Batch extends CommonText { } this.sendEnded = true; this.expectedResponseNo = this.packet.packetSend; - this.packet = null; this.emit("send_end"); } + displaySql() { + if (this.opts && this.initialValues) { + if (this.sql.length > this.opts.debugLen) { + return "sql: " + this.sql.substring(0, this.opts.debugLen) + "..."; + } + + let sqlMsg = "sql: " + this.sql + " - parameters:"; + sqlMsg += "["; + for (let i = 0; i < this.initialValues.length; i++) { + if (i !== 0) sqlMsg += ","; + let param = this.initialValues[i]; + sqlMsg = this.logParameters(sqlMsg, param); + if (sqlMsg.length > this.opts.debugLen) { + sqlMsg = sqlMsg.substr(0, this.opts.debugLen) + "..."; + break; + } + } + sqlMsg += "]"; + return sqlMsg; + } + return "sql: " + this.sql + " - parameters:[]"; + } + success(val) { this.expectedResponseNo--; - if (this.parseResults.reWritable) { + if (this.packet.haveErrorResponse) { if (this.sendEnded && this.expectedResponseNo === 0) { - let totalAffectedRows = 0; - this._rows.forEach(row => { - totalAffectedRows += row.affectedRows; - }); + this.packet = null; + this.onPacketReceive = null; + this.resolve = null; + this.reject = null; + this._columns = null; + this._rows = null; + this.emit("end", err); + return; + } + } else { + if (this.parseResults.reWritable) { + if (this.sendEnded && this.expectedResponseNo === 0) { + this.packet = null; + let totalAffectedRows = 0; + this._rows.forEach(row => { + totalAffectedRows += row.affectedRows; + }); - const rs = { - affectedRows: totalAffectedRows, - insertId: this._rows[0].insertId, - warningStatus: this._rows[this._rows.length - 1].warningStatus - }; - this.successEnd(rs); + const rs = { + affectedRows: totalAffectedRows, + insertId: this._rows[0].insertId, + warningStatus: this._rows[this._rows.length - 1].warningStatus + }; + this.successEnd(rs); + this._columns = null; + this._rows = null; + return; + } + } else if (this.sendEnded && this.expectedResponseNo === 0) { + this.successEnd(this._rows); this._columns = null; this._rows = null; return; } - } else if (this.sendEnded && this.expectedResponseNo === 0) { - this.successEnd(this._rows); - this._columns = null; - this._rows = null; - return; + this._responseIndex++; + this.onPacketReceive = this.readResponsePacket; + } + } + + throwError(err, info) { + this.expectedResponseNo--; + if (!this.packet.haveErrorResponse) { + if (this.stack) { + err = Errors.createError( + err.message, + err.fatal, + info, + err.sqlState, + err.errno, + this.stack, + false + ); + } + this.packet.endedWithError(); + process.nextTick(this.reject, err); } - this._responseIndex++; - this.onPacketReceive = this.readResponsePacket; + if (this.sendEnded && this.expectedResponseNo === 0) { + this.packet = null; + this.onPacketReceive = null; + this.resolve = null; + this.reject = null; + this.emit("end", err); + return; + } else { + this._responseIndex++; + this.onPacketReceive = this.readResponsePacket; + } } /** @@ -223,32 +281,32 @@ class Batch extends CommonText { * emitting event so next parameter can be written. */ registerStreamSendEvent(packet, info) { - const self = this; this.paramWritten = function() { while (true) { - if (self.currentParam === self.valueRow.length) { - // all parameters are written. - packet.writeString(self.parseResults.partList[self.parseResults.partList.length - 2]); - packet.mark(!this.parseResults.reWritable || self.valueIdx === self.values.length); - if (self.valueIdx < self.values.length) { - self.valueRow = self.values[self.valueIdx++]; - self.currentParam = 0; + if (this.currentParam === this.valueRow.length) { + // all parameters from row are written. + packet.writeString(this.parseResults.partList[this.parseResults.partList.length - 2]); + packet.mark(!this.parseResults.reWritable || this.valueIdx === this.values.length); + if (this.valueIdx < this.values.length) { + // still remaining rows + this.valueRow = this.values[this.valueIdx++]; + this.currentParam = 0; } else { + // all rows are written this.sendEnded = true; this.expectedResponseNo = packet.packetSend; - self.packet = null; - self.emit("send_end"); + this.emit("send_end"); return; } } - packet.writeString(self.parseResults.partList[self.currentParam + 1]); - const value = self.valueRow[self.currentParam]; + packet.writeString(this.parseResults.partList[this.currentParam + 1]); + const value = this.valueRow[this.currentParam]; if (value === null) { packet.writeStringAscii("NULL"); - process.nextTick(self.paramWritten.bind(self)); - return; + this.currentParam++; + continue; } if ( @@ -260,24 +318,28 @@ class Batch extends CommonText { // param is stream, //******************************************** packet.writeInt8(QUOTE); - value.once("end", function() { - packet.writeInt8(QUOTE); - self.currentParam++; - process.nextTick(self.paramWritten.bind(self)); - }); + value.once( + "end", + function() { + packet.writeInt8(QUOTE); + this.currentParam++; + this.paramWritten(); + }.bind(this) + ); + value.on("data", function(chunk) { packet.writeBufferEscape(chunk); }); return; - } else { - //******************************************** - // param isn't stream. directly write in buffer - //******************************************** - self.writeParam(packet, value, self.opts, info); - self.currentParam++; } + + //******************************************** + // param isn't stream. directly write in buffer + //******************************************** + this.writeParam(packet, value, this.opts, info); + this.currentParam++; } - }; + }.bind(this); } } diff --git a/lib/cmd/query.js b/lib/cmd/query.js index 18b3bf61..ed4722a9 100644 --- a/lib/cmd/query.js +++ b/lib/cmd/query.js @@ -94,7 +94,7 @@ class Query extends CommonText { function() { out.writeInt8(QUOTE); //' out.writeString(this.queryParts[this.currentParam++]); - this.emit("param_written"); + this.paramWritten(); }.bind(this) ); @@ -152,56 +152,61 @@ class Query extends CommonText { /** * Define params events. * Each parameter indicate that he is written to socket, - * emitting event so next parameter can be written. + * emitting event so next stream parameter can be written. */ registerStreamSendEvent(out, info) { // note : Implementation use recursive calls, but stack won't never get near v8 max call stack size - const self = this; - this.on("param_written", function() { - if (self.currentParam === self.queryParts.length) { - //******************************************** - // all parameters are written. - // flush packet - //******************************************** - out.flushBuffer(true); - self.emit("send_end"); - } else { - const value = self.values[self.currentParam - 1]; - - if (value === null) { - out.writeStringAscii("NULL"); - out.writeString(self.queryParts[self.currentParam++]); - self.emit("param_written"); - return; - } - - if ( - typeof value === "object" && - typeof value.pipe === "function" && - typeof value.read === "function" - ) { + //since event launched for stream parameter only + this.paramWritten = function() { + while (true) { + if (this.currentParam === this.queryParts.length) { //******************************************** - // param is stream, + // all parameters are written. + // flush packet //******************************************** - out.writeInt8(QUOTE); - value.once("end", function() { - out.writeInt8(QUOTE); - out.writeString(self.queryParts[self.currentParam++]); - self.emit("param_written"); - }); - value.on("data", function(chunk) { - out.writeBufferEscape(chunk); - }); + out.flushBuffer(true); + this.emit("send_end"); + return; } else { + const value = this.values[this.currentParam - 1]; + + if (value === null) { + out.writeStringAscii("NULL"); + out.writeString(this.queryParts[this.currentParam++]); + continue; + } + + if ( + typeof value === "object" && + typeof value.pipe === "function" && + typeof value.read === "function" + ) { + //******************************************** + // param is stream, + //******************************************** + out.writeInt8(QUOTE); + value.once( + "end", + function() { + out.writeInt8(QUOTE); + out.writeString(this.queryParts[this.currentParam++]); + this.paramWritten(); + }.bind(this) + ); + value.on("data", function(chunk) { + out.writeBufferEscape(chunk); + }); + return; + } + //******************************************** // param isn't stream. directly write in buffer //******************************************** - this.writeParam(out, value, self.opts, info); - out.writeString(self.queryParts[self.currentParam++]); - self.emit("param_written"); + this.writeParam(out, value, this.opts, info); + out.writeString(this.queryParts[this.currentParam++]); } } - }); + }.bind(this); } } diff --git a/lib/cmd/resultset.js b/lib/cmd/resultset.js index 8d17f47d..a7321873 100644 --- a/lib/cmd/resultset.js +++ b/lib/cmd/resultset.js @@ -339,46 +339,44 @@ class ResultSet extends Command { } let sqlMsg = "sql: " + this.sql + " - parameters:"; + return this.logParameters(sqlMsg, this.initialValues); + } + return "sql: " + this.sql + " - parameters:[]"; + } - if (this.opts.namedPlaceholders) { - sqlMsg += "{"; - let first = true; - for (let key in this.initialValues) { - if (first) { - first = false; - } else { - sqlMsg += ","; - } - sqlMsg += "'" + key + "':"; - let param = this.initialValues[key]; - sqlMsg = logParam(sqlMsg, param); - if (sqlMsg.length > this.opts.debugLen) { - sqlMsg = sqlMsg.substr(0, this.opts.debugLen) + "..."; - break; - } + logParameters(sqlMsg, values) { + if (this.opts.namedPlaceholders) { + sqlMsg += "{"; + let first = true; + for (let key in values) { + if (first) { + first = false; + } else { + sqlMsg += ","; } - sqlMsg += "}"; - } else { - const values = Array.isArray(this.initialValues) - ? this.initialValues - : [this.initialValues]; - sqlMsg += "["; - for (let i = 0; i < values.length; i++) { - if (i !== 0) sqlMsg += ","; - let param = values[i]; - sqlMsg = logParam(sqlMsg, param); - if (sqlMsg.length > this.opts.debugLen) { - sqlMsg = sqlMsg.substr(0, this.opts.debugLen) + "..."; - break; - } + sqlMsg += "'" + key + "':"; + let param = values[key]; + sqlMsg = this.logParam(sqlMsg, param); + if (sqlMsg.length > this.opts.debugLen) { + sqlMsg = sqlMsg.substr(0, this.opts.debugLen) + "..."; + break; } - sqlMsg += "]"; } - - return sqlMsg; + sqlMsg += "}"; + } else { + sqlMsg += "["; + for (let i = 0; i < values.length; i++) { + if (i !== 0) sqlMsg += ","; + let param = values[i]; + sqlMsg = this.logParam(sqlMsg, param); + if (sqlMsg.length > this.opts.debugLen) { + sqlMsg = sqlMsg.substr(0, this.opts.debugLen) + "..."; + break; + } + } + sqlMsg += "]"; } - - return "sql: " + this.sql + " - parameters:[]"; + return sqlMsg; } readLocalInfile(packet, opts, info, out) { @@ -418,34 +416,34 @@ class ResultSet extends Command { }); this.onPacketReceive = this.readResponsePacket; } -} - -function logParam(sqlMsg, param) { - if (!param) { - sqlMsg += param === undefined ? "undefined" : "null"; - } else { - switch (param.constructor.name) { - case "Buffer": - sqlMsg += "0x" + param.toString("hex", 0, Math.floor(1024, param.length)) + ""; - break; - - case "String": - sqlMsg += "'" + param + "'"; - break; - case "Date": - sqlMsg += getStringDate(param); - break; - - case "Object": - sqlMsg += JSON.stringify(param); - break; - - default: - sqlMsg += param.toString(); + logParam(sqlMsg, param) { + if (!param) { + sqlMsg += param === undefined ? "undefined" : "null"; + } else { + switch (param.constructor.name) { + case "Buffer": + sqlMsg += "0x" + param.toString("hex", 0, Math.floor(1024, param.length)) + ""; + break; + + case "String": + sqlMsg += "'" + param + "'"; + break; + + case "Date": + sqlMsg += getStringDate(param); + break; + + case "Object": + sqlMsg += JSON.stringify(param); + break; + + default: + sqlMsg += param.toString(); + } } + return sqlMsg; } - return sqlMsg; } function getStringDate(param) { diff --git a/lib/io/write-packet.js b/lib/io/write-packet.js index 857dae2f..eecf3576 100644 --- a/lib/io/write-packet.js +++ b/lib/io/write-packet.js @@ -35,6 +35,7 @@ class WritePacket { this.endStrLength = Buffer.byteLength(1 + this.initStr + this.endStr, this.encoding); this.packetSend = 0; this.singleQuery = false; + this.haveErrorResponse = false; if (this.encoding === "utf8") { this.writeString = this.writeDefaultBufferString; @@ -356,13 +357,16 @@ class WritePacket { //one insert is more than 16M, will continue to mono insert, hoping //that max_allowed_packet is sized accordingly to query. - this.out.buf = this.buf; - this.out.pos = this.pos; - this.out.flushBuffer(end, remainingLen); - this.pos = 4; - this.markPos = undefined; - if (!this.singleQuery) this.packetSend++; - this.singleQuery = true; + if (!this.haveErrorResponse || this.out.cmd.sequenceNo !== -1) { + this.out.buf = this.buf; + this.out.pos = this.pos; + this.out.flushBuffer(end, remainingLen); + this.pos = 4; + this.buf = Buffer.allocUnsafe(SMALL_BUFFER_SIZE); + this.markPos = undefined; + if (!this.singleQuery) this.packetSend++; + this.singleQuery = true; + } } } @@ -377,12 +381,14 @@ class WritePacket { this.pos = this.markPos; this.writeString(this.endStr); - // console.log(this.buf.slice(0, this.markPos).toString('utf8').substring(16770000)); - this.out.buf = this.buf; - this.out.pos = this.pos; - this.out.flushBuffer(); - this.out.cmd.sequenceNo = -1; - this.packetSend++; + if (!this.haveErrorResponse || this.out.cmd.sequenceNo !== -1) { + this.out.buf = this.buf; + this.out.pos = this.pos; + this.out.flushBuffer(); + this.out.cmd.sequenceNo = -1; + this.buf = Buffer.allocUnsafe(SMALL_BUFFER_SIZE); + this.packetSend++; + } this.pos = 4; this.buf[this.pos++] = 0x03; @@ -394,6 +400,10 @@ class WritePacket { this.markPos = undefined; this.singleQuery = false; } + + endedWithError() { + this.haveErrorResponse = true; + } } module.exports = WritePacket; diff --git a/lib/misc/parse.js b/lib/misc/parse.js index a6523ec3..71eb1c73 100644 --- a/lib/misc/parse.js +++ b/lib/misc/parse.js @@ -243,7 +243,28 @@ module.exports.splitQueryPlaceholder = function(sql, info, initialValues, displa /** * Split query according to parameters (question mark). - * Question mark in comment are not taken in account + * + * The only rewritten queries follow these notation: INSERT [LOW_PRIORITY | DELAYED | + * HIGH_PRIORITY] [IGNORE] [INTO] tbl_name [PARTITION (partition_list)] [(col,...)] {VALUES | + * VALUE} (...) [ ON DUPLICATE KEY UPDATE col=expr [, col=expr] ... ] With expr without + * parameter. + * + * Query with INSERT ... SELECT / containing LAST_INSERT_ID() will not be rewritten + * + * query parts will be split this way : + * - pre-value part + * - after value part + * [- after parameter part] (after each parameter) + * - ending part + * + * example : INSERT INTO MyTABLE VALUES (9, ?, 5, ?, 8) ON DUPLICATE KEY UPDATE col2=col2+10 + * will result in : + * - pre-value : "INSERT INTO MyTABLE VALUES" + * - after value : " (9, " + * - after parameter : ", 5, " + * - after parameter : ", 8)" + * - ending : " ON DUPLICATE KEY UPDATE col2=col2+10" + * * * @returns {JSON} query separated by parameters */ @@ -387,11 +408,11 @@ module.exports.splitRewritableQuery = function(sql) { //field/table name might contain 'select' if ( idx > 1 && - (sql.charAt(idx - 2) > " " && "();><=-+,".indexOf(sql.charAt(idx - 2)) == -1) + (sql.charAt(idx - 2) > " " && "();><=-+,".indexOf(sql.charAt(idx - 2)) === -1) ) { break; } - if (sql.charAt(idx + 5) > " " && "();><=-+,".indexOf(sql.charAt(idx + 5)) == -1) { + if (sql.charAt(idx + 5) > " " && "();><=-+,".indexOf(sql.charAt(idx + 5)) === -1) { break; } @@ -518,3 +539,326 @@ module.exports.splitRewritableQuery = function(sql) { multipleQueries: multipleQueriesPrepare }; }; + +/** + * Split query according to named parameters. + * + * The only rewritten queries follow these notation: INSERT [LOW_PRIORITY | DELAYED | + * HIGH_PRIORITY] [IGNORE] [INTO] tbl_name [PARTITION (partition_list)] [(col,...)] {VALUES | + * VALUE} (...) [ ON DUPLICATE KEY UPDATE col=expr [, col=expr] ... ] With expr without + * parameter. + * + * Query with INSERT ... SELECT / containing LAST_INSERT_ID() will not be rewritten + * + * query parts will be split this way : + * - pre-value part + * - after value part + * [- after parameter part] (after each parameter) + * - ending part + * + * example : INSERT INTO MyTABLE VALUES (9, :param1, 5, :param2, 8) ON DUPLICATE KEY UPDATE col2=col2+10 + * will result in : + * - pre-value : "INSERT INTO MyTABLE VALUES" + * - after value : " (9, " + * - after parameter : ", 5, " + * - after parameter : ", 8)" + * - ending : " ON DUPLICATE KEY UPDATE col2=col2+10" + * + * + * @returns {JSON} query separated by parameters + */ +module.exports.splitRewritableNamedParameterQuery = function(sql, initialValues) { + let reWritablePrepare = true; + let multipleQueriesPrepare = true; + let partList = []; + let values = new Array(initialValues.length); + for (let i = 0; i < values.length; i++) values[i] = []; + let lastChar = "\0"; + + let lastParameterPosition = 0; + + let preValuePart1 = null; + let preValuePart2 = null; + let postValuePart = null; + + let singleQuotes = false; + + let isInParenthesis = 0; + let isFirstChar = true; + let isInsert = false; + let semicolon = false; + let hasParam = false; + let placeholderName; + let state = State.Normal; + + let idx = 0; + let car = sql.charAt(idx++); + while (car !== "") { + if ( + state === State.Escape && + !((car === "'" && singleQuotes) || (car === '"' && !singleQuotes)) + ) { + state = State.String; + continue; + } + + switch (car) { + case "*": + if (state === State.Normal && lastChar === "/") { + state = State.SlashStarComment; + } + break; + + case "/": + if (state === State.SlashStarComment && lastChar === "*") { + state = State.Normal; + } + break; + + case "#": + if (state === State.Normal) { + state = State.EOLComment; + } + break; + + case "-": + if (state === State.Normal && lastChar === "-") { + state = State.EOLComment; + } + break; + + case "\n": + if (state === State.EOLComment) { + state = State.Normal; + } + break; + + case '"': + if (state === State.Normal) { + state = State.String; + singleQuotes = false; + } else if (state === State.String && !singleQuotes) { + state = State.Normal; + } else if (state === State.Escape && !singleQuotes) { + state = State.String; + } + break; + case ";": + if (state === State.Normal) { + semicolon = true; + multipleQueriesPrepare = false; + } + break; + case "'": + if (state === State.Normal) { + state = State.String; + singleQuotes = true; + } else if (state === State.String && singleQuotes) { + state = State.Normal; + } else if (state === State.Escape && singleQuotes) { + state = State.String; + } + break; + + case "\\": + if (state === State.String) { + state = State.Escape; + } + break; + + case ":": + if (state === State.Normal) { + let part = sql.substring(lastParameterPosition, idx - 1); + placeholderName = ""; + while ( + ((car = sql.charAt(idx++)) !== "" && (car >= "0" && car <= "9")) || + (car >= "A" && car <= "Z") || + (car >= "a" && car <= "z") || + car === "-" || + car === "_" + ) { + placeholderName += car; + } + idx--; + hasParam = true; + initialValues.forEach((row, idx) => { + if (row[placeholderName] !== undefined) { + values[idx].push(row[placeholderName]); + } else { + values[idx].push(null); + } + }); + + lastParameterPosition = idx; + + if (preValuePart1 === null) { + preValuePart1 = part; + preValuePart2 = ""; + } else if (preValuePart2 === null) { + preValuePart2 = part; + } else { + if (postValuePart) { + //having parameters after the last ")" of value is not rewritable + reWritablePrepare = false; + partList.push(postValuePart + part); + postValuePart = null; + } else partList.push(part); + } + } + break; + + case "`": + if (state === State.Backtick) { + state = State.Normal; + } else if (state === State.Normal) { + state = State.Backtick; + } + break; + + case "s": + case "S": + if ( + state === State.Normal && + postValuePart === null && + sql.length > idx + 5 && + (sql.charAt(idx) === "e" || sql.charAt(idx) === "E") && + (sql.charAt(idx + 1) === "l" || sql.charAt(idx + 1) === "L") && + (sql.charAt(idx + 2) === "e" || sql.charAt(idx + 2) === "E") && + (sql.charAt(idx + 3) === "c" || sql.charAt(idx + 3) === "C") && + (sql.charAt(idx + 4) === "t" || sql.charAt(idx + 4) === "T") + ) { + //field/table name might contain 'select' + if ( + idx > 1 && + (sql.charAt(idx - 2) > " " && "();><=-+,".indexOf(sql.charAt(idx - 2)) === -1) + ) { + break; + } + if (sql.charAt(idx + 5) > " " && "();><=-+,".indexOf(sql.charAt(idx + 5)) === -1) { + break; + } + + //SELECT queries, INSERT FROM SELECT not rewritable + reWritablePrepare = false; + } + break; + case "v": + case "V": + if ( + state === State.Normal && + !preValuePart1 && + (lastChar === ")" || lastChar <= " ") && + sql.length > idx + 6 && + (sql.charAt(idx) === "a" || sql.charAt(idx) === "A") && + (sql.charAt(idx + 1) === "l" || sql.charAt(idx + 1) === "L") && + (sql.charAt(idx + 2) === "u" || sql.charAt(idx + 2) === "U") && + (sql.charAt(idx + 3) === "e" || sql.charAt(idx + 3) === "E") && + (sql.charAt(idx + 4) === "s" || sql.charAt(idx + 4) === "S") && + (sql.charAt(idx + 5) === "(" || sql.charAt(idx + 5) <= " ") + ) { + idx += 5; + preValuePart1 = sql.substring(lastParameterPosition, idx); + lastParameterPosition = idx; + } + break; + case "l": + case "L": + if ( + state === State.Normal && + sql.length > idx + 13 && + (sql.charAt(idx) === "a" || sql.charAt(idx) === "A") && + (sql.charAt(idx + 1) === "s" || sql.charAt(idx + 1) === "S") && + (sql.charAt(idx + 2) === "t" || sql.charAt(idx + 2) === "T") && + sql.charAt(idx + 3) === "_" && + (sql.charAt(idx + 4) === "i" || sql.charAt(idx + 4) === "I") && + (sql.charAt(idx + 5) === "n" || sql.charAt(idx + 5) === "N") && + (sql.charAt(idx + 6) === "s" || sql.charAt(idx + 6) === "S") && + (sql.charAt(idx + 7) === "e" || sql.charAt(idx + 7) === "E") && + (sql.charAt(idx + 8) === "r" || sql.charAt(idx + 8) === "R") && + (sql.charAt(idx + 9) === "t" || sql.charAt(idx + 9) === "T") && + sql.charAt(idx + 10) === "_" && + (sql.charAt(idx + 11) === "i" || sql.charAt(idx + 11) === "I") && + (sql.charAt(idx + 12) === "d" || sql.charAt(idx + 12) === "D") && + sql.charAt(idx + 13) === "(" + ) { + reWritablePrepare = false; + idx += 13; + } + break; + case "(": + if (state === State.Normal) { + isInParenthesis++; + } + break; + case ")": + if (state === State.Normal) { + isInParenthesis--; + if (isInParenthesis == 0 && preValuePart2 !== null && postValuePart === null) { + postValuePart = sql.substring(lastParameterPosition, idx); + lastParameterPosition = idx; + } + } + break; + default: + if (state === State.Normal && isFirstChar && car > " ") { + if ( + (car === "I" || car === "i") && + sql.length > idx + 6 && + (sql.charAt(idx) === "n" || sql.charAt(idx) === "N") && + (sql.charAt(idx + 1) === "s" || sql.charAt(idx + 1) === "S") && + (sql.charAt(idx + 2) === "e" || sql.charAt(idx + 2) === "E") && + (sql.charAt(idx + 3) === "r" || sql.charAt(idx + 3) === "R") && + (sql.charAt(idx + 4) === "t" || sql.charAt(idx + 4) === "T") && + (sql.charAt(idx + 5) === "(" || sql.charAt(idx + 5) <= " ") + ) { + isInsert = true; + } + isFirstChar = false; + } + //multiple queries + if (state == State.Normal && semicolon && car >= " ") { + reWritablePrepare = false; + multipleQueriesPrepare = true; + } + break; + } + + lastChar = car; + car = sql.charAt(idx++); + } + + if (state === State.EOLComment) multipleQueriesPrepare = false; + + if (!hasParam) { + //permit to have rewrite without parameter + if (preValuePart1 === null) { + partList.unshift(""); + partList.unshift(sql); + } else { + partList.unshift(sql.substring(lastParameterPosition, idx)); + partList.unshift(preValuePart1); + } + lastParameterPosition = idx; + } else { + partList.unshift(preValuePart2 !== null ? preValuePart2 : ""); + partList.unshift(preValuePart1 !== null ? preValuePart1 : ""); + } + + if (!isInsert) { + reWritablePrepare = false; + } + + //postValuePart is the value after the last parameter and parenthesis + //if no param, don't add to the list. + if (hasParam) { + partList.push(postValuePart !== null ? postValuePart : ""); + } + partList.push(sql.substring(lastParameterPosition, idx)); + + return { + partList: partList, + reWritable: reWritablePrepare, + multipleQueries: multipleQueriesPrepare, + values: values + }; +}; diff --git a/test/integration/test-batch.js b/test/integration/test-batch.js index dec9ae1d..48f35b06 100644 --- a/test/integration/test-batch.js +++ b/test/integration/test-batch.js @@ -42,241 +42,681 @@ describe("batch", () => { fs.unlink(bigFileName, err => {}); }); - it("simple batch", done => { - base - .createConnection() - .then(conn => { - conn.query( - "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" - ); - conn - .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", [[1, "john"], [2, "jack"]]) - .then(res => { - assert.equal(res.affectedRows, 2); - conn - .query("select * from `parse`") - .then(res => { - assert.deepEqual(res, [ + describe("standard question mark", () => { + it("simple batch", done => { + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", [[1, "john"], [2, "jack"]]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn + .query("select * from `parse`") + .then(res => { + assert.deepEqual(res, [ + { + id: 1, + id2: 1, + id3: 2, + t: "john", + id4: 3 + }, + { + id: 1, + id2: 2, + id3: 2, + t: "jack", + id4: 3 + } + ]); + conn.end(); + done(); + }) + .catch(done); + }); + }) + .catch(done); + }); + + it("simple batch error message ", done => { + base + .createConnection({ trace: true }) + .then(conn => { + conn + .batch("INSERT INTO test.parse values (1, ?, 2, ?, 3)", [[1, "john"], [2, "jack"]]) + .then(() => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + assert.isTrue(err != null); + assert.isTrue(err.message.includes("Table 'test.parse' doesn't exist")); + assert.isTrue( + err.message.includes( + "INSERT INTO test.parse values (1, ?, 2, ?, 3) - parameters:[[1,'john'],[2,'jack']]" + ) + ); + assert.equal(err.errno, 1146); + assert.equal(err.sqlState, "42S02"); + assert.equal(err.code, "ER_NO_SUCH_TABLE"); + conn.end(); + done(); + }); + }) + .catch(done); + }); + + it("non rewritable batch", done => { + base + .createConnection() + .then(conn => { + conn.batch("SELECT ? as id, ? as t", [[1, "john"], [2, "jack"]]).then(res => { + assert.deepEqual(res, [ + [ + { + id: 1, + t: "john" + } + ], + [ + { + id: 2, + t: "jack" + } + ] + ]); + conn.end(); + done(); + }); + }) + .catch(done); + }); + + it("16M+ batch", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" + ); + const values = []; + for (let i = 0; i < 1000000; i++) { + values.push([i, "abcdefghijkflmnopqrtuvwxyz"]); + } + conn + .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", values) + .then(res => { + assert.equal(res.affectedRows, 1000000); + }) + .catch(done); + let currRow = 0; + conn + .queryStream("select * from `parse`") + .on("error", err => { + done(new Error("must not have thrown any error !")); + }) + .on("data", row => { + assert.deepEqual(row, { + id: 1, + id2: currRow, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: 3 + }); + currRow++; + }) + .on("end", () => { + assert.equal(1000000, currRow); + conn.end(); + done(); + }); + }) + .catch(done); + }); + + it("16M+ error batch", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" + ); + const values = []; + for (let i = 0; i < 1000000; i++) { + values.push([i, "abcdefghijkflmnopqrtuvwxyz"]); + } + conn + .batch("INSERT INTO `padddrse` values (1, ?, 2, ?, 3)", values) + .then(res => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + conn + .query("select 1") + .then(rows => { + assert.deepEqual(rows, [{ "1": 1 }]); + conn.end(); + done(); + }) + .catch(done); + }); + }) + .catch(done); + }); + + it("16M+ single insert batch", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + base + .createConnection() + .then(conn => { + conn.query("CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t longtext, id4 int)"); + conn + .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", [[1, bigBuf], [2, "john"]]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn.query("select * from `parse`").then(rows => { + assert.deepEqual(rows, [ { id: 1, id2: 1, id3: 2, - t: "john", + t: bigBuf.toString(), id4: 3 }, { id: 1, id2: 2, id3: 2, - t: "jack", + t: "john", id4: 3 } ]); conn.end(); done(); - }) - .catch(done); - }); - }) - .catch(done); - }); - - it("non rewritable batch", done => { - base - .createConnection() - .then(conn => { - conn.batch("SELECT ? as id, ? as t", [[1, "john"], [2, "jack"]]).then(res => { - assert.deepEqual(res, [ - [ - { - id: 1, - t: "john" - } - ], - [ - { - id: 2, - t: "jack" - } - ] - ]); - conn.end(); - done(); - }); - }) - .catch(done); - }); - - it("16M+ batch", function(done) { - if (maxAllowedSize <= testSize) this.skip(); - this.timeout(30000); - base - .createConnection() - .then(conn => { - conn.query( - "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" - ); - const values = []; - for (let i = 0; i < 1000000; i++) { - values.push([i, "abcdefghijkflmnopqrtuvwxyz"]); - } - conn - .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", values) - .then(res => { - assert.equal(res.affectedRows, 1000000); + }); + }) + .catch(done); + }) + .catch(done); + }); - let currRow = 0; - conn - .queryStream("select * from `parse`") - .on("error", err => { - done(new Error("must not have thrown any error !")); - }) - .on("data", row => { - assert.deepEqual(row, { - id: 1, - id2: currRow, - id3: 2, - t: "abcdefghijkflmnopqrtuvwxyz", - id4: 3 - }); - currRow++; - }) - .on("end", () => { - assert.equal(1000000, currRow); + it("batch with streams", done => { + const stream1 = fs.createReadStream(fileName); + const stream2 = fs.createReadStream(fileName); + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, ?, 2, ?, ?, 3)", [ + [1, stream1, 99], + [2, stream2, 98] + ]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn.query("select * from `parse`").then(res => { + assert.deepEqual(res, [ + { + id: 1, + id2: 1, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: 99, + id5: 3 + }, + { + id: 1, + id2: 2, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: 98, + id5: 3 + } + ]); conn.end(); done(); }); - }) - .catch(done); - }) - .catch(done); - }); + }) + .catch(done); + }) + .catch(done); + }); - it("16M+ single insert batch", function(done) { - if (maxAllowedSize <= testSize) this.skip(); - this.timeout(30000); - base - .createConnection() - .then(conn => { - conn.query("CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t longtext, id4 int)"); - conn - .batch("INSERT INTO `parse` values (1, ?, 2, ?, 3)", [[1, bigBuf], [2, "john"]]) - .then(res => { - assert.equal(res.affectedRows, 2); - conn.query("select * from `parse`").then(rows => { - assert.deepEqual(rows, [ - { - id: 1, - id2: 1, - id3: 2, - t: bigBuf.toString(), - id4: 3 - }, - { - id: 1, - id2: 2, - id3: 2, - t: "john", - id4: 3 - } - ]); + it("batch error with streams", done => { + const stream1 = fs.createReadStream(fileName); + const stream2 = fs.createReadStream(fileName); + base + .createConnection() + .then(conn => { + conn + .batch("INSERT INTO test.parse values (1, ?, 2, ?, ?, 3)", [ + [1, stream1, 99], + [2, stream2, 98] + ]) + .then(() => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + assert.isTrue(err != null); + assert.isTrue(err.message.includes("Table 'test.parse' doesn't exist")); + assert.isTrue( + err.message.includes( + "sql: INSERT INTO test.parse values (1, ?, 2, ?, ?, 3) - parameters:[[1,[object Object],99],[2,[object Object],98]]" + ) + ); + assert.equal(err.errno, 1146); + assert.equal(err.sqlState, "42S02"); + assert.equal(err.code, "ER_NO_SUCH_TABLE"); conn.end(); done(); }); - }) - .catch(done); - }) - .catch(done); + }) + .catch(done); + }); + + it("16M+ batch with streams", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + const values = []; + for (let i = 0; i < 1000000; i++) { + if (i % 100000 === 0) values.push([i, fs.createReadStream(fileName), i * 2]); + else values.push([i, "abcdefghijkflmnopqrtuvwxyz", i * 2]); + } + + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, ?, 2, ?, ?, 3)", values) + .then(res => { + assert.equal(res.affectedRows, 1000000); + let currRow = 0; + conn + .queryStream("select * from `parse`") + .on("error", err => { + done(new Error("must not have thrown any error !")); + }) + .on("data", row => { + assert.deepEqual(row, { + id: 1, + id2: currRow, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: currRow * 2, + id5: 3 + }); + currRow++; + }) + .on("end", () => { + assert.equal(1000000, currRow); + conn.end(); + done(); + }); + }) + .catch(done); + }) + .catch(done); + }); + + it("16M+ error batch with streams", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + const values = []; + for (let i = 0; i < 1000000; i++) { + if (i % 100000 === 0) values.push([i, fs.createReadStream(fileName), i * 2]); + else values.push([i, "abcdefghijkflmnopqrtuvwxyz", i * 2]); + } + + base + .createConnection() + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" + ); + conn + .batch("INSERT INTO `padrse` values (1, ?, 2, ?, ?, 3)", values) + .then(res => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + conn + .query("select 1") + .then(rows => { + assert.deepEqual(rows, [{ "1": 1 }]); + conn.end(); + done(); + }) + .catch(done); + }); + }) + .catch(done); + }); }); - it("batch with streams", done => { - const stream1 = fs.createReadStream(fileName); - const stream2 = fs.createReadStream(fileName); - base - .createConnection() - .then(conn => { - conn.query( - "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" - ); - conn - .batch("INSERT INTO `parse` values (1, ?, 2, ?, ?, 3)", [ - [1, stream1, 99], - [2, stream2, 98] - ]) - .then(res => { - assert.equal(res.affectedRows, 2); - conn.query("select * from `parse`").then(res => { + describe("named parameter", () => { + it("simple batch", done => { + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, :param_1, 2, :param_2, 3)", [ + { param_1: 1, param_2: "john" }, + { param_1: 2, param_2: "jack" } + ]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn + .query("select * from `parse`") + .then(res => { + assert.deepEqual(res, [ + { + id: 1, + id2: 1, + id3: 2, + t: "john", + id4: 3 + }, + { + id: 1, + id2: 2, + id3: 2, + t: "jack", + id4: 3 + } + ]); + conn.end(); + done(); + }) + .catch(done); + }); + }) + .catch(done); + }); + + it("simple batch error", done => { + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn + .batch("INSERT INTO test.parse values (1, :param_1, 2, :param_2, 3)", [ + { param_1: 1, param_2: "john" }, + { param_1: 2, param_2: "jack" } + ]) + .then(() => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + assert.isTrue(err != null); + assert.isTrue(err.message.includes("Table 'test.parse' doesn't exist")); + assert.isTrue( + err.message.includes( + "sql: INSERT INTO test.parse values (1, :param_1, 2, :param_2, 3) - parameters:[{'param_1':1,'param_2':'john'},{'param_1':2,'param_2':'jack'}]" + ) + ); + assert.equal(err.errno, 1146); + assert.equal(err.sqlState, "42S02"); + assert.equal(err.code, "ER_NO_SUCH_TABLE"); + conn.end(); + done(); + }); + }) + .catch(done); + }); + + it("non rewritable batch", done => { + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn + .batch("SELECT :id2 as id, :id1 as t", [ + { id2: 1, id1: "john" }, + { id1: "jack", id2: 2 } + ]) + .then(res => { assert.deepEqual(res, [ - { - id: 1, - id2: 1, - id3: 2, - t: "abcdefghijkflmnopqrtuvwxyz", - id4: 99, - id5: 3 - }, - { - id: 1, - id2: 2, - id3: 2, - t: "abcdefghijkflmnopqrtuvwxyz", - id4: 98, - id5: 3 - } + [ + { + id: 1, + t: "john" + } + ], + [ + { + id: 2, + t: "jack" + } + ] ]); conn.end(); done(); }); - }) - .catch(done); - }) - .catch(done); - }); + }) + .catch(done); + }); - it("16M+ batch with streams", function(done) { - if (maxAllowedSize <= testSize) this.skip(); - this.timeout(30000); - const values = []; - for (let i = 0; i < 1000000; i++) { - if (i % 100000 === 0) values.push([i, fs.createReadStream(fileName), i * 2]); - else values.push([i, "abcdefghijkflmnopqrtuvwxyz", i * 2]); - } + it("16M+ batch", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int)" + ); + const values = []; + for (let i = 0; i < 1000000; i++) { + values.push({ id1: i, id2: "abcdefghijkflmnopqrtuvwxyz" }); + } + conn + .batch("INSERT INTO `parse` values (1, :id1, 2, :id2, 3)", values) + .then(res => { + assert.equal(res.affectedRows, 1000000); - base - .createConnection() - .then(conn => { - conn.query( - "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" - ); - conn - .batch("INSERT INTO `parse` values (1, ?, 2, ?, ?, 3)", values) - .then(res => { - assert.equal(res.affectedRows, 1000000); - let currRow = 0; - conn - .queryStream("select * from `parse`") - .on("error", err => { - done(new Error("must not have thrown any error !")); - }) - .on("data", row => { - assert.deepEqual(row, { - id: 1, - id2: currRow, - id3: 2, - t: "abcdefghijkflmnopqrtuvwxyz", - id4: currRow * 2, - id5: 3 + let currRow = 0; + conn + .queryStream("select * from `parse`") + .on("error", err => { + done(new Error("must not have thrown any error !")); + }) + .on("data", row => { + assert.deepEqual(row, { + id: 1, + id2: currRow, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: 3 + }); + currRow++; + }) + .on("end", () => { + assert.equal(1000000, currRow); + conn.end(); + done(); }); - currRow++; - }) - .on("end", () => { - assert.equal(1000000, currRow); + }) + .catch(done); + }) + .catch(done); + }); + + it("16M+ single insert batch", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn.query("CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t longtext, id4 int)"); + conn + .batch("INSERT INTO `parse` values (1, :id, 2, :id2, 3)", [ + { id: 1, id2: bigBuf }, + { id: 2, id2: "john" } + ]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn.query("select * from `parse`").then(rows => { + assert.deepEqual(rows, [ + { + id: 1, + id2: 1, + id3: 2, + t: bigBuf.toString(), + id4: 3 + }, + { + id: 1, + id2: 2, + id3: 2, + t: "john", + id4: 3 + } + ]); conn.end(); done(); }); - }) - .catch(done); - }) - .catch(done); + }) + .catch(done); + }) + .catch(done); + }); + + it("batch with streams", done => { + const stream1 = fs.createReadStream(fileName); + const stream2 = fs.createReadStream(fileName); + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, :id1, 2, :id3, :id7, 3)", [ + { id1: 1, id3: stream1, id4: 99, id5: 6 }, + { id1: 2, id3: stream2, id4: 98 } + ]) + .then(res => { + assert.equal(res.affectedRows, 2); + conn.query("select * from `parse`").then(res => { + assert.deepEqual(res, [ + { + id: 1, + id2: 1, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: null, + id5: 3 + }, + { + id: 1, + id2: 2, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: null, + id5: 3 + } + ]); + conn.end(); + done(); + }); + }) + .catch(done); + }) + .catch(done); + }); + + it("batch error with streams", done => { + const stream1 = fs.createReadStream(fileName); + const stream2 = fs.createReadStream(fileName); + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn + .batch("INSERT INTO test.parse values (1, :id1, 2, :id3, :id7, 3)", [ + { id1: 1, id3: stream1, id4: 99, id5: 6 }, + { id1: 2, id3: stream2, id4: 98 } + ]) + .then(() => { + done(new Error("must have thrown error !")); + }) + .catch(err => { + assert.isTrue(err != null); + assert.isTrue(err.message.includes("Table 'test.parse' doesn't exist")); + assert.isTrue( + err.message.includes( + "sql: INSERT INTO test.parse values (1, :id1, 2, :id3, :id7, 3) - parameters:[{'id1':1,'id3':[object Object],'id4':99,'id5':6},{'id1':2,'id3':[object Object],'id4':98}]" + ) + ); + assert.equal(err.errno, 1146); + assert.equal(err.sqlState, "42S02"); + assert.equal(err.code, "ER_NO_SUCH_TABLE"); + conn.end(); + done(); + }); + }) + .catch(done); + }); + + it("16M+ batch with streams", function(done) { + if (maxAllowedSize <= testSize) this.skip(); + this.timeout(30000); + const values = []; + for (let i = 0; i < 1000000; i++) { + if (i % 100000 === 0) + values.push({ id1: i, id2: fs.createReadStream(fileName), id3: i * 2 }); + else values.push({ id1: i, id2: "abcdefghijkflmnopqrtuvwxyz", id3: i * 2 }); + } + + base + .createConnection({ namedPlaceholders: true }) + .then(conn => { + conn.query( + "CREATE TEMPORARY TABLE parse(id int, id2 int, id3 int, t varchar(128), id4 int, id5 int)" + ); + conn + .batch("INSERT INTO `parse` values (1, :id1, 2, :id2, :id3, 3)", values) + .then(res => { + assert.equal(res.affectedRows, 1000000); + let currRow = 0; + conn + .queryStream("select * from `parse`") + .on("error", err => { + done(new Error("must not have thrown any error !")); + }) + .on("data", row => { + assert.deepEqual(row, { + id: 1, + id2: currRow, + id3: 2, + t: "abcdefghijkflmnopqrtuvwxyz", + id4: currRow * 2, + id5: 3 + }); + currRow++; + }) + .on("end", () => { + assert.equal(1000000, currRow); + conn.end(); + done(); + }); + }) + .catch(done); + }) + .catch(done); + }); }); }); diff --git a/test/integration/test-connection.js b/test/integration/test-connection.js index ecb08aa7..22200649 100644 --- a/test/integration/test-connection.js +++ b/test/integration/test-connection.js @@ -364,7 +364,10 @@ describe("connection", () => { conn.end(); }) .catch(err => { - assert(err.message.includes("socket timeout"), err.message); + assert( + err.message.includes("socket timeout") || err.message.includes("Connection timeout"), + err.message + ); assert.equal(err.sqlState, "08S01"); assert.equal(err.errno, 45026); assert.equal(err.code, "ER_SOCKET_TIMEOUT"); diff --git a/test/integration/test-error.js b/test/integration/test-error.js index bc4c2094..555c3077 100644 --- a/test/integration/test-error.js +++ b/test/integration/test-error.js @@ -136,9 +136,7 @@ describe("Error", () => { assert.isTrue(err.message.includes("You have an error in your SQL syntax")); assert.isTrue( err.message.includes( - "sql: wrong query ?, ?, ?, ?, ?, ?, ? - " + - "parameters:[addon-bla,true,123,456.5,'long parameter that must be truncated'," + - '{"bla":4,"blou":"t"},{}]' + 'wrong query ?, ?, ?, ?, ?, ?, ? - parameters:[addon-bla,true,123,456.5,\'long parameter that must be truncated\',{"bla":4,"blou":"t"},{}]' ) ); assert.equal(err.errno, 1064); diff --git a/test/integration/test-pool-callback.js b/test/integration/test-pool-callback.js index 4a393666..e9008b0c 100644 --- a/test/integration/test-pool-callback.js +++ b/test/integration/test-pool-callback.js @@ -114,7 +114,7 @@ describe("Pool callback", () => { assert.equal(err.code, "ER_GET_CONNECTION_TIMEOUT"); const elapse = Date.now() - initTime; assert.isOk( - elapse >= 699 && elapse < 750, + elapse >= 698 && elapse < 750, "elapse time was " + elapse + " but must be just after 700" ); done(); diff --git a/test/unit/misc/test-parse.js b/test/unit/misc/test-parse.js index ab4c2a9c..9a5bf766 100644 --- a/test/unit/misc/test-parse.js +++ b/test/unit/misc/test-parse.js @@ -217,4 +217,268 @@ describe("parse", () => { }); }); }); + + describe("named parameter batch rewrite", () => { + const values = [{ id1: 1, id2: 2 }, { id3: 3, id2: 4 }, { id2: 5, id1: 6 }]; + + it("select", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "select '\\'' as a, :id2 as b, \"\\\"\" as c, :id1 as d", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["select '\\'' as a, ", "", ' as b, "\\"" as c, ', "", " as d"], + values: [[2, 1], [4, null], [5, 6]], + reWritable: false + }); + }); + + it("rewritable with constant parameters ", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO TABLE(col1,col2,col3,col4, col5) VALUES (9, :id2, 5, :id1, 8) ON DUPLICATE KEY UPDATE col2=col2+10", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: [ + "INSERT INTO TABLE(col1,col2,col3,col4, col5) VALUES", + " (9, ", + ", 5, ", + ", 8)", + " ON DUPLICATE KEY UPDATE col2=col2+10" + ], + values: [[2, 1], [4, null], [5, 6]], + reWritable: true + }); + }); + + it("test comments ", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "/* insert Select INSERT INTO tt VALUES (:id2,:id1,?,?) */" + + " INSERT into " + + "/* insert Select INSERT INTO tt VALUES (?,:id2,?,?) */" + + " tt VALUES " + + "/* insert Select INSERT INTO tt VALUES (?,?,:id2,?) */" + + " (:id2) " + + "/* insert Select INSERT INTO tt VALUES (?,?,?,?) */", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: [ + "/* insert Select INSERT INTO tt VALUES (:id2,:id1,?,?) */" + + " INSERT into " + + "/* insert Select INSERT INTO tt VALUES (?,:id2,?,?) */" + + " tt VALUES", + " /* insert Select INSERT INTO tt VALUES (?,?,:id2,?) */ (", + ")", + " /* insert Select INSERT INTO tt VALUES (?,?,?,?) */" + ], + values: [[2], [4], [5]], + reWritable: true + }); + }); + + it("rewritable with constant parameters and parameters after ", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO TABLE(col1,col2,col3,col4, col5) VALUES (9, :id2, 5, :id1, 8) ON DUPLICATE KEY UPDATE col2=:id3", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: [ + "INSERT INTO TABLE(col1,col2,col3,col4, col5) VALUES", + " (9, ", + ", 5, ", + ", 8) ON DUPLICATE KEY UPDATE col2=", + "", + "" + ], + values: [[2, 1, null], [4, null, 3], [5, 6, null]], + reWritable: false + }); + }); + + it("rewritable with multiple values ", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO TABLE(col1,col2) VALUES (:id2, :id3), (:id1, :id4)", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO TABLE(col1,col2) VALUES", " (", ", ", "), (", ", ", ")", ""], + values: [[2, null, 1, null], [4, 3, null, null], [5, null, 6, null]], + reWritable: false + }); + }); + + it("Call", () => { + const res = Parse.splitRewritableNamedParameterQuery("CALL dsdssd(:id1,:id2)", values); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["CALL dsdssd(", "", ",", ")", ""], + values: [[1, 2], [null, 4], [6, 5]], + reWritable: false + }); + }); + + it("Update", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "UPDATE MultiTestt4 SET test = :id1 WHERE test = :id2", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["UPDATE MultiTestt4 SET test = ", "", " WHERE test = ", "", ""], + values: [[1, 2], [null, 4], [6, 5]], + reWritable: false + }); + }); + + it("insert select", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "insert into test_insert_select ( field1) (select TMP.field1 from " + + "(select CAST(:id1 as binary) `field1` from dual) TMP)", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: [ + "insert into test_insert_select ( field1) (select TMP.field1 from (select CAST(", + "", + " as binary) `field1` from dual) TMP)", + "" + ], + values: [[1], [null], [6]], + reWritable: false + }); + }); + + it("select without parameter", () => { + const res = Parse.splitRewritableNamedParameterQuery("SELECT testFunction()", values); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["SELECT testFunction()", "", ""], + values: [[], [], []], + reWritable: false + }); + }); + + it("insert without parameter", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT VALUES (testFunction())", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT VALUES", " (testFunction())", ""], + values: [[], [], []], + reWritable: true + }); + }); + + it("select without parenthesis", () => { + const res = Parse.splitRewritableNamedParameterQuery("SELECT 1", values); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["SELECT 1", "", ""], + values: [[], [], []], + reWritable: false + }); + }); + + it("insert without parameters", () => { + const res = Parse.splitRewritableNamedParameterQuery("INSERT INTO tt VALUES (1)", values); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO tt VALUES", " (1)", ""], + values: [[], [], []], + reWritable: true + }); + }); + + it("semicolon", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO tt (tt) VALUES (:id1); INSERT INTO tt (tt) VALUES ('multiple')", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: [ + "INSERT INTO tt (tt) VALUES", + " (", + ")", + "; INSERT INTO tt (tt) VALUES ('multiple')" + ], + values: [[1], [null], [6]], + reWritable: false + }); + }); + + it("semicolon with empty data after", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO table (column1) VALUES (:id1); ", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO table (column1) VALUES", " (", ")", "; "], + values: [[1], [null], [6]], + reWritable: false + }); + }); + + it("semicolon not rewritable if not at end", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO table (column1) VALUES (:id1); SELECT 1", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO table (column1) VALUES", " (", ")", "; SELECT 1"], + values: [[1], [null], [6]], + reWritable: false + }); + }); + + it("line end comment", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO tt (tt) VALUES (:id1) --fin", + values + ); + assert.deepEqual(res, { + multipleQueries: false, + partList: ["INSERT INTO tt (tt) VALUES", " (", ")", " --fin"], + values: [[1], [null], [6]], + reWritable: true + }); + }); + + it("line finished comment", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO tt (tt) VALUES --fin\n (:id1)", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO tt (tt) VALUES", " --fin\n (", ")", ""], + values: [[1], [null], [6]], + reWritable: true + }); + }); + + it("line finished comment", () => { + const res = Parse.splitRewritableNamedParameterQuery( + "INSERT INTO tt (tt, tt2) VALUES (LAST_INSERT_ID(), :id1)", + values + ); + assert.deepEqual(res, { + multipleQueries: true, + partList: ["INSERT INTO tt (tt, tt2) VALUES", " (LAST_INSERT_ID(), ", ")", ""], + values: [[1], [null], [6]], + reWritable: false + }); + }); + }); });