[CONJS-153] binary protocol support

new methods connection.prepare, connection.execute, connection.unprepare like mysql2 API. Implementation differ compared to mysql2 : * support binary batching * support streaming parameters * cache is not infinite: using a LRU cache (option `cacheLen`, default to 256, 0 meaning no cache). * Implement MariaDB 10.6 new feature permitting to skip metadata if cached, server sending only the raw data for faster results
mariadb-corporation · Jun 4, 2021 · 2d0486e · 2d0486e
1 parent 0685ec2
commit 2d0486e
Show file tree

Hide file tree

Showing 101 changed files with 6,398 additions and 8,148 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 os: linux
 language: node_js
 services: docker
-node_js: 12
+node_js: 16
 addons:
   hosts:
   - mariadb.example.com
@@ -10,6 +10,7 @@ addons:
 before_install:
   - git clone https://github.com/mariadb-corporation/connector-test-machine.git
 
+
 install:
   - |-
     case $TRAVIS_OS_NAME in
@@ -31,8 +32,8 @@ env:
 jobs:
   fast_finish: true
   allow_failures:
-    - env: srv=skysql
-    - env: srv=skysql-ha
+    - env: srv=skysql RUN_LONG_TEST=0
+    - env: srv=skysql-ha RUN_LONG_TEST=0
     - env: srv=build v=10.6
   include:
     - env: srv=mariadb v=10.5
@@ -45,12 +46,12 @@ jobs:
     - env: srv=mariadb v=10.5
       node_js: 14
     - env: srv=mariadb v=10.5
-      node_js: 15
+      node_js: 12
     - env: srv=mariadb-es v=10.5
-    - env: srv=mariadb v=10.5 BENCH=1
+    - env: srv=mariadb v=10.6 BENCH=1
     - env: srv=maxscale
-    - env: srv=skysql
-    - env: srv=skysql-ha
+    - env: srv=skysql RUN_LONG_TEST=0
+    - env: srv=skysql-ha RUN_LONG_TEST=0
     - env: srv=build v=10.6
     - env: srv=mysql v=5.7
     - env: srv=mysql v=8.0

diff --git a/benchmarks/benchmarkOne.js b/benchmarks/benchmarkOne.js
@@ -10,7 +10,7 @@ const launchBenchs = function (path) {
   const test = 'bench_promise_select_param.js';
   const m = require(path + '/' + test);
   bench.initFcts.push([m.initFct, m.promise]);
-  bench.add(m.title, m.displaySql, m.benchFct, m.onComplete, m.promise, m.pool); //, bench.CONN.MYSQL);
+  bench.add(m.title, m.displaySql, m.benchFct, m.onComplete, m.promise, m.pool, m.requireExecute); //, bench.CONN.MYSQL);
 
   bench.suiteReady();
 };

diff --git a/benchmarks/benchmarks.js b/benchmarks/benchmarks.js
@@ -14,7 +14,15 @@ const launchBenchs = function (path) {
       console.log('benchmark: ./benchs/' + list[i]);
       const m = require('./benchs/' + list[i]);
       bench.initFcts.push([m.initFct, m.promise]);
-      bench.add(m.title, m.displaySql, m.benchFct, m.onComplete, m.promise, m.pool);
+      bench.add(
+        m.title,
+        m.displaySql,
+        m.benchFct,
+        m.onComplete,
+        m.promise,
+        m.pool,
+        m.requireExecute
+      );
     }
     bench.suiteReady();
   });

diff --git a/benchmarks/benchs/bench_promise_select_one_user_execute.js b/benchmarks/benchs/bench_promise_select_one_user_execute.js
@@ -0,0 +1,17 @@
+const assert = require('assert');
+
+module.exports.title = 'select one mysql.user using execute promise';
+module.exports.displaySql = 'select <all mysql.user fields> from mysql.user u LIMIT 1';
+module.exports.promise = true;
+module.exports.requireExecute = true;
+module.exports.benchFct = function (conn, deferred) {
+  conn
+    .execute('select * from mysql.user u LIMIT 1')
+    .then((rows) => {
+      // assert.equal(50000000, rows[0]["t"]);
+      deferred.resolve();
+    })
+    .catch((err) => {
+      throw err;
+    });
+};
diff --git a/benchmarks/common_benchmarks.js b/benchmarks/common_benchmarks.js
@@ -296,6 +296,7 @@ Bench.prototype.displayReport = function () {
   const keys = Object.keys(this.reportData);
   for (let i = 0; i < keys.length; i++) {
     let base = 0;
+    let base2 = 0;
     let best = 0;
     let data = this.reportData[keys[i]];
 
@@ -304,11 +305,16 @@ Bench.prototype.displayReport = function () {
       if (o.drvType === 'mysql' || o.drvType === 'promise-mysql') {
         base = o.iteration;
       }
+      if (o.drvType === 'mysql2' || o.drvType === 'promise mysql2') {
+        base2 = o.iteration;
+      }
       if (o.iteration > best) {
         best = o.iteration;
       }
     }
-
+    if (base === 0) {
+      base = base2;
+    }
     //display results
     console.log('');
     console.log('bench : ' + keys[i]);
@@ -352,7 +358,16 @@ Bench.prototype.fill = function (val, length, right) {
   return val;
 };
 
-Bench.prototype.add = function (title, displaySql, fct, onComplete, isPromise, usePool, conn) {
+Bench.prototype.add = function (
+  title,
+  displaySql,
+  fct,
+  onComplete,
+  isPromise,
+  usePool,
+  requireExecute,
+  conn
+) {
   const self = this;
   const addTest = getAddTest(
     self,
@@ -374,11 +389,11 @@ Bench.prototype.add = function (title, displaySql, fct, onComplete, isPromise, u
       addTest(self.CONN.MARIADB, 'warmup');
     }
 
-    if (!isPromise && mysql) {
+    if (!requireExecute && !isPromise && mysql) {
       addTest(self.CONN.MYSQL, self.CONN.MYSQL.desc);
     }
 
-    if (isPromise && promiseMysql) {
+    if (!requireExecute && isPromise && promiseMysql) {
       addTest(self.CONN.PROMISE_MYSQL, self.CONN.PROMISE_MYSQL.desc);
     }
 
@@ -396,7 +411,7 @@ Bench.prototype.add = function (title, displaySql, fct, onComplete, isPromise, u
       addTest(self.CONN.MARIADB, self.CONN.MARIADB.desc);
     }
 
-    if (isPromise && promiseMariasql) {
+    if (!requireExecute && isPromise && promiseMariasql) {
       addTest(self.CONN.PROMISE_MARIASQL, self.CONN.PROMISE_MARIASQL.desc);
     }
 

diff --git a/documentation/connection-options.md b/documentation/connection-options.md
@@ -25,6 +25,9 @@
 | **socketTimeout** | Socket timeout in milliseconds after the connection is established |*integer* | 0|
 | **rowsAsArray** | Return result-sets as array, rather than a JSON object. This is a faster way to get results.  For more information, see the [Promise](../README.md#querysql-values---promise) and [Callback](callback-api.md#querysql-values-callback---emoitter) query implementations.|*boolean* | false|
 | **maxAllowedPacket** | permit to indicate server global variable [max_allowed_packet](https://mariadb.com/kb/en/library/server-system-variables/#max_allowed_packet) value to ensure efficient batching. default is 4Mb. see [batch documentation](./batch.md)|*integer* | 4196304|
+| **insertIdAsNumber** | Whether the query should return last insert id from INSERT/UPDATE command as BigInt or Number. default return BigInt |*boolean* | false |
+| **decimalAsNumber** | Whether the query should return decimal as Number. If enable, this might return approximate values. |*boolean* | false |
+| **bigIntAsNumber** | Whether the query should return BigInt data type as Number. If enable, this might return approximate values. |*boolean* | false |
 
 
 ### JSON or String configuration
@@ -54,25 +57,6 @@ mariadb.createConnection('mariadb://root:pass@localhost:3307/db?metaAsArray=fals
 ```
 
 
-
-## Big Integer Support 
-
-Integers in JavaScript use IEEE-754 representation.  This means that Node.js cannot exactly represent integers in the ±9,007,199,254,740,991 range.  However, MariaDB does support larger integers. 
-
-This means that when the value set on a column is not in the [safe](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isSafeInteger) range, the default implementation receives an inexact representation of the number.
-
-The Connector provides 3 options to address this issue. 
-
-|option|description|type|default| 
-|---:|---|:---:|:---:| 
-| **bigNumberStrings** | When an integer is not in the safe range, the Connector interprets the value as a string. |*boolean* |false| 
-| **supportBigNumbers** | When an integer is not in the safe range, the Connector interprets the value as a [Long](https://www.npmjs.com/package/long) object. |*boolean* |false|
-| **supportBigInt** | Whether resultset should return javascript ES2020 [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) for [BIGINT](https://mariadb.com/kb/en/bigint/) data type. This ensures having expected value even for value > 2^53 (see [safe](#big-integer-support) range). |*boolean* | false |
-
-
-Native `supportBigInt` implementation is recommended over `supportBigNumbers` (remains for compability with older version). `supportBigInt` is not enabled by default for compatibilty to avoid major regression. 
-It will be in a future 3.x version.
-
 ## SSL
 
 The Connector can encrypt data during transfer using the Transport Layer Security (TLS) protocol.  TLS/SSL allows for transfer encryption, and can optionally use identity validation for the server and client.

diff --git a/documentation/promise-api.md b/documentation/promise-api.md
@@ -52,6 +52,30 @@ const mariadb = require('mariadb');
 
 ## Recommendation
 
+### Exact number consideration
+
+Integers in JavaScript use IEEE-754 representation.  This means that Node.js cannot exactly represent integers in the ±9,007,199,254,740,991 range.  
+However, MariaDB does support larger integers and exact big decimal.
+This means that when the value set on a BIGINT is not in the [safe](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isSafeInteger) range, javascript Number Object cannot represent exact value.
+Same with DECIMAL type that might not be exact in IEEE 754 floating number.  
+
+Driver automatically return a JSON with type depending on database data types :
+* DECIMAL => javascript String
+* BIGINT => javascript BigInt 
+
+Default implementation return exact values, but might cause some incompatibility.
+Driver provides 3 options to address this issue.
+
+|option|description|type|default| 
+|---:|---|:---:|:---:| 
+| **insertIdAsNumber** | Whether the query should return last insert id from INSERT/UPDATE command as BigInt or Number. default return BigInt |*boolean* | false |
+| **decimalAsNumber** | Whether the query should return decimal as Number. If enable, this might return approximate values. |*boolean* | false |
+| **bigIntAsNumber** | Whether the query should return BigInt data type as Number. If enable, this might return approximate values. |*boolean* | false |
+
+This is a breaking change from 3.0 compare to previous version and mysql/mysql2 drivers. 
+
+If wanting compatibility with previous version those values can be set to true / use [`typeCast`](#typeCast) to convert DECIMAL/BIGINT to expect value. 
+
 ### Timezone consideration
 
 Client and database can have a different timezone.
@@ -158,6 +182,8 @@ BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global
 
 * [`connection.query(sql[, values]) → Promise`](#connectionquerysql-values---promise): Executes a query.
 * [`connection.queryStream(sql[, values]) → Emitter`](#connectionquerystreamsql-values--emitter): Executes a query, returning an emitter object to stream rows.
+* [`connection.prepare(sql) → Promise`](#connectionpreparesql---promise): Prepares a query.
+* [`connection.execute(sql[, values]) → Promise`](#connectionexecutesql-values--promise): Prepare and Executes a query.
 * [`connection.batch(sql, values) → Promise`](#connectionbatchsql-values--promise): fast batch processing.
 * [`connection.beginTransaction() → Promise`](#connectionbegintransaction--promise): Begins a transaction.
 * [`connection.commit() → Promise`](#connectioncommit--promise): Commits the current transaction, if any.
@@ -522,9 +548,7 @@ connection.query('select * from animals')
 * [`rowsAsArray`](#rowsAsArray)
 * [`nestTables`](#nestTables)
 * [`dateStrings`](#dateStrings)
-* [`supportBigNumbers`](#supportBigNumbers)
-* [`supportBigint`](#supportBigint)
-* [`bigNumberStrings`](#bigNumberStrings)
+* [`decimalAsNumber`](#decimalAsNumber)
 
 Those options can be set on the query level, but are usually set at the connection level, and will then apply to all queries. 
 
@@ -553,7 +577,7 @@ connection
           // sql: select * from information_schema.columns as c1, information_schema.tables, information_schema.tables as t2 - parameters:[]
           // at Object.module.exports.createError (C:\projets\mariadb-connector-nodejs.git\lib\misc\errors.js:55:10)
           // at PacketNodeEncoded.readError (C:\projets\mariadb-connector-nodejs.git\lib\io\packet.js:510:19)
-          // at Query.readResponsePacket (C:\projets\mariadb-connector-nodejs.git\lib\cmd\resultset.js:46:28)
+          // at Query.readResponsePacket (C:\projets\mariadb-connector-nodejs.git\lib\cmd\parser.js:46:28)
           // at PacketInputStream.receivePacketBasic (C:\projets\mariadb-connector-nodejs.git\lib\io\packet-input-stream.js:104:9)
           // at PacketInputStream.onData (C:\projets\mariadb-connector-nodejs.git\lib\io\packet-input-stream.js:160:20)
           // at Socket.emit (events.js:210:5)
@@ -663,30 +687,33 @@ Whether the query should return integers as [`Long`](https://www.npmjs.com/packa
  the [safe](/documentation/connection-options.md#big-integer-support) range.
 
 
-#### `supportBigInt`
+#### `bigIntAsNumber`
 
-*boolean, default: false*
+*boolean, default: true*
 
 Whether the query should return javascript ES2020 [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) 
 for [BIGINT](https://mariadb.com/kb/en/bigint/) data type. 
 This ensures having expected value even for value > 2^53 (see [safe](/documentation/connection-options.md#big-integer-support) range).
 This option can be set to query level, supplanting connection option `supportBigInt` value. 
 
+this option is for compatibility for driver version < 3
+
 ```javascript
 await shareConn.query('CREATE TEMPORARY TABLE bigIntTable(id BIGINT)');
 await shareConn.query("INSERT INTO bigIntTable value ('9007199254740993')");
 const res = await shareConn.query('select * from bigIntTable');
-// res :  [{ id: 9007199254740992 }] (not exact value)
-const res2 = await shareConn.query({sql: 'select * from bigIntTable', supportBigInt: true});
 // res :  [{ id: 9007199254740993n }] (exact value)
+const res2 = await shareConn.query({sql: 'select * from bigIntTable', supportBigInt: false});
+// res :  [{ id: 9007199254740992 }] (not exact value)
 ```
 
 
-#### `bigNumberStrings`
+#### `decimalAsNumber`
 
 *boolean, default: false*
 
-Whether the query should return integers as strings when they are not in the [safe](documentation/connection-options.md#big-integer-support) range.
+Whether the query should return decimal as Number. 
+If enable, this might return approximate values. 
 
 
 #### `typeCast`
@@ -829,6 +856,67 @@ const queryStream = connection.queryStream("SELECT * FROM mysql.user");
 stream.pipeline(queryStream, transformStream, someWriterStream);
 
 ```
+## `connection.prepare(sql) → Promise`
+> * `sql`: *string | JSON* SQL string value or JSON object to supersede default connections options.  JSON objects must have an `"sql"` property.  For instance, `{ dateStrings: true, sql: 'SELECT now()' }`
+>
+> Returns a promise that :
+> * resolves with a [Prepare](#prepareobject) object.
+> * rejects with an [Error](#error).
+
+This permit to [PREPARE](https://mariadb.com/kb/en/prepare-statement/) a command that permits to be executed many times. 
+After use, prepare.close() method MUST be call, in order to properly close object. 
+
+
+### Prepare object
+
+Public variables : 
+* `id`: Prepare statement Identifier
+* `query`: sql command
+* `database`: database it applies to. 
+* `parameters`: parameter array information.
+* `columns`: columns array information. 
+
+Public methods :
+#### `execute(values) → Promise`
+> * `values`: *array | object* Defines placeholder values. This is usually an array, but in cases of only one placeholder, it can be given as a string.
+>
+> Returns a promise that :
+> * resolves with a JSON object for update/insert/delete or a [result-set](#result-set-array) object for result-set.
+> * rejects with an [Error](#error).
+
+#### `close() → void`
+This close the prepared statement. 
+Each time a Prepared object is used, it must be closed. 
+
+In case prepare cache is enabled (having option `prepareCacheLength` > 0 (default)), 
+Driver will either really close Prepare or keep it in cache. 
+
+
+```javascript
+const prepare = await conn.prepare('INSERT INTO mytable(id,val) VALUES (?,?)');
+await prepare.execute([1, 'val1'])
+prepare.close();
+```
+
+## `connection.execute(sql[, values]) → Promise`
+> * `sql`: *string | JSON* SQL string or JSON object to supersede default connection options.  When using JSON object, object must have a "sql" key. For instance, `{ dateStrings: true, sql: 'SELECT now()' }`
+> * `values`: *array | object* Placeholder values. Usually an array, but in cases of only one placeholder, it can be given as is.
+>
+> Returns a promise that :
+> * resolves with a JSON object for update/insert/delete or a [result-set](#result-set-array) object for result-set.
+> * rejects with an [Error](#error).
+
+This is quite similar to [`connection.query(sql[, values]) → Promise`](#connectionquerysql-values---promise) method, with a few differences : 
+Execute will in fact [PREPARE](https://mariadb.com/kb/en/prepare-statement/) + [EXECUTE](https://mariadb.com/kb/en/execute-statement/) + [CLOSE](https://mariadb.com/kb/en/deallocate-drop-prepare/) command.
+
+It makes sense to use this only if the command will often be used and if prepare cache is enabled (default).
+If PREPARE result is already in cache, only [EXECUTE](https://mariadb.com/kb/en/execute-statement/) command is executed.
+MariaDB server 10.6 even avoid resending result-set metadata if not changed since, permitting even faster results.
+
+
+```javascript
+const res = await conn.execute('SELECT * FROM mytable WHERE someVal = ? and otherVal = ?', [1, 'val1']);
+```
 
 
 ## `connection.batch(sql, values) → Promise`