Issue #5769: Sqlite support multi-value JSON insert

lib/.gitignore

@@ -5,3 +5,7 @@
 
 # Do not include tsc source maps
 **/*.js.map
+
+# Do not include .js files from .ts files
+util/version.js
+dialects/index.js

lib/client.js

@@ -194,6 +194,7 @@ class Client extends EventEmitter {
   }
 
   initializeDriver() {
+    if (this.driver) return;
     try {
       this.driver = this._driver();
     } catch (e) {

lib/dialects/better-sqlite3/index.js

@@ -7,6 +7,19 @@ class Client_BetterSQLite3 extends Client_SQLite3 {
     return require('better-sqlite3');
   }
 
+  /**
+   * Get `Version` from client or return `[0, 0, 0]` on failure.
+   *
+   * @returns {[number, number, number]}
+   */
+  _clientVersion() {
+    // hard coding 3.25.1 as better-sqlite3 offers no way to get Sqlite3 version it was built with
+    // and better-sqlite3 1.0.0 shipped with 3.25.1 in 2016.
+    //
+    // Issue opened for this: https://github.com/WiseLibs/better-sqlite3/issues/1122
+    return [3, 25, 1];
+  }
+
   // Get a raw connection from the database, returning a promise with the connection object.
   async acquireRawConnection() {
     const options = this.connectionSettings.options || {};

lib/dialects/sqlite3/index.js

@@ -16,6 +16,7 @@ const ViewCompiler = require('./schema/sqlite-viewcompiler');
 const SQLite3_DDL = require('./schema/ddl');
 const Formatter = require('../../formatter');
 const QueryBuilder = require('./query/sqlite-querybuilder');
+const { parseVersion, satisfiesVersion } = require('../../util/version');
 
 class Client_SQLite3 extends Client {
   constructor(config) {
@@ -42,6 +43,29 @@ class Client_SQLite3 extends Client {
     return require('sqlite3');
   }
 
+  /**
+   * Get `Version` from client or return `[0, 0, 0]` on failure.
+   *
+   * @returns {[number, number, number]}
+   */
+  _clientVersion() {
+    this.initializeDriver();
+    const version = parseVersion(this.driver.VERSION);
+    if (!version) return [0, 0, 0];
+    return version;
+  }
+
+  /**
+   * Takes min, max, or both and returns if given client satisfies version
+   *
+   * @param {undefined | [number, number, number]} minVersion
+   * @param {undefined | [number, number, number]} maxVersion
+   * @returns {boolean}
+   */
+  _satisfiesVersion(minVersion, maxVersion) {
+    return satisfiesVersion(this._clientVersion(), minVersion, maxVersion);
+  }
+
   schemaCompiler() {
     return new SchemaCompiler(this, ...arguments);
   }

lib/dialects/sqlite3/query/sqlite-querycompiler.js

@@ -34,6 +34,7 @@ class QueryCompiler_SQLite3 extends QueryCompiler {
     const insertValues = this.single.insert || [];
     let sql = this.with() + `insert into ${this.tableName} `;
 
+    // Handle "empty cases" of: [], [{}], {}
     if (Array.isArray(insertValues)) {
       if (insertValues.length === 0) {
         return '';
@@ -66,6 +67,7 @@ class QueryCompiler_SQLite3 extends QueryCompiler {
       };
     }
 
+    // 'insert into TABLE_NAME (column1, column2, ...)'
     sql += `(${this.formatter.columnize(insertData.columns)})`;
 
     // backwards compatible error
@@ -82,62 +84,34 @@ class QueryCompiler_SQLite3 extends QueryCompiler {
       });
     }
 
-    if (insertData.values.length === 1) {
-      const parameters = this.client.parameterize(
-        insertData.values[0],
-        this.client.valueForUndefined,
-        this.builder,
-        this.bindingsHolder
-      );
-      sql += ` values (${parameters})`;
-
-      const { onConflict, ignore, merge } = this.single;
-      if (onConflict && ignore) sql += this._ignore(onConflict);
-      else if (onConflict && merge) {
-        sql += this._merge(merge.updates, onConflict, insertValues);
-        const wheres = this.where();
-        if (wheres) sql += ` ${wheres}`;
-      }
-
-      const { returning } = this.single;
-      if (returning) {
-        sql += this._returning(returning);
-      }
-
-      return {
-        sql,
-        returning,
-      };
+    if (
+      !this.client._satisfiesVersion([3, 7, 11]) &&
+      insertData.values.length > 1
+    ) {
+      throw new Error('Requires Sqlite 3.7.11 or newer to do multi-insert');
     }
 
-    const blocks = [];
+    // Holds parameters wrapped in `()`
+    const parametersArray = [];
     let i = -1;
     while (++i < insertData.values.length) {
-      let i2 = -1;
-      const block = (blocks[i] = []);
-      let current = insertData.values[i];
-      current = current === undefined ? this.client.valueForUndefined : current;
-      while (++i2 < insertData.columns.length) {
-        block.push(
-          this.client.alias(
-            this.client.parameter(
-              current[i2],
-              this.builder,
-              this.bindingsHolder
-            ),
-            this.formatter.wrap(insertData.columns[i2])
-          )
-        );
-      }
-      blocks[i] = block.join(', ');
+      const parameter = this.client.parameterize(
+        insertData.values[i],
+        this.client.valueForUndefined,
+        this.builder,
+        this.bindingsHolder
+      );
+      parametersArray.push(`(${parameter})`);
     }
-    sql += ' select ' + blocks.join(' union all select ');
+    // 'insert into TABLE_NAME (column1, column2, ...) values (v1, v2, ...), (v3, v4, ...), ...'
+    sql += ` values ${parametersArray.join(', ')}`;
 
     const { onConflict, ignore, merge } = this.single;
-    if (onConflict && ignore) sql += ' where true' + this._ignore(onConflict);
+    if (onConflict && ignore) sql += this._ignore(onConflict);
     else if (onConflict && merge) {
-      sql +=
-        ' where true' + this._merge(merge.updates, onConflict, insertValues);
+      sql += this._merge(merge.updates, onConflict, insertValues);
+      const wheres = this.where();
+      if (wheres) sql += ` ${wheres}`;
     }
 
     const { returning } = this.single;

lib/query/querycompiler.js

@@ -1271,6 +1271,16 @@ class QueryCompiler {
     return str;
   }
 
+  /**
+   * Takes:
+   *
+   * - Single object or array of objects.
+   *
+   * Does:
+   *
+   * - If a raw value is given return it.
+   * - Else returns `{ columns, values }` object
+   */
   _prepInsert(data) {
     const isRaw = rawOrFn_(
       data,
@@ -1284,24 +1294,36 @@ class QueryCompiler {
     const values = [];
     if (!Array.isArray(data)) data = data ? [data] : [];
     let i = -1;
+    // Iterate over each item given ...
     while (++i < data.length) {
+      // ... ignoring null values ...
       if (data[i] == null) break;
+      // ... if this is the first item populate `columns` from its sorted keys ...
       if (i === 0) columns = Object.keys(data[i]).sort();
+      // ... create row value array based on `columns` ...
       const row = new Array(columns.length);
+      // ... populate `keys` with items keys ...
       const keys = Object.keys(data[i]);
+      // ... then for each key ...
       let j = -1;
       while (++j < keys.length) {
         const key = keys[j];
+        // ... attempt to get the `column` index that key goes to ...
         let idx = columns.indexOf(key);
+        // ... if no index found ...
         if (idx === -1) {
+          // ... then retro-actively update existing columns and existing `values`
+          // to incorporate newly discovered key ...
           columns = columns.concat(key).sort();
           idx = columns.indexOf(key);
           let k = -1;
           while (++k < values.length) {
             values[k].splice(idx, 0, undefined);
           }
+          // ... and then update row being actively built.
           row.splice(idx, 0, undefined);
         }
+        // ... else if column index found populate it with value.
         row[idx] = data[i][key];
       }
       values.push(row);

lib/util/version.ts

@@ -0,0 +1,102 @@
+/** Tuple of three non-negative integers representing a semantic version */
+export type Version = [number, number, number];
+
+/** Helper function to check `x` a integer where `x >= 0`  */
+export function isNonNegInt(x: unknown): x is number {
+  return typeof x === 'number' && Number.isInteger(x) && x >= 0;
+}
+
+/** Type-guard for `Version` type */
+export function isVersion(x: unknown): x is Version {
+  return (
+    Array.isArray(x) &&
+    x.length === 3 &&
+    x.findIndex((y) => !isNonNegInt(y)) === -1
+  );
+}
+
+/** Parses given string into `Version` or returns `undefined` */
+export function parseVersion(x: string): Version | undefined {
+  const versionRegex = /^(\d+)\.(\d+)\.(\d+)/m;
+  const versionNumbers = (versionRegex.exec(x) ?? [])
+    .slice(1, 4)
+    .map((x) => parseInt(x));
+  if (!isVersion(versionNumbers)) return undefined;
+  return versionNumbers;
+}
+
+/** Parses given string into `Version` or throws an error */
+export function parseVersionOrError(x: string): Version {
+  const version = parseVersion(x);
+  if (version === undefined) {
+    throw new Error('Could not parse string to Version');
+  }
+  return version;
+}
+
+/**
+ * Compares two versions, returning a number to represent the result:
+ *
+ * - `1` means `v1 > v2`
+ * - `0` means `v1 == v2`
+ * - `-1` means `v1 < v2`
+ */
+export function compareVersions(v1: Version, v2: Version): 1 | 0 | -1 {
+  // Check major
+  if (v1[0] < v2[0]) return -1;
+  else if (v1[0] > v2[0]) return 1;
+  else {
+    // Check minor
+    if (v1[1] < v2[1]) return -1;
+    else if (v1[1] > v2[1]) return 1;
+    else {
+      // Check patch
+      if (v1[2] < v2[2]) return -1;
+      else if (v1[2] > v2[2]) return 1;
+      else return 0;
+    }
+  }
+}
+
+/**
+ * Returns `boolean` for if a given `version` satisfies the given `min` (inclusive) and `max` (exclusive).
+ *
+ * This will throw an error if:
+ *
+ * - Given `version` is NOT a valid `Version`
+ * - Neither `min` nor `max` is given
+ * - `min` is given but is NOT a valid `Version`
+ * - `max` is given but is NOT a valid `Version`
+ */
+export function satisfiesVersion(
+  version: Version,
+  min?: Version,
+  max?: Version
+): boolean {
+  if (!min && !max) {
+    throw new Error('Must pass at least one version constraint');
+  }
+  if (!isVersion(version)) {
+    throw new Error('Invalid value given for: version');
+  }
+
+  // Check Min
+  let satisfiesMin = true;
+  if (min) {
+    if (!isVersion(min)) {
+      throw new Error('Invalid value given for: min');
+    }
+    satisfiesMin = compareVersions(version, min) > -1;
+  }
+
+  // Check max
+  let satisfiesMax = true;
+  if (max) {
+    if (!isVersion(max)) {
+      throw new Error('Invalid value given for: max');
+    }
+    satisfiesMax = compareVersions(version, max) === -1;
+  }
+
+  return satisfiesMin && satisfiesMax;
+}

test/db-less-test-suite.js

@@ -11,6 +11,7 @@ describe('Util Tests', function () {
   require('./unit/util/save-async-stack');
   require('./unit/util/comma-no-paren-regex');
   require('./unit/util/security');
+  require('./unit/util/version');
 });
 
 describe('Query Building Tests', function () {