feat(mssql): add json operations support (#15832)

Co-authored-by: Zoé <zoe@ephys.dev>
sequelize · Jun 17, 2023 · b0ee419 · b0ee419
1 parent 5c1c7ff
commit b0ee419
Show file tree

Hide file tree

Showing 17 changed files with 425 additions and 310 deletions.
diff --git a/packages/core/src/dialects/abstract/index.ts b/packages/core/src/dialects/abstract/index.ts
@@ -193,6 +193,11 @@ export type DialectSupports = {
   IREGEXP: boolean,
   /** Whether this dialect supports SQL JSON functions */
   jsonOperations: boolean,
+  /** Whether this dialect supports returning quoted & unquoted JSON strings  */
+  jsonExtraction: {
+    unquoted: boolean,
+    quoted: boolean,
+  },
   tmpTableTrigger: boolean,
   indexHints: boolean,
   searchPath: boolean,
@@ -330,6 +335,10 @@ export abstract class AbstractDialect {
       },
     },
     jsonOperations: false,
+    jsonExtraction: {
+      unquoted: false,
+      quoted: false,
+    },
     REGEXP: false,
     IREGEXP: false,
     deferrableConstraints: false,

diff --git a/packages/core/src/dialects/mariadb/index.ts b/packages/core/src/dialects/mariadb/index.ts
@@ -55,6 +55,10 @@ export class MariaDbDialect extends AbstractDialect {
       },
       REGEXP: true,
       jsonOperations: true,
+      jsonExtraction: {
+        unquoted: true,
+        quoted: true,
+      },
       globalTimeZoneConfig: true,
     },
   );

diff --git a/packages/core/src/dialects/mssql/index.ts b/packages/core/src/dialects/mssql/index.ts
@@ -43,8 +43,11 @@ export class MssqlDialect extends AbstractDialect {
       // TODO: https://learn.microsoft.com/en-us/sql/t-sql/spatial-geometry/spatial-types-geometry-transact-sql?view=sql-server-ver16
       GEOMETRY: false,
     },
-    // TODO: add support for JSON queries https://learn.microsoft.com/en-us/sql/relational-databases/json/json-data-sql-server?view=sql-server-ver16
-    jsonOperations: false,
+    jsonOperations: true,
+    jsonExtraction: {
+      unquoted: true,
+      quoted: false,
+    },
   });
 
   readonly connectionManager: MsSqlConnectionManager;

diff --git a/packages/core/src/dialects/mssql/query-generator-typescript.ts b/packages/core/src/dialects/mssql/query-generator-typescript.ts
@@ -1,9 +1,11 @@
+import type { Expression } from '../../sequelize';
 import { rejectInvalidOptions } from '../../utils/check';
 import { joinSQLFragments } from '../../utils/join-sql-fragments';
+import { buildJsonPath } from '../../utils/json';
 import { generateIndexName } from '../../utils/string';
 import { AbstractQueryGenerator } from '../abstract/query-generator';
 import { REMOVE_INDEX_QUERY_SUPPORTABLE_OPTIONS } from '../abstract/query-generator-typescript';
-import type { RemoveIndexQueryOptions, TableNameOrModel } from '../abstract/query-generator-typescript';
+import type { EscapeOptions, RemoveIndexQueryOptions, TableNameOrModel } from '../abstract/query-generator-typescript';
 
 const REMOVE_INDEX_QUERY_SUPPORTED_OPTIONS = new Set<keyof RemoveIndexQueryOptions>(['ifExists']);
 
@@ -100,4 +102,16 @@ export class MsSqlQueryGeneratorTypeScript extends AbstractQueryGenerator {
       this.quoteTable(tableName),
     ]);
   }
+
+  jsonPathExtractionQuery(sqlExpression: string, path: ReadonlyArray<number | string>, unquote: boolean): string {
+    if (!unquote) {
+      throw new Error(`JSON Paths are not supported in ${this.dialect.name} without unquoting the JSON value.`);
+    }
+
+    return `JSON_VALUE(${sqlExpression}, ${this.escape(buildJsonPath(path))})`;
+  }
+
+  formatUnquoteJson(arg: Expression, options?: EscapeOptions) {
+    return `JSON_VALUE(${this.escape(arg, options)})`;
+  }
 }
diff --git a/packages/core/src/dialects/mssql/query-generator.js b/packages/core/src/dialects/mssql/query-generator.js
@@ -420,10 +420,12 @@ export class MsSqlQueryGenerator extends MsSqlQueryGeneratorTypeScript {
       for (const attrValueHash of attrValueHashes) {
         tuples.push(`(${
           allAttributes.map(key => {
-            // TODO: pass "type"
             // TODO: bindParam
             // TODO: pass "model"
-            return this.escape(attrValueHash[key] ?? null, options);
+            return this.escape(attrValueHash[key] ?? null, {
+              type: attributes[key]?.type,
+              replacements: options.replacements,
+            });
           }).join(',')
         })`);
       }

diff --git a/packages/core/src/dialects/mysql/index.ts b/packages/core/src/dialects/mysql/index.ts
@@ -53,6 +53,10 @@ export class MysqlDialect extends AbstractDialect {
         JSON: true,
       },
       jsonOperations: true,
+      jsonExtraction: {
+        unquoted: true,
+        quoted: true,
+      },
       REGEXP: true,
       globalTimeZoneConfig: true,
       maxExecutionTimeHint: {

diff --git a/packages/core/src/dialects/mysql/query-generator-typescript.ts b/packages/core/src/dialects/mysql/query-generator-typescript.ts
@@ -1,6 +1,7 @@
 import { Op } from '../../operators.js';
 import type { Expression } from '../../sequelize.js';
 import { rejectInvalidOptions } from '../../utils/check';
+import { buildJsonPath } from '../../utils/json.js';
 import { generateIndexName } from '../../utils/string';
 import { AbstractQueryGenerator } from '../abstract/query-generator';
 import { REMOVE_INDEX_QUERY_SUPPORTABLE_OPTIONS } from '../abstract/query-generator-typescript';
@@ -63,16 +64,7 @@ export class MySqlQueryGeneratorTypeScript extends AbstractQueryGenerator {
   }
 
   jsonPathExtractionQuery(sqlExpression: string, path: ReadonlyArray<number | string>, unquote: boolean): string {
-    let jsonPathStr = '$';
-    for (const pathElement of path) {
-      if (typeof pathElement === 'number') {
-        jsonPathStr += `[${pathElement}]`;
-      } else {
-        jsonPathStr += `.${this.#quoteJsonPathIdentifier(pathElement)}`;
-      }
-    }
-
-    const extractQuery = `json_extract(${sqlExpression},${this.escape(jsonPathStr)})`;
+    const extractQuery = `json_extract(${sqlExpression},${this.escape(buildJsonPath(path))})`;
     if (unquote) {
       return `json_unquote(${extractQuery})`;
     }
@@ -83,13 +75,4 @@ export class MySqlQueryGeneratorTypeScript extends AbstractQueryGenerator {
   formatUnquoteJson(arg: Expression, options?: EscapeOptions) {
     return `json_unquote(${this.escape(arg, options)})`;
   }
-
-  #quoteJsonPathIdentifier(identifier: string): string {
-    if (/^[a-z_][a-z0-9_]*$/i.test(identifier)) {
-      return identifier;
-    }
-
-    // Escape backslashes and double quotes
-    return `"${identifier.replace(/["\\]/g, s => `\\${s}`)}"`;
-  }
 }
diff --git a/packages/core/src/dialects/postgres/index.ts b/packages/core/src/dialects/postgres/index.ts
@@ -59,6 +59,10 @@ export class PostgresDialect extends AbstractDialect {
       INET: true,
     },
     jsonOperations: true,
+    jsonExtraction: {
+      unquoted: true,
+      quoted: true,
+    },
     REGEXP: true,
     IREGEXP: true,
     deferrableConstraints: true,

diff --git a/packages/core/src/dialects/sqlite/index.ts b/packages/core/src/dialects/sqlite/index.ts
@@ -43,6 +43,10 @@ export class SqliteDialect extends AbstractDialect {
     // TODO: add support for JSON operations https://www.sqlite.org/json1.html (bundled in sqlite3)
     //  be careful: json_extract, ->, and ->> don't have the exact same meanings as mysql & mariadb
     jsonOperations: false,
+    jsonExtraction: {
+      unquoted: false,
+      quoted: false,
+    },
   });
 
   readonly defaultVersion = '3.8.0';

diff --git a/packages/core/src/utils/json.ts b/packages/core/src/utils/json.ts
@@ -0,0 +1,31 @@
+/**
+ * Returns a JSON path identifier that is safe to use in a JSON path.
+ *
+ * @param identifier - The identifier to quote.
+ */
+function quoteJsonPathIdentifier(identifier: string): string {
+  if (/^[a-z_][a-z0-9_]*$/i.test(identifier)) {
+    return identifier;
+  }
+
+  // Escape backslashes and double quotes
+  return `"${identifier.replace(/["\\]/g, s => `\\${s}`)}"`;
+}
+
+/**
+ * Builds a JSON path expression from a path.
+ *
+ * @param path - The path to build the expression from.
+ */
+export function buildJsonPath(path: ReadonlyArray<number | string>): string {
+  let jsonPathStr = '$';
+  for (const pathElement of path) {
+    if (typeof pathElement === 'number') {
+      jsonPathStr += `[${pathElement}]`;
+    } else {
+      jsonPathStr += `.${quoteJsonPathIdentifier(pathElement)}`;
+    }
+  }
+
+  return jsonPathStr;
+}
diff --git a/packages/core/test/integration/json.test.ts b/packages/core/test/integration/json.test.ts
@@ -156,72 +156,105 @@ describe('JSON Querying', () => {
     expect(posts[0].metaOldJSONtype).to.equal(posts[0].metaNewJSONtype);
   });
 
-  if (dialect.supports.jsonOperations) {
-    it('should be able to retrieve element of array by index', async () => {
-      const user = await vars.User.findOne({
-        attributes: [[sql.attribute('objectJsonAttr.phones[1]'), 'firstEmergencyNumber']],
-        rejectOnEmpty: true,
+  describe('JSON quoted', () => {
+    if (dialect.supports.jsonExtraction.quoted) {
+      it('should be able to retrieve element of array by index', async () => {
+        const user = await vars.User.findOne({
+          attributes: [[sql.attribute('objectJsonAttr.phones[1]'), 'firstEmergencyNumber']],
+          rejectOnEmpty: true,
+        });
+
+        // @ts-expect-error -- typings are not currently designed to handle custom attributes
+        const firstNumber: string = user.getDataValue('firstEmergencyNumber');
+
+        expect(Number.parseInt(firstNumber, 10)).to.equal(42);
       });
 
-      // @ts-expect-error -- typings are not currently designed to handle custom attributes
-      const firstNumber: string = user.getDataValue('firstEmergencyNumber');
-
-      expect(Number.parseInt(firstNumber, 10)).to.equal(42);
-    });
+      it('should be able to query using JSON path objects', async () => {
+        // JSON requires casting to text in postgres. There is no "json = json" operator
+        // No-cast version is tested higher up in this suite
+        const comparison = dialectName === 'postgres' ? { 'name::text': '"swen"' } : { name: 'swen' };
 
-    it('should be able to query using JSON path objects', async () => {
-      // JSON requires casting to text in postgres. There is no "json = json" operator
-      // No-cast version is tested higher up in this suite
-      const comparison = dialectName === 'postgres' ? { 'name::text': '"swen"' } : { name: 'swen' };
+        const user = await vars.User.findOne({
+          where: { objectJsonAttr: comparison },
+        });
 
-      const user = await vars.User.findOne({
-        where: { objectJsonAttr: comparison },
+        expect(user).to.exist;
       });
 
-      expect(user).to.exist;
-    });
+      it('should be able to query using JSON path dot notation', async () => {
+        // JSON requires casting to text in postgres. There is no "json = json" operator
+        // No-cast version is tested higher up in this suite
+        const comparison = dialectName === 'postgres' ? { 'objectJsonAttr.name::text': '"swen"' } : { 'objectJsonAttr.name': 'swen' };
 
-    it('should be able to query using JSON path dot notation', async () => {
-      // JSON requires casting to text in postgres. There is no "json = json" operator
-      // No-cast version is tested higher up in this suite
-      const comparison = dialectName === 'postgres' ? { 'objectJsonAttr.name::text': '"swen"' } : { 'objectJsonAttr.name': 'swen' };
+        const user = await vars.User.findOne({
+          where: comparison,
+        });
 
-      const user = await vars.User.findOne({
-        where: comparison,
+        expect(user).to.exist;
       });
 
-      expect(user).to.exist;
-    });
-
-    it('should be able to query using the JSON unquote syntax', async () => {
-      const user = await vars.User.findOne({
-        // JSON unquote does not require casting to text, as it already returns text
-        where: { 'objectJsonAttr.name:unquote': 'swen' },
+      it('should be able retrieve json value with nested include', async () => {
+        const orders = await vars.Order.findAll({
+          attributes: ['id'],
+          include: [{
+            model: vars.User,
+            attributes: [
+              [sql.attribute('objectJsonAttr.name'), 'name'],
+            ],
+          }],
+        });
+
+        // we can't automatically detect that the output is JSON type in mariadb < 10.4.3,
+        // and we don't yet support specifying (nor inferring) the type of custom attributes,
+        // so for now the output is different in this specific case
+        const expectedResult = dialectName === 'mariadb' && semver.lt(sequelize.getDatabaseVersion(), '10.4.3') ? '"swen"' : 'swen';
+
+        // @ts-expect-error -- getDataValue does not support custom attributes
+        expect(orders[0].user.getDataValue('name')).to.equal(expectedResult);
       });
+    }
+  });
 
-      expect(user).to.exist;
-    });
+  describe('JSON unquoted', () => {
+    if (dialect.supports.jsonExtraction.unquoted) {
+      it('should be able to retrieve element of array by index', async () => {
+        const user = await vars.User.findOne({
+          attributes: [[sql.attribute('objectJsonAttr.phones[1]:unquote'), 'firstEmergencyNumber']],
+          rejectOnEmpty: true,
+        });
 
-    it('should be able retrieve json value with nested include', async () => {
-      const orders = await vars.Order.findAll({
-        attributes: ['id'],
-        include: [{
-          model: vars.User,
-          attributes: [
-            [sql.attribute('objectJsonAttr.name'), 'name'],
-          ],
-        }],
+        // @ts-expect-error -- typings are not currently designed to handle custom attributes
+        const firstNumber: string = user.getDataValue('firstEmergencyNumber');
+
+        expect(Number.parseInt(firstNumber, 10)).to.equal(42);
       });
 
-      // we can't automatically detect that the output is JSON type in mariadb < 10.4.3,
-      // and we don't yet support specifying (nor inferring) the type of custom attributes,
-      // so for now the output is different in this specific case
-      const expectedResult = dialectName === 'mariadb' && semver.lt(sequelize.getDatabaseVersion(), '10.4.3') ? '"swen"' : 'swen';
+      it('should be able to query using JSON path dot notation', async () => {
+        const user = await vars.User.findOne({
+          // JSON unquote does not require casting to text, as it already returns text
+          where: { 'objectJsonAttr.name:unquote': 'swen' },
+        });
 
-      // @ts-expect-error -- getDataValue does not support custom attributes
-      expect(orders[0].user.getDataValue('name')).to.equal(expectedResult);
-    });
-  }
+        expect(user).to.exist;
+      });
+
+      it('should be able retrieve json value with nested include', async () => {
+        const orders = await vars.Order.findAll({
+          attributes: ['id'],
+          include: [{
+            model: vars.User,
+            attributes: [
+              [sql.attribute('objectJsonAttr.name:unquote'), 'name'],
+            ],
+          }],
+        });
+
+        // @ts-expect-error -- getDataValue does not support custom attributes
+        expect(orders[0].user.getDataValue('name')).to.equal('swen');
+      });
+    }
+  });
 });
 
 describe('JSON Casting', () => {
@@ -250,7 +283,11 @@ describe('JSON Casting', () => {
       },
     });
 
-    const cast = dialectName === 'mysql' || dialectName === 'mariadb' ? 'DATETIME' : 'TIMESTAMPTZ';
+    const cast = dialectName === 'mysql' || dialectName === 'mariadb'
+      ? 'DATETIME'
+      : dialectName === 'mssql'
+      ? 'DATETIMEOFFSET'
+      : 'TIMESTAMPTZ';
 
     const user = await vars.User.findOne({
       where: {
@@ -273,7 +310,7 @@ describe('JSON Casting', () => {
 
   it('supports casting to boolean', async () => {
     // These dialects do not have a native BOOLEAN type
-    if (dialectName === 'mariadb' || dialectName === 'mysql') {
+    if (['mariadb', 'mysql', 'mssql'].includes(dialectName)) {
       return;
     }