BUG#33740190: indexes for upstream document Date instances

When adding a document to a collection, applications can use JavaScript
Date instances to represent JSON date string values. The client simply
calls the ".toJSON()" method which returns a UTC date string using the
Zulu Time indicator "Z", as specified by the ISO 8601 standard.

However, the Zulu Time indicator is not covered by the SQL standard
and thus is not allowed by the MySQL optimiser. This means that it is
not possible to create a collection index using a virtual column with
the DATETIME (or similar) data type. Trying to do so yields the
following error:

  Error: Incorrect datetime value

This patch addresses the issue by replacing the Zulu Time indicator by
an explicit displacement indicator of "+00:00" which has the exact same
meaning from the ISO 8601 standpoint and is also allowed by the MySQL
optimiser.

Author: ruiquelhas

lib/Protocol/Wrappers/Messages/Datatypes/Scalar.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -368,7 +368,17 @@ Scalar.create = function (value) {
     }
 
     if (isDate(value)) {
-        return Scalar(createString(value.toJSON()));
+        // The MySQL optimiser does not allow the Zulu Time indicator because
+        // it is not covered by the SQL standard.
+        // We need to replace it by the corresponding explicit time zone
+        // displacement indicator (+00:00).
+        // Although we do not have to worry about downstream convertion (from
+        // JSON to a Date instance), the Date constructor still works fine
+        // using the explicit displacement indicator.
+        const date = value.toJSON();
+        const displacement = '+00:00';
+
+        return Scalar(createString(date.substring(0, date.length - 1).concat(displacement)));
     }
 
     if (isString(value)) {

test/functional/default/document-store/collection-add.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -228,11 +228,12 @@ describe('adding documents to a collection', () => {
 
     context('BUG#29179767 JavaScript Date converted to empty object', () => {
         it('saves a JavaScript Date as a valid JSON value', () => {
-            const now = new Date();
-            const expected = [{ createdAt: now.toJSON() }];
+            const now = (new Date()).toJSON();
+            const createdAt = now.substring(0, now.length - 1).concat('+00:00');
+            const expected = [{ createdAt }];
             const actual = [];
 
-            return collection.add({ name: 'foo', createdAt: now })
+            return collection.add({ name: 'foo', createdAt })
                 .execute()
                 .then(() => {
                     return collection.find('name = :name')

test/functional/default/document-store/collection-find.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -518,11 +518,12 @@ describe('finding documents in collections', () => {
         });
 
         it('does not fail when a placeholder is assigned a JavaScript Date instance', () => {
-            const expected = [{ _id: '1', name: 'foo', updatedAt: now.toJSON() }];
+            const updatedAt = now.toJSON().substring(0, now.toJSON().length - 1).concat('+00:00');
+            const expected = [{ _id: '1', name: 'foo', updatedAt }];
             const actual = [];
 
             return collection.find('updatedAt = :date')
-                .bind('date', now)
+                .bind('date', updatedAt)
                 .execute(doc => actual.push(doc))
                 .then(() => expect(actual).to.deep.equal(expected));
         });

test/functional/default/document-store/collection-index.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -170,4 +170,18 @@ describe('collection indexes', () => {
                 .then(actual => expect(actual).to.deep.equal(expected));
         });
     });
+
+    context('BUG#33740190 collection DATETIME index', () => {
+        it('works with a document field containing a JavaScript Date instance', () => {
+            const field = '$.createdAt';
+            const name = 'createdAt_idx';
+            const definition = { fields: [{ field, type: 'DATETIME' }] };
+
+            return collection.add({ name: 'foo', createdAt: new Date() })
+                .execute()
+                .then(() => {
+                    return collection.createIndex(name, definition);
+                });
+        });
+    });
 });

test/functional/default/document-store/collection-modify.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -513,13 +513,14 @@ describe('modifying documents in a collection', () => {
         });
 
         it('updates a single field using a valid JSON value from a JavaScript Date object', () => {
-            const now = new Date();
-            const expected = [{ createdAt: now.toJSON() }];
+            const now = (new Date()).toJSON();
+            const createdAt = now.substring(0, now.length - 1).concat('+00:00');
+            const expected = [{ createdAt }];
             const actual = [];
 
             return collection.modify('_id = :id')
                 .bind('id', '1')
-                .set('createdAt', now)
+                .set('createdAt', createdAt)
                 .execute()
                 .then(() => {
                     return collection.find('_id = :id')
@@ -533,13 +534,14 @@ describe('modifying documents in a collection', () => {
         });
 
         it('updates a multiple fields using valid JSON values from JavaScript Date objects', () => {
-            const now = new Date();
-            const expected = [{ createdAt: now.toJSON(), updatedAt: now.toJSON() }];
+            const now = (new Date()).toJSON();
+            const utcDateString = now.substring(0, now.length - 1).concat('+00:00');
+            const expected = [{ createdAt: utcDateString, updatedAt: utcDateString }];
             const actual = [];
 
             return collection.modify('_id = :id')
                 .bind('id', '1')
-                .patch({ createdAt: now, updatedAt: now })
+                .patch({ createdAt: utcDateString, updatedAt: utcDateString })
                 .execute()
                 .then(() => {
                     return collection.find('_id = :id')

test/unit/Protocol/Wrappers/Messages/Datatypes/Scalar.js

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2.0, as
@@ -166,10 +166,16 @@ describe('Mysqlx.Datatypes.Scalar wrapper', () => {
 
             it('creates a wrapper of Mysqlx.Datatypes.Scalar.Type.V_STRING for Date instances', () => {
                 const now = new Date();
+                const dateString = now.toJSON();
                 const proto = scalar.create(now).valueOf();
 
                 expect(proto.getType()).to.equal(ScalarStub.Type.V_STRING);
-                expect(Buffer.from(proto.getVString().getValue()).toString()).to.equal(now.toJSON());
+                // MySQL does not support the Zulu Time indicator used in the
+                // ISO 8601 convention for Date instances.
+                // 'Z' should be replaced by '+00:00' which has similar
+                // meaning.
+                const supportedDateString = dateString.substring(0, dateString.length - 1).concat('+00:00');
+                expect(Buffer.from(proto.getVString().getValue()).toString()).to.equal(supportedDateString);
             });
 
             it('creates a wrapper of Mysqlx.Datatypes.Scalar.Type.V_STRING for strings', () => {