fix(import): COMPASS-4164 Preview table values always strings

package.json

@@ -152,7 +152,7 @@
     "csv-parser": "^2.3.1",
     "fast-csv": "^3.4.0",
     "flat": "cipacda/flat",
-    "javascript-stringify": "^1.6.0",
+    "javascript-stringify": "^2.0.1",
     "lodash.isobjectlike": "^4.0.0",
     "lodash.isplainobject": "^4.0.6",
     "lodash.throttle": "^4.1.1",

src/components/import-preview/import-preview.jsx

@@ -69,12 +69,6 @@ class PreviewValues extends PureComponent {
   }
 }
 
-// const FieldShape = PropTypes.shape({
-//   path: PropTypes.string,
-//   checked: PropTypes.bool,
-//   type: PropTypes.string
-// });
-
 class PreviewFields extends PureComponent {
   static propTypes = {
     fields: PropTypes.array,

src/modules/import.js

@@ -366,7 +366,7 @@ const loadPreviewDocs = (
      * errors/faults hard so we can figure out edge cases that
      * actually need it.
      */
-    const source = fs.createReadStream(fileName, 'utf8');
+    const source = fs.createReadStream(fileName, {encoding: 'utf8', end: 20 * 1024});
     const dest = createPreviewWritable();
     stream.pipeline(
       source,

src/utils/bson-csv.js

@@ -6,6 +6,8 @@
  * 1. All bson type defs had a consistent `.fromString()` * method
  * 2. Castings/detection used by fromString() today were exposed
  * (e.g. JS Number float -> bson.Double).
+ * 
+ * Related: https://github.com/mongodb-js/hadron-type-checker/blob/master/src/type-checker.js
  */
 
 /**
@@ -169,13 +171,26 @@ export const serialize = function(doc) {
 
       // BSON values
       if (isBSON) {
-        output[newKey] = value.toString('hex');
+        if (type === 'BSONRegExp') {
+          /**
+           * TODO (lucas) Upstream to `bson` as `BSONRegExp` toString()
+           * returns `'[object Object]'` today.
+           */
+          output[newKey] = `/${value.pattern}/${value.options}`;
+        } else {
+          output[newKey] = value.toString();
+        }
         return;
       }
 
       // Embedded arrays
       if (type === 'Array') {
-        output[newKey] = bson.EJSON.serialize(value);
+        output[newKey] = bson.EJSON.stringify(value, null, null);
+        return;
+      }
+
+      if (type === 'Date') {
+        output[newKey] = value.toISOString();
         return;
       }
 
@@ -189,9 +204,40 @@ export const serialize = function(doc) {
       }
 
       // All other values
-      output[newKey] = value;
+      output[newKey] = '' + value;
     });
   }
   step(doc);
   return output;
 };
+
+/**
+ * TODO (lucas) Consolidate valueToString with dupe logic in serialize() later.
+ */
+export const valueToString = function(value) {
+  const { type, isBSON } = getTypeDescriptorForValue(value);
+
+  // BSON values
+  if (isBSON) {
+    if (type === 'BSONRegExp') {
+      /**
+       * TODO (lucas) Upstream to `bson` as `BSONRegExp` toString()
+       * returns `'[object Object]'` today.
+       */
+      return `/${value.pattern}/${value.options}`;
+    }
+    return value.toString();
+  }
+
+  // Embedded arrays
+  if (type === 'Array') {
+    return bson.EJSON.stringify(value, null, null);
+  }
+
+  if (type === 'Date') {
+    return value.toISOString();
+  }
+
+  // All other values
+  return '' + value;
+};

src/utils/bson-csv.spec.js

@@ -1,38 +1,147 @@
-import bsonCSV from './bson-csv';
-// import bson from 'bson';
+import bsonCSV, { serialize } from './bson-csv';
+import { EJSON, ObjectId, Long, BSONRegExp } from 'bson';
 
 // TODO: lucas: probably dumb but think about that later.
 
 describe('bson-csv', () => {
-  describe('String', () => {
-    it('should work', () => {
-      expect(bsonCSV.String.fromString(1)).to.equal('1');
+  describe('Native', () => {
+    describe('String', () => {
+      it('should work', () => {
+        expect(bsonCSV.String.fromString(1)).to.equal('1');
+      });
     });
-  });
-  describe('Boolean', () => {
-    it('should deserialize falsy values', () => {
-      expect(bsonCSV.Boolean.fromString('')).to.equal(false);
-      expect(bsonCSV.Boolean.fromString('false')).to.equal(false);
-      expect(bsonCSV.Boolean.fromString('FALSE')).to.equal(false);
-      // expect(bsonCSV.Boolean.fromString('0')).to.equal(false);
-    });
-    it('should deserialize non-falsy values', () => {
-      // expect(bsonCSV.Boolean.fromString('1')).to.equal(true);
-      expect(bsonCSV.Boolean.fromString('true')).to.equal(true);
-      expect(bsonCSV.Boolean.fromString('TRUE')).to.equal(true);
+    describe('Boolean', () => {
+      it('should deserialize falsy values', () => {
+        expect(bsonCSV.Boolean.fromString('')).to.equal(false);
+        expect(bsonCSV.Boolean.fromString('false')).to.equal(false);
+        expect(bsonCSV.Boolean.fromString('FALSE')).to.equal(false);
+        // expect(bsonCSV.Boolean.fromString('0')).to.equal(false);
+      });
+      it('should deserialize non-falsy values', () => {
+        // expect(bsonCSV.Boolean.fromString('1')).to.equal(true);
+        expect(bsonCSV.Boolean.fromString('true')).to.equal(true);
+        expect(bsonCSV.Boolean.fromString('TRUE')).to.equal(true);
+      });
     });
-  });
-  describe('Number', () => {
-    it('should work', () => {
-      expect(bsonCSV.Number.fromString('1')).to.equal(1);
+    describe('Number', () => {
+      it('should work', () => {
+        expect(bsonCSV.Number.fromString('1')).to.equal(1);
+      });
+    });
+    describe('Date', () => {
+      /**
+       * Regression test for https://jira.mongodb.org/browse/COMPASS-4164
+       */
+      it('should serialize dates as ISO strings', () => {
+        const doc = EJSON.deserialize({
+          _id: '{47844C7F-544C-8986-E050-A8C063056488}',
+          Price: 925000,
+          'Date of Transfer': '2017-01-13T00:00:00Z'
+        });
+        expect(serialize(doc)).to.deep.equal({
+          _id: '{47844C7F-544C-8986-E050-A8C063056488}',
+          Price: '925000',
+          'Date of Transfer': '2017-01-13T00:00:00Z'
+        });
+      });
+    });
+    describe('Undefined', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: undefined })).to.deep.equal({
+          value: 'undefined'
+        });
+      });
+    });
+    describe('Null', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: null })).to.deep.equal({
+          value: 'null'
+        });
+      });
+    });
+    describe('RegExp', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: /^mongodb/ })).to.deep.equal({
+          value: '/^mongodb/'
+        });
+      });
+    });
+    describe('Array', () => {
+      it('should serialize as a string of extended JSON', () => {
+        expect(
+          serialize({
+            value: [
+              new ObjectId('5e6652f22c09c775463d70f1'),
+              new ObjectId('5e6652f62c09c775463d70f2')
+            ]
+          })
+        ).to.deep.equal({
+          value:
+            '[{"$oid":"5e6652f22c09c775463d70f1"},{"$oid":"5e6652f62c09c775463d70f2"}]'
+        });
+      });
+    });
+    describe('Object', () => {
+      it('should serialize plain objects in dot notation', () => {
+        const doc = {
+          _id: 'arlo',
+          name: 'Arlo',
+          location: {
+            activity: {
+              sleeping: 'true',
+              is: 'on the couch'
+            }
+          }
+        };
+        expect(serialize(doc)).to.deep.equal({
+          _id: 'arlo',
+          name: 'Arlo',
+          'location.activity.sleeping': 'true',
+          'location.activity.is': 'on the couch'
+        });
+      });
+    });
+
+    describe('Boolean', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: false })).to.deep.equal({
+          value: 'false'
+        });
+
+        expect(serialize({ value: true })).to.deep.equal({
+          value: 'true'
+        });
+      });
     });
   });
-  describe('ObjectId', () => {
-    it('should work', () => {
-      const oid = '5dd080acc15c0d5ee3ab6ad2';
-      const deserialized = bsonCSV.ObjectId.fromString(oid);
-      expect(deserialized._bsontype).to.equal('ObjectID');
-      expect(deserialized.toString()).to.equal('5dd080acc15c0d5ee3ab6ad2');
+  describe('bson', () => {
+    describe('ObjectId', () => {
+      it('should serialize ObjectId as the hex string value', () => {
+        const oid = '5dd080acc15c0d5ee3ab6ad2';
+        const deserialized = bsonCSV.ObjectId.fromString(oid);
+        expect(deserialized._bsontype).to.equal('ObjectID');
+        expect(deserialized.toString()).to.equal('5dd080acc15c0d5ee3ab6ad2');
+      });
+    });
+    describe('Long', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: Long.fromNumber(245) })).to.deep.equal({
+          value: '245'
+        });
+      });
+    });
+    describe('BSONRegExp', () => {
+      it('should serialize as a string', () => {
+        expect(serialize({ value: new BSONRegExp('^mongodb') })).to.deep.equal({
+          value: '/^mongodb/'
+        });
+
+        expect(
+          serialize({ value: new BSONRegExp('^mongodb', 'm') })
+        ).to.deep.equal({
+          value: '/^mongodb/m'
+        });
+      });
     });
   });
 });

src/utils/get-shell-js.js

@@ -1,4 +1,4 @@
-import toJavascriptString from 'javascript-stringify';
+import { stringify as toJavascriptString } from 'javascript-stringify';
 import toNS from 'mongodb-ns';
 
 export default function(ns, spec) {

src/utils/import-preview.js

@@ -1,9 +1,9 @@
 import { Writable } from 'stream';
 import peek from 'peek-stream';
 import createParser from './import-parser';
-import dotnotation from './dotnotation';
 
-import { detectType } from './bson-csv';
+import { detectType, valueToString } from './bson-csv';
+import dotnotation from './dotnotation';
 import { createLogger } from './logger';
 const debug = createLogger('import-preview');
 
@@ -21,6 +21,7 @@ export const createPeekStream = function(
   fileIsMultilineJSON
 ) {
   return peek({ maxBuffer: 20 * 1024 }, function(data, swap) {
+    debugger;
     return swap(
       null,
       createParser({
@@ -49,6 +50,7 @@ export default function({ MAX_SIZE = 10 } = {}) {
       }
 
       if (this.docs.length >= MAX_SIZE) {
+        debug('noop');
         return next();
       }
       this.docs.push(doc);
@@ -77,7 +79,9 @@ export default function({ MAX_SIZE = 10 } = {}) {
           got: keys
         });
       }
-      this.values.push(Object.values(docAsDotnotation));
+      const values = Object.values(docAsDotnotation).map(value => valueToString(value));
+      debug('set values', values);
+      this.values.push(values);
 
       return next(null);
     }