diff --git a/package-lock.json b/package-lock.json index 8224539..b3a613c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18297,9 +18297,9 @@ "dev": true }, "javascript-stringify": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/javascript-stringify/-/javascript-stringify-1.6.0.tgz", - "integrity": "sha1-FC0RHzpuPa6PSpr9d9RYVbWpzOM=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/javascript-stringify/-/javascript-stringify-2.0.1.tgz", + "integrity": "sha512-yV+gqbd5vaOYjqlbk16EG89xB5udgjqQF3C5FAORDg4f/IS1Yc5ERCv5e/57yBcfJYw05V5JyIXabhwb75Xxow==" }, "jest-worker": { "version": "25.1.0", diff --git a/package.json b/package.json index 71571ac..8cb4d48 100644 --- a/package.json +++ b/package.json @@ -152,7 +152,7 @@ "csv-parser": "^2.3.1", "fast-csv": "^3.4.0", "flat": "cipacda/flat", - "javascript-stringify": "^1.6.0", + "javascript-stringify": "^2.0.1", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", "lodash.throttle": "^4.1.1", diff --git a/src/components/import-preview/import-preview.jsx b/src/components/import-preview/import-preview.jsx index ee8c738..a42b4de 100644 --- a/src/components/import-preview/import-preview.jsx +++ b/src/components/import-preview/import-preview.jsx @@ -69,12 +69,6 @@ class PreviewValues extends PureComponent { } } -// const FieldShape = PropTypes.shape({ -// path: PropTypes.string, -// checked: PropTypes.bool, -// type: PropTypes.string -// }); - class PreviewFields extends PureComponent { static propTypes = { fields: PropTypes.array, diff --git a/src/modules/import.js b/src/modules/import.js index 4b709fc..84feab0 100644 --- a/src/modules/import.js +++ b/src/modules/import.js @@ -366,7 +366,7 @@ const loadPreviewDocs = ( * errors/faults hard so we can figure out edge cases that * actually need it. */ - const source = fs.createReadStream(fileName, 'utf8'); + const source = fs.createReadStream(fileName, {encoding: 'utf8', end: 20 * 1024}); const dest = createPreviewWritable(); stream.pipeline( source, diff --git a/src/utils/bson-csv.js b/src/utils/bson-csv.js index 73d08a2..5c6e082 100644 --- a/src/utils/bson-csv.js +++ b/src/utils/bson-csv.js @@ -6,6 +6,8 @@ * 1. All bson type defs had a consistent `.fromString()` * method * 2. Castings/detection used by fromString() today were exposed * (e.g. JS Number float -> bson.Double). + * + * Related: https://github.com/mongodb-js/hadron-type-checker/blob/master/src/type-checker.js */ /** @@ -169,13 +171,26 @@ export const serialize = function(doc) { // BSON values if (isBSON) { - output[newKey] = value.toString('hex'); + if (type === 'BSONRegExp') { + /** + * TODO (lucas) Upstream to `bson` as `BSONRegExp` toString() + * returns `'[object Object]'` today. + */ + output[newKey] = `/${value.pattern}/${value.options}`; + } else { + output[newKey] = value.toString(); + } return; } // Embedded arrays if (type === 'Array') { - output[newKey] = bson.EJSON.serialize(value); + output[newKey] = bson.EJSON.stringify(value, null, null); + return; + } + + if (type === 'Date') { + output[newKey] = value.toISOString(); return; } @@ -189,9 +204,40 @@ export const serialize = function(doc) { } // All other values - output[newKey] = value; + output[newKey] = '' + value; }); } step(doc); return output; }; + +/** + * TODO (lucas) Consolidate valueToString with dupe logic in serialize() later. + */ +export const valueToString = function(value) { + const { type, isBSON } = getTypeDescriptorForValue(value); + + // BSON values + if (isBSON) { + if (type === 'BSONRegExp') { + /** + * TODO (lucas) Upstream to `bson` as `BSONRegExp` toString() + * returns `'[object Object]'` today. + */ + return `/${value.pattern}/${value.options}`; + } + return value.toString(); + } + + // Embedded arrays + if (type === 'Array') { + return bson.EJSON.stringify(value, null, null); + } + + if (type === 'Date') { + return value.toISOString(); + } + + // All other values + return '' + value; +}; diff --git a/src/utils/bson-csv.spec.js b/src/utils/bson-csv.spec.js index 68eac3b..7f47743 100644 --- a/src/utils/bson-csv.spec.js +++ b/src/utils/bson-csv.spec.js @@ -1,38 +1,147 @@ -import bsonCSV from './bson-csv'; -// import bson from 'bson'; +import bsonCSV, { serialize } from './bson-csv'; +import { EJSON, ObjectId, Long, BSONRegExp } from 'bson'; // TODO: lucas: probably dumb but think about that later. describe('bson-csv', () => { - describe('String', () => { - it('should work', () => { - expect(bsonCSV.String.fromString(1)).to.equal('1'); + describe('Native', () => { + describe('String', () => { + it('should work', () => { + expect(bsonCSV.String.fromString(1)).to.equal('1'); + }); }); - }); - describe('Boolean', () => { - it('should deserialize falsy values', () => { - expect(bsonCSV.Boolean.fromString('')).to.equal(false); - expect(bsonCSV.Boolean.fromString('false')).to.equal(false); - expect(bsonCSV.Boolean.fromString('FALSE')).to.equal(false); - // expect(bsonCSV.Boolean.fromString('0')).to.equal(false); - }); - it('should deserialize non-falsy values', () => { - // expect(bsonCSV.Boolean.fromString('1')).to.equal(true); - expect(bsonCSV.Boolean.fromString('true')).to.equal(true); - expect(bsonCSV.Boolean.fromString('TRUE')).to.equal(true); + describe('Boolean', () => { + it('should deserialize falsy values', () => { + expect(bsonCSV.Boolean.fromString('')).to.equal(false); + expect(bsonCSV.Boolean.fromString('false')).to.equal(false); + expect(bsonCSV.Boolean.fromString('FALSE')).to.equal(false); + // expect(bsonCSV.Boolean.fromString('0')).to.equal(false); + }); + it('should deserialize non-falsy values', () => { + // expect(bsonCSV.Boolean.fromString('1')).to.equal(true); + expect(bsonCSV.Boolean.fromString('true')).to.equal(true); + expect(bsonCSV.Boolean.fromString('TRUE')).to.equal(true); + }); }); - }); - describe('Number', () => { - it('should work', () => { - expect(bsonCSV.Number.fromString('1')).to.equal(1); + describe('Number', () => { + it('should work', () => { + expect(bsonCSV.Number.fromString('1')).to.equal(1); + }); + }); + describe('Date', () => { + /** + * Regression test for https://jira.mongodb.org/browse/COMPASS-4164 + */ + it('should serialize dates as ISO strings', () => { + const doc = EJSON.deserialize({ + _id: '{47844C7F-544C-8986-E050-A8C063056488}', + Price: 925000, + 'Date of Transfer': '2017-01-13T00:00:00Z' + }); + expect(serialize(doc)).to.deep.equal({ + _id: '{47844C7F-544C-8986-E050-A8C063056488}', + Price: '925000', + 'Date of Transfer': '2017-01-13T00:00:00Z' + }); + }); + }); + describe('Undefined', () => { + it('should serialize as a string', () => { + expect(serialize({ value: undefined })).to.deep.equal({ + value: 'undefined' + }); + }); + }); + describe('Null', () => { + it('should serialize as a string', () => { + expect(serialize({ value: null })).to.deep.equal({ + value: 'null' + }); + }); + }); + describe('RegExp', () => { + it('should serialize as a string', () => { + expect(serialize({ value: /^mongodb/ })).to.deep.equal({ + value: '/^mongodb/' + }); + }); + }); + describe('Array', () => { + it('should serialize as a string of extended JSON', () => { + expect( + serialize({ + value: [ + new ObjectId('5e6652f22c09c775463d70f1'), + new ObjectId('5e6652f62c09c775463d70f2') + ] + }) + ).to.deep.equal({ + value: + '[{"$oid":"5e6652f22c09c775463d70f1"},{"$oid":"5e6652f62c09c775463d70f2"}]' + }); + }); + }); + describe('Object', () => { + it('should serialize plain objects in dot notation', () => { + const doc = { + _id: 'arlo', + name: 'Arlo', + location: { + activity: { + sleeping: 'true', + is: 'on the couch' + } + } + }; + expect(serialize(doc)).to.deep.equal({ + _id: 'arlo', + name: 'Arlo', + 'location.activity.sleeping': 'true', + 'location.activity.is': 'on the couch' + }); + }); + }); + + describe('Boolean', () => { + it('should serialize as a string', () => { + expect(serialize({ value: false })).to.deep.equal({ + value: 'false' + }); + + expect(serialize({ value: true })).to.deep.equal({ + value: 'true' + }); + }); }); }); - describe('ObjectId', () => { - it('should work', () => { - const oid = '5dd080acc15c0d5ee3ab6ad2'; - const deserialized = bsonCSV.ObjectId.fromString(oid); - expect(deserialized._bsontype).to.equal('ObjectID'); - expect(deserialized.toString()).to.equal('5dd080acc15c0d5ee3ab6ad2'); + describe('bson', () => { + describe('ObjectId', () => { + it('should serialize ObjectId as the hex string value', () => { + const oid = '5dd080acc15c0d5ee3ab6ad2'; + const deserialized = bsonCSV.ObjectId.fromString(oid); + expect(deserialized._bsontype).to.equal('ObjectID'); + expect(deserialized.toString()).to.equal('5dd080acc15c0d5ee3ab6ad2'); + }); + }); + describe('Long', () => { + it('should serialize as a string', () => { + expect(serialize({ value: Long.fromNumber(245) })).to.deep.equal({ + value: '245' + }); + }); + }); + describe('BSONRegExp', () => { + it('should serialize as a string', () => { + expect(serialize({ value: new BSONRegExp('^mongodb') })).to.deep.equal({ + value: '/^mongodb/' + }); + + expect( + serialize({ value: new BSONRegExp('^mongodb', 'm') }) + ).to.deep.equal({ + value: '/^mongodb/m' + }); + }); }); }); }); diff --git a/src/utils/get-shell-js.js b/src/utils/get-shell-js.js index ae01648..21a0393 100644 --- a/src/utils/get-shell-js.js +++ b/src/utils/get-shell-js.js @@ -1,4 +1,4 @@ -import toJavascriptString from 'javascript-stringify'; +import { stringify as toJavascriptString } from 'javascript-stringify'; import toNS from 'mongodb-ns'; export default function(ns, spec) { diff --git a/src/utils/import-preview.js b/src/utils/import-preview.js index 8bb26ed..96eea60 100644 --- a/src/utils/import-preview.js +++ b/src/utils/import-preview.js @@ -1,9 +1,9 @@ import { Writable } from 'stream'; import peek from 'peek-stream'; import createParser from './import-parser'; -import dotnotation from './dotnotation'; -import { detectType } from './bson-csv'; +import { detectType, valueToString } from './bson-csv'; +import dotnotation from './dotnotation'; import { createLogger } from './logger'; const debug = createLogger('import-preview'); @@ -21,6 +21,7 @@ export const createPeekStream = function( fileIsMultilineJSON ) { return peek({ maxBuffer: 20 * 1024 }, function(data, swap) { + debugger; return swap( null, createParser({ @@ -49,6 +50,7 @@ export default function({ MAX_SIZE = 10 } = {}) { } if (this.docs.length >= MAX_SIZE) { + debug('noop'); return next(); } this.docs.push(doc); @@ -77,7 +79,9 @@ export default function({ MAX_SIZE = 10 } = {}) { got: keys }); } - this.values.push(Object.values(docAsDotnotation)); + const values = Object.values(docAsDotnotation).map(value => valueToString(value)); + debug('set values', values); + this.values.push(values); return next(null); }