From 34078f3c49f6f857dbc91de07c3d2aae5a213e40 Mon Sep 17 00:00:00 2001 From: Leonardo Rossi Date: Thu, 17 Feb 2022 18:06:01 +0100 Subject: [PATCH 1/4] Picks only specified columns in JSON export --- .../src/modules/export.js | 2 +- .../src/modules/export.spec.js | 80 ++++++++++++++----- .../src/utils/formatters.js | 7 +- .../src/utils/formatters.spec.js | 4 +- 4 files changed, 65 insertions(+), 28 deletions(-) diff --git a/packages/compass-import-export/src/modules/export.js b/packages/compass-import-export/src/modules/export.js index 3b59744b19d..ad9bdab643c 100644 --- a/packages/compass-import-export/src/modules/export.js +++ b/packages/compass-import-export/src/modules/export.js @@ -481,7 +481,7 @@ export const startExport = () => { if (exportData.fileType === 'csv') { formatter = createCSVFormatter({ columns }); } else { - formatter = createJSONFormatter(); + formatter = createJSONFormatter({ columns }); } const dest = fs.createWriteStream(exportData.fileName); diff --git a/packages/compass-import-export/src/modules/export.spec.js b/packages/compass-import-export/src/modules/export.spec.js index c9d6d855b2e..07a074f0e51 100644 --- a/packages/compass-import-export/src/modules/export.spec.js +++ b/packages/compass-import-export/src/modules/export.spec.js @@ -13,16 +13,11 @@ import configureExportStore from '../stores/export-store'; describe('export [module]', () => { describe('#reducer', () => { context('#startExport', () => { - let tempFile; let store; const localAppRegistry = new AppRegistry(); const globalAppRegistry = new AppRegistry(); beforeEach(async function() { - tempFile = path.join( - os.tmpdir(), - `test-${Date.now()}.csv` - ); const mockDocuments = [ { _id: 'foo', @@ -50,24 +45,65 @@ describe('export [module]', () => { }); }); - afterEach(function(done) { - rimraf(tempFile, done); + describe('CSV Export', () => { + let tempFile; + beforeEach(() => { + tempFile = path.join( + os.tmpdir(), + `test-${Date.now()}.csv` + ); + }); + afterEach((afterEachDone) => { + rimraf(tempFile, afterEachDone); + }); + it('should set the correct fields to CSV export', (done) => { + const fields = { 'first_name': 1, '_id': 1, 'foobar': 1, 'last_name': 0}; + store.dispatch(actions.updateSelectedFields(fields)); + store.dispatch(actions.selectExportFileName(tempFile)); + store.dispatch(actions.selectExportFileType('csv')); + store.dispatch(actions.toggleFullCollection()); + store.dispatch(actions.startExport()); + localAppRegistry.addListener('export-finished', function checkWrittenData() { + fs.readFile(tempFile, 'utf-8', function(err, data) { + if (err) { return done(err); } + const writtenData = data.split('\n'); + expect(writtenData[0]).to.equal('first_name,_id,foobar'); + expect(writtenData[1]).to.equal('John,foo,'); + localAppRegistry.removeListener('export-finished', checkWrittenData); + done(); + }); + }); + }); }); - it('should set the correct fields to export', (done) => { - const fields = { 'first_name': 1, '_id': 1, 'foobar': 1, 'last_name': 0}; - store.dispatch(actions.updateSelectedFields(fields)); - store.dispatch(actions.selectExportFileName(tempFile)); - store.dispatch(actions.selectExportFileType('csv')); - store.dispatch(actions.toggleFullCollection()); - - store.dispatch(actions.startExport()); - localAppRegistry.addListener('export-finished', function() { - fs.readFile(tempFile, 'utf-8', function(err, data) { - if (err) { return done(err); } - const writtenData = data.split('\n'); - expect(writtenData[0]).to.equal('first_name,_id,foobar'); - expect(writtenData[1]).to.equal('John,foo,'); - done(); + describe('JSON Export', () => { + let tempFile; + beforeEach(() => { + tempFile = path.join( + os.tmpdir(), + `test-${Date.now()}.json` + ); + }); + afterEach((done) => { + rimraf(tempFile, done); + }); + it('should not include _id if omitted', (done) => { + const fields = { 'first_name': 1, 'last_name': 0}; + store.dispatch(actions.updateSelectedFields(fields)); + store.dispatch(actions.selectExportFileName(tempFile)); + store.dispatch(actions.selectExportFileType('json')); + store.dispatch(actions.toggleFullCollection()); + store.dispatch(actions.startExport()); + localAppRegistry.addListener('export-finished', function() { + fs.readFile(tempFile, 'utf-8', function(err, data) { + if (err) { return done(err); } + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + first_name: 'John', + } + ]); + done(); + }); }); }); }); diff --git a/packages/compass-import-export/src/utils/formatters.js b/packages/compass-import-export/src/utils/formatters.js index c6f398e48ab..015c6b797fb 100644 --- a/packages/compass-import-export/src/utils/formatters.js +++ b/packages/compass-import-export/src/utils/formatters.js @@ -7,15 +7,16 @@ import { EJSON } from 'bson'; import { serialize as flatten } from './bson-csv'; import { Transform } from 'stream'; import { EOL } from 'os'; - +import pick from 'lodash.pick'; /** * @returns {Stream.Transform} */ -export const createJSONFormatter = function({ brackets = true } = {}) { +export const createJSONFormatter = function({ brackets = true, columns = [] } = {}) { return new Transform({ readableObjectMode: false, writableObjectMode: true, transform: function(doc, encoding, callback) { + const docToBeSerialized = columns.length > 0 ? pick(doc, columns) : doc; if (this._counter >= 1) { if (brackets) { this.push(','); @@ -23,7 +24,7 @@ export const createJSONFormatter = function({ brackets = true } = {}) { this.push(EOL); } } - const s = EJSON.stringify(doc, null, brackets ? 2 : null); + const s = EJSON.stringify(docToBeSerialized, null, brackets ? 2 : null); if (this._counter === undefined) { this._counter = 0; if (brackets) { diff --git a/packages/compass-import-export/src/utils/formatters.spec.js b/packages/compass-import-export/src/utils/formatters.spec.js index 395541aee7f..7ad6e94b181 100644 --- a/packages/compass-import-export/src/utils/formatters.spec.js +++ b/packages/compass-import-export/src/utils/formatters.spec.js @@ -28,7 +28,7 @@ const FIXTURES = { describe('formatters', () => { describe('json', () => { - it('should format a single docment in an array', () => { + it('should format a single document in an array', () => { const source = stream.Readable.from([{_id: new ObjectID('5e5ea7558d35931a05eafec0')}]); const formatter = createJSONFormatter({brackets: true}); const dest = fs.createWriteStream(FIXTURES.JSON_SINGLE_DOC); @@ -41,7 +41,7 @@ describe('formatters', () => { }) .then(() => rm(FIXTURES.JSON_SINGLE_DOC)); }); - it('should format more than 2 docments in an array', () => { + it('should format more than 2 documents in an array', () => { const docs = [ {_id: new ObjectID('5e5ea7558d35931a05eafec0')}, {_id: new ObjectID('5e6bafc438e060f695591713')}, From 3eb3af197119192dca323bfcff2129322b79a424 Mon Sep 17 00:00:00 2001 From: Leonardo Rossi Date: Fri, 18 Feb 2022 12:11:32 +0100 Subject: [PATCH 2/4] adds missing lodash.pick dependency --- package-lock.json | 2 ++ packages/compass-import-export/package.json | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 916024cd545..167526f2e87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -69969,6 +69969,7 @@ "JSONStream": "^1.3.5", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", + "lodash.pick": "^4.4.0", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", @@ -119942,6 +119943,7 @@ "lodash": "^4.17.21", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", + "lodash.pick": "*", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", diff --git a/packages/compass-import-export/package.json b/packages/compass-import-export/package.json index 6e47e41dfc4..f5261742a66 100644 --- a/packages/compass-import-export/package.json +++ b/packages/compass-import-export/package.json @@ -146,15 +146,16 @@ }, "dependencies": { "@mongodb-js/compass-logging": "^0.7.0", - "JSONStream": "^1.3.5", "ansi-to-html": "^0.6.11", "bson": "*", "csv-parser": "^2.3.1", "fast-csv": "^3.4.0", "flat": "cipacda/flat", "javascript-stringify": "^2.0.1", + "JSONStream": "^1.3.5", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", + "lodash.pick": "^4.4.0", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", From a8361b770b14dd9582e9c144ec902d4e441324c0 Mon Sep 17 00:00:00 2001 From: Leonardo Rossi Date: Tue, 22 Feb 2022 16:06:29 +0100 Subject: [PATCH 3/4] refactors test and fixes bug with real dataService --- package-lock.json | 2 - packages/compass-import-export/package.json | 1 - .../src/modules/export.js | 6 +- .../src/modules/export.spec.js | 172 ++++++++++++------ .../src/utils/formatters.js | 6 +- 5 files changed, 118 insertions(+), 69 deletions(-) diff --git a/package-lock.json b/package-lock.json index 167526f2e87..916024cd545 100644 --- a/package-lock.json +++ b/package-lock.json @@ -69969,7 +69969,6 @@ "JSONStream": "^1.3.5", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", - "lodash.pick": "^4.4.0", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", @@ -119943,7 +119942,6 @@ "lodash": "^4.17.21", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", - "lodash.pick": "*", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", diff --git a/packages/compass-import-export/package.json b/packages/compass-import-export/package.json index f5261742a66..2ee624bac5d 100644 --- a/packages/compass-import-export/package.json +++ b/packages/compass-import-export/package.json @@ -155,7 +155,6 @@ "JSONStream": "^1.3.5", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", - "lodash.pick": "^4.4.0", "lodash.throttle": "^4.1.1", "marky": "^1.2.1", "mime-types": "^2.1.24", diff --git a/packages/compass-import-export/src/modules/export.js b/packages/compass-import-export/src/modules/export.js index ad9bdab643c..1bda213feb0 100644 --- a/packages/compass-import-export/src/modules/export.js +++ b/packages/compass-import-export/src/modules/export.js @@ -440,7 +440,6 @@ export const startExport = () => { const spec = exportData.isFullCollection ? { filter: {} } : exportData.query; - const numDocsToExport = exportData.isFullCollection ? await fetchDocumentCount(dataService, ns, spec) : exportData.count; @@ -448,6 +447,9 @@ export const startExport = () => { const projection = Object.fromEntries( Object.entries(exportData.fields) .filter((keyAndValue) => keyAndValue[1] === 1)); + if (Object.keys(projection).length > 0 && (undefined === exportData.fields._id || exportData.fields._id === 0)) { + projection._id = 0; + } log.info(mongoLogId(1001000083), 'Export', 'Start reading from collection', { ns, numDocsToExport, @@ -481,7 +483,7 @@ export const startExport = () => { if (exportData.fileType === 'csv') { formatter = createCSVFormatter({ columns }); } else { - formatter = createJSONFormatter({ columns }); + formatter = createJSONFormatter(); } const dest = fs.createWriteStream(exportData.fileName); diff --git a/packages/compass-import-export/src/modules/export.spec.js b/packages/compass-import-export/src/modules/export.spec.js index 07a074f0e51..f5f5eee563f 100644 --- a/packages/compass-import-export/src/modules/export.spec.js +++ b/packages/compass-import-export/src/modules/export.spec.js @@ -9,42 +9,60 @@ import AppRegistry from 'hadron-app-registry'; import FILE_TYPES from '../constants/file-types'; import reducer, * as actions from './export'; import configureExportStore from '../stores/export-store'; - +import { ConnectionOptions, DataService } from 'mongodb-data-service'; +import { promisify } from 'util'; +import { once } from 'events'; describe('export [module]', () => { describe('#reducer', () => { context('#startExport', () => { let store; const localAppRegistry = new AppRegistry(); const globalAppRegistry = new AppRegistry(); + const dataService = new DataService({ connectionString: 'mongodb://localhost:27018/local'}); + const createCollection = promisify(dataService.createCollection).bind(dataService); + const dropCollection = promisify(dataService.dropCollection).bind(dataService); + const insertMany = promisify(dataService.insertMany).bind(dataService); + const TEST_COLLECTION_NAME = 'local.foobar'; + + afterEach(async function() { + await dropCollection(TEST_COLLECTION_NAME); + }); beforeEach(async function() { - const mockDocuments = [ - { - _id: 'foo', - first_name: 'John', - last_name: 'Appleseed' - } - ]; + await dataService.connect(); + try { + await createCollection(TEST_COLLECTION_NAME); + await insertMany(TEST_COLLECTION_NAME, [ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); + } catch (err) { + console.log(err); + } + store = configureExportStore({ localAppRegistry: localAppRegistry, globalAppRegistry: globalAppRegistry, + namespace: TEST_COLLECTION_NAME, dataProvider: { error: function(err) { throw err; }, - dataProvider: { - estimatedCount: function(ns, options, callback) { return callback(null, mockDocuments.length); }, - count: function(ns, filter, options, callback ) { return callback(null, mockDocuments.length); }, - fetch: function() { - return { - stream: function() { - return Readable.from(mockDocuments); - } - }; - } - } + dataProvider: dataService } }); }); - + async function configureAndStartExport(selectedFields, fileType, tempFile) { + store.dispatch(actions.updateSelectedFields(selectedFields)); + store.dispatch(actions.selectExportFileName(tempFile)); + store.dispatch(actions.selectExportFileType(fileType)); + store.dispatch(actions.toggleFullCollection()); + store.dispatch(actions.startExport()); + await once(localAppRegistry, 'export-finished'); + const writtenData = fs.readFileSync(tempFile, 'utf-8'); + return writtenData; + } describe('CSV Export', () => { let tempFile; beforeEach(() => { @@ -53,26 +71,22 @@ describe('export [module]', () => { `test-${Date.now()}.csv` ); }); - afterEach((afterEachDone) => { - rimraf(tempFile, afterEachDone); + afterEach(() => { + fs.unlinkSync(tempFile); }); - it('should set the correct fields to CSV export', (done) => { - const fields = { 'first_name': 1, '_id': 1, 'foobar': 1, 'last_name': 0}; - store.dispatch(actions.updateSelectedFields(fields)); - store.dispatch(actions.selectExportFileName(tempFile)); - store.dispatch(actions.selectExportFileType('csv')); - store.dispatch(actions.toggleFullCollection()); - store.dispatch(actions.startExport()); - localAppRegistry.addListener('export-finished', function checkWrittenData() { - fs.readFile(tempFile, 'utf-8', function(err, data) { - if (err) { return done(err); } - const writtenData = data.split('\n'); - expect(writtenData[0]).to.equal('first_name,_id,foobar'); - expect(writtenData[1]).to.equal('John,foo,'); - localAppRegistry.removeListener('export-finished', checkWrittenData); - done(); - }); - }); + it('should set the correct fields to CSV export', async() => { + const fields = { 'first_name': 1, 'foobar': 1, 'last_name': 0}; + const data = await configureAndStartExport(fields, 'csv', tempFile); + const writtenData = data.split('\n'); + expect(writtenData[0]).to.equal('first_name,foobar'); + expect(writtenData[1]).to.equal('John,'); + }); + it('should not include _id if not specified', async() => { + const fields = { 'first_name': 1, 'foobar': 1 }; + const data = await configureAndStartExport(fields, 'csv', tempFile); + const writtenData = data.split('\n'); + expect(writtenData[0]).to.equal('first_name,foobar'); + expect(writtenData[1]).to.equal('John,'); }); }); describe('JSON Export', () => { @@ -83,28 +97,66 @@ describe('export [module]', () => { `test-${Date.now()}.json` ); }); - afterEach((done) => { - rimraf(tempFile, done); + afterEach(() => { + fs.unlinkSync(tempFile); }); - it('should not include _id if omitted', (done) => { - const fields = { 'first_name': 1, 'last_name': 0}; - store.dispatch(actions.updateSelectedFields(fields)); - store.dispatch(actions.selectExportFileName(tempFile)); - store.dispatch(actions.selectExportFileType('json')); - store.dispatch(actions.toggleFullCollection()); - store.dispatch(actions.startExport()); - localAppRegistry.addListener('export-finished', function() { - fs.readFile(tempFile, 'utf-8', function(err, data) { - if (err) { return done(err); } - const writtenData = JSON.parse(data); - expect(writtenData).to.deep.equal([ - { - first_name: 'John', - } - ]); - done(); - }); - }); + it('should not include _id if omitted', async() => { + const fields = { 'first_name': 1, 'last_name': 0 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + first_name: 'John', + } + ]); + }); + + it('should not include _id if is set to 0', async() => { + const fields = { 'first_name': 1, 'last_name': 0, _id: 0 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + first_name: 'John', + } + ]); + }); + + it('should include _id if is set to 1', async() => { + const fields = { 'first_name': 1, 'last_name': 0, _id: 1 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + } + ]); + }); + + it('should include all fields if projection is empty', async() => { + const fields = {}; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); + }); + it('should include all fields all fields are set to 0', async() => { + const fields = { first_name: 0, last_name: 0, _id: 0}; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); }); }); }); diff --git a/packages/compass-import-export/src/utils/formatters.js b/packages/compass-import-export/src/utils/formatters.js index 015c6b797fb..1ec9100e3a0 100644 --- a/packages/compass-import-export/src/utils/formatters.js +++ b/packages/compass-import-export/src/utils/formatters.js @@ -7,16 +7,14 @@ import { EJSON } from 'bson'; import { serialize as flatten } from './bson-csv'; import { Transform } from 'stream'; import { EOL } from 'os'; -import pick from 'lodash.pick'; /** * @returns {Stream.Transform} */ -export const createJSONFormatter = function({ brackets = true, columns = [] } = {}) { +export const createJSONFormatter = function({ brackets = true } = {}) { return new Transform({ readableObjectMode: false, writableObjectMode: true, transform: function(doc, encoding, callback) { - const docToBeSerialized = columns.length > 0 ? pick(doc, columns) : doc; if (this._counter >= 1) { if (brackets) { this.push(','); @@ -24,7 +22,7 @@ export const createJSONFormatter = function({ brackets = true, columns = [] } = this.push(EOL); } } - const s = EJSON.stringify(docToBeSerialized, null, brackets ? 2 : null); + const s = EJSON.stringify(doc, null, brackets ? 2 : null); if (this._counter === undefined) { this._counter = 0; if (brackets) { From 482dc9844015ccf9d65508831d3f3a4f1fc37de1 Mon Sep 17 00:00:00 2001 From: Leonardo Rossi Date: Tue, 22 Feb 2022 16:10:25 +0100 Subject: [PATCH 4/4] adds pretest-posttest npm script --- packages/compass-import-export/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/compass-import-export/package.json b/packages/compass-import-export/package.json index 2ee624bac5d..b186bb46703 100644 --- a/packages/compass-import-export/package.json +++ b/packages/compass-import-export/package.json @@ -24,6 +24,8 @@ "compile": "cross-env NODE_ENV=production webpack --config ./config/webpack.prod.config.js", "start": "webpack-dev-server --config ./config/webpack.dev.config.js", "test": "cross-env NODE_ENV=test mocha-webpack \"./src/**/*.spec.js\"", + "pretest": "mongodb-runner start --port=27018", + "posttest": "mongodb-runner stop --port=27018", "test:watch": "cross-env NODE_ENV=test mocha-webpack \"./src/**/*.spec.js\" --watch", "test:dev": "cross-env NODE_ENV=test mocha-webpack", "cover": "nyc npm run test",