diff --git a/packages/compass-import-export/package.json b/packages/compass-import-export/package.json index 6e47e41dfc4..b186bb46703 100644 --- a/packages/compass-import-export/package.json +++ b/packages/compass-import-export/package.json @@ -24,6 +24,8 @@ "compile": "cross-env NODE_ENV=production webpack --config ./config/webpack.prod.config.js", "start": "webpack-dev-server --config ./config/webpack.dev.config.js", "test": "cross-env NODE_ENV=test mocha-webpack \"./src/**/*.spec.js\"", + "pretest": "mongodb-runner start --port=27018", + "posttest": "mongodb-runner stop --port=27018", "test:watch": "cross-env NODE_ENV=test mocha-webpack \"./src/**/*.spec.js\" --watch", "test:dev": "cross-env NODE_ENV=test mocha-webpack", "cover": "nyc npm run test", @@ -146,13 +148,13 @@ }, "dependencies": { "@mongodb-js/compass-logging": "^0.7.0", - "JSONStream": "^1.3.5", "ansi-to-html": "^0.6.11", "bson": "*", "csv-parser": "^2.3.1", "fast-csv": "^3.4.0", "flat": "cipacda/flat", "javascript-stringify": "^2.0.1", + "JSONStream": "^1.3.5", "lodash.isobjectlike": "^4.0.0", "lodash.isplainobject": "^4.0.6", "lodash.throttle": "^4.1.1", diff --git a/packages/compass-import-export/src/modules/export.js b/packages/compass-import-export/src/modules/export.js index 3b59744b19d..1bda213feb0 100644 --- a/packages/compass-import-export/src/modules/export.js +++ b/packages/compass-import-export/src/modules/export.js @@ -440,7 +440,6 @@ export const startExport = () => { const spec = exportData.isFullCollection ? { filter: {} } : exportData.query; - const numDocsToExport = exportData.isFullCollection ? await fetchDocumentCount(dataService, ns, spec) : exportData.count; @@ -448,6 +447,9 @@ export const startExport = () => { const projection = Object.fromEntries( Object.entries(exportData.fields) .filter((keyAndValue) => keyAndValue[1] === 1)); + if (Object.keys(projection).length > 0 && (undefined === exportData.fields._id || exportData.fields._id === 0)) { + projection._id = 0; + } log.info(mongoLogId(1001000083), 'Export', 'Start reading from collection', { ns, numDocsToExport, diff --git a/packages/compass-import-export/src/modules/export.spec.js b/packages/compass-import-export/src/modules/export.spec.js index c9d6d855b2e..f5f5eee563f 100644 --- a/packages/compass-import-export/src/modules/export.spec.js +++ b/packages/compass-import-export/src/modules/export.spec.js @@ -9,66 +9,154 @@ import AppRegistry from 'hadron-app-registry'; import FILE_TYPES from '../constants/file-types'; import reducer, * as actions from './export'; import configureExportStore from '../stores/export-store'; - +import { ConnectionOptions, DataService } from 'mongodb-data-service'; +import { promisify } from 'util'; +import { once } from 'events'; describe('export [module]', () => { describe('#reducer', () => { context('#startExport', () => { - let tempFile; let store; const localAppRegistry = new AppRegistry(); const globalAppRegistry = new AppRegistry(); + const dataService = new DataService({ connectionString: 'mongodb://localhost:27018/local'}); + const createCollection = promisify(dataService.createCollection).bind(dataService); + const dropCollection = promisify(dataService.dropCollection).bind(dataService); + const insertMany = promisify(dataService.insertMany).bind(dataService); + const TEST_COLLECTION_NAME = 'local.foobar'; + + afterEach(async function() { + await dropCollection(TEST_COLLECTION_NAME); + }); beforeEach(async function() { - tempFile = path.join( - os.tmpdir(), - `test-${Date.now()}.csv` - ); - const mockDocuments = [ - { - _id: 'foo', - first_name: 'John', - last_name: 'Appleseed' - } - ]; + await dataService.connect(); + try { + await createCollection(TEST_COLLECTION_NAME); + await insertMany(TEST_COLLECTION_NAME, [ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); + } catch (err) { + console.log(err); + } + store = configureExportStore({ localAppRegistry: localAppRegistry, globalAppRegistry: globalAppRegistry, + namespace: TEST_COLLECTION_NAME, dataProvider: { error: function(err) { throw err; }, - dataProvider: { - estimatedCount: function(ns, options, callback) { return callback(null, mockDocuments.length); }, - count: function(ns, filter, options, callback ) { return callback(null, mockDocuments.length); }, - fetch: function() { - return { - stream: function() { - return Readable.from(mockDocuments); - } - }; - } - } + dataProvider: dataService } }); }); - - afterEach(function(done) { - rimraf(tempFile, done); - }); - it('should set the correct fields to export', (done) => { - const fields = { 'first_name': 1, '_id': 1, 'foobar': 1, 'last_name': 0}; - store.dispatch(actions.updateSelectedFields(fields)); + async function configureAndStartExport(selectedFields, fileType, tempFile) { + store.dispatch(actions.updateSelectedFields(selectedFields)); store.dispatch(actions.selectExportFileName(tempFile)); - store.dispatch(actions.selectExportFileType('csv')); + store.dispatch(actions.selectExportFileType(fileType)); store.dispatch(actions.toggleFullCollection()); - store.dispatch(actions.startExport()); - localAppRegistry.addListener('export-finished', function() { - fs.readFile(tempFile, 'utf-8', function(err, data) { - if (err) { return done(err); } - const writtenData = data.split('\n'); - expect(writtenData[0]).to.equal('first_name,_id,foobar'); - expect(writtenData[1]).to.equal('John,foo,'); - done(); - }); + await once(localAppRegistry, 'export-finished'); + const writtenData = fs.readFileSync(tempFile, 'utf-8'); + return writtenData; + } + describe('CSV Export', () => { + let tempFile; + beforeEach(() => { + tempFile = path.join( + os.tmpdir(), + `test-${Date.now()}.csv` + ); + }); + afterEach(() => { + fs.unlinkSync(tempFile); + }); + it('should set the correct fields to CSV export', async() => { + const fields = { 'first_name': 1, 'foobar': 1, 'last_name': 0}; + const data = await configureAndStartExport(fields, 'csv', tempFile); + const writtenData = data.split('\n'); + expect(writtenData[0]).to.equal('first_name,foobar'); + expect(writtenData[1]).to.equal('John,'); + }); + it('should not include _id if not specified', async() => { + const fields = { 'first_name': 1, 'foobar': 1 }; + const data = await configureAndStartExport(fields, 'csv', tempFile); + const writtenData = data.split('\n'); + expect(writtenData[0]).to.equal('first_name,foobar'); + expect(writtenData[1]).to.equal('John,'); + }); + }); + describe('JSON Export', () => { + let tempFile; + beforeEach(() => { + tempFile = path.join( + os.tmpdir(), + `test-${Date.now()}.json` + ); + }); + afterEach(() => { + fs.unlinkSync(tempFile); + }); + it('should not include _id if omitted', async() => { + const fields = { 'first_name': 1, 'last_name': 0 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + first_name: 'John', + } + ]); + }); + + it('should not include _id if is set to 0', async() => { + const fields = { 'first_name': 1, 'last_name': 0, _id: 0 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + first_name: 'John', + } + ]); + }); + + it('should include _id if is set to 1', async() => { + const fields = { 'first_name': 1, 'last_name': 0, _id: 1 }; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + } + ]); + }); + + it('should include all fields if projection is empty', async() => { + const fields = {}; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); + }); + it('should include all fields all fields are set to 0', async() => { + const fields = { first_name: 0, last_name: 0, _id: 0}; + const data = await configureAndStartExport(fields, 'json', tempFile); + const writtenData = JSON.parse(data); + expect(writtenData).to.deep.equal([ + { + _id: 'foo', + first_name: 'John', + last_name: 'Appleseed' + } + ]); }); }); }); diff --git a/packages/compass-import-export/src/utils/formatters.js b/packages/compass-import-export/src/utils/formatters.js index c6f398e48ab..1ec9100e3a0 100644 --- a/packages/compass-import-export/src/utils/formatters.js +++ b/packages/compass-import-export/src/utils/formatters.js @@ -7,7 +7,6 @@ import { EJSON } from 'bson'; import { serialize as flatten } from './bson-csv'; import { Transform } from 'stream'; import { EOL } from 'os'; - /** * @returns {Stream.Transform} */ diff --git a/packages/compass-import-export/src/utils/formatters.spec.js b/packages/compass-import-export/src/utils/formatters.spec.js index 395541aee7f..7ad6e94b181 100644 --- a/packages/compass-import-export/src/utils/formatters.spec.js +++ b/packages/compass-import-export/src/utils/formatters.spec.js @@ -28,7 +28,7 @@ const FIXTURES = { describe('formatters', () => { describe('json', () => { - it('should format a single docment in an array', () => { + it('should format a single document in an array', () => { const source = stream.Readable.from([{_id: new ObjectID('5e5ea7558d35931a05eafec0')}]); const formatter = createJSONFormatter({brackets: true}); const dest = fs.createWriteStream(FIXTURES.JSON_SINGLE_DOC); @@ -41,7 +41,7 @@ describe('formatters', () => { }) .then(() => rm(FIXTURES.JSON_SINGLE_DOC)); }); - it('should format more than 2 docments in an array', () => { + it('should format more than 2 documents in an array', () => { const docs = [ {_id: new ObjectID('5e5ea7558d35931a05eafec0')}, {_id: new ObjectID('5e6bafc438e060f695591713')},