diff --git a/.gitignore b/.gitignore index dc0fea0..05d759f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ coverage *.iml .nvmrc .nyc_output +.electron/ \ No newline at end of file diff --git a/.npmignore b/.npmignore index a476449..43edeee 100644 --- a/.npmignore +++ b/.npmignore @@ -18,3 +18,4 @@ src/ .zuul.yml lib/index.html examples/ +.electron/ diff --git a/package-lock.json b/package-lock.json index d78c201..020ec1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28039,6 +28039,15 @@ "symbol-observable": "^1.0.3" } }, + "redux-mock-store": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/redux-mock-store/-/redux-mock-store-1.5.4.tgz", + "integrity": "sha512-xmcA0O/tjCLXhh9Fuiq6pMrJCwFRaouA8436zcikdIpYWWCjU76CRk+i2bHx8EeiSiMGnB85/lZdU3wIJVXHTA==", + "dev": true, + "requires": { + "lodash.isplainobject": "^4.0.6" + } + }, "redux-thunk": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.3.0.tgz", diff --git a/package.json b/package.json index e7715ab..d9562bd 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "react-hot-loader": "^4.12.14", "react-redux": "^5.0.6", "redux": "^3.7.2", + "redux-mock-store": "^1.5.4", "reflux": "^0.4.1", "reflux-state-mixin": "^0.7.0", "resolve": "^1.12.0", diff --git a/src/modules/import.js b/src/modules/import.js index d64bada..539a27f 100644 --- a/src/modules/import.js +++ b/src/modules/import.js @@ -52,22 +52,22 @@ const debug = createLogger('import'); * ## Action names */ const PREFIX = 'import-export/import'; -const STARTED = `${PREFIX}/STARTED`; -const CANCELED = `${PREFIX}/CANCELED`; -const PROGRESS = `${PREFIX}/PROGRESS`; -const FINISHED = `${PREFIX}/FINISHED`; -const FAILED = `${PREFIX}/FAILED`; -const FILE_TYPE_SELECTED = `${PREFIX}/FILE_TYPE_SELECTED`; -const FILE_SELECTED = `${PREFIX}/FILE_SELECTED`; -const OPEN = `${PREFIX}/OPEN`; -const CLOSE = `${PREFIX}/CLOSE`; -const SET_PREVIEW = `${PREFIX}/SET_PREVIEW`; -const SET_DELIMITER = `${PREFIX}/SET_DELIMITER`; -const SET_GUESSTIMATED_TOTAL = `${PREFIX}/SET_GUESSTIMATED_TOTAL`; -const SET_STOP_ON_ERRORS = `${PREFIX}/SET_STOP_ON_ERRORS`; -const SET_IGNORE_BLANKS = `${PREFIX}/SET_IGNORE_BLANKS`; -const TOGGLE_INCLUDE_FIELD = `${PREFIX}/TOGGLE_INCLUDE_FIELD`; -const SET_FIELD_TYPE = `${PREFIX}/SET_FIELD_TYPE`; +export const STARTED = `${PREFIX}/STARTED`; +export const CANCELED = `${PREFIX}/CANCELED`; +export const PROGRESS = `${PREFIX}/PROGRESS`; +export const FINISHED = `${PREFIX}/FINISHED`; +export const FAILED = `${PREFIX}/FAILED`; +export const FILE_TYPE_SELECTED = `${PREFIX}/FILE_TYPE_SELECTED`; +export const FILE_SELECTED = `${PREFIX}/FILE_SELECTED`; +export const OPEN = `${PREFIX}/OPEN`; +export const CLOSE = `${PREFIX}/CLOSE`; +export const SET_PREVIEW = `${PREFIX}/SET_PREVIEW`; +export const SET_DELIMITER = `${PREFIX}/SET_DELIMITER`; +export const SET_GUESSTIMATED_TOTAL = `${PREFIX}/SET_GUESSTIMATED_TOTAL`; +export const SET_STOP_ON_ERRORS = `${PREFIX}/SET_STOP_ON_ERRORS`; +export const SET_IGNORE_BLANKS = `${PREFIX}/SET_IGNORE_BLANKS`; +export const TOGGLE_INCLUDE_FIELD = `${PREFIX}/TOGGLE_INCLUDE_FIELD`; +export const SET_FIELD_TYPE = `${PREFIX}/SET_FIELD_TYPE`; /** * ## Initial state. @@ -92,7 +92,8 @@ export const INITIAL_STATE = { values: [], previewLoaded: false, exclude: [], - transform: [] + transform: [], + fileType: '' }; /** @@ -255,8 +256,9 @@ export const startImport = () => { progress, dest, function(err) { + debugger; console.timeEnd('import:start'); - console.groupEnd('import:start'); + console.groupEnd(); /** * Refresh data (docs, aggregations) regardless of whether we have a * partial import or full import @@ -446,6 +448,7 @@ export const selectImportFileName = fileName => { return promisify(detectImportFile)(fileName); }) .then(detected => { + debug('get detection results'); dispatch({ type: FILE_SELECTED, fileName: fileName, @@ -467,7 +470,10 @@ export const selectImportFileName = fileName => { ) ); }) - .catch(err => dispatch(onError(err))); + .catch(err => { + debug('dispatching error', err); + dispatch(onError(err)); + }); }; }; @@ -596,6 +602,7 @@ export const closeImport = () => ({ */ // eslint-disable-next-line complexity const reducer = (state = INITIAL_STATE, action) => { + debug('reducer handling action', action); if (action.type === FILE_SELECTED) { return { ...state, diff --git a/src/modules/import.spec.js b/src/modules/import.spec.js index cd849a3..0a810f4 100644 --- a/src/modules/import.spec.js +++ b/src/modules/import.spec.js @@ -1,222 +1,401 @@ -import reducer, * as actions from './import'; +import reducer, { + STARTED, + CANCELED, + PROGRESS, + FINISHED, + FAILED, + FILE_TYPE_SELECTED, + selectImportFileType, + FILE_SELECTED, + selectImportFileName, + OPEN, + CLOSE, + SET_PREVIEW, + SET_DELIMITER, + SET_GUESSTIMATED_TOTAL, + SET_STOP_ON_ERRORS, + SET_IGNORE_BLANKS, + TOGGLE_INCLUDE_FIELD, + SET_FIELD_TYPE, + INITIAL_STATE +} from './import'; import PROCESS_STATUS from 'constants/process-status'; -describe.skip('import [module]', () => { - describe('#reducer', () => { - context('when the action type is FINISHED', () => { - context('when the state has an error', () => { - const action = actions.onFinished(); - - it('returns the new state and stays open', () => { - expect(reducer({ error: true, isOpen: false }, action)).to.deep.equal({ - isOpen: true, - progress: 100, - error: true, - status: undefined - }); - }); - }); +import configureMockStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; +import path from 'path'; - context('when the state has no error', () => { - const action = actions.onFinished(); +const middlewares = [thunk]; +const mockStore = configureMockStore(middlewares); - it('returns the new state and closes', () => { - expect(reducer({ isOpen: true }, action)).to.deep.equal({ - isOpen: false, - progress: 100, - status: undefined - }); - }); - }); +/** + * Sets up a fresh store using 'redux-mock-store' + * providing a shortcut that does not use block scoped variables. + * + * @param {Object} test `this` inside an `it()` block. + */ +function setupMockStore(test) { + const state = { + importData: { + ...INITIAL_STATE + } + }; + const store = mockStore(state); + test.store = store; + test.state = test.state; +} - context('when the status is started', () => { - const action = actions.onFinished(); +/** + * Boiler plate that I can set file types. + * + * @param {Object} test `this` inside an `it()` block. + * @param {String} fileType json or csv. + * @returns {Promise} + */ +function testSetFileType(test, fileType) { + return new Promise(function(resolve) { + // See https://github.com/dmitry-zaets/redux-mock-store/issues/71#issuecomment-369546064 + // redux-mock-store does not update state automatically. + test.store.subscribe(() => { + const expected = { + fileType: fileType, + type: FILE_TYPE_SELECTED + }; - it('sets the status to completed', () => { - expect(reducer({ status: PROCESS_STATUS.STARTED }, action)).to.deep.equal({ - isOpen: false, - progress: 100, - status: PROCESS_STATUS.COMPLETED - }); - }); - }); - - context('when the status is canceled', () => { - const action = actions.onFinished(); - - it('keeps the same status', () => { - expect(reducer({ status: PROCESS_STATUS.CANCELED }, action)).to.deep.equal({ - isOpen: true, - progress: 100, - status: PROCESS_STATUS.CANCELED - }); - }); - }); + expect(reducer(test.state, expected).fileType).to.be.deep.equal(fileType); + }); + test.store.dispatch(selectImportFileType(fileType)); - context('when the status is failed', () => { - const action = actions.onFinished(); + expect(test.store.getActions()).to.deep.equal([ + { + fileType: fileType, + type: FILE_TYPE_SELECTED + } + ]); + resolve(test); + }); +} - it('keeps the same status', () => { - expect(reducer({ status: PROCESS_STATUS.FAILED }, action)).to.deep.equal({ - isOpen: true, - progress: 100, - status: PROCESS_STATUS.FAILED - }); - }); - }); +describe('import [module]', () => { + describe('selectImportFileType', () => { + beforeEach(function() { + setupMockStore(this); }); - context('when the action type is PROGRESS', () => { - const action = actions.onProgress(55); - - it('returns the new state', () => { - expect(reducer(undefined, action)).to.deep.equal({ - isOpen: false, - progress: 55, - error: null, - fileName: '', - fileType: 'json', - status: 'UNSPECIFIED' - }); - }); + it('dispatch a FILE_TYPE_SELECTED action and the reducer should update fileType to csv', function() { + return testSetFileType(this, 'csv'); }); - - context('when the action type is SELECT_FILE_TYPE', () => { - const action = actions.selectImportFileType('csv'); - - it('returns the new state', () => { - expect(reducer(undefined, action)).to.deep.equal({ - isOpen: false, - progress: 0, - error: null, - fileName: '', - fileType: 'csv', - status: 'UNSPECIFIED' - }); - }); + it('dispatch a FILE_TYPE_SELECTED action and the reducer should update fileType to json', function() { + return testSetFileType(this, 'JSON'); }); - context('when the action type is SELECT_FILE_NAME', () => { - const action = actions.selectImportFileName('test.json'); - - it('returns the new state', () => { - expect(reducer(undefined, action)).to.deep.equal({ - isOpen: false, - progress: 0, - error: null, - fileName: 'test.json', - fileType: 'json', - status: 'UNSPECIFIED' - }); - }); + afterEach(function() { + this.store.clearActions(); }); + }); - context('when the action type is OPEN', () => { - const action = actions.openImport(); - - it('returns the new state', () => { - expect(reducer(undefined, action)).to.deep.equal({ - isOpen: true, - progress: 0, - error: null, - fileName: '', - fileType: 'json', - status: 'UNSPECIFIED' - }); - }); + describe('#selectImportFileName', () => { + beforeEach(function() { + setupMockStore(this); }); - context('when the action type is CLOSE', () => { - const action = actions.closeImport(); + it('dispatch a FILE_SELECTED action', function() { + const test = this; + const fileName = path.join( + __dirname, + '..', + '..', + '..', + 'test', + 'docs.json' + ); + return new Promise(function(resolve, reject) { + // See https://github.com/dmitry-zaets/redux-mock-store/issues/71#issuecomment-369546064 + // redux-mock-store does not update state automatically. + test.store.subscribe(() => { + // const expected = { + // fileName: fileName, + // fileIsMultilineJSON: false, + // fileType: 'json', + // status: PROCESS_STATUS.UNSPECIFIED, + // progress: 0, + // docsWritten: 0, + // source: undefined, + // dest: undefined + // }; + console.log('subscribe touched', { args: arguments, actions: test.store.getActions()}); + const expected = { + isOpen: false, + progress: 0, + error: null, + fileName: '', + fileIsMultilineJSON: false, + useHeaderLines: true, + status: 'UNSPECIFIED', + fileStats: null, + docsWritten: 0, + guesstimatedDocsTotal: 0, + delimiter: ',', + stopOnErrors: false, + ignoreBlanks: true, + fields: [], + values: [], + previewLoaded: false, + exclude: [], + transform: [], + fileType: '' + }; - it('returns the new state', () => { - expect(reducer({}, action)).to.deep.equal({ isOpen: false }); - }); - }); + const result = reducer(test.state, expected); + + expect(result).to.be.deep.equal({ + isOpen: false, + progress: 0, + error: null, + fileName: '', + fileIsMultilineJSON: false, + useHeaderLines: true, + status: 'UNSPECIFIED', + fileStats: null, + docsWritten: 0, + guesstimatedDocsTotal: 0, + delimiter: ',', + stopOnErrors: false, + ignoreBlanks: true, + fields: [], + values: [], + previewLoaded: false, + exclude: [], + transform: [], + fileType: '' + }); - context('when the action type is FAILED', () => { - const error = new Error('failed'); - const action = actions.onError(error); - - it('returns the new state', () => { - expect(reducer(undefined, action)).to.deep.equal({ - isOpen: false, - progress: 100, - error: error, - fileName: '', - fileType: 'json', - status: 'FAILED' + resolve(test); + // done(); }); - }); - }); + test.store.dispatch(selectImportFileName(fileName)); - context('when the action type is not defined', () => { - it('returns the initial state', () => { - expect(reducer('', {})).to.equal(''); + expect(test.store.getActions()).to.deep.equal([ + // { + // fileName: fileName, + // fileType: 'json', + // fileStats: {}, + // fileIsMultilineJSON: false, + // type: FILE_SELECTED + // } + ]); }); }); - }); - describe('#openImport', () => { - it('returns the action', () => { - expect(actions.openImport()).to.deep.equal({ - type: actions.OPEN - }); + afterEach(function() { + this.store.clearActions(); }); }); - describe('#closeImport', () => { - it('returns the action', () => { - expect(actions.closeImport()).to.deep.equal({ - type: actions.CLOSE - }); - }); - }); + // describe('#reducer', () => { + // context('when the action type is FINISHED', () => { + // context('when the state has an error', () => { + // it('returns the new state and stays open', () => { + // expect(reducer({ error: true, isOpen: false }, action)).to.deep.equal({ + // isOpen: true, + // progress: 100, + // error: true, + // status: undefined + // }); + // }); + // }); - describe('#onError', () => { - const error = new Error('failed'); + // context('when the state has no error', () => { + // const action = actions.onFinished(); - it('returns the action', () => { - expect(actions.onError(error)).to.deep.equal({ - type: actions.FAILED, - error: error - }); - }); - }); + // it('returns the new state and closes', () => { + // expect(reducer({ isOpen: true }, action)).to.deep.equal({ + // isOpen: false, + // progress: 100, + // status: undefined + // }); + // }); + // }); - describe('#onFinished', () => { - it('returns the action', () => { - expect(actions.onFinished()).to.deep.equal({ - type: actions.FINISHED - }); - }); - }); + // context('when the status is started', () => { + // const action = actions.onFinished(); - describe('#onProgress', () => { - it('returns the action', () => { - expect(actions.onProgress(34)).to.deep.equal({ - type: actions.PROGRESS, - progress: 34, - error: null - }); - }); - }); + // it('sets the status to completed', () => { + // expect(reducer({ status: PROCESS_STATUS.STARTED }, action)).to.deep.equal({ + // isOpen: false, + // progress: 100, + // status: PROCESS_STATUS.COMPLETED + // }); + // }); + // }); - describe('#selectImportFileName', () => { - it('returns the action', () => { - expect(actions.selectImportFileName('test.json')).to.deep.equal({ - type: actions.SELECT_FILE_NAME, - fileName: 'test.json' - }); - }); - }); + // context('when the status is canceled', () => { + // const action = actions.onFinished(); - describe('#selectImportFileType', () => { - it('returns the action', () => { - expect(actions.selectImportFileType('csv')).to.deep.equal({ - type: actions.SELECT_FILE_TYPE, - fileType: 'csv' - }); - }); - }); + // it('keeps the same status', () => { + // expect(reducer({ status: PROCESS_STATUS.CANCELED }, action)).to.deep.equal({ + // isOpen: true, + // progress: 100, + // status: PROCESS_STATUS.CANCELED + // }); + // }); + // }); + + // context('when the status is failed', () => { + // const action = actions.onFinished(); + + // it('keeps the same status', () => { + // expect(reducer({ status: PROCESS_STATUS.FAILED }, action)).to.deep.equal({ + // isOpen: true, + // progress: 100, + // status: PROCESS_STATUS.FAILED + // }); + // }); + // }); + // }); + + // context('when the action type is PROGRESS', () => { + // const action = actions.onProgress(55); + + // it('returns the new state', () => { + // expect(reducer(undefined, action)).to.deep.equal({ + // isOpen: false, + // progress: 55, + // error: null, + // fileName: '', + // fileType: 'json', + // status: 'UNSPECIFIED' + // }); + // }); + // }); + + // context('when the action type is SELECT_FILE_TYPE', () => { + // const action = actions.selectImportFileType('csv'); + + // it('returns the new state', () => { + // expect(reducer(undefined, action)).to.deep.equal({ + // isOpen: false, + // progress: 0, + // error: null, + // fileName: '', + // fileType: 'csv', + // status: 'UNSPECIFIED' + // }); + // }); + // }); + + // context('when the action type is SELECT_FILE_NAME', () => { + // const action = actions.selectImportFileName('test.json'); + + // it('returns the new state', () => { + // expect(reducer(undefined, action)).to.deep.equal({ + // isOpen: false, + // progress: 0, + // error: null, + // fileName: 'test.json', + // fileType: 'json', + // status: 'UNSPECIFIED' + // }); + // }); + // }); + + // context('when the action type is OPEN', () => { + // const action = actions.openImport(); + + // it('returns the new state', () => { + // expect(reducer(undefined, action)).to.deep.equal({ + // isOpen: true, + // progress: 0, + // error: null, + // fileName: '', + // fileType: 'json', + // status: 'UNSPECIFIED' + // }); + // }); + // }); + + // context('when the action type is CLOSE', () => { + // const action = actions.closeImport(); + + // it('returns the new state', () => { + // expect(reducer({}, action)).to.deep.equal({ isOpen: false }); + // }); + // }); + + // context('when the action type is FAILED', () => { + // const error = new Error('failed'); + // const action = actions.onError(error); + + // it('returns the new state', () => { + // expect(reducer(undefined, action)).to.deep.equal({ + // isOpen: false, + // progress: 100, + // error: error, + // fileName: '', + // fileType: 'json', + // status: 'FAILED' + // }); + // }); + // }); + + // context('when the action type is not defined', () => { + // it('returns the initial state', () => { + // expect(reducer('', {})).to.equal(''); + // }); + // }); + // }); + + // describe('#openImport', () => { + // it('returns the action', () => { + // expect(actions.openImport()).to.deep.equal({ + // type: actions.OPEN + // }); + // }); + // }); + + // describe('#closeImport', () => { + // it('returns the action', () => { + // expect(actions.closeImport()).to.deep.equal({ + // type: actions.CLOSE + // }); + // }); + // }); + + // describe('#onError', () => { + // const error = new Error('failed'); + + // it('returns the action', () => { + // expect(actions.onError(error)).to.deep.equal({ + // type: actions.FAILED, + // error: error + // }); + // }); + // }); + + // describe('#onFinished', () => { + // it('returns the action', () => { + // expect(actions.onFinished()).to.deep.equal({ + // type: actions.FINISHED + // }); + // }); + // }); + + // describe('#onProgress', () => { + // it('returns the action', () => { + // expect(actions.onProgress(34)).to.deep.equal({ + // type: actions.PROGRESS, + // progress: 34, + // error: null + // }); + // }); + // }); + + // describe('#selectImportFileName', () => { + // it('returns the action', () => { + // expect(actions.selectImportFileName('test.json')).to.deep.equal({ + // type: actions.SELECT_FILE_NAME, + // fileName: 'test.json' + // }); + // }); + // }); }); diff --git a/src/utils/bson-csv.js b/src/utils/bson-csv.js index 1efc56a..8ad188e 100644 --- a/src/utils/bson-csv.js +++ b/src/utils/bson-csv.js @@ -30,7 +30,11 @@ export default { }, Number: { fromString: function(s) { - return parseFloat(s); + s = '' + s; + if (s.includes('.')) { + return parseFloat(s); + } + return parseInt(s, 10); } }, Boolean: { @@ -48,12 +52,15 @@ export default { }, Date: { fromString: function(s) { - return new Date(s); + return new Date('' + s); } }, ObjectId: { fromString: function(s) { - // eslint-disable-next-line new-cap + if (s instanceof bson.ObjectId) { + // EJSON being imported + return s; + } return new bson.ObjectId(s); } }, @@ -129,14 +136,16 @@ const TYPE_FOR_TO_STRING = new Map([ ]); export function detectType(value) { - return TYPE_FOR_TO_STRING.get(Object.prototype.toString.call(value)); + const l = Object.prototype.toString.call(value); + const t = TYPE_FOR_TO_STRING.get(l); + return t; } export function getTypeDescriptorForValue(value) { const t = detectType(value); const _bsontype = t === 'Object' && value._bsontype; return { - type: _bsontype || t, + type: _bsontype ? _bsontype : t, isBSON: !!_bsontype }; } diff --git a/src/utils/import-apply-types-and-projection.js b/src/utils/import-apply-types-and-projection.js index e8c61ec..40247e9 100644 --- a/src/utils/import-apply-types-and-projection.js +++ b/src/utils/import-apply-types-and-projection.js @@ -1,8 +1,6 @@ import { Transform, PassThrough } from 'stream'; - -import bsonCSV from './bson-csv'; -import isPlainObject from 'lodash.isplainobject'; -import isObjectLike from 'lodash.isobjectlike'; +import bsonCSV, { getTypeDescriptorForValue } from './bson-csv'; +import _ from 'lodash'; import { createLogger } from './logger'; @@ -24,36 +22,55 @@ const debug = createLogger('apply-import-type-and-projection'); */ function transformProjectedTypes(spec, data, keyPrefix = '') { if (Array.isArray(data)) { + debug('data is an array'); return data.map(transformProjectedTypes.bind(null, spec)); - } else if (!isPlainObject(data) || data === null || data === undefined) { + } else if (data === null || data === undefined) { + debug('data is null or undefined'); return data; } const keys = Object.keys(data); if (keys.length === 0) { + debug('empty doc'); return data; } - return keys.reduce(function(doc, key) { - const fullKey = `${keyPrefix}${key}`; + const result = data; - /** - * TODO: lucas: Relocate removeEmptyStrings() here? - * Avoid yet another recursive traversal of every document. - */ - if (spec.exclude.includes(fullKey)) { - // Drop the key if unchecked - return doc; - } + _.forEach( + spec.exclude, + function(d) { + if (spec.exclude.indexOf(d) > -1) { + _.unset(result, [d]); + debug('dropped', d); + return false; + } + return true; + }, + {} + ); - const toBSON = bsonCSV[spec.transform[fullKey]]; + const lookup = _.fromPairs(spec.transform); + const lookupKeys = _.keys(lookup); + lookupKeys.forEach(function(keyPath) { + const targetType = _.get(lookup, keyPath); + const typeDescriptor = getTypeDescriptorForValue(_.get(data, keyPath)); + const sourceType = typeDescriptor.t; + const isBSON = typeDescriptor.isBSON; - if (toBSON && !isObjectLike(data[key])) { - doc[key] = toBSON.fromString(data[key]); - } else { - doc[key] = transformProjectedTypes(spec, data[key], `${fullKey}.`); - } - return doc; - }, {}); + const casted = bsonCSV[targetType].fromString(_.get(data, keyPath)); + _.set(result, keyPath, casted); + + debug(`${keyPath} casted`, { + result: casted, + to: targetType, + from: sourceType, + isBSON, + lookupKeys + }); + }); + + debug('result', result); + return result; } export default transformProjectedTypes; @@ -65,15 +82,23 @@ export default transformProjectedTypes; * @returns {TransformStream} */ export function transformProjectedTypesStream(spec) { - if (Object.keys(spec.transform).length === 0 && spec.exclude.length === 0) { + if (!Array.isArray(spec.transform)) { + throw new TypeError('spec.transform must be an array'); + } + if (!Array.isArray(spec.exclude)) { + throw new TypeError('spec.exclude must be an array'); + } + if (spec.transform.length === 0 && spec.exclude.length === 0) { debug('spec is a noop. passthrough stream'); return new PassThrough({ objectMode: true }); } + debug('creating transform stream for spec', spec); return new Transform({ objectMode: true, transform: function(doc, encoding, cb) { - cb(null, transformProjectedTypes(spec, doc)); + const result = transformProjectedTypes(spec, doc); + cb(null, result); } }); } diff --git a/src/utils/import-apply-types-and-projection.spec.js b/src/utils/import-apply-types-and-projection.spec.js index aa5fa26..8fba4cb 100644 --- a/src/utils/import-apply-types-and-projection.spec.js +++ b/src/utils/import-apply-types-and-projection.spec.js @@ -2,10 +2,12 @@ import apply, { transformProjectedTypesStream } from './import-apply-types-and-projection'; +import bson from 'bson'; + describe('import-apply-types-and-projection', () => { it('should include all fields by default', () => { const res = apply( - { exclude: [], transform: {} }, + { exclude: [], transform: [] }, { _id: 'arlo' } @@ -18,7 +20,7 @@ describe('import-apply-types-and-projection', () => { const res = apply( { exclude: ['name'], - transform: {} + transform: [] }, { _id: 'arlo', @@ -34,9 +36,9 @@ describe('import-apply-types-and-projection', () => { const res = apply( { exclude: [], - transform: { - birthday: 'Date' - } + transform: [ + ['birthday', 'Date'] + ] }, { _id: 'arlo', @@ -67,10 +69,10 @@ describe('import-apply-types-and-projection', () => { const spec = { exclude: [], - transform: { - age: 'Number', - 'location.activity.sleeping': 'Boolean' - } + transform: [ + ['age', 'Number'], + ['location.activity.sleeping', 'Boolean'] + ] }; const res = apply(spec, doc); @@ -90,7 +92,7 @@ describe('import-apply-types-and-projection', () => { }); describe('transformProjectedTypesStream', () => { it('should return a passthrough if nothing to actually transform', () => { - const res = transformProjectedTypesStream({ exclude: [], transform: {} }); + const res = transformProjectedTypesStream({ exclude: [], transform: [] }); expect(res.constructor.name).to.equal('PassThrough'); }); }); @@ -104,12 +106,12 @@ describe('import-apply-types-and-projection', () => { */ const spec = { exclude: ['type'], - transform: { - sourceVersion: 'Number', - creationDate: 'Date', - startDate: 'Date', - endDate: 'Date' - } + transform: [ + ['sourceVersion', 'Number'], + ['creationDate', 'Date'], + ['startDate', 'Date'], + ['endDate', 'Date'] + ] }; const data = { @@ -125,4 +127,214 @@ describe('import-apply-types-and-projection', () => { expect(apply.bind(null, spec, data)).to.not.throw(); }); }); + describe('bson', () => { + it('should preserve an ObjectId to an ObjectId', () => { + const res = apply( + { exclude: [], transform: [] }, + { + _id: new bson.ObjectId('5e739e27a4c96922d4435c59') + } + ); + expect(res).to.deep.equal({ + _id: new bson.ObjectId('5e739e27a4c96922d4435c59') + }); + }); + it('should preserve a Date', () => { + const res = apply( + { exclude: [], transform: [] }, + { + _id: new Date('2020-03-19T16:40:38.010Z') + } + ); + expect(res).to.deep.equal({ + _id: new Date('2020-03-19T16:40:38.010Z') + }); + }); + }); + describe('Regression Tests', () => { + // COMPASS-4204 Data type is not being set during import + it('should transform csv strings to Number', () => { + const res = apply( + { + exclude: [], + transform: [ + ['age', 'Number'] + ] + }, + { + _id: 'arlo', + name: 'Arlo', + age: '5' + } + ); + + expect(res).to.deep.equal({ + _id: 'arlo', + name: 'Arlo', + age: 5 + }); + }); + it('should transform floats if Number specified', () => { + const spec = { + exclude: [], + transform: [ + [ 'Bin_#', 'Number'], + [ 'House_#', 'Number'], + [ 'Job_#', 'Number'], + [ 'Job_doc_#', 'String'], + [ 'Block', 'Number'], + [ 'Lot', 'String'], + [ 'Community_Board', 'Number'], + [ 'Zip_Code', 'Number'], + [ 'Permit_Sequence_#', 'String'] + ] + }; + + const doc = { + BOROUGH: 'QUEENS', + 'Bin_#': '4297149', + 'House_#': '17', + Street_Name: 'WEST 16 ROAD', + 'Job_#': '440325738', + 'Job_doc_#': '01', + Job_Type: 'A2', + Self_Cert: 'N', + Block: '15320', + Lot: '00048', + Community_Board: '414', + Zip_Code: '11693', + Bldg_Type: '1', + Residential: 'YES', + Special_District_1: '', + Special_District_2: '', + Work_Type: 'OT', + Permit_Status: 'ISSUED', + Filing_Status: 'RENEWAL', + Permit_Type: 'EW', + 'Permit_Sequence_#': '04', + Permit_Subtype: 'OT', + Oil_Gas: '', + Site_Fill: 'NOT APPLICABLE', + Filing_Date: '05/21/2018 12:00:00 AM', + Issuance_Date: '05/21/2018 12:00:00 AM', + Expiration_Date: '05/15/2019 12:00:00 AM', + Job_Start_Date: '04/07/2017 12:00:00 AM', + "Permittee's_First_Name": 'DONALD', + "Permittee's_Last_Name": "O'SULLIVAN", + "Permittee's_Business_Name": 'NAVILLUS TILE INC', + "Permittee's_Phone_#": '2127501808', + "Permittee's_License_Type": 'GC', + "Permittee's_License_#": '0015163', + Act_as_Superintendent: '', + "Permittee's_Other_Title": '', + HIC_License: '', + "Site_Safety_Mgr's_First_Name": '', + "Site_Safety_Mgr's_Last_Name": '', + Site_Safety_Mgr_Business_Name: '', + 'Superintendent_First_&_Last_Name': '', + Superintendent_Business_Name: '', + "Owner's_Business_Type": 'INDIVIDUAL', + 'Non-Profit': 'N', + "Owner's_Business_Name": 'TERENCE HAIRSTON ARCHITECT, PLLC', + "Owner's_First_Name": 'TERENCE', + "Owner's_Last_Name": 'HAIRSTON', + "Owner's_House_#": '16', + "Owner's_House_Street_Name": 'WEST 36TH STREET', + 'Owner’s_House_City': 'NEW YORK', + 'Owner’s_House_State': 'NY', + 'Owner’s_House_Zip_Code': '10018', + "Owner's_Phone_#": '9176924778', + DOBRunDate: '05/22/2018 12:00:00 AM', + PERMIT_SI_NO: '3463269', + LATITUDE: '40.601732', + LONGITUDE: '-73.821199', + COUNCIL_DISTRICT: '32', + CENSUS_TRACT: '107201', + NTA_NAME: 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' + }; + const res = apply(spec, doc); + expect(res).to.deep.equal({ + BOROUGH: 'QUEENS', + 'Bin_#': 4297149, + 'House_#': 17, + Street_Name: 'WEST 16 ROAD', + 'Job_#': 440325738, + 'Job_doc_#': '01', + Job_Type: 'A2', + Self_Cert: 'N', + Block: 15320, + Lot: '00048', + Community_Board: 414, + Zip_Code: 11693, + Bldg_Type: '1', + Residential: 'YES', + Special_District_1: '', + Special_District_2: '', + Work_Type: 'OT', + Permit_Status: 'ISSUED', + Filing_Status: 'RENEWAL', + Permit_Type: 'EW', + 'Permit_Sequence_#': '04', + Permit_Subtype: 'OT', + Oil_Gas: '', + Site_Fill: 'NOT APPLICABLE', + Filing_Date: '05/21/2018 12:00:00 AM', + Issuance_Date: '05/21/2018 12:00:00 AM', + Expiration_Date: '05/15/2019 12:00:00 AM', + Job_Start_Date: '04/07/2017 12:00:00 AM', + "Permittee's_First_Name": 'DONALD', + "Permittee's_Last_Name": "O'SULLIVAN", + "Permittee's_Business_Name": 'NAVILLUS TILE INC', + "Permittee's_Phone_#": '2127501808', + "Permittee's_License_Type": 'GC', + "Permittee's_License_#": '0015163', + Act_as_Superintendent: '', + "Permittee's_Other_Title": '', + HIC_License: '', + "Site_Safety_Mgr's_First_Name": '', + "Site_Safety_Mgr's_Last_Name": '', + Site_Safety_Mgr_Business_Name: '', + 'Superintendent_First_&_Last_Name': '', + Superintendent_Business_Name: '', + "Owner's_Business_Type": 'INDIVIDUAL', + 'Non-Profit': 'N', + "Owner's_Business_Name": 'TERENCE HAIRSTON ARCHITECT, PLLC', + "Owner's_First_Name": 'TERENCE', + "Owner's_Last_Name": 'HAIRSTON', + "Owner's_House_#": '16', + "Owner's_House_Street_Name": 'WEST 36TH STREET', + 'Owner’s_House_City': 'NEW YORK', + 'Owner’s_House_State': 'NY', + 'Owner’s_House_Zip_Code': '10018', + "Owner's_Phone_#": '9176924778', + DOBRunDate: '05/22/2018 12:00:00 AM', + PERMIT_SI_NO: '3463269', + LATITUDE: '40.601732', + LONGITUDE: '-73.821199', + COUNCIL_DISTRICT: '32', + CENSUS_TRACT: '107201', + NTA_NAME: 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' + }); + }); + it('should transform strings to floats', () => { + const res = apply( + { + exclude: [], + transform: [ + ['LATITUDE', 'Number'], + ['LONGITUDE', 'Number'] + ] + }, + { + LATITUDE: '40.601732', + LONGITUDE: '-73.821199' + } + ); + + expect(res).to.deep.equal({ + LATITUDE: 40.601732, + LONGITUDE: -73.821199 + }); + }); + }); }); diff --git a/src/utils/import-parser.js b/src/utils/import-parser.js index d03250d..66d1466 100644 --- a/src/utils/import-parser.js +++ b/src/utils/import-parser.js @@ -33,7 +33,8 @@ export const createJSONParser = function({ selector = '*', fileName = 'import.json' } = {}) { - debug('creating json parser with selector', selector); + debug('creating json parser with selector', { selector, fileName }); + // return new JSONParser(selector); let lastChunk = ''; const parser = new JSONParser(selector); const stream = new Transform({ @@ -47,6 +48,7 @@ export const createJSONParser = function({ }); parser.on('data', (d) => { + debug('JSON parser on data'); const doc = EJSON.deserialize(d, { promoteValues: true, bsonRegExp: true @@ -144,4 +146,14 @@ function createParser({ }); } +// export const createEJSONDeserializer = function() { +// return new Transform({ +// objectMode: true, + +// const doc = EJSON.deserialize(d, { +// promoteValues: true, +// bsonRegExp: true +// }); +// }; + export default createParser; diff --git a/src/utils/import-parser.spec.js b/src/utils/import-parser.spec.js index 44f789c..8df0fc4 100644 --- a/src/utils/import-parser.spec.js +++ b/src/utils/import-parser.spec.js @@ -14,7 +14,8 @@ const FIXTURES = { LINE_DELIMITED_JSON_EXTRA_LINE: path.join( TEST_DIR, 'docs-with-newline-ending.jsonl' - ) + ), + NUMBER_TRANSFORM_CSV: path.join(TEST_DIR, 'number-transform.csv') }; function runParser(src, parser) { @@ -94,6 +95,78 @@ describe('import-parser', () => { expect(docs).to.have.length(3); }); }); + it('should parse number-transform', () => { + return runParser( + FIXTURES.NUMBER_TRANSFORM_CSV, + createParser({ fileType: 'csv' }) + ).then((docs) => { + expect(docs).to.have.length(1); + expect(docs).to.deep.equal([ + { + BOROUGH: 'QUEENS', + 'Bin_#': '4297149', + 'House_#': '17', + Street_Name: 'WEST 16 ROAD', + 'Job_#': '440325738', + 'Job_doc_#': '01', + Job_Type: 'A2', + Self_Cert: 'N', + Block: '15320', + Lot: '00048', + Community_Board: '414', + Zip_Code: '11693', + Bldg_Type: '1', + Residential: 'YES', + Special_District_1: '', + Special_District_2: '', + Work_Type: 'OT', + Permit_Status: 'ISSUED', + Filing_Status: 'RENEWAL', + Permit_Type: 'EW', + 'Permit_Sequence_#': '04', + Permit_Subtype: 'OT', + Oil_Gas: '', + Site_Fill: 'NOT APPLICABLE', + Filing_Date: '05/21/2018 12:00:00 AM', + Issuance_Date: '05/21/2018 12:00:00 AM', + Expiration_Date: '05/15/2019 12:00:00 AM', + Job_Start_Date: '04/07/2017 12:00:00 AM', + "Permittee's_First_Name": 'DONALD', + "Permittee's_Last_Name": "O'SULLIVAN", + "Permittee's_Business_Name": 'NAVILLUS TILE INC', + "Permittee's_Phone_#": '2127501808', + "Permittee's_License_Type": 'GC', + "Permittee's_License_#": '0015163', + Act_as_Superintendent: '', + "Permittee's_Other_Title": '', + HIC_License: '', + "Site_Safety_Mgr's_First_Name": '', + "Site_Safety_Mgr's_Last_Name": '', + Site_Safety_Mgr_Business_Name: '', + 'Superintendent_First_&_Last_Name': '', + Superintendent_Business_Name: '', + "Owner's_Business_Type": 'INDIVIDUAL', + 'Non-Profit': 'N', + "Owner's_Business_Name": 'TERENCE HAIRSTON ARCHITECT, PLLC', + "Owner's_First_Name": 'TERENCE', + "Owner's_Last_Name": 'HAIRSTON', + "Owner's_House_#": '16', + "Owner's_House_Street_Name": 'WEST 36TH STREET', + 'Owner’s_House_City': 'NEW YORK', + 'Owner’s_House_State': 'NY', + 'Owner’s_House_Zip_Code': '10018', + "Owner's_Phone_#": '9176924778', + DOBRunDate: '05/22/2018 12:00:00 AM', + PERMIT_SI_NO: '3463269', + LATITUDE: '40.601732', + LONGITUDE: '-73.821199', + COUNCIL_DISTRICT: '32', + CENSUS_TRACT: '107201', + NTA_NAME: 'Breezy Point-Belle Harbor-Rockaway Park-Broad Channel' + } + ]); + }); + }); /** * TODO: lucas: Revisit and unskip if we really want csv to be strict. */ diff --git a/src/utils/import-preview.js b/src/utils/import-preview.js index 72ea0f7..749ff56 100644 --- a/src/utils/import-preview.js +++ b/src/utils/import-preview.js @@ -38,7 +38,7 @@ export const createPeekStream = function( * @option {Number} MAX_SIZE The number of documents/rows we want to preview [Default `10`] * @returns {stream.Writable} */ -export default function({ MAX_SIZE = 10 } = {}) { +export default function({ MAX_SIZE = 10, fileType, delimiter, fileIsMultilineJSON} = {}) { return new Writable({ objectMode: true, write: function(doc, encoding, next) { @@ -59,7 +59,6 @@ export default function({ MAX_SIZE = 10 } = {}) { if (this.fields.length === 0) { // eslint-disable-next-line prefer-const for (let [key, value] of Object.entries(docAsDotnotation)) { - // TODO: lucas: Document this weird bug I found with my apple health data. // eslint-disable-next-line no-control-regex key = key.replace(/[^\x00-\x7F]/g, ''); this.fields.push({ diff --git a/src/utils/remove-blanks.js b/src/utils/remove-blanks.js index d848e16..b0905ae 100644 --- a/src/utils/remove-blanks.js +++ b/src/utils/remove-blanks.js @@ -1,4 +1,6 @@ import { Transform, PassThrough } from 'stream'; +import { createLogger } from './logger'; +const debug = createLogger('remove-blanks-preview'); /** * Based on mongoimport implementation. @@ -28,11 +30,13 @@ function removeBlanks(data) { export function removeBlanksStream(ignoreEmptyFields) { if (!ignoreEmptyFields) { + debug('Ignore empty fields is no op'); return new PassThrough({ objectMode: true }); } return new Transform({ objectMode: true, transform: function(doc, encoding, cb) { + debug('removing balnks from doc'); cb(null, removeBlanks(doc)); } }); diff --git a/test/number-transform.csv b/test/number-transform.csv new file mode 100644 index 0000000..af880d8 --- /dev/null +++ b/test/number-transform.csv @@ -0,0 +1,2 @@ +BOROUGH,Bin_#,House_#,Street_Name,Job_#,Job_doc_#,Job_Type,Self_Cert,Block,Lot,Community_Board,Zip_Code,Bldg_Type,Residential,Special_District_1,Special_District_2,Work_Type,Permit_Status,Filing_Status,Permit_Type,Permit_Sequence_#,Permit_Subtype,Oil_Gas,Site_Fill,Filing_Date,Issuance_Date,Expiration_Date,Job_Start_Date,Permittee's_First_Name,Permittee's_Last_Name,Permittee's_Business_Name,Permittee's_Phone_#,Permittee's_License_Type,Permittee's_License_#,Act_as_Superintendent,Permittee's_Other_Title,HIC_License,Site_Safety_Mgr's_First_Name,Site_Safety_Mgr's_Last_Name,Site_Safety_Mgr_Business_Name,Superintendent_First_&_Last_Name,Superintendent_Business_Name,Owner's_Business_Type,Non-Profit,Owner's_Business_Name,Owner's_First_Name,Owner's_Last_Name,Owner's_House_#,Owner's_House_Street_Name,Owner’s_House_City,Owner’s_House_State,Owner’s_House_Zip_Code,Owner's_Phone_#,DOBRunDate,PERMIT_SI_NO,LATITUDE,LONGITUDE,COUNCIL_DISTRICT,CENSUS_TRACT,NTA_NAME +QUEENS,4297149,17,WEST 16 ROAD,440325738,01,A2,N,15320,00048,414,11693,1,YES,,,OT,ISSUED,RENEWAL,EW,04,OT,,NOT APPLICABLE,05/21/2018 12:00:00 AM,05/21/2018 12:00:00 AM,05/15/2019 12:00:00 AM,04/07/2017 12:00:00 AM,DONALD,O'SULLIVAN,NAVILLUS TILE INC,2127501808,GC,0015163,,,,,,,,,INDIVIDUAL,N,"TERENCE HAIRSTON ARCHITECT, PLLC",TERENCE,HAIRSTON,16,WEST 36TH STREET,NEW YORK,NY,10018,9176924778,05/22/2018 12:00:00 AM,3463269,40.601732,-73.821199,32,107201,Breezy Point-Belle Harbor-Rockaway Park-Broad Channel \ No newline at end of file