diff --git a/packages/compass-import-export/src/utils/dotnotation.spec.js b/packages/compass-import-export/src/utils/dotnotation.spec.js index aa80bbe965f..0d79ee78dcb 100644 --- a/packages/compass-import-export/src/utils/dotnotation.spec.js +++ b/packages/compass-import-export/src/utils/dotnotation.spec.js @@ -90,4 +90,177 @@ describe('dotnotation', function () { foo: ['a', 'b'], }); }); + + it('should parse deeply nested objects without including objects', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + a: 123, + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', + }, + }, + test: '123', + }, + { includeObjects: false } + ); + + expect(serializedObject).to.deep.equal({ + 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, + 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, + 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, + 'supermarket.fruits.oranges.amount.2022-04-14': 7.69, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.a': 123, + 'supermarket.fruits.currency': 'usd', + test: '123', + }); + }); + + it('should parse deeply nested objects without overriding arrays', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }, + { includeObjects: true } + ); + + expect(serializedObject).to.deep.equal({ + supermarket: {}, + 'supermarket.17': 76, + 'supermarket.fruits': {}, + 'supermarket.fruits.apples': {}, + 'supermarket.fruits.apples.12': '34', + 'supermarket.fruits.apples.amount': {}, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.a': 123, + 'supermarket.fruits.currency': 'usd', + test: '123', + a: {}, + 'a.b': {}, + 'a.b.c': {}, + 'a.b.c.17': 76, + 'a.b.c.d': {}, + 'a.b.c.d.a': 'ok', + 'a.b.c.d.99': 'test', + 'a.b.c.f': [ + { + aa: { + 4: 5, + bb: { + 123: 'test', + }, + }, + }, + ], + }); + }); + + it('should parse deeply nested objects', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + fruits: { + oranges: { + aTest: ['test'], + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + a: 123, + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + arrayTest: ['test'], + }, + currency: 'usd', + }, + }, + test: '123', + }, + { includeObjects: true } + ); + + expect(serializedObject).to.deep.equal({ + supermarket: {}, + 'supermarket.fruits': {}, + 'supermarket.fruits.oranges': {}, + 'supermarket.fruits.oranges.aTest': ['test'], + 'supermarket.fruits.oranges.amount': {}, + 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, + 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, + 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, + 'supermarket.fruits.oranges.amount.2022-04-14': 7.69, + 'supermarket.fruits.apples': {}, + 'supermarket.fruits.apples.amount': {}, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.arrayTest': ['test'], + 'supermarket.fruits.apples.a': 123, + 'supermarket.fruits.currency': 'usd', + test: '123', + }); + }); }); diff --git a/packages/compass-import-export/src/utils/dotnotation.ts b/packages/compass-import-export/src/utils/dotnotation.ts index d36b414490c..0cbc2b694ed 100644 --- a/packages/compass-import-export/src/utils/dotnotation.ts +++ b/packages/compass-import-export/src/utils/dotnotation.ts @@ -45,43 +45,60 @@ export function serialize( }, }); - if (includeObjects) { - /* - Make sure that paths to objects exist in the returned value before the paths - to properties inside those objects. - ie. for { foo: { 1: 'one', two: 'two' } } we will return - { foo: {}, 'foo.1': 'one', 'foo.two': 'two' } rather than - { 'foo.1': 'one', 'foo.two': 'two'}. - - This way when we walk the return value later by the time we encounter - 'foo.1' we already created foo, initialised to {}. Then _.set(result, - 'foo.1', 'one') will not create foo as an array because 1 looks like an - index. This is because at that point result will already contain { foo: {} } - - The use-case for this came about because paths that end with numbers are - ambiguous and _.set() will assume it is an array index by default. By - ensuring that there is already an object at the target the ambiguity is - removed. - */ - const withObjects: Record = {}; - const knownParents: Record = {}; - for (const [path, value] of Object.entries(flattened)) { - const parentPath = path.includes('.') - ? path.slice(0, path.lastIndexOf('.')) - : null; - if (parentPath && !knownParents[parentPath]) { + if (!includeObjects) { + return flattened; + } + + /* + Make sure that paths to objects exist in the returned value before the paths + to properties inside those objects. + ie. for { foo: { 1: 'one', two: 'two' } } we will return + { foo: {}, 'foo.1': 'one', 'foo.two': 'two' } rather than + { 'foo.1': 'one', 'foo.two': 'two'}. + + This way when we walk the return value later by the time we encounter + 'foo.1' we already created foo, initialized to {}. Then _.set(result, + 'foo.1', 'one') will not create foo as an array because 1 looks like an + index. This is because at that point result will already contain { foo: {} } + + The use-case for this came about because paths that end with numbers are + ambiguous and _.set() will assume it is an array index by default. By + ensuring that there is already an object at the target the ambiguity is + removed. + */ + const withObjects: Record = {}; + const knownParents: Record = {}; + + for (const [path, value] of Object.entries(flattened)) { + let currentIndex = path.indexOf('.'); + + let parentPath: string | null = + currentIndex > -1 ? path.slice(0, currentIndex) : null; + + // Build all of the parent objects that contain the current path. + // (a.b.c -> a = {} a.b = {}) + while (parentPath) { + // Leave arrays alone because they already got handled by safe: true above. + if (!knownParents[parentPath] && !Array.isArray(_.get(obj, parentPath))) { knownParents[parentPath] = true; - // Leave arrays alone because they already got handled by safe: true above. - if (!Array.isArray(_.get(obj, parentPath))) { - withObjects[parentPath] = {}; - } + + withObjects[parentPath] = {}; + } + + currentIndex = path.indexOf('.', currentIndex + 1); + if (currentIndex === -1) { + // No more parents. + break; } - withObjects[path] = value; + + // Continue to the next parent if there is one. + parentPath = path.slice(0, currentIndex); } - return withObjects; + + withObjects[path] = value; } - return flattened; + return withObjects; } /** diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js index 70dcfaec14f..c75a82585ac 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js @@ -74,6 +74,7 @@ function transformProjectedTypes( _.set(result, keyPath, value); return; } + const sourceType = getTypeDescriptorForValue(value).type; let casted = value; @@ -87,7 +88,7 @@ function transformProjectedTypes( // sourceType, // value, // keyPath, - // casted + // casted, // }); } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js index a18322d766e..0de53165582 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js @@ -173,6 +173,149 @@ describe('import-apply-types-and-projection', function () { }); }); describe('Regression Tests', function () { + // COMPASS-5971 Importing JSON document from file drops deeply nested fields + it('should parse deeply nested objects', function () { + const res = apply( + { + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', + }, + }, + test: '123', + }, + { + exclude: [], + transform: [], + } + ); + + expect(res).to.deep.equal({ + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', + }, + }, + test: '123', + }); + }); + it('should parse deeply nested objects without overriding arrays', function () { + const res = apply( + { + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }, + { + exclude: [], + transform: [], + } + ); + expect(res).to.deep.equal({ + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }); + }); + // COMPASS-4204 Data type is not being set during import it('should transform csv strings to Number', function () { const res = apply(