From 5aa3c664f3ae2ee66185d83ba09fda60b842fadb Mon Sep 17 00:00:00 2001 From: Anemy Date: Tue, 23 Aug 2022 12:04:53 -0400 Subject: [PATCH 1/5] stash for laptop switch --- .../src/utils/dotnotation.ts | 2 +- .../import-apply-types-and-projection.js | 29 +++++++--- .../import-apply-types-and-projection.spec.js | 57 +++++++++++++++++++ .../test/deep-nested.json | 25 ++++++++ .../compass-import-export/test/fixtures.js | 1 + 5 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 packages/compass-import-export/test/deep-nested.json diff --git a/packages/compass-import-export/src/utils/dotnotation.ts b/packages/compass-import-export/src/utils/dotnotation.ts index d36b414490c..b505341d27a 100644 --- a/packages/compass-import-export/src/utils/dotnotation.ts +++ b/packages/compass-import-export/src/utils/dotnotation.ts @@ -54,7 +54,7 @@ export function serialize( { 'foo.1': 'one', 'foo.two': 'two'}. This way when we walk the return value later by the time we encounter - 'foo.1' we already created foo, initialised to {}. Then _.set(result, + 'foo.1' we already created foo, initialized to {}. Then _.set(result, 'foo.1', 'one') will not create foo as an array because 1 looks like an index. This is because at that point result will already contain { foo: {} } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js index 70dcfaec14f..bfeca4ff59f 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js @@ -43,8 +43,12 @@ function transformProjectedTypes( return data; } + console.log('ayo ayo'); + console.log('data', data); + const dotted = serialize(data, { includeObjects: true }); const result = {}; + console.log('data123', dotted); _.forEach( exclude, @@ -61,11 +65,13 @@ function transformProjectedTypes( const allPaths = _.keys(dotted); allPaths.forEach(function (keyPath) { const value = _.get(dotted, keyPath); + console.log('value', keyPath, value); if (ignoreBlanks === true && value === '') { // debug('dropped blank field', value); _.unset(result, [keyPath]); return false; } + console.log('1') // debug('targetType', {keyPathToTransform, keyPath}); const targetType = _.get(keyPathToTransform, keyPath); @@ -74,6 +80,8 @@ function transformProjectedTypes( _.set(result, keyPath, value); return; } + console.log('22') + const sourceType = getTypeDescriptorForValue(value).type; let casted = value; @@ -82,18 +90,25 @@ function transformProjectedTypes( throw new TypeError('Cant find lookup for ' + targetType); } casted = bsonCSV[targetType].fromString(value); - // debug('Target type differs from source type. Casting.', { - // targetType, - // sourceType, - // value, - // keyPath, - // casted - // }); + debug('Target type differs from source type. Casting.', { + targetType, + sourceType, + value, + keyPath, + casted + }); } + console.log('\n\n\nsetset', keyPath); + console.log('\nto', casted); + console.log('\n\n\n') + _.set(result, keyPath, casted); }); + console.log('\n\n123result', result); + console.log('\n\n\n') + debug('result', result); return result; } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js index a18322d766e..471faf75bf4 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js @@ -173,6 +173,63 @@ describe('import-apply-types-and-projection', function () { }); }); describe('Regression Tests', function () { + + // COMPASS-5971 Importing JSON document from file drops deeply nested fields + it('should parse deeply nested objects', function () { + const res = apply({ + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69 + } + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18 + } + }, + currency: 'usd' + } + }, + test: '123' + }, { + exclude: [], + transform: [ + + ], + }); + + expect(res).to.deep.equal({ + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69 + } + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18 + } + }, + currency: 'usd' + } + }, + test: '123' + }); + }); + // COMPASS-4204 Data type is not being set during import it('should transform csv strings to Number', function () { const res = apply( diff --git a/packages/compass-import-export/test/deep-nested.json b/packages/compass-import-export/test/deep-nested.json new file mode 100644 index 00000000000..ddcde286d44 --- /dev/null +++ b/packages/compass-import-export/test/deep-nested.json @@ -0,0 +1,25 @@ +[ + { + "supermarket": { + "fruits": { + "oranges": { + "amount": { + "2022-01-15": 1.66, + "2022-02-16": 1.22, + "2022-03-13": 1.11, + "2022-04-14": 7.69 + } + }, + "apples": { + "amount": { + "2022-01-15": 3.47, + "2022-02-14": 4.18, + "2022-03-15": 4.18 + } + }, + "currency": "usd" + } + }, + "test": "123" + } +] diff --git a/packages/compass-import-export/test/fixtures.js b/packages/compass-import-export/test/fixtures.js index 0906bcfb6bd..f88f90e877a 100644 --- a/packages/compass-import-export/test/fixtures.js +++ b/packages/compass-import-export/test/fixtures.js @@ -6,6 +6,7 @@ const FIXTURES = { BAD_CSV: path.join(__dirname, 'test_bad.csv'), JS_I_THINK_IS_JSON: path.join(__dirname, 'js-i-think-is-json'), GOOD_JSON: path.join(__dirname, 'docs.json'), + DEEP_NESTED_JSON: path.join(__dirname, 'deep-nested.json'), LINE_DELIMITED_JSON: path.join(__dirname, 'docs.jsonl'), LINE_DELIMITED_JSON_EXTRA_LINE: path.join( __dirname, From 333e13d3dadba432ab33dedd5a34fb5f1a7fd0e2 Mon Sep 17 00:00:00 2001 From: Rhys Howell Date: Mon, 29 Aug 2022 14:19:46 -0400 Subject: [PATCH 2/5] Add fix for dot notation, ensure entire parent is formed when adding object --- .../src/utils/dotnotation.spec.js | 83 +++++++++++++++++++ .../src/utils/dotnotation.ts | 21 ++++- .../import-apply-types-and-projection.js | 16 +--- .../import-apply-types-and-projection.spec.js | 68 +++++++-------- 4 files changed, 137 insertions(+), 51 deletions(-) diff --git a/packages/compass-import-export/src/utils/dotnotation.spec.js b/packages/compass-import-export/src/utils/dotnotation.spec.js index aa80bbe965f..c39206d2e85 100644 --- a/packages/compass-import-export/src/utils/dotnotation.spec.js +++ b/packages/compass-import-export/src/utils/dotnotation.spec.js @@ -90,4 +90,87 @@ describe('dotnotation', function () { foo: ['a', 'b'], }); }); + + it('should parse deeply nested objects without including objects', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', + }, + }, + test: '123', + }, + { includeObjects: false } + ); + + expect(serializedObject).to.deep.equal({ + 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, + 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, + 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, + 'supermarket.fruits.oranges.amount.2022-04-14': 7.69, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.currency': 'usd', + test: '123', + }); + }); + + it('should parse deeply nested objects', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', + }, + }, + test: '123', + }, + { includeObjects: true } + ); + + expect(serializedObject).to.deep.equal({ + supermarket: {}, + 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, + 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, + 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, + 'supermarket.fruits.oranges.amount.2022-04-14': 7.69, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.currency': 'usd', + test: '123', + }); + }); }); diff --git a/packages/compass-import-export/src/utils/dotnotation.ts b/packages/compass-import-export/src/utils/dotnotation.ts index b505341d27a..00057258285 100644 --- a/packages/compass-import-export/src/utils/dotnotation.ts +++ b/packages/compass-import-export/src/utils/dotnotation.ts @@ -66,8 +66,8 @@ export function serialize( const withObjects: Record = {}; const knownParents: Record = {}; for (const [path, value] of Object.entries(flattened)) { - const parentPath = path.includes('.') - ? path.slice(0, path.lastIndexOf('.')) + let parentPath = path.includes('.') + ? path.slice(0, path.indexOf('.')) : null; if (parentPath && !knownParents[parentPath]) { knownParents[parentPath] = true; @@ -75,9 +75,26 @@ export function serialize( if (!Array.isArray(_.get(obj, parentPath))) { withObjects[parentPath] = {}; } + + // Build all of the parent objects that contain the current path. + // (a.b.c -> a = {} a.b = {}) + while (parentPath && parentPath.includes('.')) { + knownParents[parentPath] = true; + + // Leave arrays alone because they already got handled by safe: true above. + if (!Array.isArray(_.get(obj, parentPath))) { + withObjects[parentPath] = {}; + } + + // Continue to the next parent if there is one. + parentPath = parentPath.includes('.') + ? parentPath.slice(0, path.indexOf('.')) + : null; + } } withObjects[path] = value; } + return withObjects; } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js index bfeca4ff59f..19613ebf9f4 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js @@ -43,12 +43,8 @@ function transformProjectedTypes( return data; } - console.log('ayo ayo'); - console.log('data', data); - const dotted = serialize(data, { includeObjects: true }); const result = {}; - console.log('data123', dotted); _.forEach( exclude, @@ -65,13 +61,11 @@ function transformProjectedTypes( const allPaths = _.keys(dotted); allPaths.forEach(function (keyPath) { const value = _.get(dotted, keyPath); - console.log('value', keyPath, value); if (ignoreBlanks === true && value === '') { // debug('dropped blank field', value); _.unset(result, [keyPath]); return false; } - console.log('1') // debug('targetType', {keyPathToTransform, keyPath}); const targetType = _.get(keyPathToTransform, keyPath); @@ -80,7 +74,6 @@ function transformProjectedTypes( _.set(result, keyPath, value); return; } - console.log('22') const sourceType = getTypeDescriptorForValue(value).type; @@ -95,20 +88,13 @@ function transformProjectedTypes( sourceType, value, keyPath, - casted + casted, }); } - console.log('\n\n\nsetset', keyPath); - console.log('\nto', casted); - console.log('\n\n\n') - _.set(result, keyPath, casted); }); - console.log('\n\n123result', result); - console.log('\n\n\n') - debug('result', result); return result; } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js index 471faf75bf4..7c92378a009 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js @@ -173,37 +173,37 @@ describe('import-apply-types-and-projection', function () { }); }); describe('Regression Tests', function () { - // COMPASS-5971 Importing JSON document from file drops deeply nested fields it('should parse deeply nested objects', function () { - const res = apply({ - supermarket: { - fruits: { - oranges: { - amount: { - '2022-01-15': 1.66, - '2022-02-16': 1.22, - '2022-03-13': 1.11, - '2022-04-14': 7.69 - } - }, - apples: { - amount: { - '2022-01-15': 3.47, - '2022-02-14': 4.18, - '2022-03-15': 4.18 - } + const res = apply( + { + supermarket: { + fruits: { + oranges: { + amount: { + '2022-01-15': 1.66, + '2022-02-16': 1.22, + '2022-03-13': 1.11, + '2022-04-14': 7.69, + }, + }, + apples: { + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + }, + currency: 'usd', }, - currency: 'usd' - } + }, + test: '123', }, - test: '123' - }, { - exclude: [], - transform: [ - - ], - }); + { + exclude: [], + transform: [], + } + ); expect(res).to.deep.equal({ supermarket: { @@ -213,20 +213,20 @@ describe('import-apply-types-and-projection', function () { '2022-01-15': 1.66, '2022-02-16': 1.22, '2022-03-13': 1.11, - '2022-04-14': 7.69 - } + '2022-04-14': 7.69, + }, }, apples: { amount: { '2022-01-15': 3.47, '2022-02-14': 4.18, - '2022-03-15': 4.18 - } + '2022-03-15': 4.18, + }, }, - currency: 'usd' - } + currency: 'usd', + }, }, - test: '123' + test: '123', }); }); From ff180734b8c4bdb8ca1b7b7dfa9ee9b0dc02ff9d Mon Sep 17 00:00:00 2001 From: Rhys Howell Date: Mon, 29 Aug 2022 14:23:12 -0400 Subject: [PATCH 3/5] return early style, remove extra text fixture --- .../src/utils/dotnotation.ts | 84 +++++++++---------- .../import-apply-types-and-projection.js | 14 ++-- .../test/deep-nested.json | 25 ------ .../compass-import-export/test/fixtures.js | 1 - 4 files changed, 49 insertions(+), 75 deletions(-) delete mode 100644 packages/compass-import-export/test/deep-nested.json diff --git a/packages/compass-import-export/src/utils/dotnotation.ts b/packages/compass-import-export/src/utils/dotnotation.ts index 00057258285..e2c7fff103b 100644 --- a/packages/compass-import-export/src/utils/dotnotation.ts +++ b/packages/compass-import-export/src/utils/dotnotation.ts @@ -45,60 +45,60 @@ export function serialize( }, }); - if (includeObjects) { - /* - Make sure that paths to objects exist in the returned value before the paths - to properties inside those objects. - ie. for { foo: { 1: 'one', two: 'two' } } we will return - { foo: {}, 'foo.1': 'one', 'foo.two': 'two' } rather than - { 'foo.1': 'one', 'foo.two': 'two'}. + if (!includeObjects) { + return flattened; + } + + /* + Make sure that paths to objects exist in the returned value before the paths + to properties inside those objects. + ie. for { foo: { 1: 'one', two: 'two' } } we will return + { foo: {}, 'foo.1': 'one', 'foo.two': 'two' } rather than + { 'foo.1': 'one', 'foo.two': 'two'}. - This way when we walk the return value later by the time we encounter - 'foo.1' we already created foo, initialized to {}. Then _.set(result, - 'foo.1', 'one') will not create foo as an array because 1 looks like an - index. This is because at that point result will already contain { foo: {} } + This way when we walk the return value later by the time we encounter + 'foo.1' we already created foo, initialized to {}. Then _.set(result, + 'foo.1', 'one') will not create foo as an array because 1 looks like an + index. This is because at that point result will already contain { foo: {} } - The use-case for this came about because paths that end with numbers are - ambiguous and _.set() will assume it is an array index by default. By - ensuring that there is already an object at the target the ambiguity is - removed. - */ - const withObjects: Record = {}; - const knownParents: Record = {}; - for (const [path, value] of Object.entries(flattened)) { - let parentPath = path.includes('.') - ? path.slice(0, path.indexOf('.')) - : null; - if (parentPath && !knownParents[parentPath]) { + The use-case for this came about because paths that end with numbers are + ambiguous and _.set() will assume it is an array index by default. By + ensuring that there is already an object at the target the ambiguity is + removed. + */ + const withObjects: Record = {}; + const knownParents: Record = {}; + for (const [path, value] of Object.entries(flattened)) { + let parentPath = path.includes('.') + ? path.slice(0, path.indexOf('.')) + : null; + if (parentPath && !knownParents[parentPath]) { + knownParents[parentPath] = true; + // Leave arrays alone because they already got handled by safe: true above. + if (!Array.isArray(_.get(obj, parentPath))) { + withObjects[parentPath] = {}; + } + + // Build all of the parent objects that contain the current path. + // (a.b.c -> a = {} a.b = {}) + while (parentPath && parentPath.includes('.')) { knownParents[parentPath] = true; + // Leave arrays alone because they already got handled by safe: true above. if (!Array.isArray(_.get(obj, parentPath))) { withObjects[parentPath] = {}; } - // Build all of the parent objects that contain the current path. - // (a.b.c -> a = {} a.b = {}) - while (parentPath && parentPath.includes('.')) { - knownParents[parentPath] = true; - - // Leave arrays alone because they already got handled by safe: true above. - if (!Array.isArray(_.get(obj, parentPath))) { - withObjects[parentPath] = {}; - } - - // Continue to the next parent if there is one. - parentPath = parentPath.includes('.') - ? parentPath.slice(0, path.indexOf('.')) - : null; - } + // Continue to the next parent if there is one. + parentPath = parentPath.includes('.') + ? parentPath.slice(0, path.indexOf('.')) + : null; } - withObjects[path] = value; } - - return withObjects; + withObjects[path] = value; } - return flattened; + return withObjects; } /** diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js index 19613ebf9f4..c75a82585ac 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.js @@ -83,13 +83,13 @@ function transformProjectedTypes( throw new TypeError('Cant find lookup for ' + targetType); } casted = bsonCSV[targetType].fromString(value); - debug('Target type differs from source type. Casting.', { - targetType, - sourceType, - value, - keyPath, - casted, - }); + // debug('Target type differs from source type. Casting.', { + // targetType, + // sourceType, + // value, + // keyPath, + // casted, + // }); } _.set(result, keyPath, casted); diff --git a/packages/compass-import-export/test/deep-nested.json b/packages/compass-import-export/test/deep-nested.json deleted file mode 100644 index ddcde286d44..00000000000 --- a/packages/compass-import-export/test/deep-nested.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "supermarket": { - "fruits": { - "oranges": { - "amount": { - "2022-01-15": 1.66, - "2022-02-16": 1.22, - "2022-03-13": 1.11, - "2022-04-14": 7.69 - } - }, - "apples": { - "amount": { - "2022-01-15": 3.47, - "2022-02-14": 4.18, - "2022-03-15": 4.18 - } - }, - "currency": "usd" - } - }, - "test": "123" - } -] diff --git a/packages/compass-import-export/test/fixtures.js b/packages/compass-import-export/test/fixtures.js index f88f90e877a..0906bcfb6bd 100644 --- a/packages/compass-import-export/test/fixtures.js +++ b/packages/compass-import-export/test/fixtures.js @@ -6,7 +6,6 @@ const FIXTURES = { BAD_CSV: path.join(__dirname, 'test_bad.csv'), JS_I_THINK_IS_JSON: path.join(__dirname, 'js-i-think-is-json'), GOOD_JSON: path.join(__dirname, 'docs.json'), - DEEP_NESTED_JSON: path.join(__dirname, 'deep-nested.json'), LINE_DELIMITED_JSON: path.join(__dirname, 'docs.jsonl'), LINE_DELIMITED_JSON_EXTRA_LINE: path.join( __dirname, From 231ec32526c0bbadea6507adaa988b89ba4562f0 Mon Sep 17 00:00:00 2001 From: Rhys Howell Date: Tue, 30 Aug 2022 01:04:09 -0400 Subject: [PATCH 4/5] Add more test cases, streamline code --- .../src/utils/dotnotation.spec.js | 91 +++++++++++++++++++ .../src/utils/dotnotation.ts | 40 ++++---- .../import-apply-types-and-projection.spec.js | 86 ++++++++++++++++++ 3 files changed, 197 insertions(+), 20 deletions(-) diff --git a/packages/compass-import-export/src/utils/dotnotation.spec.js b/packages/compass-import-export/src/utils/dotnotation.spec.js index c39206d2e85..223d7134c96 100644 --- a/packages/compass-import-export/src/utils/dotnotation.spec.js +++ b/packages/compass-import-export/src/utils/dotnotation.spec.js @@ -105,6 +105,7 @@ describe('dotnotation', function () { }, }, apples: { + a: 123, amount: { '2022-01-15': 3.47, '2022-02-14': 4.18, @@ -127,17 +128,96 @@ describe('dotnotation', function () { 'supermarket.fruits.apples.amount.2022-01-15': 3.47, 'supermarket.fruits.apples.amount.2022-02-14': 4.18, 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.a': 123, 'supermarket.fruits.currency': 'usd', test: '123', }); }); + it('should parse deeply nested objects without overriding arrays', function () { + const serializedObject = dotnotation.serialize( + { + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }, + { includeObjects: true } + ); + + expect(serializedObject).to.deep.equal({ + supermarket: {}, + 'supermarket.17': 76, + 'supermarket.fruits': {}, + 'supermarket.fruits.apples': {}, + 'supermarket.fruits.apples.12': '34', + 'supermarket.fruits.apples.amount': {}, + 'supermarket.fruits.apples.amount.2022-01-15': 3.47, + 'supermarket.fruits.apples.amount.2022-02-14': 4.18, + 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.a': 123, + 'supermarket.fruits.currency': 'usd', + test: '123', + a: {}, + 'a.b': {}, + 'a.b.c': {}, + 'a.b.c.17': 76, + 'a.b.c.d': {}, + 'a.b.c.d.a': 'ok', + 'a.b.c.d.99': 'test', + 'a.b.c.f': [ + { + aa: { + 4: 5, + bb: { + 123: 'test', + }, + }, + }, + ], + }); + }); + it('should parse deeply nested objects', function () { const serializedObject = dotnotation.serialize( { supermarket: { fruits: { oranges: { + aTest: ['test'], amount: { '2022-01-15': 1.66, '2022-02-16': 1.22, @@ -146,11 +226,13 @@ describe('dotnotation', function () { }, }, apples: { + a: 123, amount: { '2022-01-15': 3.47, '2022-02-14': 4.18, '2022-03-15': 4.18, }, + arrayTest: ['test'], }, currency: 'usd', }, @@ -162,13 +244,22 @@ describe('dotnotation', function () { expect(serializedObject).to.deep.equal({ supermarket: {}, + 'supermarket.fruits': {}, + 'supermarket.fruits.oranges': {}, + 'supermarket.fruits.oranges.aTest': ['test'], + 'supermarket.fruits.oranges.amount': {}, + 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, 'supermarket.fruits.oranges.amount.2022-04-14': 7.69, + 'supermarket.fruits.apples': {}, + 'supermarket.fruits.apples.amount': {}, 'supermarket.fruits.apples.amount.2022-01-15': 3.47, 'supermarket.fruits.apples.amount.2022-02-14': 4.18, 'supermarket.fruits.apples.amount.2022-03-15': 4.18, + 'supermarket.fruits.apples.arrayTest': ['test'], + 'supermarket.fruits.apples.a': 123, 'supermarket.fruits.currency': 'usd', test: '123', }); diff --git a/packages/compass-import-export/src/utils/dotnotation.ts b/packages/compass-import-export/src/utils/dotnotation.ts index e2c7fff103b..0cbc2b694ed 100644 --- a/packages/compass-import-export/src/utils/dotnotation.ts +++ b/packages/compass-import-export/src/utils/dotnotation.ts @@ -68,33 +68,33 @@ export function serialize( */ const withObjects: Record = {}; const knownParents: Record = {}; + for (const [path, value] of Object.entries(flattened)) { - let parentPath = path.includes('.') - ? path.slice(0, path.indexOf('.')) - : null; - if (parentPath && !knownParents[parentPath]) { - knownParents[parentPath] = true; - // Leave arrays alone because they already got handled by safe: true above. - if (!Array.isArray(_.get(obj, parentPath))) { - withObjects[parentPath] = {}; - } + let currentIndex = path.indexOf('.'); + + let parentPath: string | null = + currentIndex > -1 ? path.slice(0, currentIndex) : null; - // Build all of the parent objects that contain the current path. - // (a.b.c -> a = {} a.b = {}) - while (parentPath && parentPath.includes('.')) { + // Build all of the parent objects that contain the current path. + // (a.b.c -> a = {} a.b = {}) + while (parentPath) { + // Leave arrays alone because they already got handled by safe: true above. + if (!knownParents[parentPath] && !Array.isArray(_.get(obj, parentPath))) { knownParents[parentPath] = true; - // Leave arrays alone because they already got handled by safe: true above. - if (!Array.isArray(_.get(obj, parentPath))) { - withObjects[parentPath] = {}; - } + withObjects[parentPath] = {}; + } - // Continue to the next parent if there is one. - parentPath = parentPath.includes('.') - ? parentPath.slice(0, path.indexOf('.')) - : null; + currentIndex = path.indexOf('.', currentIndex + 1); + if (currentIndex === -1) { + // No more parents. + break; } + + // Continue to the next parent if there is one. + parentPath = path.slice(0, currentIndex); } + withObjects[path] = value; } diff --git a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js index 7c92378a009..0de53165582 100644 --- a/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js +++ b/packages/compass-import-export/src/utils/import-apply-types-and-projection.spec.js @@ -229,6 +229,92 @@ describe('import-apply-types-and-projection', function () { test: '123', }); }); + it('should parse deeply nested objects without overriding arrays', function () { + const res = apply( + { + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }, + { + exclude: [], + transform: [], + } + ); + expect(res).to.deep.equal({ + supermarket: { + 17: 76, + fruits: { + apples: { + 12: '34', + amount: { + '2022-01-15': 3.47, + '2022-02-14': 4.18, + '2022-03-15': 4.18, + }, + a: 123, + }, + currency: 'usd', + }, + }, + test: '123', + a: { + b: { + c: { + 17: 76, + d: { + a: 'ok', + 99: 'test', + }, + f: [ + { + aa: { + bb: { + 123: 'test', + }, + 4: 5, + }, + }, + ], + }, + }, + }, + }); + }); // COMPASS-4204 Data type is not being set during import it('should transform csv strings to Number', function () { From fed354df72a15313e8f797050b48b6da13c000c3 Mon Sep 17 00:00:00 2001 From: Rhys Howell Date: Tue, 30 Aug 2022 03:08:50 -0400 Subject: [PATCH 5/5] fix lint --- packages/compass-import-export/src/utils/dotnotation.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/compass-import-export/src/utils/dotnotation.spec.js b/packages/compass-import-export/src/utils/dotnotation.spec.js index 223d7134c96..0d79ee78dcb 100644 --- a/packages/compass-import-export/src/utils/dotnotation.spec.js +++ b/packages/compass-import-export/src/utils/dotnotation.spec.js @@ -249,7 +249,6 @@ describe('dotnotation', function () { 'supermarket.fruits.oranges.aTest': ['test'], 'supermarket.fruits.oranges.amount': {}, 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, - 'supermarket.fruits.oranges.amount.2022-01-15': 1.66, 'supermarket.fruits.oranges.amount.2022-02-16': 1.22, 'supermarket.fruits.oranges.amount.2022-03-13': 1.11, 'supermarket.fruits.oranges.amount.2022-04-14': 7.69,