diff --git a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index ef0c812f..f90c2a9e 100644 --- a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -113,11 +113,76 @@ describe('test create saved search report', () => { test('create report for small data set by single search', async () => { const hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -142,17 +207,160 @@ describe('test create saved search report', () => { test('create report for large data set by scroll', async () => { const hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), - hit({ category: 'c6', customer_gender: 'Female' }), - hit({ category: 'c7', customer_gender: 'Female' }), - hit({ category: 'c8', customer_gender: 'Female' }), - hit({ category: 'c9', customer_gender: 'Female' }), - hit({ category: 'c10', customer_gender: 'Female' }), - hit({ category: 'c11', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c6', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c7', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c8', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c9', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c10', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c11', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -186,11 +394,76 @@ describe('test create saved search report', () => { input.report_definition.report_params.core_params.limit = 1; const hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -211,16 +484,146 @@ describe('test create saved search report', () => { input.report_definition.report_params.core_params.limit = 6; const hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), - hit({ category: 'c6', customer_gender: 'Female' }), - hit({ category: 'c7', customer_gender: 'Female' }), - hit({ category: 'c8', customer_gender: 'Female' }), - hit({ category: 'c9', customer_gender: 'Female' }), - hit({ category: 'c10', customer_gender: 'Female' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c6', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c7', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c8', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c9', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c10', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -249,12 +652,90 @@ describe('test create saved search report', () => { input.report_definition.report_params.core_params.limit = 10; const hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), - hit({ category: 'c6', customer_gender: 'Female' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c6', + customer_gender: 'Female', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -280,9 +761,48 @@ describe('test create saved search report', () => { test('create report for data set with comma', async () => { const hits = [ - hit({ category: ',c1', customer_gender: 'Ma,le' }), - hit({ category: 'c2,', customer_gender: 'M,ale' }), - hit({ category: ',,c3', customer_gender: 'Male,,,' }), + hit( + { + category: ',c1', + customer_gender: 'Ma,le', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2,', + customer_gender: 'M,ale', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',,c3', + customer_gender: 'Male,,,', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -305,9 +825,48 @@ describe('test create saved search report', () => { test('create report for data set with comma and custom separator', async () => { const hits = [ - hit({ category: ',c1', customer_gender: 'Ma,le' }), - hit({ category: 'c2,', customer_gender: 'M,ale' }), - hit({ category: ',,c3', customer_gender: 'Male,,,' }), + hit( + { + category: ',c1', + customer_gender: 'Ma,le', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2,', + customer_gender: 'M,ale', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',,c3', + customer_gender: 'Male,,,', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -330,15 +889,35 @@ describe('test create saved search report', () => { test('create report for data set with nested fields', async () => { const hits = [ - hit({ - 'geoip.country_iso_code': 'GB', - 'geoip.location': { lon: -0.1, lat: 51.5 }, - }), - hit({ - 'geoip.country_iso_code': 'US', - 'geoip.city_name': 'New York', - 'geoip.location': { lon: -74, lat: 40.8 }, - }), + hit( + { + 'geoip.country_iso_code': 'GB', + 'geoip.location': { lon: -0.1, lat: 51.5 }, + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + 'geoip.country_iso_code': 'US', + 'geoip.city_name': 'New York', + 'geoip.location': { lon: -74, lat: 40.8 }, + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient( hits, @@ -363,11 +942,76 @@ describe('test create saved search report', () => { test('create report by sanitizing data set for Excel', async () => { const hits = [ - hit({ category: 'c1', customer_gender: '=Male' }), - hit({ category: 'c2', customer_gender: 'Male=' }), - hit({ category: 'c3', customer_gender: '+Ma,le' }), - hit({ category: ',-c4', customer_gender: 'Male' }), - hit({ category: ',,,@c5', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: '=Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male=', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: '+Ma,le', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',-c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',,,@c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -395,11 +1039,76 @@ describe('test create saved search report', () => { input.report_definition.report_params.core_params.excel = false; const hits = [ - hit({ category: 'c1', customer_gender: '=Male' }), - hit({ category: 'c2', customer_gender: 'Male=' }), - hit({ category: 'c3', customer_gender: '+Ma,le' }), - hit({ category: ',-c4', customer_gender: 'Male' }), - hit({ category: ',,,@c5', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: '=Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male=', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: '+Ma,le', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',-c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: ',,,@c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -425,9 +1134,48 @@ describe('test create saved search report', () => { test('create report for data set contains null field value', async () => { const hits = [ - hit({ category: 'c1', customer_gender: 'Ma' }), - hit({ category: 'c2', customer_gender: 'le' }), - hit({ category: 'c3', customer_gender: null }), + hit( + { + category: 'c1', + customer_gender: 'Ma', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'le', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: null, + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; const client = mockOpenSearchClient(hits); const { dataUrl } = await createSavedSearchReport( @@ -448,11 +1196,76 @@ test('create report for data set contains null field value', async () => { test('create report for data set with metadata fields', async () => { const metadataFields = { _index: 'nameofindex', _id: 'someid' }; let hits = [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), + hit( + { + category: 'c1', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c2', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c3', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c4', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), + hit( + { + category: 'c5', + customer_gender: 'Male', + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + customer_birth_date: '2023-04-26T04:34:32Z', + order_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } + ), ]; hits.forEach((i) => { _.merge(i, metadataFields); @@ -485,32 +1298,60 @@ test('create report for data set with metadata fields', async () => { test('create report with empty/one/multiple(list) date values', async () => { const hits = [ hit( - { category: 'c1', customer_gender: 'Ma', order_date: [] }, - { order_date: [] } + { + category: 'c1', + customer_gender: 'Ma', + order_date: [], + customer_birth_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, + }, + { + order_date: [], + customer_birth_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } ), hit( { category: 'c2', customer_gender: 'le', order_date: ['2021-12-16T14:04:55'], + customer_birth_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, }, - { order_date: ['2021-12-16T14:04:55'] } + { + order_date: ['2021-12-16T14:04:55'], + customer_birth_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } ), hit( { category: 'c3', customer_gender: 'he', order_date: ['2021-12-17T14:04:55', '2021-12-18T14:04:55'], + customer_birth_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, }, - { order_date: ['2021-12-17T14:04:55', '2021-12-18T14:04:55'] } + { + order_date: ['2021-12-17T14:04:55', '2021-12-18T14:04:55'], + customer_birth_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } ), hit( { category: 'c4', customer_gender: 'te', order_date: '2021-12-19T14:04:55', + customer_birth_date: '2023-04-26T04:34:32Z', + products: { created_on: '2023-04-26T04:34:32Z' }, }, - { order_date: ['2021-12-19T14:04:55'] } + { + order_date: ['2021-12-19T14:04:55'], + customer_birth_date: '2023-04-26T04:34:32Z', + 'products.created_on': '2023-04-26T04:34:32Z', + } ), ]; const client = mockOpenSearchClient( @@ -526,7 +1367,6 @@ test('create report with empty/one/multiple(list) date values', async () => { undefined, mockLogger ); - expect(dataUrl).toEqual( 'category,customer_gender,order_date\n' + 'c1,Ma,[]\n' + @@ -540,7 +1380,7 @@ test('create report with empty/one/multiple(list) date values', async () => { * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ function mockOpenSearchClient( - mockHits: Array<{ _source: any }>, + mockHits: Array<{ _source: any; fields: any }>, columns = '"category", "customer_gender"' ) { let call = 0; diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts index a7bee2b9..718af5f3 100644 --- a/server/routes/utils/dataReportHelpers.ts +++ b/server/routes/utils/dataReportHelpers.ts @@ -14,6 +14,7 @@ import { Query, OpenSearchQueryConfig, } from '../../../../../src/plugins/data/common'; +import { string } from 'joi'; export var metaData = { saved_search_id: null, @@ -50,7 +51,11 @@ export const getSelectedFields = async (columns) => { // Build the OpenSearch query from the meta data // is_count is set to 1 if we building the count query but 0 if we building the fetch data query -export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is_count: number) => { +export const buildRequestBody = ( + report: any, + allowLeadingWildcards: boolean, + is_count: number +) => { let esbBoolQuery = esb.boolQuery(); const searchSourceJSON = report._source.searchSourceJSON; const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query; @@ -59,12 +64,12 @@ export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is allowLeadingWildcards: allowLeadingWildcards, queryStringOptions: {}, ignoreFilterIfFieldNotInIndex: false, - } + }; const QueryFromSavedObject = buildOpenSearchQuery( undefined, savedObjectQuery, savedObjectFilter, - savedObjectConfig, + savedObjectConfig ); // Add time range if (report._source.timeFieldName && report._source.timeFieldName.length > 0) { @@ -114,9 +119,9 @@ export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is // Fetch the data from OpenSearch export const getOpenSearchData = ( - arrayHits, - report, - params, + arrayHits: any, + report: { _source: any }, + params: { excel: any; limit: number }, dateFormat: string ) => { let hits: any = []; @@ -124,22 +129,56 @@ export const getOpenSearchData = ( for (let data of valueRes.hits) { const fields = data.fields; // get all the fields of type date and format them to excel format + let tempKeyElement: string[] = []; for (let dateField of report._source.dateFields) { + let keys; + keys = dateField.split('.'); const dateValue = data._source[dateField]; - if (dateValue && dateValue.length !== 0) { - if (dateValue instanceof Array) { - // loop through array - dateValue.forEach((element, index) => { - data._source[dateField][index] = moment( - fields[dateField][index] - ).format(dateFormat); + const fieldDateValue = fields[dateField]; + // if its not a nested date field + if (keys.length === 1) { + // if conditions to determine if the date field's value is an array or a string + if (typeof dateValue === 'string') { + data._source[keys] = moment(dateValue).format(dateFormat); + } else if ( + fieldDateValue.length !== 0 && + fieldDateValue instanceof Array + ) { + fieldDateValue.forEach((element, index) => { + data._source[keys][index] = moment(element).format(dateFormat); + }); + } else { + data._source[keys] = []; + } + // else to cover cases with nested date fields + } else { + let keyElement = keys.shift(); + // if conditions to determine if the date field's value is an array or a string + if (typeof fieldDateValue === 'string') { + keys.push(moment(fieldDateValue).format(dateFormat)); + } else if ( + fieldDateValue.length !== 0 && + fieldDateValue instanceof Array + ) { + let tempArray: string[] = []; + fieldDateValue.forEach((index) => { + tempArray.push(moment(index).format(dateFormat)); }); + keys.push(tempArray); } else { - // The fields response always returns an array of values for each field - // https://www.elastic.co/guide/en/elasticsearch/reference/master/search-fields.html#search-fields-response - data._source[dateField] = moment(fields[dateField][0]).format( - dateFormat - ); + keys.push([]); + } + const nestedJSON = arrayToNestedJSON(keys); + let keyLength = Object.keys(data._source); + // to check if the nested field have anyother keys apart from date field + if (tempKeyElement.includes(keyElement) || keyLength.length > 1) { + data._source[keyElement] = { + ...data._source[keyElement], + ...nestedJSON, + }; + } else { + data._source[keyElement] = nestedJSON; + tempKeyElement.push(keyElement); } } } @@ -226,6 +265,18 @@ function sanitize(doc: any) { return doc; } +function arrayToNestedJSON(arr: string[]) { + if (arr.length === 0) { + return null; + } else if (arr.length === 1) { + return arr[0]; + } else { + const key = arr[0]; + const rest = arr.slice(1); + return { [key]: arrayToNestedJSON(rest) }; + } +} + const addDocValueFields = (report: any, requestBody: any) => { const docValues = []; for (const dateType of report._source.dateFields) {