From ced5484356779837302ecabf53fad41a6cc924dd Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 29 Oct 2025 15:53:31 +0530 Subject: [PATCH 01/38] phase 1 changes --- .vscode/launch.json | 44 + .../sql-expression-modifiers.ts | 97 +- .../src/resolution/resolution.spec.ts | 1602 +++++++++-------- meerkat-core/src/resolution/types.ts | 2 + meerkat-core/src/types/cube-types/table.ts | 1 + .../cube-to-sql-with-resolution-array.spec.ts | 238 +++ .../cube-to-sql-with-resolution.ts | 161 ++ 7 files changed, 1399 insertions(+), 746 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..64487837 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,44 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/meerkat-core/src/resolution/resolution.spec.ts", + "outFiles": [ + "${workspaceFolder}/**/*.js" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Jest Tests", + "runtimeExecutable": "node", + "runtimeArgs": [ + "--inspect-brk", + "${workspaceFolder}/node_modules/.bin/jest", + "--runInBand", + "--no-cache", + "--testPathPattern=meerkat-core/src/resolution/resolution.spec.ts" + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "cwd": "${workspaceFolder}", + "env": { + "NODE_ENV": "test" + }, + "sourceMaps": true, + "resolveSourceMapLocations": [ + "${workspaceFolder}/**", + "!**/node_modules/**" + ] + }, + ] +} \ No newline at end of file diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts index 2f2a7326..693a6e7a 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts @@ -1,46 +1,87 @@ -import { Dimension, Query } from "../types/cube-types"; -import { isArrayTypeMember } from "../utils/is-array-member-type"; +import { Dimension, Query } from '../types/cube-types'; +import { isArrayTypeMember } from '../utils/is-array-member-type'; export interface DimensionModifier { - sqlExpression: string, - dimension: Dimension, - key: string, - query: Query + sqlExpression: string; + dimension: Dimension; + key: string; + query: Query; } -export const arrayFieldUnNestModifier = ({ sqlExpression }: DimensionModifier): string => { - return `array[unnest(${sqlExpression})]`; -} +export const arrayFieldUnNestModifier = ({ + sqlExpression, +}: DimensionModifier): string => { + return `array[unnest(${sqlExpression})]`; +}; + +export const arrayUnnestModifier = ({ + sqlExpression, +}: DimensionModifier): string => { + return `unnest(${sqlExpression})`; +}; -export const shouldUnnest = ({ dimension, query }: DimensionModifier): boolean => { - const isArrayType = isArrayTypeMember(dimension.type); +export const shouldUnnest = ({ + dimension, + query, +}: DimensionModifier): boolean => { + const isArrayType = isArrayTypeMember(dimension.type); const hasUnNestedGroupBy = dimension.modifier?.shouldUnnestGroupBy; return !!(isArrayType && hasUnNestedGroupBy && query.measures.length > 0); -} +}; +export const shouldUnnestArray = ({ + dimension, + query, +}: DimensionModifier): boolean => { + const isArrayType = isArrayTypeMember(dimension.type); + const hasUnNestedGroupBy = dimension.modifier?.shouldUnnestArray; + return !!(isArrayType && hasUnNestedGroupBy); +}; export type Modifier = { - name: string, - matcher: (modifier: DimensionModifier) => boolean, - modifier: (modifier: DimensionModifier) => string -} - -export const MODIFIERS: Modifier[] = [{ - name: 'shouldUnnestGroupBy', - matcher: shouldUnnest, - modifier: arrayFieldUnNestModifier -}] + name: string; + matcher: (modifier: DimensionModifier) => boolean; + modifier: (modifier: DimensionModifier) => string; +}; +export const MODIFIERS: Modifier[] = [ + { + name: 'shouldUnnestGroupBy', + matcher: shouldUnnest, + modifier: arrayFieldUnNestModifier, + }, + { + name: 'shouldUnnestArray', + matcher: shouldUnnestArray, + modifier: arrayUnnestModifier, + }, +]; -export const getModifiedSqlExpression = ({ sqlExpression, dimension, key, modifiers, query }: DimensionModifier & { - modifiers: Modifier[] +export const getModifiedSqlExpression = ({ + sqlExpression, + dimension, + key, + modifiers, + query, +}: DimensionModifier & { + modifiers: Modifier[]; }) => { let finalDimension: string = sqlExpression; modifiers.forEach(({ modifier, matcher }) => { - const shouldModify = matcher({ sqlExpression: finalDimension, dimension, key, query }); + const shouldModify = matcher({ + sqlExpression: finalDimension, + dimension, + key, + query, + }); if (shouldModify) { - finalDimension = modifier({ sqlExpression: finalDimension, dimension, key, query }); + finalDimension = modifier({ + sqlExpression: finalDimension, + dimension, + key, + query, + }); } - }) + }); return finalDimension; -} \ No newline at end of file +}; diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index 45170199..1fd11261 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -1,13 +1,7 @@ -import { - createBaseTableSchema, - generateResolutionJoinPaths, - generateResolutionSchemas, - generateResolvedDimensions, -} from './resolution'; -import { ResolutionConfig } from './types'; - +import { cubeQueryToSQLWithResolution } from '../../../meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution'; describe('Create base table schema', () => { - it('dimensions and measures are converted to dimensions', () => { + it('testing single field with count flow', async () => { + debugger; const sql = 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; const tableSchemas = [ @@ -18,375 +12,27 @@ describe('Create base table schema', () => { { name: 'count', sql: 'COUNT(*)', - type: 'number', - }, - ], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', - }, - { - name: 'column2', - sql: 'base_table.column2', - type: 'string', - }, - ], - }, - ]; - const resolutionConfig: ResolutionConfig = { - columnConfigs: [], - tableSchemas: [], - }; - const measures = ['base_table.count']; - const dimensions = ['base_table.column1', 'base_table.column2']; - - const baseTableSchema = createBaseTableSchema( - sql, - tableSchemas, - resolutionConfig, - measures, - dimensions - ); - - expect(baseTableSchema).toEqual({ - name: '__base_query', - sql: 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2', - measures: [], - dimensions: [ - { - name: 'base_table__count', - sql: '__base_query.base_table__count', - type: 'number', - alias: 'base_table__count', - }, - { - name: 'base_table__column1', - sql: '__base_query.base_table__column1', - type: 'string', - alias: 'base_table__column1', - }, - { - name: 'base_table__column2', - sql: '__base_query.base_table__column2', - type: 'string', - alias: 'base_table__column2', - }, - ], - joins: [], - }); - }); - - it('create join config', () => { - const sql = 'SELECT * FROM base_table'; - const tableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', - }, - { - name: 'column2', - sql: 'base_table.column2', - type: 'string', - }, - ], - }, - ]; - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, - ], - tableSchemas: [], - }; - const dimensions = ['base_table.column1', 'base_table.column2']; - - const baseTableSchema = createBaseTableSchema( - sql, - tableSchemas, - resolutionConfig, - [], - dimensions - ); - - expect(baseTableSchema).toEqual({ - name: '__base_query', - sql: 'SELECT * FROM base_table', - measures: [], - dimensions: [ - { - name: 'base_table__column1', - sql: '__base_query.base_table__column1', - type: 'string', - alias: 'base_table__column1', - }, - { - name: 'base_table__column2', - sql: '__base_query.base_table__column2', - type: 'string', - alias: 'base_table__column2', - }, - ], - joins: [ - { - sql: '__base_query.base_table__column1 = base_table__column1.id', - }, - { - sql: '__base_query.base_table__column2 = base_table__column2.id', - }, - ], - }); - }); - - it('dimension not found', () => { - const sql = 'SELECT * FROM base_table'; - const tableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', + type: 'number' as const, }, ], - }, - ]; - const resolutionConfig = { - columnConfigs: [], - tableSchemas: [], - }; - const dimensions = ['base_table.column1', 'base_table.column2']; // column2 does not exist - - expect(() => { - createBaseTableSchema( - sql, - tableSchemas, - resolutionConfig, - [], - dimensions - ); - }).toThrow('Not found: base_table.column2'); - }); - - it('handle aliases', () => { - const sql = 'SELECT * FROM base_table'; - const tableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', - alias: 'Column 1', - }, - { - name: 'column2', - sql: 'base_table.column2', - type: 'string', - alias: 'Column 2', - }, - ], - }, - ]; - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, - ], - tableSchemas: [], - }; - const dimensions = ['base_table.column1', 'base_table.column2']; - - const baseTableSchema = createBaseTableSchema( - sql, - tableSchemas, - resolutionConfig, - [], - dimensions - ); - - expect(baseTableSchema).toEqual({ - name: '__base_query', - sql: 'SELECT * FROM base_table', - measures: [], - dimensions: [ - { - name: 'base_table__column1', - sql: '__base_query."Column 1"', - type: 'string', - alias: 'Column 1', - }, - { - name: 'base_table__column2', - sql: '__base_query."Column 2"', - type: 'string', - alias: 'Column 2', - }, - ], - joins: [ - { - sql: '__base_query."Column 1" = base_table__column1.id', - }, - { - sql: '__base_query."Column 2" = base_table__column2.id', - }, - ], - }); - }); -}); - -describe('Generate resolution schemas', () => { - it('multiple columns using same table', () => { - const baseTableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], dimensions: [ { name: 'column1', sql: 'base_table.column1', - type: 'string', - }, - { - name: 'column2', - sql: 'base_table.column2', - type: 'string', + type: 'string' as const, }, ], }, ]; - - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['id', 'display_name'], - }, - ], - tableSchemas: [ - { - name: 'resolution_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'id', - sql: 'resolution_table.id', - type: 'string', - }, - { - name: 'display_id', - sql: 'resolution_table.display_id', - type: 'string', - }, - { - name: 'display_name', - sql: 'resolution_table.display_name', - type: 'string', - }, - ], - }, - ], - }; - - const schemas = generateResolutionSchemas( - resolutionConfig, - baseTableSchemas - ); - - expect(schemas).toEqual([ - { - name: 'base_table__column1', - sql: '', - measures: [], - dimensions: [ - { - name: 'base_table__column1__display_id', - sql: 'base_table__column1.display_id', - type: 'string', - alias: 'base_table__column1 - display_id', - }, - ], - }, - { - name: 'base_table__column2', - sql: '', - measures: [], - dimensions: [ - { - name: 'base_table__column2__id', - sql: 'base_table__column2.id', - type: 'string', - alias: 'base_table__column2 - id', - }, - { - name: 'base_table__column2__display_name', - sql: 'base_table__column2.display_name', - type: 'string', - alias: 'base_table__column2 - display_name', - }, - ], - }, - ]); - }); - - it('table does not exist', () => { const resolutionConfig = { columnConfigs: [ { name: 'base_table.column1', + isArrayType: false, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, - { - name: 'base_table.column2', - source: 'resolution_table1', // does not exist - joinColumn: 'id', - resolutionColumns: ['id', 'display_name'], - }, ], tableSchemas: [ { @@ -397,378 +43,898 @@ describe('Generate resolution schemas', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, }, { name: 'display_name', sql: 'resolution_table.display_name', - type: 'string', + type: 'string' as const, }, ], }, ], }; - - expect(() => { - generateResolutionSchemas(resolutionConfig, []); - }).toThrow('Table schema not found for resolution_table1'); - }); - - it('resolution column does not exist', () => { - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - ], - tableSchemas: [ - { - name: 'resolution_table', - sql: '', - measures: [ - { - name: 'display_id', - sql: 'resolution_table.display_id', - type: 'string', - }, - ], - dimensions: [ - { - name: 'id', - sql: 'resolution_table.id', - type: 'string', - }, - ], - }, - ], - }; - - expect(() => { - generateResolutionSchemas(resolutionConfig, []); - }).toThrow('Dimension not found: display_id'); - }); - - it('column does not exist in base query', () => { - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - ], - tableSchemas: [ - { - name: 'resolution_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'id', - sql: 'resolution_table.id', - type: 'string', - }, - { - name: 'display_id', - sql: 'resolution_table.display_id', - type: 'string', - }, - ], - }, - ], - }; - - const schemas = generateResolutionSchemas(resolutionConfig, []); - expect(schemas).toEqual([ - { - name: 'base_table__column1', - sql: '', - measures: [], - dimensions: [ - { - name: 'base_table__column1__display_id', - sql: 'base_table__column1.display_id', - type: 'string', - alias: 'base_table__column1 - display_id', - }, - ], - }, - ]); - }); - - it('handle aliases', () => { - const baseTableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', - alias: 'Column 1', - }, - ], - }, - ]; - - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - ], - tableSchemas: [ - { - name: 'resolution_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'id', - sql: 'resolution_table.id', - type: 'string', - alias: 'ID', - }, - { - name: 'display_id', - sql: 'resolution_table.display_id', - type: 'string', - alias: 'Display ID', - }, - ], - }, - ], - }; - - const schemas = generateResolutionSchemas( - resolutionConfig, - baseTableSchemas - ); - expect(schemas).toEqual([ - { - name: 'base_table__column1', - sql: '', - measures: [], - dimensions: [ - { - name: 'base_table__column1__display_id', - sql: 'base_table__column1.display_id', - type: 'string', - alias: 'Column 1 - Display ID', - }, - ], + const measures = ['base_table.count']; + const dimensions = ['base_table.column1']; + // Initialize DuckDB and create a connection + debugger; + const result = await cubeQueryToSQLWithResolution({ + query: { + measures: measures, + dimensions: dimensions, }, - ]); - }); -}); - -describe('Generate resolved dimensions', () => { - it('resolves dimensions based on resolution config', () => { - const query = { - measures: [], - dimensions: ['base_table.column1', 'base_table.column2'], - }; - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, - ], - tableSchemas: [], - }; - - const resolvedDimensions = generateResolvedDimensions( - query, - resolutionConfig - ); - - expect(resolvedDimensions).toEqual([ - 'base_table__column1.base_table__column1__display_id', - 'base_table__column2.base_table__column2__display_name', - ]); - }); - - it('unresolved columns', () => { - const query = { - measures: ['base_table.count'], - dimensions: ['base_table.column1'], - }; - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column3', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - ], - tableSchemas: [], - }; - - const resolvedDimensions = generateResolvedDimensions( - query, - resolutionConfig - ); - - expect(resolvedDimensions).toEqual([ - '__base_query.base_table__count', - '__base_query.base_table__column1', - ]); - }); - - it('only include projected columns', () => { - const query = { - measures: ['base_table.count', 'base_table.total'], - dimensions: ['base_table.column1', 'base_table.column2'], - }; - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['id', 'display_name'], - }, - ], - tableSchemas: [], - }; - const projections = [ - 'base_table.count', - 'base_table.column2', - 'base_table.total', - ]; - - const resolvedDimensions = generateResolvedDimensions( - query, - resolutionConfig, - projections - ); - - expect(resolvedDimensions).toEqual([ - '__base_query.base_table__count', - 'base_table__column2.base_table__column2__id', - 'base_table__column2.base_table__column2__display_name', - '__base_query.base_table__total', - ]); + tableSchemas: tableSchemas, + resolutionConfig: resolutionConfig, + }); + console.log('Generated SQL:', result); + expect(result).toBeDefined(); }); + // it('testing entire flow', async () => { + // debugger; + // const sql = + // 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; + // const tableSchemas = [ + // { + // name: 'base_table', + // sql: '', + // measures: [ + // { + // name: 'count', + // sql: 'COUNT(*)', + // type: 'number' as const, + // }, + // ], + // dimensions: [ + // { + // name: 'column1', + // sql: 'base_table.column1', + // type: 'string' as const, + // }, + // { + // name: 'column2', + // sql: 'base_table.column2', + // type: 'string_array' as const, + // }, + // ], + // }, + // ]; + // const resolutionConfig = { + // columnConfigs: [ + // { + // name: 'base_table.column1', + // isArrayType: false, + // source: 'resolution_table', + // joinColumn: 'id', + // resolutionColumns: ['display_id'], + // }, + // { + // name: 'base_table.column2', + // isArrayType: true, + // source: 'resolution_table_2', + // joinColumn: 'id', + // resolutionColumns: ['fullname'], + // }, + // ], + // tableSchemas: [ + // { + // name: 'resolution_table', + // sql: '', + // measures: [], + // dimensions: [ + // { + // name: 'id', + // sql: 'resolution_table.id', + // type: 'string' as const, + // }, + // { + // name: 'display_id', + // sql: 'resolution_table.display_id', + // type: 'string' as const, + // }, + // { + // name: 'display_name', + // sql: 'resolution_table.display_name', + // type: 'string' as const, + // }, + // ], + // }, + // { + // name: 'resolution_table_2', + // sql: '', + // measures: [], + // dimensions: [ + // { + // name: 'id', + // sql: 'resolution_table_2.id', + // type: 'string' as const, + // }, + // { + // name: 'display_id', + // sql: 'resolution_table_2.display_id', + // type: 'string' as const, + // }, + // { + // name: 'fullname', + // sql: 'resolution_table_2.fullname', + // type: 'string' as const, + // }, + // ], + // }, + // ], + // }; + // const measures = ['base_table.count']; + // const dimensions = ['base_table.column1', 'base_table.column2']; + // // Initialize DuckDB and create a connection + // debugger; + // const result = await cubeQueryToSQLWithResolutionWithArray({ + // query: { + // measures: measures, + // dimensions: dimensions, + // }, + // tableSchemas: tableSchemas, + // resolutionConfig: resolutionConfig, + // }); + // console.log('Generated SQL:', result); + // expect(result).toBeDefined(); + // }); + // it('dimensions and measures are converted to dimensions', () => { + // debugger; + // const sql = + // 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; + // const tableSchemas = [ + // { + // name: 'base_table', + // sql: '', + // measures: [ + // { + // name: 'count', + // sql: 'COUNT(*)', + // type: 'number', + // }, + // ], + // dimensions: [ + // { + // name: 'column1', + // sql: 'base_table.column1', + // type: 'string', + // }, + // { + // name: 'column2', + // sql: 'base_table.column2', + // type: 'string', + // }, + // ], + // }, + // ]; + // const resolutionConfig: ResolutionConfig = { + // columnConfigs: [], + // tableSchemas: [], + // }; + // const measures = ['base_table.count']; + // const dimensions = ['base_table.column1', 'base_table.column2']; + // const baseTableSchema = createBaseTableSchema( + // sql, + // tableSchemas, + // resolutionConfig, + // measures, + // dimensions + // ); + // expect(baseTableSchema).toEqual({ + // name: '__base_query', + // sql: 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2', + // measures: [], + // dimensions: [ + // { + // name: 'base_table__count', + // sql: '__base_query.base_table__count', + // type: 'number', + // alias: 'base_table__count', + // }, + // { + // name: 'base_table__column1', + // sql: '__base_query.base_table__column1', + // type: 'string', + // alias: 'base_table__column1', + // }, + // { + // name: 'base_table__column2', + // sql: '__base_query.base_table__column2', + // type: 'string', + // alias: 'base_table__column2', + // }, + // ], + // joins: [], + // }); + // }); + // it('create join config', () => { + // const sql = 'SELECT * FROM base_table'; + // const tableSchemas = [ + // { + // name: 'base_table', + // sql: '', + // measures: [], + // dimensions: [ + // { + // name: 'column1', + // sql: 'base_table.column1', + // type: 'string', + // }, + // { + // name: 'column2', + // sql: 'base_table.column2', + // type: 'string', + // }, + // ], + // }, + // ]; + // const resolutionConfig = { + // columnConfigs: [ + // { + // name: 'base_table.column1', + // source: 'resolution_table', + // joinColumn: 'id', + // resolutionColumns: ['display_id'], + // }, + // { + // name: 'base_table.column2', + // source: 'resolution_table', + // joinColumn: 'id', + // resolutionColumns: ['display_name'], + // }, + // ], + // tableSchemas: [], + // }; + // const dimensions = ['base_table.column1', 'base_table.column2']; + // const baseTableSchema = createBaseTableSchema( + // sql, + // tableSchemas, + // resolutionConfig, + // [], + // dimensions + // ); + // expect(baseTableSchema).toEqual({ + // name: '__base_query', + // sql: 'SELECT * FROM base_table', + // measures: [], + // dimensions: [ + // { + // name: 'base_table__column1', + // sql: '__base_query.base_table__column1', + // type: 'string', + // alias: 'base_table__column1', + // }, + // { + // name: 'base_table__column2', + // sql: '__base_query.base_table__column2', + // type: 'string', + // alias: 'base_table__column2', + // }, + // ], + // joins: [ + // { + // sql: '__base_query.base_table__column1 = base_table__column1.id', + // }, + // { + // sql: '__base_query.base_table__column2 = base_table__column2.id', + // }, + // ], + // }); + // }); + // it('dimension not found', () => { + // const sql = 'SELECT * FROM base_table'; + // const tableSchemas = [ + // { + // name: 'base_table', + // sql: '', + // measures: [], + // dimensions: [ + // { + // name: 'column1', + // sql: 'base_table.column1', + // type: 'string', + // }, + // ], + // }, + // ]; + // const resolutionConfig = { + // columnConfigs: [], + // tableSchemas: [], + // }; + // const dimensions = ['base_table.column1', 'base_table.column2']; // column2 does not exist + // expect(() => { + // createBaseTableSchema( + // sql, + // tableSchemas, + // resolutionConfig, + // [], + // dimensions + // ); + // }).toThrow('Not found: base_table.column2'); + // }); + // it('handle aliases', () => { + // const sql = 'SELECT * FROM base_table'; + // const tableSchemas = [ + // { + // name: 'base_table', + // sql: '', + // measures: [], + // dimensions: [ + // { + // name: 'column1', + // sql: 'base_table.column1', + // type: 'string', + // alias: 'Column 1', + // }, + // { + // name: 'column2', + // sql: 'base_table.column2', + // type: 'string', + // alias: 'Column 2', + // }, + // ], + // }, + // ]; + // const resolutionConfig = { + // columnConfigs: [ + // { + // name: 'base_table.column1', + // source: 'resolution_table', + // joinColumn: 'id', + // resolutionColumns: ['display_id'], + // }, + // { + // name: 'base_table.column2', + // source: 'resolution_table', + // joinColumn: 'id', + // resolutionColumns: ['display_name'], + // }, + // ], + // tableSchemas: [], + // }; + // const dimensions = ['base_table.column1', 'base_table.column2']; + // const baseTableSchema = createBaseTableSchema( + // sql, + // tableSchemas, + // resolutionConfig, + // [], + // dimensions + // ); + // expect(baseTableSchema).toEqual({ + // name: '__base_query', + // sql: 'SELECT * FROM base_table', + // measures: [], + // dimensions: [ + // { + // name: 'base_table__column1', + // sql: '__base_query."Column 1"', + // type: 'string', + // alias: 'Column 1', + // }, + // { + // name: 'base_table__column2', + // sql: '__base_query."Column 2"', + // type: 'string', + // alias: 'Column 2', + // }, + // ], + // joins: [ + // { + // sql: '__base_query."Column 1" = base_table__column1.id', + // }, + // { + // sql: '__base_query."Column 2" = base_table__column2.id', + // }, + // ], + // }); + // }); }); -describe('Generate resolution join paths', () => { - it('generate join paths for resolution columns', () => { - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - { - name: 'base_table.column2', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, - ], - tableSchemas: [], - }; - - const joinPaths = generateResolutionJoinPaths(resolutionConfig, []); - - expect(joinPaths).toEqual([ - [ - { - left: '__base_query', - right: 'base_table__column1', - on: 'base_table__column1', - }, - ], - [ - { - left: '__base_query', - right: 'base_table__column2', - on: 'base_table__column2', - }, - ], - ]); - }); - - it('join paths with aliases', () => { - const baseTableSchemas = [ - { - name: 'base_table', - sql: '', - measures: [], - dimensions: [ - { - name: 'column1', - sql: 'base_table.column1', - type: 'string', - alias: 'Column 1', - }, - { - name: 'column2', - sql: 'base_table.column2', - type: 'string', - alias: 'Column 2', - }, - ], - }, - ]; - - const resolutionConfig = { - columnConfigs: [ - { - name: 'base_table.column1', - source: 'resolution_table', - joinColumn: 'id', - resolutionColumns: ['display_id'], - }, - ], - tableSchemas: [], - }; - - const joinPaths = generateResolutionJoinPaths( - resolutionConfig, - baseTableSchemas - ); - expect(joinPaths).toEqual([ - [ - { - left: '__base_query', - right: 'base_table__column1', - on: '"Column 1"', - }, - ], - ]); - }); -}); +// describe('Generate resolution schemas', () => { +// it('multiple columns using same table', () => { +// const baseTableSchemas = [ +// { +// name: 'base_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'column1', +// sql: 'base_table.column1', +// type: 'string', +// }, +// { +// name: 'column2', +// sql: 'base_table.column2', +// type: 'string', +// }, +// ], +// }, +// ]; + +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// { +// name: 'base_table.column2', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['id', 'display_name'], +// }, +// ], +// tableSchemas: [ +// { +// name: 'resolution_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'id', +// sql: 'resolution_table.id', +// type: 'string', +// }, +// { +// name: 'display_id', +// sql: 'resolution_table.display_id', +// type: 'string', +// }, +// { +// name: 'display_name', +// sql: 'resolution_table.display_name', +// type: 'string', +// }, +// ], +// }, +// ], +// }; + +// const schemas = generateResolutionSchemas( +// resolutionConfig, +// baseTableSchemas +// ); + +// expect(schemas).toEqual([ +// { +// name: 'base_table__column1', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'base_table__column1__display_id', +// sql: 'base_table__column1.display_id', +// type: 'string', +// alias: 'base_table__column1 - display_id', +// }, +// ], +// }, +// { +// name: 'base_table__column2', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'base_table__column2__id', +// sql: 'base_table__column2.id', +// type: 'string', +// alias: 'base_table__column2 - id', +// }, +// { +// name: 'base_table__column2__display_name', +// sql: 'base_table__column2.display_name', +// type: 'string', +// alias: 'base_table__column2 - display_name', +// }, +// ], +// }, +// ]); +// }); + +// it('table does not exist', () => { +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// { +// name: 'base_table.column2', +// source: 'resolution_table1', // does not exist +// joinColumn: 'id', +// resolutionColumns: ['id', 'display_name'], +// }, +// ], +// tableSchemas: [ +// { +// name: 'resolution_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'id', +// sql: 'resolution_table.id', +// type: 'string', +// }, +// { +// name: 'display_id', +// sql: 'resolution_table.display_id', +// type: 'string', +// }, +// { +// name: 'display_name', +// sql: 'resolution_table.display_name', +// type: 'string', +// }, +// ], +// }, +// ], +// }; + +// expect(() => { +// generateResolutionSchemas(resolutionConfig, []); +// }).toThrow('Table schema not found for resolution_table1'); +// }); + +// it('resolution column does not exist', () => { +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// ], +// tableSchemas: [ +// { +// name: 'resolution_table', +// sql: '', +// measures: [ +// { +// name: 'display_id', +// sql: 'resolution_table.display_id', +// type: 'string', +// }, +// ], +// dimensions: [ +// { +// name: 'id', +// sql: 'resolution_table.id', +// type: 'string', +// }, +// ], +// }, +// ], +// }; + +// expect(() => { +// generateResolutionSchemas(resolutionConfig, []); +// }).toThrow('Dimension not found: display_id'); +// }); + +// it('column does not exist in base query', () => { +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// ], +// tableSchemas: [ +// { +// name: 'resolution_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'id', +// sql: 'resolution_table.id', +// type: 'string', +// }, +// { +// name: 'display_id', +// sql: 'resolution_table.display_id', +// type: 'string', +// }, +// ], +// }, +// ], +// }; + +// const schemas = generateResolutionSchemas(resolutionConfig, []); +// expect(schemas).toEqual([ +// { +// name: 'base_table__column1', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'base_table__column1__display_id', +// sql: 'base_table__column1.display_id', +// type: 'string', +// alias: 'base_table__column1 - display_id', +// }, +// ], +// }, +// ]); +// }); + +// it('handle aliases', () => { +// const baseTableSchemas = [ +// { +// name: 'base_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'column1', +// sql: 'base_table.column1', +// type: 'string', +// alias: 'Column 1', +// }, +// ], +// }, +// ]; + +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// ], +// tableSchemas: [ +// { +// name: 'resolution_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'id', +// sql: 'resolution_table.id', +// type: 'string', +// alias: 'ID', +// }, +// { +// name: 'display_id', +// sql: 'resolution_table.display_id', +// type: 'string', +// alias: 'Display ID', +// }, +// ], +// }, +// ], +// }; + +// const schemas = generateResolutionSchemas( +// resolutionConfig, +// baseTableSchemas +// ); +// expect(schemas).toEqual([ +// { +// name: 'base_table__column1', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'base_table__column1__display_id', +// sql: 'base_table__column1.display_id', +// type: 'string', +// alias: 'Column 1 - Display ID', +// }, +// ], +// }, +// ]); +// }); +// }); + +// describe('Generate resolved dimensions', () => { +// it('resolves dimensions based on resolution config', () => { +// const query = { +// measures: [], +// dimensions: ['base_table.column1', 'base_table.column2'], +// }; +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// { +// name: 'base_table.column2', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_name'], +// }, +// ], +// tableSchemas: [], +// }; + +// const resolvedDimensions = generateResolvedDimensions( +// query, +// resolutionConfig +// ); + +// expect(resolvedDimensions).toEqual([ +// 'base_table__column1.base_table__column1__display_id', +// 'base_table__column2.base_table__column2__display_name', +// ]); +// }); + +// it('unresolved columns', () => { +// const query = { +// measures: ['base_table.count'], +// dimensions: ['base_table.column1'], +// }; +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column3', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// ], +// tableSchemas: [], +// }; + +// const resolvedDimensions = generateResolvedDimensions( +// query, +// resolutionConfig +// ); + +// expect(resolvedDimensions).toEqual([ +// '__base_query.base_table__count', +// '__base_query.base_table__column1', +// ]); +// }); + +// it('only include projected columns', () => { +// const query = { +// measures: ['base_table.count', 'base_table.total'], +// dimensions: ['base_table.column1', 'base_table.column2'], +// }; +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// { +// name: 'base_table.column2', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['id', 'display_name'], +// }, +// ], +// tableSchemas: [], +// }; +// const projections = [ +// 'base_table.count', +// 'base_table.column2', +// 'base_table.total', +// ]; + +// const resolvedDimensions = generateResolvedDimensions( +// query, +// resolutionConfig, +// projections +// ); + +// expect(resolvedDimensions).toEqual([ +// '__base_query.base_table__count', +// 'base_table__column2.base_table__column2__id', +// 'base_table__column2.base_table__column2__display_name', +// '__base_query.base_table__total', +// ]); +// }); +// }); + +// describe('Generate resolution join paths', () => { +// it('generate join paths for resolution columns', () => { +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// { +// name: 'base_table.column2', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_name'], +// }, +// ], +// tableSchemas: [], +// }; + +// const joinPaths = generateResolutionJoinPaths(resolutionConfig, []); + +// expect(joinPaths).toEqual([ +// [ +// { +// left: '__base_query', +// right: 'base_table__column1', +// on: 'base_table__column1', +// }, +// ], +// [ +// { +// left: '__base_query', +// right: 'base_table__column2', +// on: 'base_table__column2', +// }, +// ], +// ]); +// }); + +// it('join paths with aliases', () => { +// const baseTableSchemas = [ +// { +// name: 'base_table', +// sql: '', +// measures: [], +// dimensions: [ +// { +// name: 'column1', +// sql: 'base_table.column1', +// type: 'string', +// alias: 'Column 1', +// }, +// { +// name: 'column2', +// sql: 'base_table.column2', +// type: 'string', +// alias: 'Column 2', +// }, +// ], +// }, +// ]; + +// const resolutionConfig = { +// columnConfigs: [ +// { +// name: 'base_table.column1', +// source: 'resolution_table', +// joinColumn: 'id', +// resolutionColumns: ['display_id'], +// }, +// ], +// tableSchemas: [], +// }; + +// const joinPaths = generateResolutionJoinPaths( +// resolutionConfig, +// baseTableSchemas +// ); +// expect(joinPaths).toEqual([ +// [ +// { +// left: '__base_query', +// right: 'base_table__column1', +// on: '"Column 1"', +// }, +// ], +// ]); +// }); +// }); diff --git a/meerkat-core/src/resolution/types.ts b/meerkat-core/src/resolution/types.ts index 0a0c1981..4f1581e0 100644 --- a/meerkat-core/src/resolution/types.ts +++ b/meerkat-core/src/resolution/types.ts @@ -4,6 +4,8 @@ export interface ResolutionColumnConfig { // Name of the column that needs resolution. // Should match a measure or dimension in the query. name: string; + // is array type + isArrayType: boolean; // Name of the data source to use for resolution. source: string; // Name of the column in the data source to join on. diff --git a/meerkat-core/src/types/cube-types/table.ts b/meerkat-core/src/types/cube-types/table.ts index 4578a4e7..8c214f29 100644 --- a/meerkat-core/src/types/cube-types/table.ts +++ b/meerkat-core/src/types/cube-types/table.ts @@ -27,6 +27,7 @@ export type Dimension = { type: DimensionType; modifier?: { shouldUnnestGroupBy?: boolean; + shouldUnnestArray?: boolean; }; alias?: string; }; diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts new file mode 100644 index 00000000..70178cb6 --- /dev/null +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -0,0 +1,238 @@ +import { Query, ResolutionConfig, TableSchema } from '@devrev/meerkat-core'; +import { cubeQueryToSQLWithResolutionWithArray } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; +import { duckdbExec } from '../duckdb-exec'; + +const CREATE_TEST_TABLE = `CREATE TABLE tickets ( + id INTEGER, + owners VARCHAR[], + tags VARCHAR[], + created_by VARCHAR, + subscribers_count INTEGER +)`; + +const INPUT_DATA_QUERY = `INSERT INTO tickets VALUES +(1, ['owner1', 'owner2'], ['tag1'], 'user1', 30), +(2, ['owner2', 'owner3'], ['tag2', 'tag3'], 'user2', 10), +(3, ['owner4'], ['tag1', 'tag4', 'tag3'], 'user3', 80)`; + +const CREATE_RESOLUTION_TABLE = `CREATE TABLE owners_lookup ( + id VARCHAR, + display_name VARCHAR, + email VARCHAR +)`; + +const RESOLUTION_DATA_QUERY = `INSERT INTO owners_lookup VALUES +('owner1', 'Alice Smith', 'alice@example.com'), +('owner2', 'Bob Jones', 'bob@example.com'), +('owner3', 'Charlie Brown', 'charlie@example.com'), +('owner4', 'Diana Prince', 'diana@example.com')`; + +const TICKETS_TABLE_SCHEMA: TableSchema = { + name: 'tickets', + sql: 'select * from tickets', + measures: [ + { + name: 'count', + sql: 'COUNT(*)', + type: 'number', + }, + ], + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'number', + }, + { + name: 'created_by', + sql: 'created_by', + type: 'string', + }, + { + name: 'owners', + sql: 'owners', + type: 'string_array', + }, + { + name: 'tags', + sql: 'tags', + type: 'string_array', + }, + ], +}; + +const OWNERS_LOOKUP_SCHEMA: TableSchema = { + name: 'owners_lookup', + sql: 'select * from owners_lookup', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'string', + }, + { + name: 'display_name', + sql: 'display_name', + type: 'string', + }, + { + name: 'email', + sql: 'email', + type: 'string', + }, + ], +}; + +describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { + jest.setTimeout(1000000); + beforeAll(async () => { + // Create test tables + await duckdbExec(CREATE_TEST_TABLE); + await duckdbExec(INPUT_DATA_QUERY); + await duckdbExec(CREATE_RESOLUTION_TABLE); + await duckdbExec(RESOLUTION_DATA_QUERY); + }); + + it('Should add row_id and unnest array fields that need resolution', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: ['tickets.id', 'tickets.owners'], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.owners', + isArrayType: true, + source: 'owners_lookup', + joinColumn: 'id', + resolutionColumns: ['display_name', 'email'], + }, + ], + tableSchemas: [OWNERS_LOOKUP_SCHEMA], + }; + + const sql = await cubeQueryToSQLWithResolutionWithArray({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + }); + + console.log('Phase 1 SQL (with row_id and unnest):', sql); + + // Verify the SQL includes row_id + expect(sql).toContain('row_id'); + + // Verify the SQL unnests the owners array + expect(sql).toContain('unnest'); + + // Verify it includes the owners dimension + expect(sql).toContain('owners'); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + // console.log('Phase 1 Result:', JSON.stringify(result, null, 2)); + + // The result should have unnested rows (more rows than the original 3) + // Original: 3 rows, but ticket 1 has 2 owners, ticket 2 has 2 owners, ticket 3 has 1 owner + // Expected: 5 unnested rows + expect(result.length).toBe(5); + + // Each row should have a row_id + expect(result[0]).toHaveProperty('__base_query_with_row_id__row_id'); + expect(result[0]).toHaveProperty( + '__base_query_with_row_id__tickets__count' + ); + expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__id'); + expect(result[0]).toHaveProperty( + '__base_query_with_row_id__tickets__owners' + ); + + debugger; + // Verify row_ids are preserved (rows with same original row should have same row_id) + const rowIds = result.map((r) => r.row_id); + expect(rowIds.length).toBe(5); // 5 unnested rows total + }); + + it('Should handle multiple array fields that need unnesting', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: ['tickets.id', 'tickets.owners', 'tickets.tags'], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.owners', + isArrayType: true, + source: 'owners_lookup', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + { + name: 'tickets.tags', + isArrayType: true, + source: 'tags_lookup', + joinColumn: 'id', + resolutionColumns: ['tag_name'], + }, + ], + tableSchemas: [OWNERS_LOOKUP_SCHEMA], + }; + + const sql = await cubeQueryToSQLWithResolutionWithArray({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + }); + + console.log('Phase 1 SQL (multiple arrays):', sql); + + // Verify row_id is included + expect(sql).toContain('row_id'); + + // Both arrays should be unnested + expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(2); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + + expect(result.length).toBe(7); + + // Each row should have a row_id + expect(result[0]).toHaveProperty('__base_query_with_row_id__row_id'); + expect(result[0]).toHaveProperty( + '__base_query_with_row_id__tickets__count' + ); + expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__id'); + expect(result[0]).toHaveProperty( + '__base_query_with_row_id__tickets__owners' + ); + expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__tags'); + }); + + it('Should return regular SQL when no array fields need resolution', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: ['tickets.id', 'tickets.created_by'], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + + const sql = await cubeQueryToSQLWithResolutionWithArray({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + }); + + console.log('SQL without resolution:', sql); + + // Should not have row_id or unnest when no array resolution is needed + expect(sql).not.toContain('row_id'); + expect(sql).not.toContain('unnest'); + }); +}); diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 8ef3de8e..2b04f61a 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -69,3 +69,164 @@ export const cubeQueryToSQLWithResolution = async ({ return sql; }; + +/** + * Helper function to get array-type columns from resolution config + */ +const getArrayTypeColumns = (resolutionConfig: ResolutionConfig) => { + return resolutionConfig.columnConfigs.filter( + (config) => config.isArrayType === true + ); +}; + +export const cubeQueryToSQLWithResolutionWithArray = async ({ + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, +}: CubeQueryToSQLWithResolutionParams) => { + debugger; + if (resolutionConfig.columnConfigs.length === 0) { + // If no resolution is needed, return the base SQL. + const baseSql = await cubeQueryToSQL({ + query, + tableSchemas, + contextParams, + }); + return baseSql; + } + + // Phase 1: Setup and Unnest + const arrayColumns = getArrayTypeColumns(resolutionConfig); + + if (arrayColumns.length === 0) { + // No array columns to process, return base SQL + const baseSql = await cubeQueryToSQL({ + query, + tableSchemas, + contextParams, + }); + return baseSql; + } + + // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) + const modifiedTableSchemasWithRowId = tableSchemas.map((schema, index) => { + // Add row_id to the first table only + if (index !== 0) { + return schema; + } + + // Add row_id dimension (no unnest modifier yet) + // TODO: Will this cause a problem of adding row_id to the first schema ? + const newDimensions = [ + { + name: 'row_id', + sql: 'row_number() OVER ()', + type: 'number' as const, + alias: '__row_id', + }, + ...schema.dimensions, + ]; + + return { + ...schema, + dimensions: newDimensions, + }; + }); + + // Use the first table for row_id reference + const firstTable = tableSchemas[0]; + + const queryWithRowId: Query = { + measures: query.measures, + dimensions: [`${firstTable.name}.row_id`, ...(query.dimensions || [])], + joinPaths: query.joinPaths, + filters: query.filters, + order: query.order, + limit: query.limit, + offset: query.offset, + }; + + // Generate base SQL with row_id + const baseSqlWithRowId = await cubeQueryToSQL({ + query: queryWithRowId, + tableSchemas: modifiedTableSchemasWithRowId, + contextParams, + }); + + // Step 2: Create a new table schema from the base SQL with row_id + // This will be used to apply unnesting + const baseTableName = '__base_query_with_row_id'; + + // Create dimensions from the base SQL output (all measures become dimensions now) + const baseDimensions: TableSchema['dimensions'] = [ + { + name: 'row_id', + sql: `${baseTableName}.__row_id`, + type: 'number', + }, + ...query.measures.map((measure) => ({ + name: measure.replace('.', '__'), + sql: `${baseTableName}.${measure.replace('.', '__')}`, + type: 'number' as const, + })), + ...(query.dimensions || []).map((dimension) => { + // Check if this dimension is an array that needs unnesting + const isArrayDimension = arrayColumns.some( + (col) => col.name === dimension + ); + + const originalSchema = tableSchemas.find((s) => + dimension.startsWith(`${s.name}.`) + ); + const dimName = dimension.split('.')[1]; + const originalDimension = originalSchema?.dimensions.find( + (d) => d.name === dimName + ); + + return { + name: dimension.replace('.', '__'), + sql: `${baseTableName}.${dimension.replace('.', '__')}`, + type: originalDimension?.type || ('string' as const), + ...(isArrayDimension && { + modifier: { + shouldUnnestArray: true, + }, + }), + }; + }), + ]; + + const baseTableSchema: TableSchema = { + name: baseTableName, + sql: baseSqlWithRowId, + measures: [], + dimensions: baseDimensions, + }; + + // Step 3: Create query with unnest modifiers applied + const unnestQuery: Query = { + measures: [], + dimensions: [ + `${baseTableName}.row_id`, + ...query.measures.map((m) => `${baseTableName}.${m.replace('.', '__')}`), + ...(query.dimensions || []).map( + (d) => `${baseTableName}.${d.replace('.', '__')}` + ), + ], + filters: query.filters, + order: query.order, + limit: query.limit, + offset: query.offset, + }; + + // Generate the final SQL with unnesting applied + const unnestedBaseSql = await cubeQueryToSQL({ + query: unnestQuery, + tableSchemas: [baseTableSchema], + contextParams, + }); + + return unnestedBaseSql; +}; From d065d22f6793fa59b5819a29baefe919d761f4c4 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 29 Oct 2025 19:07:12 +0530 Subject: [PATCH 02/38] phase 2 changes --- meerkat-core/src/resolution/resolution.ts | 29 +- .../cube-to-sql-with-resolution-array.spec.ts | 276 ++++++++++++------ .../cube-to-sql-with-resolution.ts | 204 +++++++++---- 3 files changed, 357 insertions(+), 152 deletions(-) diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 3afe29ce..28d75af6 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -9,10 +9,21 @@ import { findInDimensionSchemas, findInSchemas, } from '../utils/find-in-table-schema'; -import { BASE_DATA_SOURCE_NAME, ResolutionConfig } from './types'; +import { + BASE_DATA_SOURCE_NAME, + ResolutionColumnConfig, + ResolutionConfig, +} from './types'; -const constructBaseDimension = (name: string, schema: Measure | Dimension) => { - return { +const constructBaseDimension = ( + name: string, + schema: Measure | Dimension, + resolutionColumnConfigs: ResolutionColumnConfig[] +) => { + const shouldPerformUnnest = resolutionColumnConfigs.some( + (config) => config.name == name + ); + const dimension: Dimension = { name: memberKeyToSafeKey(name), sql: `${BASE_DATA_SOURCE_NAME}.${constructAlias({ name, @@ -27,6 +38,12 @@ const constructBaseDimension = (name: string, schema: Measure | Dimension) => { aliasContext: { isTableSchemaAlias: true }, }), }; + if (shouldPerformUnnest) { + dimension.modifier = { + shouldUnnestArray: true, + }; + } + return dimension; }; export const createBaseTableSchema = ( @@ -54,7 +71,11 @@ export const createBaseTableSchema = ( dimensions: [...measures, ...(dimensions || [])].map((member) => { const schema = schemaByName[member]; if (schema) { - return constructBaseDimension(member, schema); + return constructBaseDimension( + member, + schema, + resolutionConfig.columnConfigs + ); } else { throw new Error(`Not found: ${member}`); } diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index 70178cb6..0333412e 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -27,6 +27,17 @@ const RESOLUTION_DATA_QUERY = `INSERT INTO owners_lookup VALUES ('owner3', 'Charlie Brown', 'charlie@example.com'), ('owner4', 'Diana Prince', 'diana@example.com')`; +const CREATE_TAGS_LOOKUP_TABLE = `CREATE TABLE tags_lookup ( + id VARCHAR, + tag_name VARCHAR +)`; + +const TAGS_LOOKUP_DATA_QUERY = `INSERT INTO tags_lookup VALUES +('tag1', 'Tag 1'), +('tag2', 'Tag 2'), +('tag3', 'Tag 3'), +('tag4', 'Tag 4')`; + const TICKETS_TABLE_SCHEMA: TableSchema = { name: 'tickets', sql: 'select * from tickets', @@ -84,6 +95,23 @@ const OWNERS_LOOKUP_SCHEMA: TableSchema = { ], }; +const TAGS_LOOKUP_SCHEMA: TableSchema = { + name: 'tags_lookup', + sql: 'select * from tags_lookup', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'string', + }, + { + name: 'tag_name', + sql: 'tag_name', + type: 'string', + }, + ], +}; describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { jest.setTimeout(1000000); beforeAll(async () => { @@ -92,68 +120,66 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { await duckdbExec(INPUT_DATA_QUERY); await duckdbExec(CREATE_RESOLUTION_TABLE); await duckdbExec(RESOLUTION_DATA_QUERY); + await duckdbExec(CREATE_TAGS_LOOKUP_TABLE); }); - it('Should add row_id and unnest array fields that need resolution', async () => { - const query: Query = { - measures: ['tickets.count'], - dimensions: ['tickets.id', 'tickets.owners'], - }; - - const resolutionConfig: ResolutionConfig = { - columnConfigs: [ - { - name: 'tickets.owners', - isArrayType: true, - source: 'owners_lookup', - joinColumn: 'id', - resolutionColumns: ['display_name', 'email'], - }, - ], - tableSchemas: [OWNERS_LOOKUP_SCHEMA], - }; - - const sql = await cubeQueryToSQLWithResolutionWithArray({ - query, - tableSchemas: [TICKETS_TABLE_SCHEMA], - resolutionConfig, - }); - - console.log('Phase 1 SQL (with row_id and unnest):', sql); - - // Verify the SQL includes row_id - expect(sql).toContain('row_id'); - - // Verify the SQL unnests the owners array - expect(sql).toContain('unnest'); - - // Verify it includes the owners dimension - expect(sql).toContain('owners'); - - // Execute the SQL to verify it works - const result = (await duckdbExec(sql)) as any[]; - // console.log('Phase 1 Result:', JSON.stringify(result, null, 2)); - - // The result should have unnested rows (more rows than the original 3) - // Original: 3 rows, but ticket 1 has 2 owners, ticket 2 has 2 owners, ticket 3 has 1 owner - // Expected: 5 unnested rows - expect(result.length).toBe(5); - - // Each row should have a row_id - expect(result[0]).toHaveProperty('__base_query_with_row_id__row_id'); - expect(result[0]).toHaveProperty( - '__base_query_with_row_id__tickets__count' - ); - expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__id'); - expect(result[0]).toHaveProperty( - '__base_query_with_row_id__tickets__owners' - ); - - debugger; - // Verify row_ids are preserved (rows with same original row should have same row_id) - const rowIds = result.map((r) => r.row_id); - expect(rowIds.length).toBe(5); // 5 unnested rows total - }); + // it('Should add row_id and unnest array fields that need resolution', async () => { + // const query: Query = { + // measures: ['tickets.count'], + // dimensions: ['tickets.id', 'tickets.owners'], + // }; + + // const resolutionConfig: ResolutionConfig = { + // columnConfigs: [ + // { + // name: 'tickets.owners', + // isArrayType: true, + // source: 'owners_lookup', + // joinColumn: 'id', + // resolutionColumns: ['display_name', 'email'], + // }, + // ], + // tableSchemas: [OWNERS_LOOKUP_SCHEMA], + // }; + + // debugger; + // const sql = await cubeQueryToSQLWithResolutionWithArray({ + // query, + // tableSchemas: [TICKETS_TABLE_SCHEMA], + // resolutionConfig, + // }); + + // console.log('Phase 1 SQL (with row_id and unnest):', sql); + + // // Verify the SQL includes row_id + // expect(sql).toContain('row_id'); + + // // Verify the SQL unnests the owners array + // expect(sql).toContain('unnest'); + + // // Verify it includes the owners dimension + // expect(sql).toContain('owners'); + + // // Execute the SQL to verify it works + // const result = (await duckdbExec(sql)) as any[]; + // // console.log('Phase 1 Result:', JSON.stringify(result, null, 2)); + + // // The result should have unnested rows (more rows than the original 3) + // // Original: 3 rows, but ticket 1 has 2 owners, ticket 2 has 2 owners, ticket 3 has 1 owner + // // Expected: 5 unnested rows + // expect(result.length).toBe(5); + + // // Each row should have a row_id + // expect(result[0]).toHaveProperty('__row_id'); + // expect(result[0]).toHaveProperty('tickets__count'); + // expect(result[0]).toHaveProperty('tickets__id'); + // expect(result[0]).toHaveProperty('tickets__owners'); + + // debugger; + // // Verify row_ids are preserved (rows with same original row should have same row_id) + // const rowIds = result.map((r) => r.row_id); + // expect(rowIds.length).toBe(5); // 5 unnested rows total + // }); it('Should handle multiple array fields that need unnesting', async () => { const query: Query = { @@ -178,7 +204,7 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { resolutionColumns: ['tag_name'], }, ], - tableSchemas: [OWNERS_LOOKUP_SCHEMA], + tableSchemas: [OWNERS_LOOKUP_SCHEMA, TAGS_LOOKUP_SCHEMA], }; const sql = await cubeQueryToSQLWithResolutionWithArray({ @@ -201,38 +227,102 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { expect(result.length).toBe(7); // Each row should have a row_id - expect(result[0]).toHaveProperty('__base_query_with_row_id__row_id'); - expect(result[0]).toHaveProperty( - '__base_query_with_row_id__tickets__count' - ); - expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__id'); - expect(result[0]).toHaveProperty( - '__base_query_with_row_id__tickets__owners' - ); - expect(result[0]).toHaveProperty('__base_query_with_row_id__tickets__tags'); + expect(result[0]).toHaveProperty('__row_id'); + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('tickets__id'); + expect(result[0]).toHaveProperty('tickets__owners - display_name'); + expect(result[0]).toHaveProperty('tickets__tags - tag_name'); }); - it('Should return regular SQL when no array fields need resolution', async () => { - const query: Query = { - measures: ['tickets.count'], - dimensions: ['tickets.id', 'tickets.created_by'], - }; - - const resolutionConfig: ResolutionConfig = { - columnConfigs: [], - tableSchemas: [], - }; - - const sql = await cubeQueryToSQLWithResolutionWithArray({ - query, - tableSchemas: [TICKETS_TABLE_SCHEMA], - resolutionConfig, - }); - - console.log('SQL without resolution:', sql); - - // Should not have row_id or unnest when no array resolution is needed - expect(sql).not.toContain('row_id'); - expect(sql).not.toContain('unnest'); - }); + // it('Should return regular SQL when no array fields need resolution', async () => { + // const query: Query = { + // measures: ['tickets.count'], + // dimensions: ['tickets.id', 'tickets.created_by'], + // }; + + // const resolutionConfig: ResolutionConfig = { + // columnConfigs: [], + // tableSchemas: [], + // }; + + // const sql = await cubeQueryToSQLWithResolutionWithArray({ + // query, + // tableSchemas: [TICKETS_TABLE_SCHEMA], + // resolutionConfig, + // }); + + // console.log('SQL without resolution:', sql); + + // // Should not have row_id or unnest when no array resolution is needed + // expect(sql).not.toContain('row_id'); + // expect(sql).not.toContain('unnest'); + // }); }); + +// describe('cubeQueryToSQLWithResolutionWithArray - Phase 2: Resolution', () => { +// jest.setTimeout(1000000); + +// beforeAll(async () => { +// // Tables are already created in Phase 1 tests +// }); + +// it('Should resolve array values by joining with lookup tables', async () => { +// const query: Query = { +// measures: ['tickets.count'], +// dimensions: ['tickets.id', 'tickets.owners'], +// }; + +// const resolutionConfig: ResolutionConfig = { +// columnConfigs: [ +// { +// name: 'tickets.owners', +// isArrayType: true, +// source: 'owners_lookup', +// joinColumn: 'id', +// resolutionColumns: ['display_name', 'email'], +// }, +// ], +// tableSchemas: [OWNERS_LOOKUP_SCHEMA], +// }; + +// const sql = await cubeQueryToSQLWithResolutionWithArray({ +// query, +// tableSchemas: [TICKETS_TABLE_SCHEMA], +// resolutionConfig, +// }); + +// console.log('Phase 2 SQL (with resolution):', sql); + +// // Verify the SQL includes row_id +// expect(sql).toContain('row_id'); + +// // Verify the SQL has joins to resolution tables +// expect(sql).toContain('owners_lookup'); + +// // Verify it includes resolution columns +// expect(sql).toContain('display_name'); +// expect(sql).toContain('email'); + +// // Execute the SQL to verify it works +// const result = (await duckdbExec(sql)) as any[]; +// console.log( +// 'Phase 2 Result (first 3 rows):', +// JSON.stringify(result.slice(0, 3), null, 2) +// ); + +// // The result should have 5 unnested + resolved rows +// expect(result.length).toBe(5); + +// // Each row should have resolution columns +// expect(result[0]).toHaveProperty('__unnested_base_query__row_id'); +// expect(result[0]).toHaveProperty('tickets__owners__display_name'); +// expect(result[0]).toHaveProperty('tickets__owners__email'); + +// // Verify actual resolved values +// const owner1Rows = result.filter( +// (r: any) => r['__unnested_base_query__tickets__owners'] === 'owner1' +// ); +// expect(owner1Rows[0]['tickets__owners__display_name']).toBe('Alice Smith'); +// expect(owner1Rows[0]['tickets__owners__email']).toBe('alice@example.com'); +// }); +// }); diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 2b04f61a..e1106181 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -1,6 +1,7 @@ import { ContextParams, createBaseTableSchema, + Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, @@ -86,7 +87,49 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ columnProjections, contextParams, }: CubeQueryToSQLWithResolutionParams) => { - debugger; + // Phase 1: Generate SQL with row_id and unnested arrays + const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, + }); + + // Get array columns for Phase 2 + const arrayColumns = getArrayTypeColumns(resolutionConfig); + + if (arrayColumns.length === 0) { + // No resolution needed + return unnestBaseSql; + } + + // // // Phase 2: Apply resolution (join with lookup tables) + const resolvedSql = await getResolvedSql({ + unnestedSql: unnestBaseSql, + baseTableSchema, + query, + tableSchemas, + resolutionConfig, + contextParams, + columnProjections, + }); + + // TODO: Phase 3 - Re-aggregate to reverse the unnest + + return resolvedSql; +}; + +export const getUnnestBaseSql = async ({ + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, +}: CubeQueryToSQLWithResolutionParams): Promise<{ + sql: string; + baseTableSchema: TableSchema; +}> => { if (resolutionConfig.columnConfigs.length === 0) { // If no resolution is needed, return the base SQL. const baseSql = await cubeQueryToSQL({ @@ -94,7 +137,11 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ tableSchemas, contextParams, }); - return baseSql; + // Return a dummy schema since we won't use it + return { + sql: baseSql, + baseTableSchema: { name: '', sql: '', measures: [], dimensions: [] }, + }; } // Phase 1: Setup and Unnest @@ -107,7 +154,11 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ tableSchemas, contextParams, }); - return baseSql; + // Return a dummy schema since we won't use it + return { + sql: baseSql, + baseTableSchema: { name: '', sql: '', measures: [], dimensions: [] }, + }; } // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) @@ -157,53 +208,22 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ // Step 2: Create a new table schema from the base SQL with row_id // This will be used to apply unnesting - const baseTableName = '__base_query_with_row_id'; - - // Create dimensions from the base SQL output (all measures become dimensions now) - const baseDimensions: TableSchema['dimensions'] = [ - { - name: 'row_id', - sql: `${baseTableName}.__row_id`, - type: 'number', - }, - ...query.measures.map((measure) => ({ - name: measure.replace('.', '__'), - sql: `${baseTableName}.${measure.replace('.', '__')}`, - type: 'number' as const, - })), - ...(query.dimensions || []).map((dimension) => { - // Check if this dimension is an array that needs unnesting - const isArrayDimension = arrayColumns.some( - (col) => col.name === dimension - ); - - const originalSchema = tableSchemas.find((s) => - dimension.startsWith(`${s.name}.`) - ); - const dimName = dimension.split('.')[1]; - const originalDimension = originalSchema?.dimensions.find( - (d) => d.name === dimName - ); - - return { - name: dimension.replace('.', '__'), - sql: `${baseTableName}.${dimension.replace('.', '__')}`, - type: originalDimension?.type || ('string' as const), - ...(isArrayDimension && { - modifier: { - shouldUnnestArray: true, - }, - }), - }; - }), - ]; - - const baseTableSchema: TableSchema = { - name: baseTableName, - sql: baseSqlWithRowId, - measures: [], - dimensions: baseDimensions, - }; + const baseTableName = '__base_query'; // Use standard name to work with helpers + + const baseTableSchema: TableSchema = createBaseTableSchema( + baseSqlWithRowId, + tableSchemas, + resolutionConfig, + query.measures, + query.dimensions + ); + + baseTableSchema.dimensions.push({ + name: 'row_id', + sql: '__row_id', + type: 'number', + alias: '__row_id', + } as Dimension); // Step 3: Create query with unnest modifiers applied const unnestQuery: Query = { @@ -215,10 +235,6 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ (d) => `${baseTableName}.${d.replace('.', '__')}` ), ], - filters: query.filters, - order: query.order, - limit: query.limit, - offset: query.offset, }; // Generate the final SQL with unnesting applied @@ -228,5 +244,83 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ contextParams, }); - return unnestedBaseSql; + return { + sql: unnestedBaseSql, + baseTableSchema, + }; +}; + +/** + * Phase 2: Apply resolution (join with lookup tables) + * + * This function: + * 1. Uses the base table schema from Phase 1 (source of truth) + * 2. Generates resolution schemas for array fields + * 3. Sets up join paths between unnested data and resolution tables + * 4. Generates SQL with resolved values + * + * @param unnestedSql - SQL output from Phase 1 (with row_id and unnested arrays) + * @param baseTableSchema - Schema from Phase 1 that describes the unnested SQL + * @returns SQL with row_id, unnested arrays, and resolved values from lookup tables + */ +export const getResolvedSql = async ({ + unnestedSql, + baseTableSchema, + query, + tableSchemas, + resolutionConfig, + contextParams, + columnProjections, +}: { + unnestedSql: string; + baseTableSchema: TableSchema; + query: Query; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; + columnProjections?: string[]; +}): Promise => { + // Step 1: Use the base table schema from Phase 1 as source of truth + // Update the SQL to point to the unnested SQL + const updatedBaseTableSchema: TableSchema = { + ...baseTableSchema, + sql: unnestedSql, + }; + + debugger; + // Step 2: Generate resolution schemas for array fields only + // Use the existing generateResolutionSchemas helper + + const resolutionSchemas = generateResolutionSchemas( + resolutionConfig, + tableSchemas + ); + + // Step 3: Generate join paths using existing helper + // Note: Pass the base table schema (from Phase 1) to generate correct join paths + const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); + + debugger; + query.dimensions?.push('row_id'); + // Step 4: Generate resolved dimensions using existing helper + const resolvedDimensions = generateResolvedDimensions( + query, + resolutionConfig, + columnProjections + ); + + // Step 5: Create query and generate SQL + const resolutionQuery: Query = { + measures: [], + dimensions: resolvedDimensions, + joinPaths, + }; + + const resolvedSql = await cubeQueryToSQL({ + query: resolutionQuery, + tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], + contextParams, + }); + + return resolvedSql; }; From c1f0d408e7926e81d682ee3b5c9102e29cd0e505 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 29 Oct 2025 19:54:59 +0530 Subject: [PATCH 03/38] phase 3 complete and working --- .../cube-to-sql-with-resolution-array.spec.ts | 75 +++++++ .../cube-to-sql-with-resolution.ts | 202 +++++++++++++++++- 2 files changed, 272 insertions(+), 5 deletions(-) diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index 0333412e..3e62a673 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -326,3 +326,78 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { // expect(owner1Rows[0]['tickets__owners__email']).toBe('alice@example.com'); // }); // }); + +// describe('cubeQueryToSQLWithResolutionWithArray - Phase 3: Re-aggregation', () => { +// jest.setTimeout(1000000); + +// beforeAll(async () => { +// // Tables are already created in Phase 1 tests +// }); + +// it('Should re-aggregate unnested rows back to original count with resolved arrays', async () => { +// const query: Query = { +// measures: ['tickets.count'], +// dimensions: ['tickets.id', 'tickets.owners'], +// }; + +// const resolutionConfig: ResolutionConfig = { +// columnConfigs: [ +// { +// name: 'tickets.owners', +// isArrayType: true, +// source: 'owners_lookup', +// joinColumn: 'id', +// resolutionColumns: ['display_name', 'email'], +// }, +// ], +// tableSchemas: [OWNERS_LOOKUP_SCHEMA], +// }; + +// const sql = await cubeQueryToSQLWithResolutionWithArray({ +// query, +// tableSchemas: [TICKETS_TABLE_SCHEMA], +// resolutionConfig, +// }); + +// console.log('Phase 3 SQL (re-aggregated):', sql); + +// // Verify the SQL includes aggregation functions +// expect(sql).toContain('GROUP BY'); +// expect(sql).toContain('row_id'); + +// // Execute the SQL to verify it works +// const result = (await duckdbExec(sql)) as any[]; +// console.log('Phase 3 Result:', JSON.stringify(result, null, 2)); + +// // Should have 3 rows (back to original count) +// expect(result.length).toBe(3); + +// // Each row should have aggregated data +// expect(result[0]).toHaveProperty('__aggregation_base__row_id'); +// expect(result[0]).toHaveProperty('__aggregation_base__tickets__count'); +// expect(result[0]).toHaveProperty('__aggregation_base__tickets__id'); + +// // Should have arrays with resolved values +// expect(result[0]).toHaveProperty( +// '__aggregation_base__tickets__owners__display_name' +// ); +// expect(result[0]).toHaveProperty( +// '__aggregation_base__tickets__owners__email' +// ); + +// // Verify the resolved arrays contain the correct values +// // Ticket 1 has owners: owner1, owner2 +// const ticket1 = result.find( +// (r: any) => r['__aggregation_base__tickets__id'] === 1 +// ); +// expect(ticket1).toBeDefined(); + +// // The display names should be aggregated into an array +// const displayNames = +// ticket1['__aggregation_base__tickets__owners__display_name']; +// console.log('Ticket 1 display names:', displayNames); + +// // Note: The actual format depends on how cubeQueryToSQL handles ARRAY_AGG +// // It should be an array containing the resolved values +// }); +// }); diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index e1106181..b68c2340 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -105,7 +105,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ } // // // Phase 2: Apply resolution (join with lookup tables) - const resolvedSql = await getResolvedSql({ + const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ unnestedSql: unnestBaseSql, baseTableSchema, query, @@ -115,9 +115,16 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ columnProjections, }); - // TODO: Phase 3 - Re-aggregate to reverse the unnest + // Phase 3: Re-aggregate to reverse the unnest + const aggregatedSql = await getAggregatedSql({ + resolvedSql, + resolvedTableSchema, + query, + resolutionConfig, + contextParams, + }); - return resolvedSql; + return aggregatedSql; }; export const getUnnestBaseSql = async ({ @@ -279,7 +286,10 @@ export const getResolvedSql = async ({ resolutionConfig: ResolutionConfig; contextParams?: ContextParams; columnProjections?: string[]; -}): Promise => { +}): Promise<{ + sql: string; + resolvedTableSchema: TableSchema; +}> => { // Step 1: Use the base table schema from Phase 1 as source of truth // Update the SQL to point to the unnested SQL const updatedBaseTableSchema: TableSchema = { @@ -322,5 +332,187 @@ export const getResolvedSql = async ({ contextParams, }); - return resolvedSql; + // Create a simple schema describing Phase 2's output columns + // cubeQueryToSQL outputs columns using their alias field, not table-prefixed names + const resolvedTableSchema: TableSchema = { + name: '__resolved_query', + sql: resolvedSql, + measures: [], + dimensions: [ + // Row ID - comes from baseTableSchema with alias '__row_id' + { + name: 'row_id', + sql: '__resolved_query."__row_id"', + type: 'number', + alias: '__row_id', + }, + // Original measures from base query - use their aliases + ...query.measures.map((measure) => { + const measureName = measure.replace('.', '__'); + return { + name: measureName, + sql: `__resolved_query."${measureName}"`, + type: 'number' as const, + alias: measureName, + }; + }), + // Non-array dimensions from base query - use their aliases + ...(query.dimensions || []) + .filter( + (dim) => + !getArrayTypeColumns(resolutionConfig).some( + (ac) => ac.name === dim + ) && dim !== 'row_id' + ) + .map((dimension) => { + const dimName = dimension.replace('.', '__'); + return { + name: dimName, + sql: `__resolved_query."${dimName}"`, + type: 'string' as const, + alias: dimName, + }; + }), + // Resolved array columns - these use the alias pattern "colname - rescolname" + ...getArrayTypeColumns(resolutionConfig).flatMap((arrayColumn) => { + return arrayColumn.resolutionColumns.map((resCol) => { + // Find the resolution dimension to get its alias + const resolutionDimName = `${arrayColumn.name.replace( + '.', + '__' + )}__${resCol}`; + const resolutionSchema = resolutionSchemas.find( + (s) => s.name === arrayColumn.name.replace('.', '__') + ); + const resDim = resolutionSchema?.dimensions.find((d) => + d.name.includes(resCol) + ); + + // The alias from the resolution schema already has the full format + const aliasName = resDim?.alias || resolutionDimName; + + return { + name: resolutionDimName, + sql: `__resolved_query."${aliasName}"`, + type: 'string' as const, + alias: aliasName, + }; + }); + }), + ], + }; + + return { + sql: resolvedSql, + resolvedTableSchema, + }; +}; + +/** + * Phase 3: Re-aggregate to reverse the unnest + * + * This function: + * 1. Wraps Phase 2 SQL as a new base table + * 2. Groups by row_id + * 3. Uses MAX for non-array columns (they're duplicated) + * 4. Uses ARRAY_AGG for resolved array columns + * + * @param resolvedSql - SQL output from Phase 2 (with resolved values) + * @param query - Original query + * @param resolutionConfig - Resolution configuration + * @param contextParams - Optional context parameters + * @returns Final SQL with arrays containing resolved values + */ +export const getAggregatedSql = async ({ + resolvedSql, + resolvedTableSchema, + query, + resolutionConfig, + contextParams, +}: { + resolvedSql: string; + resolvedTableSchema: TableSchema; + query: Query; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + // Step 1: Use the resolved table schema from Phase 2 as source of truth + // Update the SQL to point to the resolved SQL + const aggregationBaseTableSchema: TableSchema = { + ...resolvedTableSchema, + sql: resolvedSql, + }; + + // Step 2: Identify which columns need ARRAY_AGG vs MAX + const arrayColumns = getArrayTypeColumns(resolutionConfig); + const baseTableName = aggregationBaseTableSchema.name; + + // Helper to check if a dimension is a resolved array column + const isResolvedArrayColumn = (dimName: string) => { + // Check if the dimension name matches a resolved column pattern + // e.g., "tickets__owners__display_name" + return arrayColumns.some((arrayCol) => { + const arrayColPrefix = arrayCol.name.replace('.', '__'); + return ( + dimName.includes(`${arrayColPrefix}__`) && + !dimName.startsWith('__base_query__') + ); + }); + }; + + // Step 3: Create aggregation measures with proper aggregation functions + // Get row_id dimension for GROUP BY + const rowIdDimension = aggregationBaseTableSchema.dimensions.find( + (d) => d.name === 'row_id' || d.name.endsWith('__row_id') + ); + + // Create measures with MAX or ARRAY_AGG based on column type + const aggregationMeasures: TableSchema['measures'] = []; + + aggregationBaseTableSchema.dimensions + .filter((dim) => dim.name !== rowIdDimension?.name) + .forEach((dim) => { + const isArrayColumn = isResolvedArrayColumn(dim.name); + + // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") + // We just need to wrap it in the aggregation function + const columnRef = + dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; + + // Use ARRAY_AGG for resolved array columns, MAX for others + const aggregationFn = isArrayColumn + ? `ARRAY_AGG(DISTINCT ${columnRef})` + : `MAX(${columnRef})`; + + aggregationMeasures.push({ + name: dim.name, + sql: aggregationFn, + type: dim.type, + alias: dim.alias, + }); + }); + + // Update the schema with aggregation measures + const schemaWithAggregation: TableSchema = { + ...aggregationBaseTableSchema, + measures: aggregationMeasures, + dimensions: rowIdDimension ? [rowIdDimension] : [], + }; + + // Step 4: Create the aggregation query + const aggregationQuery: Query = { + measures: aggregationMeasures.map((m) => `${baseTableName}.${m.name}`), + dimensions: rowIdDimension + ? [`${baseTableName}.${rowIdDimension.name}`] + : [], + }; + + // Step 5: Generate the final SQL + const aggregatedSql = await cubeQueryToSQL({ + query: aggregationQuery, + tableSchemas: [schemaWithAggregation], + contextParams, + }); + + return aggregatedSql; }; From 0396184a06ddbbd14492148989ebd56e7d229658 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 08:06:24 +0530 Subject: [PATCH 04/38] working with array + scalar resolution --- .../cube-to-sql-with-resolution-array.spec.ts | 50 +++++++++++++++++-- .../cube-to-sql-with-resolution.ts | 21 ++++---- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index 3e62a673..e1e546de 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -31,7 +31,14 @@ const CREATE_TAGS_LOOKUP_TABLE = `CREATE TABLE tags_lookup ( id VARCHAR, tag_name VARCHAR )`; - +const CREATE_CREATED_BY_LOOKUP_TABLE = `CREATE TABLE created_by_lookup ( + id VARCHAR, + name VARCHAR +)`; +const CREATED_BY_LOOKUP_DATA_QUERY = `INSERT INTO created_by_lookup VALUES +('user1', 'User 1'), +('user2', 'User 2'), +('user3', 'User 3')`; const TAGS_LOOKUP_DATA_QUERY = `INSERT INTO tags_lookup VALUES ('tag1', 'Tag 1'), ('tag2', 'Tag 2'), @@ -112,6 +119,24 @@ const TAGS_LOOKUP_SCHEMA: TableSchema = { }, ], }; + +const CREATED_BY_LOOKUP_SCHEMA: TableSchema = { + name: 'created_by_lookup', + sql: 'select * from created_by_lookup', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'id', + type: 'string', + }, + { + name: 'name', + sql: 'name', + type: 'string', + }, + ], +}; describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { jest.setTimeout(1000000); beforeAll(async () => { @@ -121,6 +146,9 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { await duckdbExec(CREATE_RESOLUTION_TABLE); await duckdbExec(RESOLUTION_DATA_QUERY); await duckdbExec(CREATE_TAGS_LOOKUP_TABLE); + await duckdbExec(TAGS_LOOKUP_DATA_QUERY); + await duckdbExec(CREATE_CREATED_BY_LOOKUP_TABLE); + await duckdbExec(CREATED_BY_LOOKUP_DATA_QUERY); }); // it('Should add row_id and unnest array fields that need resolution', async () => { @@ -184,7 +212,12 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { it('Should handle multiple array fields that need unnesting', async () => { const query: Query = { measures: ['tickets.count'], - dimensions: ['tickets.id', 'tickets.owners', 'tickets.tags'], + dimensions: [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + ], }; const resolutionConfig: ResolutionConfig = { @@ -203,8 +236,19 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { joinColumn: 'id', resolutionColumns: ['tag_name'], }, + { + name: 'tickets.created_by', + isArrayType: false, + source: 'created_by_lookup', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [ + OWNERS_LOOKUP_SCHEMA, + TAGS_LOOKUP_SCHEMA, + CREATED_BY_LOOKUP_SCHEMA, ], - tableSchemas: [OWNERS_LOOKUP_SCHEMA, TAGS_LOOKUP_SCHEMA], }; const sql = await cubeQueryToSQLWithResolutionWithArray({ diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index b68c2340..03b3a004 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -92,7 +92,6 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ query, tableSchemas, resolutionConfig, - columnProjections, contextParams, }); @@ -131,7 +130,6 @@ export const getUnnestBaseSql = async ({ query, tableSchemas, resolutionConfig, - columnProjections, contextParams, }: CubeQueryToSQLWithResolutionParams): Promise<{ sql: string; @@ -310,7 +308,6 @@ export const getResolvedSql = async ({ // Note: Pass the base table schema (from Phase 1) to generate correct join paths const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); - debugger; query.dimensions?.push('row_id'); // Step 4: Generate resolved dimensions using existing helper const resolvedDimensions = generateResolvedDimensions( @@ -360,9 +357,8 @@ export const getResolvedSql = async ({ ...(query.dimensions || []) .filter( (dim) => - !getArrayTypeColumns(resolutionConfig).some( - (ac) => ac.name === dim - ) && dim !== 'row_id' + !resolutionConfig.columnConfigs.some((ac) => ac.name === dim) && + dim !== 'row_id' ) .map((dimension) => { const dimName = dimension.replace('.', '__'); @@ -373,16 +369,16 @@ export const getResolvedSql = async ({ alias: dimName, }; }), - // Resolved array columns - these use the alias pattern "colname - rescolname" - ...getArrayTypeColumns(resolutionConfig).flatMap((arrayColumn) => { - return arrayColumn.resolutionColumns.map((resCol) => { + // ALL resolved columns (both array and scalar) - these use the alias pattern "colname - rescolname" + ...resolutionConfig.columnConfigs.flatMap((columnConfig) => { + return columnConfig.resolutionColumns.map((resCol) => { // Find the resolution dimension to get its alias - const resolutionDimName = `${arrayColumn.name.replace( + const resolutionDimName = `${columnConfig.name.replace( '.', '__' )}__${resCol}`; const resolutionSchema = resolutionSchemas.find( - (s) => s.name === arrayColumn.name.replace('.', '__') + (s) => s.name === columnConfig.name.replace('.', '__') ); const resDim = resolutionSchema?.dimensions.find((d) => d.name.includes(resCol) @@ -514,5 +510,6 @@ export const getAggregatedSql = async ({ contextParams, }); - return aggregatedSql; + const rowIdExcludedSql = `select * exclude(__row_id) from (${aggregatedSql})`; + return rowIdExcludedSql; }; From 11f30ecddbd9cb9717b91ff4793107f3a27574fc Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 08:17:14 +0530 Subject: [PATCH 05/38] working with only a scalar resolution field --- .../cube-to-sql-with-resolution-array.spec.ts | 84 +++++++++++++++---- .../cube-to-sql-with-resolution.ts | 43 ++-------- 2 files changed, 74 insertions(+), 53 deletions(-) diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index e1e546de..00da125b 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -222,20 +222,20 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { const resolutionConfig: ResolutionConfig = { columnConfigs: [ - { - name: 'tickets.owners', - isArrayType: true, - source: 'owners_lookup', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, - { - name: 'tickets.tags', - isArrayType: true, - source: 'tags_lookup', - joinColumn: 'id', - resolutionColumns: ['tag_name'], - }, + // { + // name: 'tickets.owners', + // isArrayType: true, + // source: 'owners_lookup', + // joinColumn: 'id', + // resolutionColumns: ['display_name'], + // }, + // { + // name: 'tickets.tags', + // isArrayType: true, + // source: 'tags_lookup', + // joinColumn: 'id', + // resolutionColumns: ['tag_name'], + // }, { name: 'tickets.created_by', isArrayType: false, @@ -245,8 +245,8 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { }, ], tableSchemas: [ - OWNERS_LOOKUP_SCHEMA, - TAGS_LOOKUP_SCHEMA, + // OWNERS_LOOKUP_SCHEMA, + // TAGS_LOOKUP_SCHEMA, CREATED_BY_LOOKUP_SCHEMA, ], }; @@ -278,6 +278,58 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { expect(result[0]).toHaveProperty('tickets__tags - tag_name'); }); + it('Should handle only scalar field resolution without unnesting', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + ], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.created_by', + isArrayType: false, + source: 'created_by_lookup', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], + }; + + const sql = await cubeQueryToSQLWithResolutionWithArray({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + }); + + console.log('Phase 1 SQL (multiple arrays):', sql); + + // Verify row_id is included + expect(sql).toContain('row_id'); + + // Both arrays should be unnested + expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(0); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + + expect(result.length).toBe(7); + + // Each row should have a row_id + expect(result[0]).toHaveProperty('__row_id'); + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('tickets__id'); + expect(result[0]).toHaveProperty('tickets__owners'); + expect(result[0]).toHaveProperty('tickets__tags'); + expect(result[0]).toHaveProperty('tickets__created_by - name'); + }); + // it('Should return regular SQL when no array fields need resolution', async () => { // const query: Query = { // measures: ['tickets.count'], diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 03b3a004..1bc2b532 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -95,13 +95,13 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ contextParams, }); - // Get array columns for Phase 2 - const arrayColumns = getArrayTypeColumns(resolutionConfig); + // // Get array columns for Phase 2 + // const arrayColumns = getArrayTypeColumns(resolutionConfig); - if (arrayColumns.length === 0) { - // No resolution needed - return unnestBaseSql; - } + // if (arrayColumns.length === 0) { + // // No resolution needed + // return unnestBaseSql; + // } // // // Phase 2: Apply resolution (join with lookup tables) const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ @@ -135,37 +135,6 @@ export const getUnnestBaseSql = async ({ sql: string; baseTableSchema: TableSchema; }> => { - if (resolutionConfig.columnConfigs.length === 0) { - // If no resolution is needed, return the base SQL. - const baseSql = await cubeQueryToSQL({ - query, - tableSchemas, - contextParams, - }); - // Return a dummy schema since we won't use it - return { - sql: baseSql, - baseTableSchema: { name: '', sql: '', measures: [], dimensions: [] }, - }; - } - - // Phase 1: Setup and Unnest - const arrayColumns = getArrayTypeColumns(resolutionConfig); - - if (arrayColumns.length === 0) { - // No array columns to process, return base SQL - const baseSql = await cubeQueryToSQL({ - query, - tableSchemas, - contextParams, - }); - // Return a dummy schema since we won't use it - return { - sql: baseSql, - baseTableSchema: { name: '', sql: '', measures: [], dimensions: [] }, - }; - } - // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) const modifiedTableSchemasWithRowId = tableSchemas.map((schema, index) => { // Add row_id to the first table only From c24c8a105133987751a99f2551342cab2ecb3af8 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 16:27:02 +0530 Subject: [PATCH 06/38] updating in meerkat browser --- .../browser-cube-to-sql-with-resolution.ts | 418 +++++ .../src/resolution/resolution.spec.ts | 1602 ++++++++--------- .../cube-to-sql-with-resolution-array.spec.ts | 124 +- .../cube-to-sql-with-resolution.ts | 8 - 4 files changed, 1198 insertions(+), 954 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index c06d8a75..99a902dc 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -1,6 +1,7 @@ import { ContextParams, createBaseTableSchema, + Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, @@ -23,6 +24,15 @@ export interface CubeQueryToSQLWithResolutionParams { contextParams?: ContextParams; } +/** + * Helper function to get array-type columns from resolution config + */ +const getArrayTypeColumns = (resolutionConfig: ResolutionConfig) => { + return resolutionConfig.columnConfigs.filter( + (config) => config.isArrayType === true + ); +}; + export const cubeQueryToSQLWithResolution = async ({ connection, query, @@ -74,3 +84,411 @@ export const cubeQueryToSQLWithResolution = async ({ return sql; }; + +export const cubeQueryToSQLWithResolutionWithArray = async ({ + connection, + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, +}: CubeQueryToSQLWithResolutionParams) => { + // Phase 1: Generate SQL with row_id and unnested arrays + const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ + connection, + query, + tableSchemas, + resolutionConfig, + contextParams, + }); + + // // // Phase 2: Apply resolution (join with lookup tables) + const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ + connection, + unnestedSql: unnestBaseSql, + baseTableSchema, + query, + tableSchemas, + resolutionConfig, + contextParams, + columnProjections, + }); + + // Phase 3: Re-aggregate to reverse the unnest + const aggregatedSql = await getAggregatedSql({ + connection, + resolvedSql, + resolvedTableSchema, + query, + resolutionConfig, + contextParams, + }); + + return aggregatedSql; +}; + +export const getUnnestBaseSql = async ({ + connection, + query, + tableSchemas, + resolutionConfig, + contextParams, +}: CubeQueryToSQLWithResolutionParams): Promise<{ + sql: string; + baseTableSchema: TableSchema; +}> => { + // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) + const modifiedTableSchemasWithRowId = tableSchemas.map((schema, index) => { + // Add row_id to the first table only + if (index !== 0) { + return schema; + } + + // Add row_id dimension (no unnest modifier yet) + // TODO: Will this cause a problem of adding row_id to the first schema ? + const newDimensions = [ + { + name: 'row_id', + sql: 'row_number() OVER ()', + type: 'number' as const, + alias: '__row_id', + }, + ...schema.dimensions, + ]; + + return { + ...schema, + dimensions: newDimensions, + }; + }); + + // Use the first table for row_id reference + const firstTable = tableSchemas[0]; + + const queryWithRowId: Query = { + measures: query.measures, + dimensions: [`${firstTable.name}.row_id`, ...(query.dimensions || [])], + joinPaths: query.joinPaths, + filters: query.filters, + order: query.order, + limit: query.limit, + offset: query.offset, + }; + + // Generate base SQL with row_id + const baseSqlWithRowId = await cubeQueryToSQL({ + connection, + query: queryWithRowId, + tableSchemas: modifiedTableSchemasWithRowId, + contextParams, + }); + + // Step 2: Create a new table schema from the base SQL with row_id + // This will be used to apply unnesting + const baseTableName = '__base_query'; // Use standard name to work with helpers + + const baseTableSchema: TableSchema = createBaseTableSchema( + baseSqlWithRowId, + tableSchemas, + resolutionConfig, + query.measures, + query.dimensions + ); + + baseTableSchema.dimensions.push({ + name: 'row_id', + sql: '__row_id', + type: 'number', + alias: '__row_id', + } as Dimension); + + // Step 3: Create query with unnest modifiers applied + const unnestQuery: Query = { + measures: [], + dimensions: [ + `${baseTableName}.row_id`, + ...query.measures.map((m) => `${baseTableName}.${m.replace('.', '__')}`), + ...(query.dimensions || []).map( + (d) => `${baseTableName}.${d.replace('.', '__')}` + ), + ], + }; + + // Generate the final SQL with unnesting applied + const unnestedBaseSql = await cubeQueryToSQL({ + connection, + query: unnestQuery, + tableSchemas: [baseTableSchema], + contextParams, + }); + + return { + sql: unnestedBaseSql, + baseTableSchema, + }; +}; + +/** + * Phase 2: Apply resolution (join with lookup tables) + * + * This function: + * 1. Uses the base table schema from Phase 1 (source of truth) + * 2. Generates resolution schemas for array fields + * 3. Sets up join paths between unnested data and resolution tables + * 4. Generates SQL with resolved values + * + * @param unnestedSql - SQL output from Phase 1 (with row_id and unnested arrays) + * @param baseTableSchema - Schema from Phase 1 that describes the unnested SQL + * @returns SQL with row_id, unnested arrays, and resolved values from lookup tables + */ +export const getResolvedSql = async ({ + connection, + unnestedSql, + baseTableSchema, + query, + tableSchemas, + resolutionConfig, + contextParams, + columnProjections, +}: { + connection: AsyncDuckDBConnection; + unnestedSql: string; + baseTableSchema: TableSchema; + query: Query; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; + columnProjections?: string[]; +}): Promise<{ + sql: string; + resolvedTableSchema: TableSchema; +}> => { + // Step 1: Use the base table schema from Phase 1 as source of truth + // Update the SQL to point to the unnested SQL + const updatedBaseTableSchema: TableSchema = { + ...baseTableSchema, + sql: unnestedSql, + }; + + debugger; + // Step 2: Generate resolution schemas for array fields only + // Use the existing generateResolutionSchemas helper + + const resolutionSchemas = generateResolutionSchemas( + resolutionConfig, + tableSchemas + ); + + // Step 3: Generate join paths using existing helper + // Note: Pass the base table schema (from Phase 1) to generate correct join paths + const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); + + query.dimensions?.push('row_id'); + // Step 4: Generate resolved dimensions using existing helper + const resolvedDimensions = generateResolvedDimensions( + query, + resolutionConfig, + columnProjections + ); + + // Step 5: Create query and generate SQL + const resolutionQuery: Query = { + measures: [], + dimensions: resolvedDimensions, + joinPaths, + }; + + const resolvedSql = await cubeQueryToSQL({ + connection, + query: resolutionQuery, + tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], + contextParams, + }); + + // Create a simple schema describing Phase 2's output columns + // cubeQueryToSQL outputs columns using their alias field, not table-prefixed names + const resolvedTableSchema: TableSchema = { + name: '__resolved_query', + sql: resolvedSql, + measures: [], + dimensions: [ + // Row ID - comes from baseTableSchema with alias '__row_id' + { + name: 'row_id', + sql: '__resolved_query."__row_id"', + type: 'number', + alias: '__row_id', + }, + // Original measures from base query - use their aliases + ...query.measures.map((measure) => { + const measureName = measure.replace('.', '__'); + return { + name: measureName, + sql: `__resolved_query."${measureName}"`, + type: 'number' as const, + alias: measureName, + }; + }), + // Non-array dimensions from base query - use their aliases + ...(query.dimensions || []) + .filter( + (dim) => + !resolutionConfig.columnConfigs.some((ac) => ac.name === dim) && + dim !== 'row_id' + ) + .map((dimension) => { + const dimName = dimension.replace('.', '__'); + return { + name: dimName, + sql: `__resolved_query."${dimName}"`, + type: 'string' as const, + alias: dimName, + }; + }), + // ALL resolved columns (both array and scalar) - these use the alias pattern "colname - rescolname" + ...resolutionConfig.columnConfigs.flatMap((columnConfig) => { + return columnConfig.resolutionColumns.map((resCol) => { + // Find the resolution dimension to get its alias + const resolutionDimName = `${columnConfig.name.replace( + '.', + '__' + )}__${resCol}`; + const resolutionSchema = resolutionSchemas.find( + (s) => s.name === columnConfig.name.replace('.', '__') + ); + const resDim = resolutionSchema?.dimensions.find((d) => + d.name.includes(resCol) + ); + + // The alias from the resolution schema already has the full format + const aliasName = resDim?.alias || resolutionDimName; + + return { + name: resolutionDimName, + sql: `__resolved_query."${aliasName}"`, + type: 'string' as const, + alias: aliasName, + }; + }); + }), + ], + }; + + return { + sql: resolvedSql, + resolvedTableSchema, + }; +}; + +/** + * Phase 3: Re-aggregate to reverse the unnest + * + * This function: + * 1. Wraps Phase 2 SQL as a new base table + * 2. Groups by row_id + * 3. Uses MAX for non-array columns (they're duplicated) + * 4. Uses ARRAY_AGG for resolved array columns + * + * @param resolvedSql - SQL output from Phase 2 (with resolved values) + * @param query - Original query + * @param resolutionConfig - Resolution configuration + * @param contextParams - Optional context parameters + * @returns Final SQL with arrays containing resolved values + */ +export const getAggregatedSql = async ({ + connection, + resolvedSql, + resolvedTableSchema, + query, + resolutionConfig, + contextParams, +}: { + connection: AsyncDuckDBConnection; + resolvedSql: string; + resolvedTableSchema: TableSchema; + query: Query; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + // Step 1: Use the resolved table schema from Phase 2 as source of truth + // Update the SQL to point to the resolved SQL + const aggregationBaseTableSchema: TableSchema = { + ...resolvedTableSchema, + sql: resolvedSql, + }; + + // Step 2: Identify which columns need ARRAY_AGG vs MAX + const arrayColumns = getArrayTypeColumns(resolutionConfig); + const baseTableName = aggregationBaseTableSchema.name; + + // Helper to check if a dimension is a resolved array column + const isResolvedArrayColumn = (dimName: string) => { + // Check if the dimension name matches a resolved column pattern + // e.g., "tickets__owners__display_name" + return arrayColumns.some((arrayCol) => { + const arrayColPrefix = arrayCol.name.replace('.', '__'); + return ( + dimName.includes(`${arrayColPrefix}__`) && + !dimName.startsWith('__base_query__') + ); + }); + }; + + // Step 3: Create aggregation measures with proper aggregation functions + // Get row_id dimension for GROUP BY + const rowIdDimension = aggregationBaseTableSchema.dimensions.find( + (d) => d.name === 'row_id' || d.name.endsWith('__row_id') + ); + + // Create measures with MAX or ARRAY_AGG based on column type + const aggregationMeasures: TableSchema['measures'] = []; + + aggregationBaseTableSchema.dimensions + .filter((dim) => dim.name !== rowIdDimension?.name) + .forEach((dim) => { + const isArrayColumn = isResolvedArrayColumn(dim.name); + + // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") + // We just need to wrap it in the aggregation function + const columnRef = + dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; + + // Use ARRAY_AGG for resolved array columns, MAX for others + const aggregationFn = isArrayColumn + ? `ARRAY_AGG(DISTINCT ${columnRef})` + : `MAX(${columnRef})`; + + aggregationMeasures.push({ + name: dim.name, + sql: aggregationFn, + type: dim.type, + alias: dim.alias, + }); + }); + + // Update the schema with aggregation measures + const schemaWithAggregation: TableSchema = { + ...aggregationBaseTableSchema, + measures: aggregationMeasures, + dimensions: rowIdDimension ? [rowIdDimension] : [], + }; + + // Step 4: Create the aggregation query + const aggregationQuery: Query = { + measures: aggregationMeasures.map((m) => `${baseTableName}.${m.name}`), + dimensions: rowIdDimension + ? [`${baseTableName}.${rowIdDimension.name}`] + : [], + }; + + // Step 5: Generate the final SQL + const aggregatedSql = await cubeQueryToSQL({ + connection, + query: aggregationQuery, + tableSchemas: [schemaWithAggregation], + contextParams, + }); + + const rowIdExcludedSql = `select * exclude(__row_id) from (${aggregatedSql})`; + return rowIdExcludedSql; +}; diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index 1fd11261..45170199 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -1,7 +1,13 @@ -import { cubeQueryToSQLWithResolution } from '../../../meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution'; +import { + createBaseTableSchema, + generateResolutionJoinPaths, + generateResolutionSchemas, + generateResolvedDimensions, +} from './resolution'; +import { ResolutionConfig } from './types'; + describe('Create base table schema', () => { - it('testing single field with count flow', async () => { - debugger; + it('dimensions and measures are converted to dimensions', () => { const sql = 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; const tableSchemas = [ @@ -12,27 +18,292 @@ describe('Create base table schema', () => { { name: 'count', sql: 'COUNT(*)', - type: 'number' as const, + type: 'number', + }, + ], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'string', + }, + ], + }, + ]; + const resolutionConfig: ResolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + const measures = ['base_table.count']; + const dimensions = ['base_table.column1', 'base_table.column2']; + + const baseTableSchema = createBaseTableSchema( + sql, + tableSchemas, + resolutionConfig, + measures, + dimensions + ); + + expect(baseTableSchema).toEqual({ + name: '__base_query', + sql: 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2', + measures: [], + dimensions: [ + { + name: 'base_table__count', + sql: '__base_query.base_table__count', + type: 'number', + alias: 'base_table__count', + }, + { + name: 'base_table__column1', + sql: '__base_query.base_table__column1', + type: 'string', + alias: 'base_table__column1', + }, + { + name: 'base_table__column2', + sql: '__base_query.base_table__column2', + type: 'string', + alias: 'base_table__column2', + }, + ], + joins: [], + }); + }); + + it('create join config', () => { + const sql = 'SELECT * FROM base_table'; + const tableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'string', + }, + ], + }, + ]; + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + ], + tableSchemas: [], + }; + const dimensions = ['base_table.column1', 'base_table.column2']; + + const baseTableSchema = createBaseTableSchema( + sql, + tableSchemas, + resolutionConfig, + [], + dimensions + ); + + expect(baseTableSchema).toEqual({ + name: '__base_query', + sql: 'SELECT * FROM base_table', + measures: [], + dimensions: [ + { + name: 'base_table__column1', + sql: '__base_query.base_table__column1', + type: 'string', + alias: 'base_table__column1', + }, + { + name: 'base_table__column2', + sql: '__base_query.base_table__column2', + type: 'string', + alias: 'base_table__column2', + }, + ], + joins: [ + { + sql: '__base_query.base_table__column1 = base_table__column1.id', + }, + { + sql: '__base_query.base_table__column2 = base_table__column2.id', + }, + ], + }); + }); + + it('dimension not found', () => { + const sql = 'SELECT * FROM base_table'; + const tableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', }, ], + }, + ]; + const resolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + const dimensions = ['base_table.column1', 'base_table.column2']; // column2 does not exist + + expect(() => { + createBaseTableSchema( + sql, + tableSchemas, + resolutionConfig, + [], + dimensions + ); + }).toThrow('Not found: base_table.column2'); + }); + + it('handle aliases', () => { + const sql = 'SELECT * FROM base_table'; + const tableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + alias: 'Column 1', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'string', + alias: 'Column 2', + }, + ], + }, + ]; + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + ], + tableSchemas: [], + }; + const dimensions = ['base_table.column1', 'base_table.column2']; + + const baseTableSchema = createBaseTableSchema( + sql, + tableSchemas, + resolutionConfig, + [], + dimensions + ); + + expect(baseTableSchema).toEqual({ + name: '__base_query', + sql: 'SELECT * FROM base_table', + measures: [], + dimensions: [ + { + name: 'base_table__column1', + sql: '__base_query."Column 1"', + type: 'string', + alias: 'Column 1', + }, + { + name: 'base_table__column2', + sql: '__base_query."Column 2"', + type: 'string', + alias: 'Column 2', + }, + ], + joins: [ + { + sql: '__base_query."Column 1" = base_table__column1.id', + }, + { + sql: '__base_query."Column 2" = base_table__column2.id', + }, + ], + }); + }); +}); + +describe('Generate resolution schemas', () => { + it('multiple columns using same table', () => { + const baseTableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], dimensions: [ { name: 'column1', sql: 'base_table.column1', - type: 'string' as const, + type: 'string', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'string', }, ], }, ]; + const resolutionConfig = { columnConfigs: [ { name: 'base_table.column1', - isArrayType: false, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['id', 'display_name'], + }, ], tableSchemas: [ { @@ -43,898 +314,461 @@ describe('Create base table schema', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string' as const, + type: 'string', }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string' as const, + type: 'string', }, { name: 'display_name', sql: 'resolution_table.display_name', - type: 'string' as const, + type: 'string', }, ], }, ], }; - const measures = ['base_table.count']; - const dimensions = ['base_table.column1']; - // Initialize DuckDB and create a connection - debugger; - const result = await cubeQueryToSQLWithResolution({ - query: { - measures: measures, - dimensions: dimensions, + + const schemas = generateResolutionSchemas( + resolutionConfig, + baseTableSchemas + ); + + expect(schemas).toEqual([ + { + name: 'base_table__column1', + sql: '', + measures: [], + dimensions: [ + { + name: 'base_table__column1__display_id', + sql: 'base_table__column1.display_id', + type: 'string', + alias: 'base_table__column1 - display_id', + }, + ], }, - tableSchemas: tableSchemas, - resolutionConfig: resolutionConfig, - }); - console.log('Generated SQL:', result); - expect(result).toBeDefined(); + { + name: 'base_table__column2', + sql: '', + measures: [], + dimensions: [ + { + name: 'base_table__column2__id', + sql: 'base_table__column2.id', + type: 'string', + alias: 'base_table__column2 - id', + }, + { + name: 'base_table__column2__display_name', + sql: 'base_table__column2.display_name', + type: 'string', + alias: 'base_table__column2 - display_name', + }, + ], + }, + ]); + }); + + it('table does not exist', () => { + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table1', // does not exist + joinColumn: 'id', + resolutionColumns: ['id', 'display_name'], + }, + ], + tableSchemas: [ + { + name: 'resolution_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'resolution_table.id', + type: 'string', + }, + { + name: 'display_id', + sql: 'resolution_table.display_id', + type: 'string', + }, + { + name: 'display_name', + sql: 'resolution_table.display_name', + type: 'string', + }, + ], + }, + ], + }; + + expect(() => { + generateResolutionSchemas(resolutionConfig, []); + }).toThrow('Table schema not found for resolution_table1'); + }); + + it('resolution column does not exist', () => { + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + ], + tableSchemas: [ + { + name: 'resolution_table', + sql: '', + measures: [ + { + name: 'display_id', + sql: 'resolution_table.display_id', + type: 'string', + }, + ], + dimensions: [ + { + name: 'id', + sql: 'resolution_table.id', + type: 'string', + }, + ], + }, + ], + }; + + expect(() => { + generateResolutionSchemas(resolutionConfig, []); + }).toThrow('Dimension not found: display_id'); + }); + + it('column does not exist in base query', () => { + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + ], + tableSchemas: [ + { + name: 'resolution_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'resolution_table.id', + type: 'string', + }, + { + name: 'display_id', + sql: 'resolution_table.display_id', + type: 'string', + }, + ], + }, + ], + }; + + const schemas = generateResolutionSchemas(resolutionConfig, []); + expect(schemas).toEqual([ + { + name: 'base_table__column1', + sql: '', + measures: [], + dimensions: [ + { + name: 'base_table__column1__display_id', + sql: 'base_table__column1.display_id', + type: 'string', + alias: 'base_table__column1 - display_id', + }, + ], + }, + ]); + }); + + it('handle aliases', () => { + const baseTableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + alias: 'Column 1', + }, + ], + }, + ]; + + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + ], + tableSchemas: [ + { + name: 'resolution_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'resolution_table.id', + type: 'string', + alias: 'ID', + }, + { + name: 'display_id', + sql: 'resolution_table.display_id', + type: 'string', + alias: 'Display ID', + }, + ], + }, + ], + }; + + const schemas = generateResolutionSchemas( + resolutionConfig, + baseTableSchemas + ); + expect(schemas).toEqual([ + { + name: 'base_table__column1', + sql: '', + measures: [], + dimensions: [ + { + name: 'base_table__column1__display_id', + sql: 'base_table__column1.display_id', + type: 'string', + alias: 'Column 1 - Display ID', + }, + ], + }, + ]); }); - // it('testing entire flow', async () => { - // debugger; - // const sql = - // 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; - // const tableSchemas = [ - // { - // name: 'base_table', - // sql: '', - // measures: [ - // { - // name: 'count', - // sql: 'COUNT(*)', - // type: 'number' as const, - // }, - // ], - // dimensions: [ - // { - // name: 'column1', - // sql: 'base_table.column1', - // type: 'string' as const, - // }, - // { - // name: 'column2', - // sql: 'base_table.column2', - // type: 'string_array' as const, - // }, - // ], - // }, - // ]; - // const resolutionConfig = { - // columnConfigs: [ - // { - // name: 'base_table.column1', - // isArrayType: false, - // source: 'resolution_table', - // joinColumn: 'id', - // resolutionColumns: ['display_id'], - // }, - // { - // name: 'base_table.column2', - // isArrayType: true, - // source: 'resolution_table_2', - // joinColumn: 'id', - // resolutionColumns: ['fullname'], - // }, - // ], - // tableSchemas: [ - // { - // name: 'resolution_table', - // sql: '', - // measures: [], - // dimensions: [ - // { - // name: 'id', - // sql: 'resolution_table.id', - // type: 'string' as const, - // }, - // { - // name: 'display_id', - // sql: 'resolution_table.display_id', - // type: 'string' as const, - // }, - // { - // name: 'display_name', - // sql: 'resolution_table.display_name', - // type: 'string' as const, - // }, - // ], - // }, - // { - // name: 'resolution_table_2', - // sql: '', - // measures: [], - // dimensions: [ - // { - // name: 'id', - // sql: 'resolution_table_2.id', - // type: 'string' as const, - // }, - // { - // name: 'display_id', - // sql: 'resolution_table_2.display_id', - // type: 'string' as const, - // }, - // { - // name: 'fullname', - // sql: 'resolution_table_2.fullname', - // type: 'string' as const, - // }, - // ], - // }, - // ], - // }; - // const measures = ['base_table.count']; - // const dimensions = ['base_table.column1', 'base_table.column2']; - // // Initialize DuckDB and create a connection - // debugger; - // const result = await cubeQueryToSQLWithResolutionWithArray({ - // query: { - // measures: measures, - // dimensions: dimensions, - // }, - // tableSchemas: tableSchemas, - // resolutionConfig: resolutionConfig, - // }); - // console.log('Generated SQL:', result); - // expect(result).toBeDefined(); - // }); - // it('dimensions and measures are converted to dimensions', () => { - // debugger; - // const sql = - // 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2'; - // const tableSchemas = [ - // { - // name: 'base_table', - // sql: '', - // measures: [ - // { - // name: 'count', - // sql: 'COUNT(*)', - // type: 'number', - // }, - // ], - // dimensions: [ - // { - // name: 'column1', - // sql: 'base_table.column1', - // type: 'string', - // }, - // { - // name: 'column2', - // sql: 'base_table.column2', - // type: 'string', - // }, - // ], - // }, - // ]; - // const resolutionConfig: ResolutionConfig = { - // columnConfigs: [], - // tableSchemas: [], - // }; - // const measures = ['base_table.count']; - // const dimensions = ['base_table.column1', 'base_table.column2']; - // const baseTableSchema = createBaseTableSchema( - // sql, - // tableSchemas, - // resolutionConfig, - // measures, - // dimensions - // ); - // expect(baseTableSchema).toEqual({ - // name: '__base_query', - // sql: 'SELECT COUNT(*), column1, column2 FROM base_table GROUP BY column1, column2', - // measures: [], - // dimensions: [ - // { - // name: 'base_table__count', - // sql: '__base_query.base_table__count', - // type: 'number', - // alias: 'base_table__count', - // }, - // { - // name: 'base_table__column1', - // sql: '__base_query.base_table__column1', - // type: 'string', - // alias: 'base_table__column1', - // }, - // { - // name: 'base_table__column2', - // sql: '__base_query.base_table__column2', - // type: 'string', - // alias: 'base_table__column2', - // }, - // ], - // joins: [], - // }); - // }); - // it('create join config', () => { - // const sql = 'SELECT * FROM base_table'; - // const tableSchemas = [ - // { - // name: 'base_table', - // sql: '', - // measures: [], - // dimensions: [ - // { - // name: 'column1', - // sql: 'base_table.column1', - // type: 'string', - // }, - // { - // name: 'column2', - // sql: 'base_table.column2', - // type: 'string', - // }, - // ], - // }, - // ]; - // const resolutionConfig = { - // columnConfigs: [ - // { - // name: 'base_table.column1', - // source: 'resolution_table', - // joinColumn: 'id', - // resolutionColumns: ['display_id'], - // }, - // { - // name: 'base_table.column2', - // source: 'resolution_table', - // joinColumn: 'id', - // resolutionColumns: ['display_name'], - // }, - // ], - // tableSchemas: [], - // }; - // const dimensions = ['base_table.column1', 'base_table.column2']; - // const baseTableSchema = createBaseTableSchema( - // sql, - // tableSchemas, - // resolutionConfig, - // [], - // dimensions - // ); - // expect(baseTableSchema).toEqual({ - // name: '__base_query', - // sql: 'SELECT * FROM base_table', - // measures: [], - // dimensions: [ - // { - // name: 'base_table__column1', - // sql: '__base_query.base_table__column1', - // type: 'string', - // alias: 'base_table__column1', - // }, - // { - // name: 'base_table__column2', - // sql: '__base_query.base_table__column2', - // type: 'string', - // alias: 'base_table__column2', - // }, - // ], - // joins: [ - // { - // sql: '__base_query.base_table__column1 = base_table__column1.id', - // }, - // { - // sql: '__base_query.base_table__column2 = base_table__column2.id', - // }, - // ], - // }); - // }); - // it('dimension not found', () => { - // const sql = 'SELECT * FROM base_table'; - // const tableSchemas = [ - // { - // name: 'base_table', - // sql: '', - // measures: [], - // dimensions: [ - // { - // name: 'column1', - // sql: 'base_table.column1', - // type: 'string', - // }, - // ], - // }, - // ]; - // const resolutionConfig = { - // columnConfigs: [], - // tableSchemas: [], - // }; - // const dimensions = ['base_table.column1', 'base_table.column2']; // column2 does not exist - // expect(() => { - // createBaseTableSchema( - // sql, - // tableSchemas, - // resolutionConfig, - // [], - // dimensions - // ); - // }).toThrow('Not found: base_table.column2'); - // }); - // it('handle aliases', () => { - // const sql = 'SELECT * FROM base_table'; - // const tableSchemas = [ - // { - // name: 'base_table', - // sql: '', - // measures: [], - // dimensions: [ - // { - // name: 'column1', - // sql: 'base_table.column1', - // type: 'string', - // alias: 'Column 1', - // }, - // { - // name: 'column2', - // sql: 'base_table.column2', - // type: 'string', - // alias: 'Column 2', - // }, - // ], - // }, - // ]; - // const resolutionConfig = { - // columnConfigs: [ - // { - // name: 'base_table.column1', - // source: 'resolution_table', - // joinColumn: 'id', - // resolutionColumns: ['display_id'], - // }, - // { - // name: 'base_table.column2', - // source: 'resolution_table', - // joinColumn: 'id', - // resolutionColumns: ['display_name'], - // }, - // ], - // tableSchemas: [], - // }; - // const dimensions = ['base_table.column1', 'base_table.column2']; - // const baseTableSchema = createBaseTableSchema( - // sql, - // tableSchemas, - // resolutionConfig, - // [], - // dimensions - // ); - // expect(baseTableSchema).toEqual({ - // name: '__base_query', - // sql: 'SELECT * FROM base_table', - // measures: [], - // dimensions: [ - // { - // name: 'base_table__column1', - // sql: '__base_query."Column 1"', - // type: 'string', - // alias: 'Column 1', - // }, - // { - // name: 'base_table__column2', - // sql: '__base_query."Column 2"', - // type: 'string', - // alias: 'Column 2', - // }, - // ], - // joins: [ - // { - // sql: '__base_query."Column 1" = base_table__column1.id', - // }, - // { - // sql: '__base_query."Column 2" = base_table__column2.id', - // }, - // ], - // }); - // }); }); -// describe('Generate resolution schemas', () => { -// it('multiple columns using same table', () => { -// const baseTableSchemas = [ -// { -// name: 'base_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'column1', -// sql: 'base_table.column1', -// type: 'string', -// }, -// { -// name: 'column2', -// sql: 'base_table.column2', -// type: 'string', -// }, -// ], -// }, -// ]; - -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// { -// name: 'base_table.column2', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['id', 'display_name'], -// }, -// ], -// tableSchemas: [ -// { -// name: 'resolution_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'id', -// sql: 'resolution_table.id', -// type: 'string', -// }, -// { -// name: 'display_id', -// sql: 'resolution_table.display_id', -// type: 'string', -// }, -// { -// name: 'display_name', -// sql: 'resolution_table.display_name', -// type: 'string', -// }, -// ], -// }, -// ], -// }; - -// const schemas = generateResolutionSchemas( -// resolutionConfig, -// baseTableSchemas -// ); - -// expect(schemas).toEqual([ -// { -// name: 'base_table__column1', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'base_table__column1__display_id', -// sql: 'base_table__column1.display_id', -// type: 'string', -// alias: 'base_table__column1 - display_id', -// }, -// ], -// }, -// { -// name: 'base_table__column2', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'base_table__column2__id', -// sql: 'base_table__column2.id', -// type: 'string', -// alias: 'base_table__column2 - id', -// }, -// { -// name: 'base_table__column2__display_name', -// sql: 'base_table__column2.display_name', -// type: 'string', -// alias: 'base_table__column2 - display_name', -// }, -// ], -// }, -// ]); -// }); - -// it('table does not exist', () => { -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// { -// name: 'base_table.column2', -// source: 'resolution_table1', // does not exist -// joinColumn: 'id', -// resolutionColumns: ['id', 'display_name'], -// }, -// ], -// tableSchemas: [ -// { -// name: 'resolution_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'id', -// sql: 'resolution_table.id', -// type: 'string', -// }, -// { -// name: 'display_id', -// sql: 'resolution_table.display_id', -// type: 'string', -// }, -// { -// name: 'display_name', -// sql: 'resolution_table.display_name', -// type: 'string', -// }, -// ], -// }, -// ], -// }; - -// expect(() => { -// generateResolutionSchemas(resolutionConfig, []); -// }).toThrow('Table schema not found for resolution_table1'); -// }); - -// it('resolution column does not exist', () => { -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// ], -// tableSchemas: [ -// { -// name: 'resolution_table', -// sql: '', -// measures: [ -// { -// name: 'display_id', -// sql: 'resolution_table.display_id', -// type: 'string', -// }, -// ], -// dimensions: [ -// { -// name: 'id', -// sql: 'resolution_table.id', -// type: 'string', -// }, -// ], -// }, -// ], -// }; - -// expect(() => { -// generateResolutionSchemas(resolutionConfig, []); -// }).toThrow('Dimension not found: display_id'); -// }); - -// it('column does not exist in base query', () => { -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// ], -// tableSchemas: [ -// { -// name: 'resolution_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'id', -// sql: 'resolution_table.id', -// type: 'string', -// }, -// { -// name: 'display_id', -// sql: 'resolution_table.display_id', -// type: 'string', -// }, -// ], -// }, -// ], -// }; - -// const schemas = generateResolutionSchemas(resolutionConfig, []); -// expect(schemas).toEqual([ -// { -// name: 'base_table__column1', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'base_table__column1__display_id', -// sql: 'base_table__column1.display_id', -// type: 'string', -// alias: 'base_table__column1 - display_id', -// }, -// ], -// }, -// ]); -// }); - -// it('handle aliases', () => { -// const baseTableSchemas = [ -// { -// name: 'base_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'column1', -// sql: 'base_table.column1', -// type: 'string', -// alias: 'Column 1', -// }, -// ], -// }, -// ]; - -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// ], -// tableSchemas: [ -// { -// name: 'resolution_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'id', -// sql: 'resolution_table.id', -// type: 'string', -// alias: 'ID', -// }, -// { -// name: 'display_id', -// sql: 'resolution_table.display_id', -// type: 'string', -// alias: 'Display ID', -// }, -// ], -// }, -// ], -// }; - -// const schemas = generateResolutionSchemas( -// resolutionConfig, -// baseTableSchemas -// ); -// expect(schemas).toEqual([ -// { -// name: 'base_table__column1', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'base_table__column1__display_id', -// sql: 'base_table__column1.display_id', -// type: 'string', -// alias: 'Column 1 - Display ID', -// }, -// ], -// }, -// ]); -// }); -// }); - -// describe('Generate resolved dimensions', () => { -// it('resolves dimensions based on resolution config', () => { -// const query = { -// measures: [], -// dimensions: ['base_table.column1', 'base_table.column2'], -// }; -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// { -// name: 'base_table.column2', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_name'], -// }, -// ], -// tableSchemas: [], -// }; - -// const resolvedDimensions = generateResolvedDimensions( -// query, -// resolutionConfig -// ); - -// expect(resolvedDimensions).toEqual([ -// 'base_table__column1.base_table__column1__display_id', -// 'base_table__column2.base_table__column2__display_name', -// ]); -// }); - -// it('unresolved columns', () => { -// const query = { -// measures: ['base_table.count'], -// dimensions: ['base_table.column1'], -// }; -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column3', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// ], -// tableSchemas: [], -// }; - -// const resolvedDimensions = generateResolvedDimensions( -// query, -// resolutionConfig -// ); - -// expect(resolvedDimensions).toEqual([ -// '__base_query.base_table__count', -// '__base_query.base_table__column1', -// ]); -// }); - -// it('only include projected columns', () => { -// const query = { -// measures: ['base_table.count', 'base_table.total'], -// dimensions: ['base_table.column1', 'base_table.column2'], -// }; -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// { -// name: 'base_table.column2', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['id', 'display_name'], -// }, -// ], -// tableSchemas: [], -// }; -// const projections = [ -// 'base_table.count', -// 'base_table.column2', -// 'base_table.total', -// ]; - -// const resolvedDimensions = generateResolvedDimensions( -// query, -// resolutionConfig, -// projections -// ); - -// expect(resolvedDimensions).toEqual([ -// '__base_query.base_table__count', -// 'base_table__column2.base_table__column2__id', -// 'base_table__column2.base_table__column2__display_name', -// '__base_query.base_table__total', -// ]); -// }); -// }); - -// describe('Generate resolution join paths', () => { -// it('generate join paths for resolution columns', () => { -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// { -// name: 'base_table.column2', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_name'], -// }, -// ], -// tableSchemas: [], -// }; - -// const joinPaths = generateResolutionJoinPaths(resolutionConfig, []); - -// expect(joinPaths).toEqual([ -// [ -// { -// left: '__base_query', -// right: 'base_table__column1', -// on: 'base_table__column1', -// }, -// ], -// [ -// { -// left: '__base_query', -// right: 'base_table__column2', -// on: 'base_table__column2', -// }, -// ], -// ]); -// }); - -// it('join paths with aliases', () => { -// const baseTableSchemas = [ -// { -// name: 'base_table', -// sql: '', -// measures: [], -// dimensions: [ -// { -// name: 'column1', -// sql: 'base_table.column1', -// type: 'string', -// alias: 'Column 1', -// }, -// { -// name: 'column2', -// sql: 'base_table.column2', -// type: 'string', -// alias: 'Column 2', -// }, -// ], -// }, -// ]; - -// const resolutionConfig = { -// columnConfigs: [ -// { -// name: 'base_table.column1', -// source: 'resolution_table', -// joinColumn: 'id', -// resolutionColumns: ['display_id'], -// }, -// ], -// tableSchemas: [], -// }; - -// const joinPaths = generateResolutionJoinPaths( -// resolutionConfig, -// baseTableSchemas -// ); -// expect(joinPaths).toEqual([ -// [ -// { -// left: '__base_query', -// right: 'base_table__column1', -// on: '"Column 1"', -// }, -// ], -// ]); -// }); -// }); +describe('Generate resolved dimensions', () => { + it('resolves dimensions based on resolution config', () => { + const query = { + measures: [], + dimensions: ['base_table.column1', 'base_table.column2'], + }; + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + ], + tableSchemas: [], + }; + + const resolvedDimensions = generateResolvedDimensions( + query, + resolutionConfig + ); + + expect(resolvedDimensions).toEqual([ + 'base_table__column1.base_table__column1__display_id', + 'base_table__column2.base_table__column2__display_name', + ]); + }); + + it('unresolved columns', () => { + const query = { + measures: ['base_table.count'], + dimensions: ['base_table.column1'], + }; + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column3', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + ], + tableSchemas: [], + }; + + const resolvedDimensions = generateResolvedDimensions( + query, + resolutionConfig + ); + + expect(resolvedDimensions).toEqual([ + '__base_query.base_table__count', + '__base_query.base_table__column1', + ]); + }); + + it('only include projected columns', () => { + const query = { + measures: ['base_table.count', 'base_table.total'], + dimensions: ['base_table.column1', 'base_table.column2'], + }; + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['id', 'display_name'], + }, + ], + tableSchemas: [], + }; + const projections = [ + 'base_table.count', + 'base_table.column2', + 'base_table.total', + ]; + + const resolvedDimensions = generateResolvedDimensions( + query, + resolutionConfig, + projections + ); + + expect(resolvedDimensions).toEqual([ + '__base_query.base_table__count', + 'base_table__column2.base_table__column2__id', + 'base_table__column2.base_table__column2__display_name', + '__base_query.base_table__total', + ]); + }); +}); + +describe('Generate resolution join paths', () => { + it('generate join paths for resolution columns', () => { + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + { + name: 'base_table.column2', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + ], + tableSchemas: [], + }; + + const joinPaths = generateResolutionJoinPaths(resolutionConfig, []); + + expect(joinPaths).toEqual([ + [ + { + left: '__base_query', + right: 'base_table__column1', + on: 'base_table__column1', + }, + ], + [ + { + left: '__base_query', + right: 'base_table__column2', + on: 'base_table__column2', + }, + ], + ]); + }); + + it('join paths with aliases', () => { + const baseTableSchemas = [ + { + name: 'base_table', + sql: '', + measures: [], + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + alias: 'Column 1', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'string', + alias: 'Column 2', + }, + ], + }, + ]; + + const resolutionConfig = { + columnConfigs: [ + { + name: 'base_table.column1', + source: 'resolution_table', + joinColumn: 'id', + resolutionColumns: ['display_id'], + }, + ], + tableSchemas: [], + }; + + const joinPaths = generateResolutionJoinPaths( + resolutionConfig, + baseTableSchemas + ); + expect(joinPaths).toEqual([ + [ + { + left: '__base_query', + right: 'base_table__column1', + on: '"Column 1"', + }, + ], + ]); + }); +}); diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index 00da125b..49d1cf89 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -214,28 +214,28 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { measures: ['tickets.count'], dimensions: [ 'tickets.id', - 'tickets.owners', - 'tickets.tags', - 'tickets.created_by', + 'tickets.owners', //array + 'tickets.tags', // array + 'tickets.created_by', // scalar ], }; const resolutionConfig: ResolutionConfig = { columnConfigs: [ - // { - // name: 'tickets.owners', - // isArrayType: true, - // source: 'owners_lookup', - // joinColumn: 'id', - // resolutionColumns: ['display_name'], - // }, - // { - // name: 'tickets.tags', - // isArrayType: true, - // source: 'tags_lookup', - // joinColumn: 'id', - // resolutionColumns: ['tag_name'], - // }, + { + name: 'tickets.owners', + isArrayType: true, + source: 'owners_lookup', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + { + name: 'tickets.tags', + isArrayType: true, + source: 'tags_lookup', + joinColumn: 'id', + resolutionColumns: ['tag_name'], + }, { name: 'tickets.created_by', isArrayType: false, @@ -245,8 +245,8 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { }, ], tableSchemas: [ - // OWNERS_LOOKUP_SCHEMA, - // TAGS_LOOKUP_SCHEMA, + OWNERS_LOOKUP_SCHEMA, + TAGS_LOOKUP_SCHEMA, CREATED_BY_LOOKUP_SCHEMA, ], }; @@ -278,57 +278,57 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { expect(result[0]).toHaveProperty('tickets__tags - tag_name'); }); - it('Should handle only scalar field resolution without unnesting', async () => { - const query: Query = { - measures: ['tickets.count'], - dimensions: [ - 'tickets.id', - 'tickets.owners', - 'tickets.tags', - 'tickets.created_by', - ], - }; + // it('Should handle only scalar field resolution without unnesting', async () => { + // const query: Query = { + // measures: ['tickets.count'], + // dimensions: [ + // 'tickets.id', + // 'tickets.owners', + // 'tickets.tags', + // 'tickets.created_by', + // ], + // }; - const resolutionConfig: ResolutionConfig = { - columnConfigs: [ - { - name: 'tickets.created_by', - isArrayType: false, - source: 'created_by_lookup', - joinColumn: 'id', - resolutionColumns: ['name'], - }, - ], - tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], - }; + // const resolutionConfig: ResolutionConfig = { + // columnConfigs: [ + // { + // name: 'tickets.created_by', + // isArrayType: false, + // source: 'created_by_lookup', + // joinColumn: 'id', + // resolutionColumns: ['name'], + // }, + // ], + // tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], + // }; - const sql = await cubeQueryToSQLWithResolutionWithArray({ - query, - tableSchemas: [TICKETS_TABLE_SCHEMA], - resolutionConfig, - }); + // const sql = await cubeQueryToSQLWithResolutionWithArray({ + // query, + // tableSchemas: [TICKETS_TABLE_SCHEMA], + // resolutionConfig, + // }); - console.log('Phase 1 SQL (multiple arrays):', sql); + // console.log('Phase 1 SQL (multiple arrays):', sql); - // Verify row_id is included - expect(sql).toContain('row_id'); + // // Verify row_id is included + // expect(sql).toContain('row_id'); - // Both arrays should be unnested - expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(0); + // // Both arrays should be unnested + // expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(0); - // Execute the SQL to verify it works - const result = (await duckdbExec(sql)) as any[]; + // // Execute the SQL to verify it works + // const result = (await duckdbExec(sql)) as any[]; - expect(result.length).toBe(7); + // expect(result.length).toBe(7); - // Each row should have a row_id - expect(result[0]).toHaveProperty('__row_id'); - expect(result[0]).toHaveProperty('tickets__count'); - expect(result[0]).toHaveProperty('tickets__id'); - expect(result[0]).toHaveProperty('tickets__owners'); - expect(result[0]).toHaveProperty('tickets__tags'); - expect(result[0]).toHaveProperty('tickets__created_by - name'); - }); + // // Each row should have a row_id + // expect(result[0]).toHaveProperty('__row_id'); + // expect(result[0]).toHaveProperty('tickets__count'); + // expect(result[0]).toHaveProperty('tickets__id'); + // expect(result[0]).toHaveProperty('tickets__owners'); + // expect(result[0]).toHaveProperty('tickets__tags'); + // expect(result[0]).toHaveProperty('tickets__created_by - name'); + // }); // it('Should return regular SQL when no array fields need resolution', async () => { // const query: Query = { diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 1bc2b532..3f97119e 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -95,14 +95,6 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ contextParams, }); - // // Get array columns for Phase 2 - // const arrayColumns = getArrayTypeColumns(resolutionConfig); - - // if (arrayColumns.length === 0) { - // // No resolution needed - // return unnestBaseSql; - // } - // // // Phase 2: Apply resolution (join with lookup tables) const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ unnestedSql: unnestBaseSql, From 13b7f72263f65996b8f26bc4e630af3c84979b22 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 18:30:37 +0530 Subject: [PATCH 07/38] re-using dimensions instead of re-creating it --- .../browser-cube-to-sql-with-resolution.ts | 100 +++++++----------- .../cube-to-sql-with-resolution-array.spec.ts | 11 ++ .../cube-to-sql-with-resolution.ts | 100 +++++++----------- 3 files changed, 85 insertions(+), 126 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 99a902dc..acc1cc43 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -285,10 +285,10 @@ export const getResolvedSql = async ({ query.dimensions?.push('row_id'); // Step 4: Generate resolved dimensions using existing helper + // should not be using column projections here, it doesn't have row_id. const resolvedDimensions = generateResolvedDimensions( query, - resolutionConfig, - columnProjections + resolutionConfig ); // Step 5: Create query and generate SQL @@ -305,72 +305,46 @@ export const getResolvedSql = async ({ contextParams, }); - // Create a simple schema describing Phase 2's output columns - // cubeQueryToSQL outputs columns using their alias field, not table-prefixed names + // Create a schema describing Phase 2's output by combining: + // 1. Dimensions from updatedBaseTableSchema that were actually queried + // 2. Dimensions from resolutionSchemas (resolved columns) + // The key insight: cubeQueryToSQL outputs columns using their alias field + + // Build list of dimension names that should be in output + const baseDimensionNames = new Set([ + 'row_id', + ...query.measures.map((m) => m.replace('.', '__')), + ...(query.dimensions || []) + .filter( + (d) => !resolutionConfig.columnConfigs.some((ac) => ac.name === d) + ) + .map((d) => d.replace('.', '__')), + ]); + + debugger; const resolvedTableSchema: TableSchema = { name: '__resolved_query', sql: resolvedSql, measures: [], dimensions: [ - // Row ID - comes from baseTableSchema with alias '__row_id' - { - name: 'row_id', - sql: '__resolved_query."__row_id"', - type: 'number', - alias: '__row_id', - }, - // Original measures from base query - use their aliases - ...query.measures.map((measure) => { - const measureName = measure.replace('.', '__'); - return { - name: measureName, - sql: `__resolved_query."${measureName}"`, - type: 'number' as const, - alias: measureName, - }; - }), - // Non-array dimensions from base query - use their aliases - ...(query.dimensions || []) - .filter( - (dim) => - !resolutionConfig.columnConfigs.some((ac) => ac.name === dim) && - dim !== 'row_id' - ) - .map((dimension) => { - const dimName = dimension.replace('.', '__'); - return { - name: dimName, - sql: `__resolved_query."${dimName}"`, - type: 'string' as const, - alias: dimName, - }; - }), - // ALL resolved columns (both array and scalar) - these use the alias pattern "colname - rescolname" - ...resolutionConfig.columnConfigs.flatMap((columnConfig) => { - return columnConfig.resolutionColumns.map((resCol) => { - // Find the resolution dimension to get its alias - const resolutionDimName = `${columnConfig.name.replace( - '.', - '__' - )}__${resCol}`; - const resolutionSchema = resolutionSchemas.find( - (s) => s.name === columnConfig.name.replace('.', '__') - ); - const resDim = resolutionSchema?.dimensions.find((d) => - d.name.includes(resCol) - ); - - // The alias from the resolution schema already has the full format - const aliasName = resDim?.alias || resolutionDimName; - - return { - name: resolutionDimName, - sql: `__resolved_query."${aliasName}"`, - type: 'string' as const, - alias: aliasName, - }; - }); - }), + // Dimensions from base table that were queried (row_id, measures, non-resolved dimensions) + ...updatedBaseTableSchema.dimensions + .filter((dim) => baseDimensionNames.has(dim.name)) + .map((dim) => ({ + name: dim.name, + sql: `__resolved_query."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })), + // All dimensions from resolution schemas (resolved columns from JOINs) + ...resolutionSchemas.flatMap((resSchema) => + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `__resolved_query."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ), ], }; diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index 49d1cf89..df16fa6e 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -57,21 +57,25 @@ const TICKETS_TABLE_SCHEMA: TableSchema = { ], dimensions: [ { + alias: 'ID', name: 'id', sql: 'id', type: 'number', }, { + alias: 'Created By', name: 'created_by', sql: 'created_by', type: 'string', }, { + alias: 'Owners', name: 'owners', sql: 'owners', type: 'string_array', }, { + alias: 'Tags', name: 'tags', sql: 'tags', type: 'string_array', @@ -85,16 +89,19 @@ const OWNERS_LOOKUP_SCHEMA: TableSchema = { measures: [], dimensions: [ { + alias: 'ID', name: 'id', sql: 'id', type: 'string', }, { + alias: 'Display Name', name: 'display_name', sql: 'display_name', type: 'string', }, { + alias: 'Email', name: 'email', sql: 'email', type: 'string', @@ -108,11 +115,13 @@ const TAGS_LOOKUP_SCHEMA: TableSchema = { measures: [], dimensions: [ { + alias: 'ID', name: 'id', sql: 'id', type: 'string', }, { + alias: 'Tag Name', name: 'tag_name', sql: 'tag_name', type: 'string', @@ -126,11 +135,13 @@ const CREATED_BY_LOOKUP_SCHEMA: TableSchema = { measures: [], dimensions: [ { + alias: 'ID', name: 'id', sql: 'id', type: 'string', }, { + alias: 'Name', name: 'name', sql: 'name', type: 'string', diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 3f97119e..db34ff89 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -271,10 +271,10 @@ export const getResolvedSql = async ({ query.dimensions?.push('row_id'); // Step 4: Generate resolved dimensions using existing helper + // should not be using column projections here, it doesn't have row_id. const resolvedDimensions = generateResolvedDimensions( query, - resolutionConfig, - columnProjections + resolutionConfig ); // Step 5: Create query and generate SQL @@ -290,72 +290,46 @@ export const getResolvedSql = async ({ contextParams, }); - // Create a simple schema describing Phase 2's output columns - // cubeQueryToSQL outputs columns using their alias field, not table-prefixed names + // Create a schema describing Phase 2's output by combining: + // 1. Dimensions from updatedBaseTableSchema that were actually queried + // 2. Dimensions from resolutionSchemas (resolved columns) + // The key insight: cubeQueryToSQL outputs columns using their alias field + + // Build list of dimension names that should be in output + const baseDimensionNames = new Set([ + 'row_id', + ...query.measures.map((m) => m.replace('.', '__')), + ...(query.dimensions || []) + .filter( + (d) => !resolutionConfig.columnConfigs.some((ac) => ac.name === d) + ) + .map((d) => d.replace('.', '__')), + ]); + + debugger; const resolvedTableSchema: TableSchema = { name: '__resolved_query', sql: resolvedSql, measures: [], dimensions: [ - // Row ID - comes from baseTableSchema with alias '__row_id' - { - name: 'row_id', - sql: '__resolved_query."__row_id"', - type: 'number', - alias: '__row_id', - }, - // Original measures from base query - use their aliases - ...query.measures.map((measure) => { - const measureName = measure.replace('.', '__'); - return { - name: measureName, - sql: `__resolved_query."${measureName}"`, - type: 'number' as const, - alias: measureName, - }; - }), - // Non-array dimensions from base query - use their aliases - ...(query.dimensions || []) - .filter( - (dim) => - !resolutionConfig.columnConfigs.some((ac) => ac.name === dim) && - dim !== 'row_id' - ) - .map((dimension) => { - const dimName = dimension.replace('.', '__'); - return { - name: dimName, - sql: `__resolved_query."${dimName}"`, - type: 'string' as const, - alias: dimName, - }; - }), - // ALL resolved columns (both array and scalar) - these use the alias pattern "colname - rescolname" - ...resolutionConfig.columnConfigs.flatMap((columnConfig) => { - return columnConfig.resolutionColumns.map((resCol) => { - // Find the resolution dimension to get its alias - const resolutionDimName = `${columnConfig.name.replace( - '.', - '__' - )}__${resCol}`; - const resolutionSchema = resolutionSchemas.find( - (s) => s.name === columnConfig.name.replace('.', '__') - ); - const resDim = resolutionSchema?.dimensions.find((d) => - d.name.includes(resCol) - ); - - // The alias from the resolution schema already has the full format - const aliasName = resDim?.alias || resolutionDimName; - - return { - name: resolutionDimName, - sql: `__resolved_query."${aliasName}"`, - type: 'string' as const, - alias: aliasName, - }; - }); - }), + // Dimensions from base table that were queried (row_id, measures, non-resolved dimensions) + ...updatedBaseTableSchema.dimensions + .filter((dim) => baseDimensionNames.has(dim.name)) + .map((dim) => ({ + name: dim.name, + sql: `__resolved_query."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })), + // All dimensions from resolution schemas (resolved columns from JOINs) + ...resolutionSchemas.flatMap((resSchema) => + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `__resolved_query."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ), ], }; From 32f0d90fce876b8429457a6fa6b1ef9cb1c7f158 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 20:07:12 +0530 Subject: [PATCH 08/38] minor refactoring --- .../sql-expression-modifiers.ts | 13 ++++++------- meerkat-core/src/resolution/resolution.ts | 10 +++++----- meerkat-core/src/types/cube-types/table.ts | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts index 693a6e7a..195c4b88 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts @@ -14,7 +14,7 @@ export const arrayFieldUnNestModifier = ({ return `array[unnest(${sqlExpression})]`; }; -export const arrayUnnestModifier = ({ +export const arrayFlattenModifier = ({ sqlExpression, }: DimensionModifier): string => { return `unnest(${sqlExpression})`; @@ -29,12 +29,11 @@ export const shouldUnnest = ({ return !!(isArrayType && hasUnNestedGroupBy && query.measures.length > 0); }; -export const shouldUnnestArray = ({ +export const shouldFlattenArray = ({ dimension, - query, }: DimensionModifier): boolean => { const isArrayType = isArrayTypeMember(dimension.type); - const hasUnNestedGroupBy = dimension.modifier?.shouldUnnestArray; + const hasUnNestedGroupBy = dimension.modifier?.shouldFlattenArray; return !!(isArrayType && hasUnNestedGroupBy); }; @@ -51,9 +50,9 @@ export const MODIFIERS: Modifier[] = [ modifier: arrayFieldUnNestModifier, }, { - name: 'shouldUnnestArray', - matcher: shouldUnnestArray, - modifier: arrayUnnestModifier, + name: 'shouldFlattenArray', + matcher: shouldFlattenArray, + modifier: arrayFlattenModifier, }, ]; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 28d75af6..a7ac6ddc 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -20,9 +20,6 @@ const constructBaseDimension = ( schema: Measure | Dimension, resolutionColumnConfigs: ResolutionColumnConfig[] ) => { - const shouldPerformUnnest = resolutionColumnConfigs.some( - (config) => config.name == name - ); const dimension: Dimension = { name: memberKeyToSafeKey(name), sql: `${BASE_DATA_SOURCE_NAME}.${constructAlias({ @@ -38,9 +35,12 @@ const constructBaseDimension = ( aliasContext: { isTableSchemaAlias: true }, }), }; - if (shouldPerformUnnest) { + const shouldFlattenField = resolutionColumnConfigs.some( + (config) => config.name == name && config.isArrayType + ); + if (shouldFlattenField) { dimension.modifier = { - shouldUnnestArray: true, + shouldFlattenArray: true, }; } return dimension; diff --git a/meerkat-core/src/types/cube-types/table.ts b/meerkat-core/src/types/cube-types/table.ts index 8c214f29..4bfd5a24 100644 --- a/meerkat-core/src/types/cube-types/table.ts +++ b/meerkat-core/src/types/cube-types/table.ts @@ -27,7 +27,7 @@ export type Dimension = { type: DimensionType; modifier?: { shouldUnnestGroupBy?: boolean; - shouldUnnestArray?: boolean; + shouldFlattenArray?: boolean; }; alias?: string; }; From c00155e408ef4545c2ae1a6514d5f26191c32603 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Thu, 30 Oct 2025 23:21:08 +0530 Subject: [PATCH 09/38] minor update --- .../cube-to-sql-with-resolution-array.spec.ts | 12 +- .../cube-to-sql-with-resolution.ts | 104 ++++++------------ 2 files changed, 42 insertions(+), 74 deletions(-) diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index df16fa6e..f96f2951 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -1,5 +1,5 @@ import { Query, ResolutionConfig, TableSchema } from '@devrev/meerkat-core'; -import { cubeQueryToSQLWithResolutionWithArray } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; +import { cubeQueryToSQLWithResolution } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; import { duckdbExec } from '../duckdb-exec'; const CREATE_TEST_TABLE = `CREATE TABLE tickets ( @@ -262,10 +262,18 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { ], }; - const sql = await cubeQueryToSQLWithResolutionWithArray({ + const columnProjections = [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + 'tickets.count', + ]; + const sql = await cubeQueryToSQLWithResolution({ query, tableSchemas: [TICKETS_TABLE_SCHEMA], resolutionConfig, + columnProjections, }); console.log('Phase 1 SQL (multiple arrays):', sql); diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index db34ff89..6603b0f2 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -39,6 +39,26 @@ export const cubeQueryToSQLWithResolution = async ({ // If no resolution is needed, return the base SQL. return baseSql; } + if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { + query.dimensions?.push(`${tableSchemas[0].name}.row_id`); + tableSchemas[0].dimensions.push({ + name: 'row_id', + sql: 'row_number() OVER ()', + type: 'number' as const, + alias: '__row_id', + } as Dimension); + columnProjections = [ + `${tableSchemas[0].name}.row_id`, + ...(columnProjections || []), + ]; + return cubeQueryToSQLWithResolutionWithArray({ + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, + }); + } // Create a table schema for the base query. const baseTable: TableSchema = createBaseTableSchema( @@ -87,7 +107,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ columnProjections, contextParams, }: CubeQueryToSQLWithResolutionParams) => { - // Phase 1: Generate SQL with row_id and unnested arrays + // Generate SQL with row_id and unnested arrays const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ query, tableSchemas, @@ -95,7 +115,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ contextParams, }); - // // // Phase 2: Apply resolution (join with lookup tables) + // Apply resolution (join with lookup tables) const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ unnestedSql: unnestBaseSql, baseTableSchema, @@ -127,54 +147,15 @@ export const getUnnestBaseSql = async ({ sql: string; baseTableSchema: TableSchema; }> => { - // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) - const modifiedTableSchemasWithRowId = tableSchemas.map((schema, index) => { - // Add row_id to the first table only - if (index !== 0) { - return schema; - } - - // Add row_id dimension (no unnest modifier yet) - // TODO: Will this cause a problem of adding row_id to the first schema ? - const newDimensions = [ - { - name: 'row_id', - sql: 'row_number() OVER ()', - type: 'number' as const, - alias: '__row_id', - }, - ...schema.dimensions, - ]; - - return { - ...schema, - dimensions: newDimensions, - }; - }); - - // Use the first table for row_id reference - const firstTable = tableSchemas[0]; - - const queryWithRowId: Query = { - measures: query.measures, - dimensions: [`${firstTable.name}.row_id`, ...(query.dimensions || [])], - joinPaths: query.joinPaths, - filters: query.filters, - order: query.order, - limit: query.limit, - offset: query.offset, - }; - // Generate base SQL with row_id const baseSqlWithRowId = await cubeQueryToSQL({ - query: queryWithRowId, - tableSchemas: modifiedTableSchemasWithRowId, + query: query, + tableSchemas: tableSchemas, contextParams, }); - // Step 2: Create a new table schema from the base SQL with row_id // This will be used to apply unnesting - const baseTableName = '__base_query'; // Use standard name to work with helpers + const baseTableName = '__base_query'; const baseTableSchema: TableSchema = createBaseTableSchema( baseSqlWithRowId, @@ -184,18 +165,10 @@ export const getUnnestBaseSql = async ({ query.dimensions ); - baseTableSchema.dimensions.push({ - name: 'row_id', - sql: '__row_id', - type: 'number', - alias: '__row_id', - } as Dimension); - - // Step 3: Create query with unnest modifiers applied + // Create query with unnest modifiers applied const unnestQuery: Query = { measures: [], dimensions: [ - `${baseTableName}.row_id`, ...query.measures.map((m) => `${baseTableName}.${m.replace('.', '__')}`), ...(query.dimensions || []).map( (d) => `${baseTableName}.${d.replace('.', '__')}` @@ -249,35 +222,29 @@ export const getResolvedSql = async ({ sql: string; resolvedTableSchema: TableSchema; }> => { - // Step 1: Use the base table schema from Phase 1 as source of truth // Update the SQL to point to the unnested SQL const updatedBaseTableSchema: TableSchema = { ...baseTableSchema, sql: unnestedSql, }; - debugger; - // Step 2: Generate resolution schemas for array fields only - // Use the existing generateResolutionSchemas helper - + // Generate resolution schemas for array fields only const resolutionSchemas = generateResolutionSchemas( resolutionConfig, tableSchemas ); - // Step 3: Generate join paths using existing helper - // Note: Pass the base table schema (from Phase 1) to generate correct join paths + // Generate join paths using existing helper const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); - query.dimensions?.push('row_id'); - // Step 4: Generate resolved dimensions using existing helper - // should not be using column projections here, it doesn't have row_id. + // Generate resolved dimensions const resolvedDimensions = generateResolvedDimensions( query, - resolutionConfig + resolutionConfig, + columnProjections ); - // Step 5: Create query and generate SQL + // Create query and generate SQL const resolutionQuery: Query = { measures: [], dimensions: resolvedDimensions, @@ -290,14 +257,8 @@ export const getResolvedSql = async ({ contextParams, }); - // Create a schema describing Phase 2's output by combining: - // 1. Dimensions from updatedBaseTableSchema that were actually queried - // 2. Dimensions from resolutionSchemas (resolved columns) - // The key insight: cubeQueryToSQL outputs columns using their alias field - // Build list of dimension names that should be in output const baseDimensionNames = new Set([ - 'row_id', ...query.measures.map((m) => m.replace('.', '__')), ...(query.dimensions || []) .filter( @@ -306,7 +267,6 @@ export const getResolvedSql = async ({ .map((d) => d.replace('.', '__')), ]); - debugger; const resolvedTableSchema: TableSchema = { name: '__resolved_query', sql: resolvedSql, From 7600c52c3a22619a4e5560a3a3f616211e99b58a Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Fri, 31 Oct 2025 19:30:04 +0530 Subject: [PATCH 10/38] unnest working as expected --- .../browser-cube-to-sql-with-resolution.ts | 15 +- meerkat-core/src/joins/joins.ts | 2 + meerkat-core/src/resolution/resolution.ts | 25 +- .../cube-to-sql-with-resolution.ts | 250 ++++++++++++------ 4 files changed, 203 insertions(+), 89 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index acc1cc43..2d021536 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -1,4 +1,5 @@ import { + BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, Dimension, @@ -72,11 +73,16 @@ export const cubeQueryToSQLWithResolution = async ({ query: { measures: [], dimensions: generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig, columnProjections ), - joinPaths: generateResolutionJoinPaths(resolutionConfig, tableSchemas), + joinPaths: generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + tableSchemas + ), }, tableSchemas: [baseTable, ...resolutionSchemas], }; @@ -281,12 +287,17 @@ export const getResolvedSql = async ({ // Step 3: Generate join paths using existing helper // Note: Pass the base table schema (from Phase 1) to generate correct join paths - const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); + const joinPaths = generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + tableSchemas + ); query.dimensions?.push('row_id'); // Step 4: Generate resolved dimensions using existing helper // should not be using column projections here, it doesn't have row_id. const resolvedDimensions = generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig ); diff --git a/meerkat-core/src/joins/joins.ts b/meerkat-core/src/joins/joins.ts index 46bdb7fe..81468cea 100644 --- a/meerkat-core/src/joins/joins.ts +++ b/meerkat-core/src/joins/joins.ts @@ -56,6 +56,7 @@ export function generateSqlQuery( // If visitedFrom is undefined, this is the first visit to the node visitedNodes.set(currentEdge.right, currentEdge); + debugger; query += ` LEFT JOIN (${tableSchemaSqlMap[currentEdge.right]}) AS ${ currentEdge.right @@ -72,6 +73,7 @@ export const createDirectedGraph = ( tableSchema: TableSchema[], tableSchemaSqlMap: { [key: string]: string } ) => { + debugger; const directedGraph: { [key: string]: { [key: string]: { [key: string]: string } }; } = {}; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index a7ac6ddc..8cf825b1 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -35,14 +35,14 @@ const constructBaseDimension = ( aliasContext: { isTableSchemaAlias: true }, }), }; - const shouldFlattenField = resolutionColumnConfigs.some( - (config) => config.name == name && config.isArrayType - ); - if (shouldFlattenField) { - dimension.modifier = { - shouldFlattenArray: true, - }; - } + // const shouldFlattenField = resolutionColumnConfigs.some( + // (config) => config.name == name && config.isArrayType + // ); + // if (shouldFlattenField) { + // dimension.modifier = { + // shouldFlattenArray: true, + // }; + // } return dimension; }; @@ -147,6 +147,7 @@ export const generateResolutionSchemas = ( }; export const generateResolvedDimensions = ( + baseDataSourceName: string, query: Query, config: ResolutionConfig, columnProjections?: string[] @@ -165,10 +166,7 @@ export const generateResolvedDimensions = ( if (!columnConfig) { return [ - getNamespacedKey( - BASE_DATA_SOURCE_NAME, - memberKeyToSafeKey(dimension) - ), + getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), ]; } else { return columnConfig.resolutionColumns.map((col) => @@ -184,12 +182,13 @@ export const generateResolvedDimensions = ( }; export const generateResolutionJoinPaths = ( + baseDataSourceName: string, resolutionConfig: ResolutionConfig, baseTableSchemas: TableSchema[] ): JoinPath[] => { return resolutionConfig.columnConfigs.map((config) => [ { - left: BASE_DATA_SOURCE_NAME, + left: baseDataSourceName, right: memberKeyToSafeKey(config.name), on: constructAlias({ name: config.name, diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 6603b0f2..1e91301a 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -1,10 +1,13 @@ import { + BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + getNamespacedKey, + memberKeyToSafeKey, Query, ResolutionConfig, TableSchema, @@ -39,20 +42,15 @@ export const cubeQueryToSQLWithResolution = async ({ // If no resolution is needed, return the base SQL. return baseSql; } + + // Step 2: Check if array-type resolution is needed if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - query.dimensions?.push(`${tableSchemas[0].name}.row_id`); - tableSchemas[0].dimensions.push({ - name: 'row_id', - sql: 'row_number() OVER ()', - type: 'number' as const, - alias: '__row_id', - } as Dimension); - columnProjections = [ - `${tableSchemas[0].name}.row_id`, - ...(columnProjections || []), - ]; + debugger; + // Delegate to array handler, passing baseSql instead of query return cubeQueryToSQLWithResolutionWithArray({ - query, + baseSql, + measures: query.measures, + dimensions: query.dimensions || [], tableSchemas, resolutionConfig, columnProjections, @@ -78,11 +76,16 @@ export const cubeQueryToSQLWithResolution = async ({ query: { measures: [], dimensions: generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig, columnProjections ), - joinPaths: generateResolutionJoinPaths(resolutionConfig, tableSchemas), + joinPaths: generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + tableSchemas + ), }, tableSchemas: [baseTable, ...resolutionSchemas], }; @@ -100,26 +103,40 @@ const getArrayTypeColumns = (resolutionConfig: ResolutionConfig) => { ); }; +export interface CubeQueryToSQLWithResolutionWithArrayParams { + baseSql: string; + measures: string[]; + dimensions: string[]; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + columnProjections?: string[]; + contextParams?: ContextParams; +} + export const cubeQueryToSQLWithResolutionWithArray = async ({ - query, + baseSql, + measures, + dimensions, tableSchemas, resolutionConfig, columnProjections, contextParams, -}: CubeQueryToSQLWithResolutionParams) => { - // Generate SQL with row_id and unnested arrays +}: CubeQueryToSQLWithResolutionWithArrayParams) => { + // Phase 1: Generate SQL with row_id and unnested arrays const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ - query, + baseSql, + measures, + dimensions, tableSchemas, resolutionConfig, contextParams, }); - // Apply resolution (join with lookup tables) + columnProjections?.push(`${tableSchemas[0].name}.row_id`); + // Phase 2: Apply resolution (join with lookup tables) const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ unnestedSql: unnestBaseSql, baseTableSchema, - query, tableSchemas, resolutionConfig, contextParams, @@ -130,7 +147,6 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ const aggregatedSql = await getAggregatedSql({ resolvedSql, resolvedTableSchema, - query, resolutionConfig, contextParams, }); @@ -138,54 +154,120 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ return aggregatedSql; }; +/** + * Phase 1: Add row_id and apply unnesting + * + * This function performs 3 steps: + * 1. Wrap base SQL and add row_id dimension + * 2. Create schema with unnest modifiers for array columns + * 3. Generate final unnested SQL + * + * @param baseSql - Base SQL generated from the original query (no modifications) + * @param measures - Original measures from the query + * @param dimensions - Original dimensions from the query + * @returns SQL with row_id and unnested arrays, the schema, and list of projected columns + */ export const getUnnestBaseSql = async ({ - query, + baseSql, + measures, + dimensions, tableSchemas, resolutionConfig, contextParams, -}: CubeQueryToSQLWithResolutionParams): Promise<{ +}: { + baseSql: string; + measures: string[]; + dimensions: string[]; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise<{ sql: string; baseTableSchema: TableSchema; }> => { - // Generate base SQL with row_id - const baseSqlWithRowId = await cubeQueryToSQL({ - query: query, - tableSchemas: tableSchemas, - contextParams, - }); - - // This will be used to apply unnesting - const baseTableName = '__base_query'; + const BASE_TABLE_NAME = '__base_query'; + debugger; + const emptyResolutionConfig: ResolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + // Step 1: Create schema for the base SQL const baseTableSchema: TableSchema = createBaseTableSchema( - baseSqlWithRowId, + baseSql, tableSchemas, - resolutionConfig, - query.measures, - query.dimensions + emptyResolutionConfig, + measures, + dimensions ); - // Create query with unnest modifiers applied - const unnestQuery: Query = { + const arrayColumns = getArrayTypeColumns(resolutionConfig); + for (const dimension of baseTableSchema.dimensions) { + const arrayResolvedFields = arrayColumns.map((ac) => + memberKeyToSafeKey(ac.name) + ); + if ( + arrayResolvedFields.some( + (resolvedField) => resolvedField === dimension.name + ) + ) { + dimension.modifier = { shouldFlattenArray: true }; + } + } + debugger; + + // Step 2: Add row_id dimension to schema and generate SQL with row_id + const schemaWithRowId: TableSchema = { + ...baseTableSchema, + dimensions: [ + ...baseTableSchema.dimensions, + { + name: 'row_id', + sql: 'row_number() OVER ()', + type: 'number', + alias: '__row_id', + } as Dimension, + ], + }; + + // Query that projects all original columns plus row_id + const queryWithRowId: Query = { measures: [], dimensions: [ - ...query.measures.map((m) => `${baseTableName}.${m.replace('.', '__')}`), - ...(query.dimensions || []).map( - (d) => `${baseTableName}.${d.replace('.', '__')}` + ...dimensions.map((d) => + getNamespacedKey(BASE_TABLE_NAME, memberKeyToSafeKey(d)) + ), + getNamespacedKey(BASE_TABLE_NAME, 'row_id'), + ...measures.map((m) => + getNamespacedKey(BASE_TABLE_NAME, memberKeyToSafeKey(m)) ), ], }; - // Generate the final SQL with unnesting applied - const unnestedBaseSql = await cubeQueryToSQL({ - query: unnestQuery, - tableSchemas: [baseTableSchema], + const unnestedSql = await cubeQueryToSQL({ + query: queryWithRowId, + tableSchemas: [schemaWithRowId], contextParams, }); + // Build list of projected columns (in original format, not namespaced) + // This is what was actually projected in the unnested SQL + // const projectedColumns = [...dimensions, 'row_id', ...measures]; + + const unnestedBaseTableSchema: TableSchema = { + name: '__unnested_base_query', + sql: unnestedSql, + dimensions: schemaWithRowId.dimensions.map((d) => ({ + name: d.name, + sql: `__unnested_base_query."${d.alias || d.name}"`, + type: d.type, + alias: d.alias, + })), + measures: [], + }; return { - sql: unnestedBaseSql, - baseTableSchema, + sql: unnestedSql, + baseTableSchema: unnestedBaseTableSchema, }; }; @@ -205,7 +287,6 @@ export const getUnnestBaseSql = async ({ export const getResolvedSql = async ({ unnestedSql, baseTableSchema, - query, tableSchemas, resolutionConfig, contextParams, @@ -213,7 +294,6 @@ export const getResolvedSql = async ({ }: { unnestedSql: string; baseTableSchema: TableSchema; - query: Query; tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; @@ -223,27 +303,46 @@ export const getResolvedSql = async ({ resolvedTableSchema: TableSchema; }> => { // Update the SQL to point to the unnested SQL - const updatedBaseTableSchema: TableSchema = { - ...baseTableSchema, - sql: unnestedSql, - }; + const updatedBaseTableSchema: TableSchema = baseTableSchema; - // Generate resolution schemas for array fields only - const resolutionSchemas = generateResolutionSchemas( + // Generate resolution schemas for array fields + const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ + updatedBaseTableSchema, + ]); + + for (const columnConfig of resolutionConfig.columnConfigs) { + columnConfig.name = memberKeyToSafeKey(columnConfig.name); + } + // Generate join paths using existing helper + const joinPaths = generateResolutionJoinPaths( + updatedBaseTableSchema.name, resolutionConfig, - tableSchemas + [updatedBaseTableSchema] ); - // Generate join paths using existing helper - const joinPaths = generateResolutionJoinPaths(resolutionConfig, tableSchemas); + const tempQuery: Query = { + measures: [], + dimensions: baseTableSchema.dimensions.map((d) => d.name), + }; - // Generate resolved dimensions + const updatedColumnProjections = columnProjections?.map((cp) => + memberKeyToSafeKey(cp) + ); + // Generate resolved dimensions using columnProjections const resolvedDimensions = generateResolvedDimensions( - query, + updatedBaseTableSchema.name, + tempQuery, resolutionConfig, - columnProjections + updatedColumnProjections ); + const reconstructingBaseSchema: TableSchema = createBaseTableSchema( + unnestedSql, + [baseTableSchema], + resolutionConfig, + [], + resolvedDimensions + ); // Create query and generate SQL const resolutionQuery: Query = { measures: [], @@ -258,14 +357,17 @@ export const getResolvedSql = async ({ }); // Build list of dimension names that should be in output - const baseDimensionNames = new Set([ - ...query.measures.map((m) => m.replace('.', '__')), - ...(query.dimensions || []) - .filter( - (d) => !resolutionConfig.columnConfigs.some((ac) => ac.name === d) - ) - .map((d) => d.replace('.', '__')), - ]); + // Use the baseTableSchema which already has all the column info + const baseDimensionNames = new Set( + baseTableSchema.dimensions + .filter((dim) => { + // Exclude columns that need resolution (they'll be replaced by resolved columns) + return !resolutionConfig.columnConfigs.some( + (ac) => memberKeyToSafeKey(ac.name) === dim.name + ); + }) + .map((dim) => dim.name) + ); const resolvedTableSchema: TableSchema = { name: '__resolved_query', @@ -309,7 +411,7 @@ export const getResolvedSql = async ({ * 4. Uses ARRAY_AGG for resolved array columns * * @param resolvedSql - SQL output from Phase 2 (with resolved values) - * @param query - Original query + * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) * @param resolutionConfig - Resolution configuration * @param contextParams - Optional context parameters * @returns Final SQL with arrays containing resolved values @@ -317,13 +419,11 @@ export const getResolvedSql = async ({ export const getAggregatedSql = async ({ resolvedSql, resolvedTableSchema, - query, resolutionConfig, contextParams, }: { resolvedSql: string; resolvedTableSchema: TableSchema; - query: Query; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; }): Promise => { @@ -343,7 +443,7 @@ export const getAggregatedSql = async ({ // Check if the dimension name matches a resolved column pattern // e.g., "tickets__owners__display_name" return arrayColumns.some((arrayCol) => { - const arrayColPrefix = arrayCol.name.replace('.', '__'); + const arrayColPrefix = memberKeyToSafeKey(arrayCol.name); return ( dimName.includes(`${arrayColPrefix}__`) && !dimName.startsWith('__base_query__') @@ -392,9 +492,11 @@ export const getAggregatedSql = async ({ // Step 4: Create the aggregation query const aggregationQuery: Query = { - measures: aggregationMeasures.map((m) => `${baseTableName}.${m.name}`), + measures: aggregationMeasures.map((m) => + getNamespacedKey(baseTableName, m.name) + ), dimensions: rowIdDimension - ? [`${baseTableName}.${rowIdDimension.name}`] + ? [getNamespacedKey(baseTableName, rowIdDimension.name)] : [], }; From 75889b087f2991f03692254f703b220e9a39aeec Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Sat, 1 Nov 2025 06:42:05 +0530 Subject: [PATCH 11/38] working properly --- meerkat-core/src/resolution/resolution.ts | 13 + .../cube-to-sql-with-resolution-array.spec.ts | 3108 ++++++++++++++++- .../cube-to-sql-with-resolution.ts | 80 +- 3 files changed, 3171 insertions(+), 30 deletions(-) diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 8cf825b1..96b5a95d 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -165,6 +165,19 @@ export const generateResolvedDimensions = ( ); if (!columnConfig) { + // TODO: See if this can be optimized + // In a first level join right now, we are just passing the dimension without namespacing it. + // But in a resolution level, we are already adding __unnest_query as the base name and using in + // other places like join paths as it depends on it to work. + // For generating join paths, we expect the table name to be exactly the name of memberToSafeKey(namespace, joinDimensionName) + // Hence, at second level, we are namespacing it everywhere, and we need this condition. + if (dimension.includes('.')) { + return [dimension]; + } else { + return [ + getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), + ]; + } return [ getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), ]; diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index f96f2951..e2682dbd 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -1,7 +1,6 @@ import { Query, ResolutionConfig, TableSchema } from '@devrev/meerkat-core'; import { cubeQueryToSQLWithResolution } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; import { duckdbExec } from '../duckdb-exec'; - const CREATE_TEST_TABLE = `CREATE TABLE tickets ( id INTEGER, owners VARCHAR[], @@ -297,6 +296,3113 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { expect(result[0]).toHaveProperty('tickets__tags - tag_name'); }); + // it('test an proper query from UI', async () => { + // const query = { + // dimensions: ['dim_ticket.tags_json'], + // filters: [ + // { + // and: [ + // { + // and: [ + // { + // member: 'dim_ticket.created_date', + // operator: 'inDateRange', + // values: [ + // '2025-08-02T09:30:00.000Z', + // '2025-10-31T10:29:59.999Z', + // ], + // }, + // { + // member: 'dim_ticket.work_type', + // operator: 'in', + // values: ['ticket'], + // }, + // { + // member: 'dim_ticket.stage_json', + // operator: 'in', + // values: [ + // 'don:core:dvrv-us-1:devo/787:custom_stage/514', + // 'don:core:dvrv-us-1:devo/787:custom_stage/512', + // 'don:core:dvrv-us-1:devo/787:custom_stage/501', + // 'don:core:dvrv-us-1:devo/787:custom_stage/485', + // 'don:core:dvrv-us-1:devo/787:custom_stage/440', + // 'don:core:dvrv-us-1:devo/787:custom_stage/25', + // 'don:core:dvrv-us-1:devo/787:custom_stage/24', + // 'don:core:dvrv-us-1:devo/787:custom_stage/22', + // 'don:core:dvrv-us-1:devo/787:custom_stage/21', + // 'don:core:dvrv-us-1:devo/787:custom_stage/19', + // 'don:core:dvrv-us-1:devo/787:custom_stage/13', + // 'don:core:dvrv-us-1:devo/787:custom_stage/10', + // 'don:core:dvrv-us-1:devo/787:custom_stage/8', + // 'don:core:dvrv-us-1:devo/787:custom_stage/7', + // 'don:core:dvrv-us-1:devo/787:custom_stage/6', + // 'don:core:dvrv-us-1:devo/787:custom_stage/4', + // ], + // }, + // ], + // }, + // ], + // }, + // ], + // limit: 50000, + // measures: ['dim_ticket.id_count___function__count'], + // } as any; + // const tableSchemas = [ + // { + // dimensions: [ + // { + // name: 'rev_oid', + // sql: 'dim_ticket.rev_oid', + // type: 'string', + // alias: 'Customer Workspace', + // }, + // { + // name: 'title', + // sql: 'dim_ticket.title', + // type: 'string', + // alias: 'Title', + // }, + // { + // name: 'last_internal_comment_date', + // sql: 'dim_ticket.last_internal_comment_date', + // type: 'time', + // alias: 'Last Internal Comment Date', + // }, + // { + // name: 'work_type', + // sql: "'ticket'", + // type: 'time', + // alias: 'Work Type', + // }, + // { + // name: 'is_spam', + // sql: 'dim_ticket.is_spam', + // type: 'boolean', + // alias: 'Spam', + // }, + // { + // name: 'object_type', + // sql: 'dim_ticket.object_type', + // type: 'string', + // alias: 'Object type', + // }, + // { + // name: 'modified_by_id', + // sql: 'dim_ticket.modified_by_id', + // type: 'string', + // alias: 'Modified by', + // }, + // { + // name: 'source_channel', + // sql: 'dim_ticket.source_channel', + // type: 'string', + // alias: 'Source channel', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'channels', + // sql: 'dim_ticket.channels', + // type: 'number_array', + // alias: 'Channels', + // }, + // { + // name: 'created_by_id', + // sql: 'dim_ticket.created_by_id', + // type: 'string', + // alias: 'Created by', + // }, + // { + // name: 'created_date', + // sql: 'dim_ticket.created_date', + // type: 'time', + // alias: 'Created date', + // }, + // { + // name: 'target_close_date', + // sql: 'dim_ticket.target_close_date', + // type: 'time', + // alias: 'Target close date', + // }, + // { + // name: 'applies_to_part_id', + // sql: 'dim_ticket.applies_to_part_id', + // type: 'string', + // alias: 'Part', + // }, + // { + // name: 'subtype', + // sql: 'dim_ticket.subtype', + // type: 'string', + // alias: 'Subtype', + // }, + // { + // name: 'actual_close_date', + // sql: 'dim_ticket.actual_close_date', + // type: 'time', + // alias: 'Close date', + // }, + // { + // name: 'reported_by_id', + // sql: 'dim_ticket.reported_by_id', + // type: 'string', + // alias: 'Reported by ID', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'reported_by_ids', + // sql: 'dim_ticket.reported_by_ids', + // type: 'string_array', + // alias: 'Reported by', + // }, + // { + // name: 'needs_response', + // sql: 'dim_ticket.needs_response', + // type: 'boolean', + // alias: 'Needs Response', + // }, + // { + // name: 'group', + // sql: 'dim_ticket.group', + // type: 'string', + // alias: 'Group', + // }, + // { + // name: 'modified_date', + // sql: 'dim_ticket.modified_date', + // type: 'time', + // alias: 'Modified date', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'owned_by_ids', + // sql: 'dim_ticket.owned_by_ids', + // type: 'string_array', + // alias: 'Owner', + // }, + // { + // name: 'id', + // sql: 'dim_ticket.id', + // type: 'string', + // alias: 'ID', + // }, + // { + // name: 'sla_tracker_id', + // sql: 'dim_ticket.sla_tracker_id', + // type: 'string', + // alias: 'SLA Tracker', + // }, + // { + // name: 'severity', + // sql: 'dim_ticket.severity', + // type: 'number', + // alias: 'Severity', + // }, + // { + // name: 'stage_json', + // sql: "json_extract_string(dim_ticket.stage_json, '$.stage_id')", + // type: 'string', + // alias: 'Stage', + // }, + // { + // name: 'sla_id', + // sql: 'dim_ticket.sla_id', + // type: 'string', + // alias: 'SLA Name', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'links_json', + // sql: "list_distinct(CAST(json_extract_string(dim_ticket.links_json, '$[*].target_object_type') AS VARCHAR[]))", + // type: 'string_array', + // alias: 'Links', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tags_json', + // sql: "CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[])", + // type: 'string_array', + // alias: 'Tags', + // }, + // { + // name: 'surveys_aggregation_json', + // sql: "list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min')", + // type: 'number', + // alias: 'CSAT Rating', + // }, + // { + // name: 'sla_summary_target_time', + // sql: "cast(json_extract_string(dim_ticket.sla_summary, '$.target_time') as timestamp)", + // type: 'time', + // alias: 'Next SLA Target', + // }, + // { + // name: 'staged_info', + // sql: "cast(json_extract_string(dim_ticket.staged_info, '$.is_staged') as boolean)", + // type: 'boolean', + // alias: 'Changes Need Review', + // }, + // { + // name: 'tnt__account_id', + // sql: 'tnt__account_id', + // type: 'string', + // alias: 'account id', + // }, + // { + // name: 'tnt__actual_effort_spent', + // sql: 'tnt__actual_effort_spent', + // type: 'number', + // alias: 'Actual Effort Spent', + // }, + // { + // name: 'tnt__ai_subtype', + // sql: 'tnt__ai_subtype', + // type: 'string', + // alias: 'AI Subtype', + // }, + // { + // name: 'tnt__asdfad', + // sql: 'tnt__asdfad', + // type: 'string', + // alias: 'asdfad', + // }, + // { + // name: 'tnt__bool_summary', + // sql: 'tnt__bool_summary', + // type: 'boolean', + // alias: 'bool summary', + // }, + // { + // name: 'tnt__boolyesnoresp', + // sql: 'tnt__boolyesnoresp', + // type: 'boolean', + // alias: 'boolyesnoresp', + // }, + // { + // name: 'tnt__capability_part', + // sql: 'tnt__capability_part', + // type: 'string', + // alias: 'Capability part', + // }, + // { + // name: 'tnt__card', + // sql: 'tnt__card', + // type: 'string', + // alias: 'card', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__cust_account', + // sql: 'tnt__cust_account', + // type: 'string_array', + // alias: 'cust_account', + // }, + // { + // name: 'tnt__custom_field_for_sla', + // sql: 'tnt__custom_field_for_sla', + // type: 'string', + // alias: 'custom_field_for_sla', + // }, + // { + // name: 'tnt__custom_id_field', + // sql: 'tnt__custom_id_field', + // type: 'string', + // alias: 'custom id field', + // }, + // { + // name: 'tnt__customer_tier', + // sql: 'tnt__customer_tier', + // type: 'string', + // alias: 'Customer Tier', + // }, + // { + // name: 'tnt__date_field', + // sql: 'tnt__date_field', + // type: 'time', + // alias: 'date_field', + // }, + // { + // name: 'tnt__department_list', + // sql: 'tnt__department_list', + // type: 'string', + // alias: 'Department List', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__email_references', + // sql: 'tnt__email_references', + // type: 'string_array', + // alias: 'tnt__email_references', + // }, + // { + // name: 'tnt__escalated', + // sql: 'tnt__escalated', + // type: 'boolean', + // alias: 'Escalated', + // }, + // { + // name: 'tnt__estimated_effort', + // sql: 'tnt__estimated_effort', + // type: 'number', + // alias: 'Estimated Effort', + // }, + // { + // name: 'tnt__external_source_id', + // sql: 'tnt__external_source_id', + // type: 'string', + // alias: 'tnt__external_source_id', + // }, + // { + // name: 'tnt__feature_affected', + // sql: 'tnt__feature_affected', + // type: 'string', + // alias: 'Feature Affected', + // }, + // { + // name: 'tnt__field1', + // sql: 'tnt__field1', + // type: 'string', + // alias: 'field1', + // }, + // { + // name: 'tnt__foo', + // sql: 'tnt__foo', + // type: 'string', + // alias: 'tnt__foo', + // }, + // { + // name: 'tnt__fruit', + // sql: 'tnt__fruit', + // type: 'string', + // alias: 'Fruit', + // }, + // { + // name: 'tnt__git_template_name', + // sql: 'tnt__git_template_name', + // type: 'string', + // alias: 'Github Template Name', + // }, + // { + // name: 'tnt__id_field', + // sql: 'tnt__id_field', + // type: 'string', + // alias: 'id field', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__id_list_field', + // sql: 'tnt__id_list_field', + // type: 'string_array', + // alias: 'id list field', + // }, + // { + // name: 'tnt__impact', + // sql: 'tnt__impact', + // type: 'string', + // alias: 'Impact', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__include_on_emails', + // sql: 'tnt__include_on_emails', + // type: 'string_array', + // alias: 'Include on emails', + // }, + // { + // name: 'tnt__issue_score', + // sql: 'tnt__issue_score', + // type: 'number', + // alias: 'Issue Score', + // }, + // { + // name: 'tnt__knowledge_gap', + // sql: 'tnt__knowledge_gap', + // type: 'string', + // alias: 'Knowledge Gap', + // }, + // { + // name: 'tnt__linear_assignee', + // sql: 'tnt__linear_assignee', + // type: 'string', + // alias: 'Linear Assignee', + // }, + // { + // name: 'tnt__linear_description', + // sql: 'tnt__linear_description', + // type: 'string', + // alias: 'Linear Description', + // }, + // { + // name: 'tnt__linear_id', + // sql: 'tnt__linear_id', + // type: 'string', + // alias: 'Linear Id', + // }, + // { + // name: 'tnt__linear_identifier', + // sql: 'tnt__linear_identifier', + // type: 'string', + // alias: 'Linear Identifier', + // }, + // { + // name: 'tnt__linear_priority', + // sql: 'tnt__linear_priority', + // type: 'string', + // alias: 'Linear Priority', + // }, + // { + // name: 'tnt__linear_state', + // sql: 'tnt__linear_state', + // type: 'string', + // alias: 'Linear State', + // }, + // { + // name: 'tnt__linear_team', + // sql: 'tnt__linear_team', + // type: 'string', + // alias: 'Linear Team', + // }, + // { + // name: 'tnt__linear_title', + // sql: 'tnt__linear_title', + // type: 'string', + // alias: 'Linear Title', + // }, + // { + // name: 'tnt__linear_url', + // sql: 'tnt__linear_url', + // type: 'string', + // alias: 'Linear Url', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__members', + // sql: 'tnt__members', + // type: 'string_array', + // alias: 'tnt__members', + // }, + // { + // name: 'tnt__micro_service_name', + // sql: 'tnt__micro_service_name', + // type: 'string', + // alias: 'Microservice Name', + // }, + // { + // name: 'tnt__notify_others', + // sql: 'tnt__notify_others', + // type: 'string', + // alias: 'notify others', + // }, + // { + // name: 'tnt__numeric_field', + // sql: 'tnt__numeric_field', + // type: 'number', + // alias: 'numeric field', + // }, + // { + // name: 'tnt__parent_part', + // sql: 'tnt__parent_part', + // type: 'string', + // alias: 'Parent part', + // }, + // { + // name: 'tnt__part_product', + // sql: 'tnt__part_product', + // type: 'string', + // alias: 'part product', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__paytm_customer_name', + // sql: 'tnt__paytm_customer_name', + // type: 'string_array', + // alias: 'paytm customer name', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__platform', + // sql: 'tnt__platform', + // type: 'string_array', + // alias: 'Platform', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__portal_test', + // sql: 'tnt__portal_test', + // type: 'string_array', + // alias: 'portal test', + // }, + // { + // name: 'tnt__product', + // sql: 'tnt__product', + // type: 'string', + // alias: 'Product', + // }, + // { + // name: 'tnt__project', + // sql: 'tnt__project', + // type: 'string', + // alias: 'Project', + // }, + // { + // name: 'tnt__project_stage', + // sql: 'tnt__project_stage', + // type: 'string', + // alias: 'Project Stage', + // }, + // { + // name: 'tnt__rank', + // sql: 'tnt__rank', + // type: 'number', + // alias: 'Rank', + // }, + // { + // name: 'tnt__remaining_effort', + // sql: 'tnt__remaining_effort', + // type: 'number', + // alias: 'Remaining Effort', + // }, + // { + // name: 'tnt__required_text_field', + // sql: 'tnt__required_text_field', + // type: 'string', + // alias: 'required text field', + // }, + // { + // name: 'tnt__resolution_sla_met', + // sql: 'tnt__resolution_sla_met', + // type: 'string', + // alias: 'Resolution SLA Met', + // }, + // { + // name: 'tnt__resolution_sla_target_time', + // sql: 'tnt__resolution_sla_target_time', + // type: 'string', + // alias: 'Resolution SLA Target Time', + // }, + // { + // name: 'tnt__resolution_turnaround_time', + // sql: 'tnt__resolution_turnaround_time', + // type: 'string', + // alias: 'Resolution SLA Turnaround Time', + // }, + // { + // name: 'tnt__response_sla_met', + // sql: 'tnt__response_sla_met', + // type: 'string', + // alias: 'First Response SLA Met', + // }, + // { + // name: 'tnt__response_sla_target_time', + // sql: 'tnt__response_sla_target_time', + // type: 'string', + // alias: 'First Response SLA Target Time', + // }, + // { + // name: 'tnt__response_turnaround_time', + // sql: 'tnt__response_turnaround_time', + // type: 'string', + // alias: 'First Response SLA Turnaround Time', + // }, + // { + // name: 'tnt__root_cause_analysis', + // sql: 'tnt__root_cause_analysis', + // type: 'string', + // alias: 'Root cause analysis', + // }, + // { + // name: 'tnt__search_field_query', + // sql: 'tnt__search_field_query', + // type: 'string', + // alias: 'Search Field Query', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__stakeholders', + // sql: 'tnt__stakeholders', + // type: 'string_array', + // alias: 'stakeholders', + // }, + // { + // name: 'tnt__stray_user', + // sql: 'tnt__stray_user', + // type: 'string', + // alias: 'stray user', + // }, + // { + // name: 'tnt__stray_users', + // sql: 'tnt__stray_users', + // type: 'string', + // alias: 'stray users', + // }, + // { + // name: 'tnt__test', + // sql: 'tnt__test', + // type: 'number', + // alias: 'Test', + // }, + // { + // name: 'tnt__test123', + // sql: 'tnt__test123', + // type: 'string', + // alias: 'test123', + // }, + // { + // name: 'tnt__test_capability_part', + // sql: 'tnt__test_capability_part', + // type: 'string', + // alias: 'Test Capability Part', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__test_field_for_sla', + // sql: 'tnt__test_field_for_sla', + // type: 'string_array', + // alias: 'test field for sla', + // }, + // { + // name: 'tnt__test_filed', + // sql: 'tnt__test_filed', + // type: 'string', + // alias: 'test field', + // }, + // { + // name: 'tnt__test_mikasa', + // sql: 'tnt__test_mikasa', + // type: 'string', + // alias: 'Test mikasa', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__test_multivalue_testrequiredfield', + // sql: 'tnt__test_multivalue_testrequiredfield', + // type: 'string_array', + // alias: 'test multivalue testrequiredfield', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__test_multivalue_textfield', + // sql: 'tnt__test_multivalue_textfield', + // type: 'string_array', + // alias: 'test multivalue textfield', + // }, + // { + // name: 'tnt__test_product_part', + // sql: 'tnt__test_product_part', + // type: 'string', + // alias: 'Test Product part', + // }, + // { + // name: 'tnt__testing_bool_drpdown', + // sql: 'tnt__testing_bool_drpdown', + // type: 'string', + // alias: 'Testing Bool Drpdown', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'tnt__text_list_account', + // sql: 'tnt__text_list_account', + // type: 'string_array', + // alias: 'Text list account', + // }, + // { + // name: 'tnt__ticket_custom_part', + // sql: 'tnt__ticket_custom_part', + // type: 'string', + // alias: 'ticket custom part', + // }, + // { + // name: 'tnt__ticket_dropd', + // sql: 'tnt__ticket_dropd', + // type: 'string', + // alias: 'ticket dropdown', + // }, + // { + // name: 'tnt__ticket_type', + // sql: 'tnt__ticket_type', + // type: 'string', + // alias: 'Ticket type', + // }, + // { + // name: 'tnt__tier', + // sql: 'tnt__tier', + // type: 'string', + // alias: 'Tier', + // }, + // { + // name: 'tnt__urgency', + // sql: 'tnt__urgency', + // type: 'string', + // alias: 'Urgency', + // }, + // { + // name: 'tnt__velocity_test_enum', + // sql: 'tnt__velocity_test_enum', + // type: 'string', + // alias: 'velocity test enum', + // }, + // { + // name: 'tnt__version', + // sql: 'tnt__version', + // type: 'string', + // alias: 'Version', + // }, + // { + // name: 'tnt__work_duration', + // sql: 'tnt__work_duration', + // type: 'string', + // alias: 'Work duration', + // }, + // { + // name: 'tnt__workspace_custom', + // sql: 'tnt__workspace_custom', + // type: 'string', + // alias: 'workspace_custom', + // }, + // { + // name: 'ctype_deal_registration__buying_region', + // sql: 'ctype_deal_registration__buying_region', + // type: 'string', + // alias: 'Theater', + // }, + // { + // name: 'ctype_deal_registration__deal_status', + // sql: 'ctype_deal_registration__deal_status', + // type: 'string', + // alias: 'Deal Status', + // }, + // { + // name: 'ctype_deal_registration__deal_type', + // sql: 'ctype_deal_registration__deal_type', + // type: 'string', + // alias: 'Deal Type', + // }, + // { + // name: 'ctype_deal_registration__employee_count', + // sql: 'ctype_deal_registration__employee_count', + // type: 'string', + // alias: 'Employee Count', + // }, + // { + // name: 'ctype_deal_registration__estimated_deal_value', + // sql: 'ctype_deal_registration__estimated_deal_value', + // type: 'number', + // alias: 'Estimated Deal Value', + // }, + // { + // name: 'ctype_deal_registration__expected_close_date', + // sql: 'ctype_deal_registration__expected_close_date', + // type: 'time', + // alias: 'Expected Close Date', + // }, + // { + // name: 'ctype_deal_registration__partner_type', + // sql: 'ctype_deal_registration__partner_type', + // type: 'string', + // alias: 'Partner Type', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_deal_registration__products_or_services', + // sql: 'ctype_deal_registration__products_or_services', + // type: 'string_array', + // alias: 'Product(s) or Service(s)', + // }, + // { + // name: 'ctype_deal_registration__prospect_company_name', + // sql: 'ctype_deal_registration__prospect_company_name', + // type: 'string', + // alias: 'Prospect Company Name', + // }, + // { + // name: 'ctype_deal_registration__prospect_company_website', + // sql: 'ctype_deal_registration__prospect_company_website', + // type: 'string', + // alias: 'Prospect Company Website', + // }, + // { + // name: 'ctype_deal_registration__prospect_contact_email', + // sql: 'ctype_deal_registration__prospect_contact_email', + // type: 'string', + // alias: 'Prospect Contact Email', + // }, + // { + // name: 'ctype_deal_registration__prospect_contact_name', + // sql: 'ctype_deal_registration__prospect_contact_name', + // type: 'string', + // alias: 'Prospect Contact Name', + // }, + // { + // name: 'ctype_deal_registration__sub_region', + // sql: 'ctype_deal_registration__sub_region', + // type: 'string', + // alias: 'Region ', + // }, + // { + // name: 'ctype_deal_registration__subregion', + // sql: 'ctype_deal_registration__subregion', + // type: 'string', + // alias: 'Subregion', + // }, + // { + // name: 'ctype_Events__campaign_category', + // sql: 'ctype_Events__campaign_category', + // type: 'string', + // alias: 'Campaign Category', + // }, + // { + // name: 'ctype_Events__event_end_date', + // sql: 'ctype_Events__event_end_date', + // type: 'time', + // alias: 'Event End Date', + // }, + // { + // name: 'ctype_Events__event_owner', + // sql: 'ctype_Events__event_owner', + // type: 'string', + // alias: 'Event Owner', + // }, + // { + // name: 'ctype_Events__event_start_date', + // sql: 'ctype_Events__event_start_date', + // type: 'time', + // alias: 'Event Start Date', + // }, + // { + // name: 'ctype_Events__events_test_ref', + // sql: 'ctype_Events__events_test_ref', + // type: 'string', + // alias: 'events test ref', + // }, + // { + // name: 'ctype_Events__external_budget', + // sql: 'ctype_Events__external_budget', + // type: 'number', + // alias: 'External Budget', + // }, + // { + // name: 'ctype_Events__internal_budget', + // sql: 'ctype_Events__internal_budget', + // type: 'number', + // alias: 'Internal Budget', + // }, + // { + // name: 'ctype_Events__is_devrev_event', + // sql: 'ctype_Events__is_devrev_event', + // type: 'boolean', + // alias: 'Is DevRev Event', + // }, + // { + // name: 'ctype_Events__mode', + // sql: 'ctype_Events__mode', + // type: 'string', + // alias: 'Mode', + // }, + // { + // name: 'ctype_Events__organized_by', + // sql: 'ctype_Events__organized_by', + // type: 'string', + // alias: 'Organized by', + // }, + // { + // name: 'ctype_Events__pipeline_generated', + // sql: 'ctype_Events__pipeline_generated', + // type: 'number', + // alias: 'Pipeline Generated', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_Events__representatives', + // sql: 'ctype_Events__representatives', + // type: 'string_array', + // alias: 'Representatives', + // }, + // { + // name: 'ctype_Events__source', + // sql: 'ctype_Events__source', + // type: 'string', + // alias: 'Source', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_Events__sponsors', + // sql: 'ctype_Events__sponsors', + // type: 'string_array', + // alias: 'Sponsors', + // }, + // { + // name: 'ctype_Events__sub_source', + // sql: 'ctype_Events__sub_source', + // type: 'string', + // alias: 'Sub-Source', + // }, + // { + // name: 'ctype_Events__total_budget', + // sql: 'ctype_Events__total_budget', + // type: 'number', + // alias: 'Total Budget', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', + // type: 'boolean', + // alias: 'allow_attachments', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', + // type: 'boolean', + // alias: 'allow_channelback', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', + // type: 'string_array', + // alias: 'Collaborators', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', + // type: 'string', + // alias: 'External Object Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', + // type: 'string', + // alias: 'external_id', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', + // type: 'string_array', + // alias: 'Fields modified during import', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', + // type: 'boolean', + // alias: 'from_messaging_channel', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', + // type: 'boolean', + // alias: 'has_incidents', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', + // type: 'boolean', + // alias: 'is_public', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', + // type: 'string_array', + // alias: 'Reporters', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', + // type: 'string', + // alias: 'Customer', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', + // type: 'string', + // alias: 'raw_subject', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', + // type: 'string', + // alias: 'recipient', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', + // type: 'string', + // alias: 'External Source', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', + // type: 'string', + // alias: 'url', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', + // type: 'string', + // alias: 'Hold Status', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', + // type: 'string', + // alias: 'ATI Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', + // type: 'string', + // alias: 'Event ID', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', + // type: 'boolean', + // alias: 'Do Not Close', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', + // type: 'boolean', + // alias: 'Escalated', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', + // type: 'string', + // alias: 'Partner: Sales Contact', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', + // type: 'string', + // alias: 'Partner Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', + // type: 'string', + // alias: 'Env Setup Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', + // type: 'string', + // alias: 'Additional Comments', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', + // type: 'string', + // alias: 'Remind me before', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', + // type: 'string', + // alias: ' zIPS + z3A Licenses', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', + // type: 'string', + // alias: 'zDefend Licenses', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', + // type: 'string', + // alias: 'Approval Status', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', + // type: 'string', + // alias: 'Booking Year', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', + // type: 'string', + // alias: 'Term', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', + // type: 'string', + // alias: 'End Customer : Admin Details', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', + // type: 'string_array', + // alias: 'Root Cause', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', + // type: 'string', + // alias: 'Request Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', + // type: 'string', + // alias: 'End User - Full Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', + // type: 'string', + // alias: 'End Customer: Administrator Email Address', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', + // type: 'string', + // alias: 'Services Purchased', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', + // type: 'string', + // alias: 'Support Purchased', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', + // type: 'string', + // alias: 'Cancellation Reason', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', + // type: 'string', + // alias: 'Cancellation Date', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', + // type: 'string', + // alias: 'zConsole URL', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', + // type: 'string', + // alias: 'Customer Success Representative', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', + // type: 'string', + // alias: 'VPC Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', + // type: 'string', + // alias: 'End Customer: Region', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', + // type: 'string', + // alias: 'Proof of Concept Impact ?', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', + // type: 'string', + // alias: 'Internal Status', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', + // type: 'string', + // alias: 'Problem Sub-Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', + // type: 'string', + // alias: 'Tenant Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', + // type: 'string', + // alias: 'Customer Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', + // type: 'string', + // alias: 'Channel', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', + // type: 'string', + // alias: 'Order Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', + // type: 'string', + // alias: 'zIPS Licenses', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', + // type: 'boolean', + // alias: 'z3A License', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', + // type: 'string', + // alias: 'Sub-Partner Name', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', + // type: 'string_array', + // alias: 'Ticket Region', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', + // type: 'string', + // alias: 'zShield Licenses', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', + // type: 'string', + // alias: 'zScan Apps Licensed', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', + // type: 'string', + // alias: 'KPE MTD Licenses', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', + // type: 'string', + // alias: 'Product Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', + // type: 'string', + // alias: 'whiteCryption agent', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', + // type: 'string', + // alias: 'zIPS Test Case Types', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', + // type: 'string', + // alias: 'Product Version', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', + // type: 'string', + // alias: 'Target Delivery Date', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', + // type: 'string', + // alias: 'App Version ', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', + // type: 'string', + // alias: 'End Customer: Company Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', + // type: 'string', + // alias: 'End Customer: Physical Address', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', + // type: 'string', + // alias: 'Term Start Date', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', + // type: 'string', + // alias: 'Term End Date', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', + // type: 'string', + // alias: 'Order Fulfilled', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', + // type: 'string', + // alias: 'Requested Delivery Date', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', + // type: 'string', + // alias: 'Internal Tracking Id', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', + // type: 'string', + // alias: 'Feature Status', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', + // type: 'string', + // alias: 'Case Type', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', + // type: 'string', + // alias: 'Business Impact', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', + // type: 'string', + // alias: 'Zimperium Console URL', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', + // type: 'string', + // alias: 'End Customer: Administrator Name', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', + // type: 'string', + // alias: 'zIPS Only Licenses ', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', + // type: 'string', + // alias: 'Zimperium Sales Contact: Email Address', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', + // type: 'boolean', + // alias: 'allow_attachments', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', + // type: 'boolean', + // alias: 'allow_channelback', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', + // type: 'string_array', + // alias: 'Collaborators', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', + // type: 'string', + // alias: 'External Object Type', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', + // type: 'string_array', + // alias: 'Fields modified during import', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', + // type: 'boolean', + // alias: 'from_messaging_channel', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', + // type: 'boolean', + // alias: 'has_incidents', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', + // type: 'boolean', + // alias: 'is_public', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', + // type: 'string_array', + // alias: 'Reporters', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', + // type: 'string', + // alias: 'Customer', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', + // type: 'string', + // alias: 'raw_subject', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', + // type: 'string', + // alias: 'recipient', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', + // type: 'string', + // alias: 'External Source', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', + // type: 'string', + // alias: 'url', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', + // type: 'string', + // alias: 'Hold Status', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', + // type: 'boolean', + // alias: 'Do Not Close', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', + // type: 'boolean', + // alias: 'Escalated', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', + // type: 'string', + // alias: 'Partner: Sales Contact', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', + // type: 'string', + // alias: 'Partner Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', + // type: 'string', + // alias: 'Env Setup Type', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', + // type: 'string', + // alias: 'Approval Status', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', + // type: 'string', + // alias: 'Booking Year', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', + // type: 'string_array', + // alias: 'Root Cause', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', + // type: 'string', + // alias: 'Services Purchased', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', + // type: 'string', + // alias: 'Support Purchased', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', + // type: 'string', + // alias: 'VPC Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', + // type: 'string', + // alias: 'End Customer: Region', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', + // type: 'string', + // alias: 'Proof of Concept Impact ?', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', + // type: 'string', + // alias: 'Problem Sub-Type', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', + // type: 'string', + // alias: 'Tenant Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', + // type: 'string', + // alias: 'Customer Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', + // type: 'string', + // alias: 'Channel', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', + // type: 'string', + // alias: 'Order Type', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', + // type: 'string', + // alias: 'zIPS Licenses', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', + // type: 'boolean', + // alias: 'z3A License', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', + // type: 'string', + // alias: 'Sub-Partner Name', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', + // type: 'string_array', + // alias: 'Ticket Region', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', + // type: 'string', + // alias: 'Product Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', + // type: 'string', + // alias: 'whiteCryption agent', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', + // type: 'string', + // alias: 'Product Version', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', + // type: 'string', + // alias: 'App Version ', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', + // type: 'string', + // alias: 'End Customer: Company Name', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', + // type: 'string', + // alias: 'End Customer: Physical Address', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', + // type: 'string', + // alias: 'Term Start Date', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', + // type: 'string', + // alias: 'Term End Date', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', + // type: 'string', + // alias: 'Order Fulfilled', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', + // type: 'string', + // alias: 'Internal Tracking Id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', + // type: 'string', + // alias: 'Feature Status', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', + // type: 'string', + // alias: 'Case Type', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', + // type: 'string', + // alias: 'Business Impact', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', + // type: 'string', + // alias: 'DuplicateField Console URL', + // }, + // { + // name: 'ctype_hell__test', + // sql: 'ctype_hell__test', + // type: 'string', + // alias: 'test', + // }, + // { + // name: 'ctype_hell__text_default_field', + // sql: 'ctype_hell__text_default_field', + // type: 'string', + // alias: 'text default field', + // }, + // { + // name: 'ctype_clone_ticket___clone_field_dropdown', + // sql: 'ctype_clone_ticket___clone_field_dropdown', + // type: 'string', + // alias: 'clone field dropdown', + // }, + // { + // name: 'ctype_test_subtype_sep__new_field_2', + // sql: 'ctype_test_subtype_sep__new_field_2', + // type: 'string', + // alias: 'field_harsh', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_test_subtype_sep__new_field_id_list_3', + // sql: 'ctype_test_subtype_sep__new_field_id_list_3', + // type: 'string_array', + // alias: 'field_id_list_harsh', + // }, + // { + // name: 'ctype_vgtestyesno__booltestyesno', + // sql: 'ctype_vgtestyesno__booltestyesno', + // type: 'boolean', + // alias: 'booltestyesno', + // }, + // { + // name: 'ctype_asd__test_asd', + // sql: 'ctype_asd__test_asd', + // type: 'string', + // alias: 'test asd', + // }, + // { + // name: 'ctype_Microservice__git_template_name', + // sql: 'ctype_Microservice__git_template_name', + // type: 'string', + // alias: 'Github Template Name', + // }, + // { + // name: 'ctype_Microservice__micro_service_name', + // sql: 'ctype_Microservice__micro_service_name', + // type: 'string', + // alias: 'Microservice Name', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_Microservice__platform', + // sql: 'ctype_Microservice__platform', + // type: 'string_array', + // alias: 'Platform', + // }, + // { + // name: 'ctype_Microservice__read_only', + // sql: 'ctype_Microservice__read_only', + // type: 'string', + // alias: 'read only', + // }, + // { + // name: 'ctype_helohelohelohelohelo__dfkdkl', + // sql: 'ctype_helohelohelohelohelo__dfkdkl', + // type: 'number', + // alias: 'dfkdkl', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_1122', + // sql: 'ctype_helohelohelohelohelo__hello_1122', + // type: 'string', + // alias: 'hello hello folks', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_12', + // sql: 'ctype_helohelohelohelohelo__hello_12', + // type: 'string', + // alias: 'hello_12', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_123', + // sql: 'ctype_helohelohelohelohelo__hello_123', + // type: 'string', + // alias: 'hello_123', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_12x', + // sql: 'ctype_helohelohelohelohelo__hello_12x', + // type: 'string', + // alias: 'hello_12x', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_12y', + // sql: 'ctype_helohelohelohelohelo__hello_12y', + // type: 'string', + // alias: 'hello_12y', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_wed', + // sql: 'ctype_helohelohelohelohelo__hello_wed', + // type: 'string', + // alias: 'hello_wed', + // }, + // { + // name: 'ctype_helohelohelohelohelo__hello_world', + // sql: 'ctype_helohelohelohelohelo__hello_world', + // type: 'string', + // alias: 'hello_world', + // }, + // { + // name: 'ctype_dddddddd__dkay_march_23', + // sql: 'ctype_dddddddd__dkay_march_23', + // type: 'string', + // alias: 'dkay_march_23', + // }, + // { + // name: 'ctype_dddddddd__dkay_march_26', + // sql: 'ctype_dddddddd__dkay_march_26', + // type: 'string', + // alias: 'dkay_march_26', + // }, + // { + // name: 'ctype_dddddddd__scope_approval', + // sql: 'ctype_dddddddd__scope_approval', + // type: 'string', + // alias: 'Scope Approval', + // }, + // { + // name: 'ctype_dddddddd__uat_approval', + // sql: 'ctype_dddddddd__uat_approval', + // type: 'string', + // alias: 'UAT approval', + // }, + // { + // name: 'ctype_defaults__customer_uat', + // sql: 'ctype_defaults__customer_uat', + // type: 'time', + // alias: 'customer uat', + // }, + // { + // name: 'ctype_defaults__default', + // sql: 'ctype_defaults__default', + // type: 'string', + // alias: 'default', + // }, + // { + // name: 'ctype_defaults__default_enum', + // sql: 'ctype_defaults__default_enum', + // type: 'string', + // alias: 'default enum', + // }, + // { + // name: 'ctype_defaults__enum_testing', + // sql: 'ctype_defaults__enum_testing', + // type: 'string', + // alias: 'enum testing', + // }, + // { + // name: 'ctype_defaults__uat_date', + // sql: 'ctype_defaults__uat_date', + // type: 'time', + // alias: 'uat date', + // }, + // { + // name: 'ctype_bug__abc', + // sql: 'ctype_bug__abc', + // type: 'string', + // alias: 'abc', + // }, + // { + // name: 'ctype_bug__dependent_test', + // sql: 'ctype_bug__dependent_test', + // type: 'string', + // alias: 'something', + // }, + // { + // name: 'ctype_event_request__approver', + // sql: 'ctype_event_request__approver', + // type: 'string', + // alias: 'Approver', + // }, + // { + // name: 'ctype_event_request__budget', + // sql: 'ctype_event_request__budget', + // type: 'number', + // alias: 'Budget', + // }, + // { + // name: 'ctype_event_request__event_date', + // sql: 'ctype_event_request__event_date', + // type: 'time', + // alias: 'Event Date', + // }, + // { + // name: 'ctype_event_request__event_location', + // sql: 'ctype_event_request__event_location', + // type: 'string', + // alias: 'Event Location', + // }, + // { + // name: 'ctype_event_request__event_name', + // sql: 'ctype_event_request__event_name', + // type: 'string', + // alias: 'Event Name', + // }, + // { + // name: 'ctype_event_request__event_owner', + // sql: 'ctype_event_request__event_owner', + // type: 'string', + // alias: 'Event Owner', + // }, + // { + // name: 'ctype_event_request__event_type', + // sql: 'ctype_event_request__event_type', + // type: 'string', + // alias: 'Event Type', + // }, + // { + // name: 'ctype_event_request__link_to_event', + // sql: 'ctype_event_request__link_to_event', + // type: 'string', + // alias: 'Link To Event', + // }, + // { + // name: 'ctype_event_request__region', + // sql: 'ctype_event_request__region', + // type: 'string', + // alias: 'Region', + // }, + // { + // name: 'ctype_event_request__requested_by', + // sql: 'ctype_event_request__requested_by', + // type: 'string', + // alias: 'Requested by', + // }, + // { + // name: 'ctype_event_request__status', + // sql: 'ctype_event_request__status', + // type: 'string', + // alias: 'Status', + // }, + // { + // name: 'ctype_event_request__target_pipeline', + // sql: 'ctype_event_request__target_pipeline', + // type: 'number', + // alias: 'Target Pipeline', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_event_request__target_segment', + // sql: 'ctype_event_request__target_segment', + // type: 'string_array', + // alias: 'Target Segment', + // }, + // { + // name: 'ctype_mfz_subtype_check__enum_field', + // sql: 'ctype_mfz_subtype_check__enum_field', + // type: 'string', + // alias: 'Custom Enum field', + // }, + // { + // name: 'ctype_mfz_subtype_check__text_field', + // sql: 'ctype_mfz_subtype_check__text_field', + // type: 'string', + // alias: 'Custom text field', + // }, + // { + // name: 'ctype_mfz_subtype_check__user_field', + // sql: 'ctype_mfz_subtype_check__user_field', + // type: 'string', + // alias: 'Custom user field', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_mfz_subtype_check__user_list_field', + // sql: 'ctype_mfz_subtype_check__user_list_field', + // type: 'string_array', + // alias: 'Custom user list field', + // }, + // { + // name: 'ctype_mfz_subtype_check__workspace_field', + // sql: 'ctype_mfz_subtype_check__workspace_field', + // type: 'string', + // alias: 'Custom workspace field', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_mfz_subtype_check__workspace_list_field', + // sql: 'ctype_mfz_subtype_check__workspace_list_field', + // type: 'string_array', + // alias: 'Custom workspace list field', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', + // type: 'boolean', + // alias: 'allow_attachments', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', + // type: 'boolean', + // alias: 'allow_channelback', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', + // type: 'string_array', + // alias: 'Collaborators', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', + // type: 'string', + // alias: 'External Object Type', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', + // type: 'string_array', + // alias: 'Fields modified during import', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', + // type: 'boolean', + // alias: 'from_messaging_channel', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', + // type: 'boolean', + // alias: 'has_incidents', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', + // type: 'boolean', + // alias: 'is_public', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', + // type: 'string_array', + // alias: 'Reporters', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', + // type: 'string', + // alias: 'Customer', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', + // type: 'string', + // alias: 'raw_subject', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', + // type: 'string', + // alias: 'recipient', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', + // type: 'string', + // alias: 'External Source', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', + // type: 'string', + // alias: 'url', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', + // type: 'string_array', + // alias: 'Ticket Region', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', + // type: 'string', + // alias: 'whiteCryption agent', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', + // type: 'string', + // alias: 'Feature Status', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', + // type: 'string', + // alias: 'Case Type', + // }, + // { + // name: 'ctype_mandatory_field_subtype__some_text_field', + // sql: 'ctype_mandatory_field_subtype__some_text_field', + // type: 'string', + // alias: 'Some text field', + // }, + // { + // name: 'ctype_review_subtype__date', + // sql: 'ctype_review_subtype__date', + // type: 'time', + // alias: 'Date', + // }, + // { + // name: 'ctype_review_subtype__play_store_review_url', + // sql: 'ctype_review_subtype__play_store_review_url', + // type: 'string', + // alias: 'Review URL', + // }, + // { + // name: 'ctype_review_subtype__rating', + // sql: 'ctype_review_subtype__rating', + // type: 'string', + // alias: 'Rating', + // }, + // { + // name: 'ctype_review_subtype__reply_date', + // sql: 'ctype_review_subtype__reply_date', + // type: 'time', + // alias: 'Reply date', + // }, + // { + // name: 'ctype_review_subtype__reply_text', + // sql: 'ctype_review_subtype__reply_text', + // type: 'string', + // alias: 'Reply text', + // }, + // { + // name: 'ctype_review_subtype__upvote_count', + // sql: 'ctype_review_subtype__upvote_count', + // type: 'number', + // alias: 'Upvote Count', + // }, + // { + // name: 'ctype_jkjkjkjkjkjk__dgg', + // sql: 'ctype_jkjkjkjkjkjk__dgg', + // type: 'string', + // alias: 'dgg', + // }, + // { + // name: 'ctype_jkjkjkjkjkjk__dggd', + // sql: 'ctype_jkjkjkjkjkjk__dggd', + // type: 'number', + // alias: 'dggd', + // }, + // { + // name: 'ctype_jkjkjkjkjkjk__hello', + // sql: 'ctype_jkjkjkjkjkjk__hello', + // type: 'string', + // alias: 'hello', + // }, + // { + // name: 'ctype_jkjkjkjkjkjk__hello_acl', + // sql: 'ctype_jkjkjkjkjkjk__hello_acl', + // type: 'string', + // alias: 'hellO_acl', + // }, + // { + // name: 'ctype_huryyyyyyyyyyy__all_internal_selected', + // sql: 'ctype_huryyyyyyyyyyy__all_internal_selected', + // type: 'string', + // alias: 'all_external_selected', + // }, + // { + // name: 'ctype_huryyyyyyyyyyy__all_selected', + // sql: 'ctype_huryyyyyyyyyyy__all_selected', + // type: 'string', + // alias: 'all_selected', + // }, + // { + // name: 'ctype_huryyyyyyyyyyy__test_acl_2', + // sql: 'ctype_huryyyyyyyyyyy__test_acl_2', + // type: 'string', + // alias: 'test_acl_2', + // }, + // { + // name: 'ctype_aaa__fkjdsfjkdj', + // sql: 'ctype_aaa__fkjdsfjkdj', + // type: 'string', + // alias: 'fkjdsfjkdj', + // }, + // { + // name: 'ctype_aaa__test_acl_march', + // sql: 'ctype_aaa__test_acl_march', + // type: 'string', + // alias: 'test_acl_march', + // }, + // { + // name: 'ctype_lolololololol__hell_acl_2', + // sql: 'ctype_lolololololol__hell_acl_2', + // type: 'string', + // alias: 'hell_acl_2', + // }, + // { + // name: 'ctype_lolololololol__hello_acl', + // sql: 'ctype_lolololololol__hello_acl', + // type: 'string', + // alias: 'hello_acl', + // }, + // { + // name: 'ctype_lolololololol__hello_acl_3', + // sql: 'ctype_lolololololol__hello_acl_3', + // type: 'string', + // alias: 'hello_acl_3', + // }, + // { + // name: 'ctype_zzz__dds', + // sql: 'ctype_zzz__dds', + // type: 'string', + // alias: 'DDS', + // }, + // { + // name: 'ctype_zzz__hello', + // sql: 'ctype_zzz__hello', + // type: 'string', + // alias: 'hello ', + // }, + // { + // name: 'ctype_zzz__hello_2', + // sql: 'ctype_zzz__hello_2', + // type: 'string', + // alias: 'hello_2', + // }, + // { + // name: 'ctype_zzz__hello_3', + // sql: 'ctype_zzz__hello_3', + // type: 'string', + // alias: 'hello_3', + // }, + // { + // name: 'ctype_zzz__hello_4', + // sql: 'ctype_zzz__hello_4', + // type: 'string', + // alias: 'hello_4', + // }, + // { + // name: 'ctype_zzz__sdkjdsfjk', + // sql: 'ctype_zzz__sdkjdsfjk', + // type: 'string', + // alias: 'SDKJDSFJK', + // }, + // { + // name: 'ctype_ggg__dkay_march_1234', + // sql: 'ctype_ggg__dkay_march_1234', + // type: 'string', + // alias: 'dkay_march_1234', + // }, + // { + // name: 'ctype_ggg__dkay_march_2', + // sql: 'ctype_ggg__dkay_march_2', + // type: 'string', + // alias: 'dkay_march_2', + // }, + // { + // name: 'ctype_ggg__dkay_test', + // sql: 'ctype_ggg__dkay_test', + // type: 'string', + // alias: 'dkay_test', + // }, + // { + // name: 'ctype_ooooo__dkay_march_3', + // sql: 'ctype_ooooo__dkay_march_3', + // type: 'string', + // alias: 'dkay_march_3', + // }, + // { + // name: 'ctype_ooooo__dkay_march_4', + // sql: 'ctype_ooooo__dkay_march_4', + // type: 'string', + // alias: 'dkay_march_4', + // }, + // { + // name: 'ctype_ooooo__dkay_march_5', + // sql: 'ctype_ooooo__dkay_march_5', + // type: 'string', + // alias: 'dkay_march_5', + // }, + // { + // name: 'ctype_ooooo__dkay_march_7', + // sql: 'ctype_ooooo__dkay_march_7', + // type: 'string', + // alias: 'dkay_march_7', + // }, + // { + // name: 'ctype_sync_engine_test__field1', + // sql: 'ctype_sync_engine_test__field1', + // type: 'string', + // alias: 'field1', + // }, + // { + // name: 'ctype_sync_engine_test__field2_bool', + // sql: 'ctype_sync_engine_test__field2_bool', + // type: 'boolean', + // alias: 'field2_bool', + // }, + // { + // name: 'ctype_priviliges_testing__read_only_field', + // sql: 'ctype_priviliges_testing__read_only_field', + // type: 'string', + // alias: 'read only field', + // }, + // { + // name: 'ctype_priviliges_testing__read_write_field', + // sql: 'ctype_priviliges_testing__read_write_field', + // type: 'string', + // alias: 'read/write field', + // }, + // { + // name: 'ctype_dashboard_test__pulkit', + // sql: 'ctype_dashboard_test__pulkit', + // type: 'boolean', + // alias: 'pulkit', + // }, + // { + // name: 'ctype_tttt__dropdown', + // sql: 'ctype_tttt__dropdown', + // type: 'string', + // alias: 'dropdown', + // }, + // ], + // joins: [ + // { + // sql: 'dim_ticket.sla_tracker_id = dim_sla_tracker.id', + // }, + // { + // sql: 'dim_ticket.id = dim_survey_response.object', + // }, + // { + // sql: 'dim_ticket.id = dim_link_issue_target.source_id', + // }, + // { + // sql: 'dim_ticket.id = dim_link_conversation_source.target_id', + // }, + // { + // sql: 'dim_ticket.rev_oid = dim_revo.id', + // }, + // { + // sql: 'dim_ticket.applies_to_part_id = dim_part.id', + // }, + // ], + // measures: [ + // { + // name: 'id_count', + // function: { + // type: 'count', + // }, + // sql: 'count(dim_ticket.id)', + // type: 'string', + // alias: 'Ticket Id', + // }, + // { + // name: 'created_date_max', + // function: { + // type: 'max', + // }, + // sql: 'max(dim_ticket.created_date)', + // type: 'time', + // alias: 'Created date', + // }, + // { + // name: 'actual_close_date_max', + // function: { + // type: 'max', + // }, + // sql: 'max(dim_ticket.actual_close_date)', + // type: 'time', + // alias: 'Closed date', + // }, + // { + // name: 'sla_tracker_id_count', + // function: { + // type: 'count', + // }, + // sql: 'count(dim_ticket.sla_tracker_id)', + // type: 'string', + // alias: 'Sla Tracker Id', + // }, + // { + // name: 'resolution_time', + // sql: "case WHEN actual_close_date > created_date THEN date_diff('minutes', created_date, actual_close_date) ELSE null END ", + // type: 'number', + // alias: 'Resolution Time', + // }, + // { + // name: 'surveys_aggregation_json_measure', + // function: { + // type: 'median', + // }, + // sql: "median(list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min'))", + // type: 'number', + // alias: 'CSAT Rating', + // }, + // { + // name: 'tnt__account_id', + // sql: 'tnt__account_id', + // type: 'string', + // alias: 'account id', + // }, + // { + // name: 'tnt__actual_effort_spent', + // sql: 'tnt__actual_effort_spent', + // type: 'number', + // alias: 'Actual Effort Spent', + // }, + // { + // name: 'tnt__capability_part', + // sql: 'tnt__capability_part', + // type: 'string', + // alias: 'Capability part', + // }, + // { + // name: 'tnt__custom_id_field', + // sql: 'tnt__custom_id_field', + // type: 'string', + // alias: 'custom id field', + // }, + // { + // name: 'tnt__date_field', + // sql: 'tnt__date_field', + // type: 'time', + // alias: 'date_field', + // }, + // { + // name: 'tnt__estimated_effort', + // sql: 'tnt__estimated_effort', + // type: 'number', + // alias: 'Estimated Effort', + // }, + // { + // name: 'tnt__fruit', + // sql: 'tnt__fruit', + // type: 'string', + // alias: 'Fruit', + // }, + // { + // name: 'tnt__id_field', + // sql: 'tnt__id_field', + // type: 'string', + // alias: 'id field', + // }, + // { + // name: 'tnt__issue_score', + // sql: 'tnt__issue_score', + // type: 'number', + // alias: 'Issue Score', + // }, + // { + // name: 'tnt__numeric_field', + // sql: 'tnt__numeric_field', + // type: 'number', + // alias: 'numeric field', + // }, + // { + // name: 'tnt__parent_part', + // sql: 'tnt__parent_part', + // type: 'string', + // alias: 'Parent part', + // }, + // { + // name: 'tnt__part_product', + // sql: 'tnt__part_product', + // type: 'string', + // alias: 'part product', + // }, + // { + // name: 'tnt__rank', + // sql: 'tnt__rank', + // type: 'number', + // alias: 'Rank', + // }, + // { + // name: 'tnt__remaining_effort', + // sql: 'tnt__remaining_effort', + // type: 'number', + // alias: 'Remaining Effort', + // }, + // { + // name: 'tnt__stray_user', + // sql: 'tnt__stray_user', + // type: 'string', + // alias: 'stray user', + // }, + // { + // name: 'tnt__stray_users', + // sql: 'tnt__stray_users', + // type: 'string', + // alias: 'stray users', + // }, + // { + // name: 'tnt__test', + // sql: 'tnt__test', + // type: 'number', + // alias: 'Test', + // }, + // { + // name: 'tnt__test_capability_part', + // sql: 'tnt__test_capability_part', + // type: 'string', + // alias: 'Test Capability Part', + // }, + // { + // name: 'tnt__test_product_part', + // sql: 'tnt__test_product_part', + // type: 'string', + // alias: 'Test Product part', + // }, + // { + // name: 'tnt__ticket_custom_part', + // sql: 'tnt__ticket_custom_part', + // type: 'string', + // alias: 'ticket custom part', + // }, + // { + // name: 'tnt__workspace_custom', + // sql: 'tnt__workspace_custom', + // type: 'string', + // alias: 'workspace_custom', + // }, + // { + // name: 'ctype_deal_registration__estimated_deal_value', + // sql: 'ctype_deal_registration__estimated_deal_value', + // type: 'number', + // alias: 'Estimated Deal Value', + // }, + // { + // name: 'ctype_deal_registration__expected_close_date', + // sql: 'ctype_deal_registration__expected_close_date', + // type: 'time', + // alias: 'Expected Close Date', + // }, + // { + // name: 'ctype_Events__event_end_date', + // sql: 'ctype_Events__event_end_date', + // type: 'time', + // alias: 'Event End Date', + // }, + // { + // name: 'ctype_Events__event_owner', + // sql: 'ctype_Events__event_owner', + // type: 'string', + // alias: 'Event Owner', + // }, + // { + // name: 'ctype_Events__event_start_date', + // sql: 'ctype_Events__event_start_date', + // type: 'time', + // alias: 'Event Start Date', + // }, + // { + // name: 'ctype_Events__events_test_ref', + // sql: 'ctype_Events__events_test_ref', + // type: 'string', + // alias: 'events test ref', + // }, + // { + // name: 'ctype_Events__external_budget', + // sql: 'ctype_Events__external_budget', + // type: 'number', + // alias: 'External Budget', + // }, + // { + // name: 'ctype_Events__internal_budget', + // sql: 'ctype_Events__internal_budget', + // type: 'number', + // alias: 'Internal Budget', + // }, + // { + // name: 'ctype_Events__pipeline_generated', + // sql: 'ctype_Events__pipeline_generated', + // type: 'number', + // alias: 'Pipeline Generated', + // }, + // { + // name: 'ctype_Events__total_budget', + // sql: 'ctype_Events__total_budget', + // type: 'number', + // alias: 'Total Budget', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_helohelohelohelohelo__dfkdkl', + // sql: 'ctype_helohelohelohelohelo__dfkdkl', + // type: 'number', + // alias: 'dfkdkl', + // }, + // { + // name: 'ctype_defaults__customer_uat', + // sql: 'ctype_defaults__customer_uat', + // type: 'time', + // alias: 'customer uat', + // }, + // { + // name: 'ctype_defaults__uat_date', + // sql: 'ctype_defaults__uat_date', + // type: 'time', + // alias: 'uat date', + // }, + // { + // name: 'ctype_event_request__approver', + // sql: 'ctype_event_request__approver', + // type: 'string', + // alias: 'Approver', + // }, + // { + // name: 'ctype_event_request__budget', + // sql: 'ctype_event_request__budget', + // type: 'number', + // alias: 'Budget', + // }, + // { + // name: 'ctype_event_request__event_date', + // sql: 'ctype_event_request__event_date', + // type: 'time', + // alias: 'Event Date', + // }, + // { + // name: 'ctype_event_request__event_owner', + // sql: 'ctype_event_request__event_owner', + // type: 'string', + // alias: 'Event Owner', + // }, + // { + // name: 'ctype_event_request__requested_by', + // sql: 'ctype_event_request__requested_by', + // type: 'string', + // alias: 'Requested by', + // }, + // { + // name: 'ctype_event_request__target_pipeline', + // sql: 'ctype_event_request__target_pipeline', + // type: 'number', + // alias: 'Target Pipeline', + // }, + // { + // name: 'ctype_mfz_subtype_check__user_field', + // sql: 'ctype_mfz_subtype_check__user_field', + // type: 'string', + // alias: 'Custom user field', + // }, + // { + // name: 'ctype_mfz_subtype_check__workspace_field', + // sql: 'ctype_mfz_subtype_check__workspace_field', + // type: 'string', + // alias: 'Custom workspace field', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + // type: 'number', + // alias: 'brand_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + // type: 'time', + // alias: 'created_at', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + // type: 'time', + // alias: 'generated_timestamp', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + // type: 'number', + // alias: 'group_id', + // }, + // { + // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + // type: 'number', + // alias: 'ticket_form_id', + // }, + // { + // name: 'ctype_review_subtype__date', + // sql: 'ctype_review_subtype__date', + // type: 'time', + // alias: 'Date', + // }, + // { + // name: 'ctype_review_subtype__reply_date', + // sql: 'ctype_review_subtype__reply_date', + // type: 'time', + // alias: 'Reply date', + // }, + // { + // name: 'ctype_review_subtype__upvote_count', + // sql: 'ctype_review_subtype__upvote_count', + // type: 'number', + // alias: 'Upvote Count', + // }, + // { + // name: 'ctype_jkjkjkjkjkjk__dggd', + // sql: 'ctype_jkjkjkjkjkjk__dggd', + // type: 'number', + // alias: 'dggd', + // }, + // { + // name: 'id_count___function__count', + // function: { + // type: 'count', + // }, + // sql: 'count(dim_ticket.id)', + // type: 'string', + // alias: 'Ticket Id', + // }, + // ], + // name: 'dim_ticket', + // sql: 'select * from devrev.dim_ticket', + // }, + // ] as any; + // const resolutionConfig = { + // columnConfigs: [ + // { + // joinColumn: 'id', + // name: 'dim_ticket.tags_json', + // isArrayType: true, + // resolutionColumns: ['name'], + // source: 'dim_tag', + // }, + // ], + // tableSchemas: [ + // { + // dimensions: [ + // { + // name: 'created_by_id', + // sql: 'created_by_id', + // type: 'string', + // alias: 'Created by', + // }, + // { + // name: 'dev_oid', + // sql: 'dev_oid', + // type: 'string', + // alias: 'Dev organization ID', + // }, + // { + // name: 'modified_date', + // sql: 'modified_date', + // type: 'time', + // alias: 'Modified date', + // }, + // { + // name: 'object_type', + // sql: 'object_type', + // type: 'string', + // alias: 'Object type', + // }, + // { + // name: 'tag_type', + // sql: 'tag_type', + // type: 'number', + // alias: 'Values enabled', + // }, + // { + // modifier: { + // shouldUnnestGroupBy: false, + // }, + // name: 'allowed_values', + // sql: 'allowed_values', + // type: 'string_array', + // alias: 'Allowed values', + // }, + // { + // name: 'is_deleted', + // sql: 'is_deleted', + // type: 'boolean', + // alias: 'Is deleted', + // }, + // { + // name: 'modified_by_id', + // sql: 'modified_by_id', + // type: 'string', + // alias: 'Modified by', + // }, + // { + // name: 'access_level', + // sql: 'access_level', + // type: 'number', + // alias: 'Access level', + // }, + // { + // name: 'display_id', + // sql: 'display_id', + // type: 'string', + // alias: 'Display ID', + // }, + // { + // name: 'id', + // sql: 'id', + // type: 'string', + // alias: 'ID', + // }, + // { + // name: 'name', + // sql: 'name', + // type: 'string', + // alias: 'Tag name', + // }, + // { + // name: 'object_version', + // sql: 'object_version', + // type: 'number', + // alias: 'Object Version', + // }, + // { + // name: 'created_date', + // sql: 'created_date', + // type: 'time', + // alias: 'Created date', + // }, + // ], + // measures: [], + // name: 'dim_tag', + // sql: 'SELECT * FROM devrev.dim_tag', + // }, + // ], + // } as any; + // const columnProjections = [ + // 'dim_ticket.tags_json', + // 'dim_ticket.id_count___function__count', + // ]; + + // const resultQuery = await cubeQueryToSQLWithResolution({ + // query, + // tableSchemas, + // resolutionConfig, + // columnProjections, + // }); + + // const expectedQuery = `select * exclude(__row_id) from (SELECT MAX(__resolved_query."Ticket Id") AS "Ticket Id" , ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name" , "__row_id" FROM (SELECT __resolved_query."__row_id" AS "__row_id", * FROM (SELECT "Tags - Tag name", "Ticket Id", "__row_id" FROM (SELECT __unnested_base_query."Ticket Id" AS "Ticket Id", __unnested_base_query."__row_id" AS "__row_id", * FROM (SELECT "Ticket Id", "Tags", "__row_id" FROM (SELECT __base_query."Ticket Id" AS "Ticket Id", unnest(__base_query."Tags") AS "Tags", row_number() OVER () AS "__row_id", * FROM (SELECT count("ID") AS "Ticket Id" , "Tags" FROM (SELECT dim_ticket.created_date AS "Created date", 'ticket' AS "Work Type", json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[]) AS "Tags", dim_ticket.id AS "ID", * FROM (select * from devrev.dim_ticket) AS dim_ticket) AS dim_ticket WHERE (((("Created date" >= '2025-08-02T09:30:00.000Z') AND ("Created date" <= '2025-10-31T10:29:59.999Z')) AND ("Work Type" IN ('ticket')) AND (Stage IN ('don:core:dvrv-us-1:devo/787:custom_stage/514', 'don:core:dvrv-us-1:devo/787:custom_stage/512', 'don:core:dvrv-us-1:devo/787:custom_stage/501', 'don:core:dvrv-us-1:devo/787:custom_stage/485', 'don:core:dvrv-us-1:devo/787:custom_stage/440', 'don:core:dvrv-us-1:devo/787:custom_stage/25', 'don:core:dvrv-us-1:devo/787:custom_stage/24', 'don:core:dvrv-us-1:devo/787:custom_stage/22', 'don:core:dvrv-us-1:devo/787:custom_stage/21', 'don:core:dvrv-us-1:devo/787:custom_stage/19', 'don:core:dvrv-us-1:devo/787:custom_stage/13', 'don:core:dvrv-us-1:devo/787:custom_stage/10', 'don:core:dvrv-us-1:devo/787:custom_stage/8', 'don:core:dvrv-us-1:devo/787:custom_stage/7', 'don:core:dvrv-us-1:devo/787:custom_stage/6', 'don:core:dvrv-us-1:devo/787:custom_stage/4')))) GROUP BY Tags LIMIT 50000) AS __base_query) AS __base_query) AS __unnested_base_query LEFT JOIN (SELECT __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", * FROM (SELECT * FROM devrev.dim_tag) AS __unnested_base_query__dim_ticket__tags_json) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags"=__unnested_base_query__dim_ticket__tags_json.id) AS MEERKAT_GENERATED_TABLE) AS __resolved_query) AS __resolved_query GROUP BY __row_id)`; + // expect(resultQuery).toBe(expectedQuery); + // }); // it('Should handle only scalar field resolution without unnesting', async () => { // const query: Query = { // measures: ['tickets.count'], diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 1e91301a..581b162c 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -185,18 +185,11 @@ export const getUnnestBaseSql = async ({ sql: string; baseTableSchema: TableSchema; }> => { - const BASE_TABLE_NAME = '__base_query'; - - debugger; - const emptyResolutionConfig: ResolutionConfig = { - columnConfigs: [], - tableSchemas: [], - }; // Step 1: Create schema for the base SQL const baseTableSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, - emptyResolutionConfig, + resolutionConfig, measures, dimensions ); @@ -222,24 +215,21 @@ export const getUnnestBaseSql = async ({ dimensions: [ ...baseTableSchema.dimensions, { - name: 'row_id', + name: memberKeyToSafeKey(`${tableSchemas[0].name}.row_id`), sql: 'row_number() OVER ()', type: 'number', alias: '__row_id', } as Dimension, ], + joins: baseTableSchema.joins, }; // Query that projects all original columns plus row_id const queryWithRowId: Query = { measures: [], dimensions: [ - ...dimensions.map((d) => - getNamespacedKey(BASE_TABLE_NAME, memberKeyToSafeKey(d)) - ), - getNamespacedKey(BASE_TABLE_NAME, 'row_id'), - ...measures.map((m) => - getNamespacedKey(BASE_TABLE_NAME, memberKeyToSafeKey(m)) + ...schemaWithRowId.dimensions.map((d) => + getNamespacedKey(schemaWithRowId.name, d.name) ), ], }; @@ -264,7 +254,40 @@ export const getUnnestBaseSql = async ({ alias: d.alias, })), measures: [], + joins: schemaWithRowId.joins, }; + for (const join of unnestedBaseTableSchema.joins || []) { + const leftJoin = join.sql.split('=')[0]; + const namespace = leftJoin.split('.')[0]; + const name = leftJoin.split('.')[1]; + const toFind = `${namespace}.${name}`.trim(); + const dimensionToJoinOn = schemaWithRowId.dimensions.filter( + (d) => d.sql.trim() === toFind + ); + if (dimensionToJoinOn.length === 0) { + throw new Error(`Dimension not found: ${namespace}.${name}`); + } + if (dimensionToJoinOn.length > 1) { + throw new Error(`Multiple dimensions found: ${namespace}.${name}`); + } + const rightJoin = join.sql.split('=')[1]; + const rightJoinNamespace = rightJoin.split('.')[0].trim(); + const rightJoinField = rightJoin.split('.')[1].trim(); + join.sql = join.sql.replace( + leftJoin, + `${unnestedBaseTableSchema.name}.${ + dimensionToJoinOn[0].alias || dimensionToJoinOn[0].name + }` + ); + // TODO: Confirm if name also needs "" like this. + join.sql = `${unnestedBaseTableSchema.name}.${ + dimensionToJoinOn[0].alias + ? `"${dimensionToJoinOn[0].alias}"` + : dimensionToJoinOn[0].name + }=${memberKeyToSafeKey( + getNamespacedKey(unnestedBaseTableSchema.name, rightJoinNamespace) + )}.${rightJoinField}`; + } return { sql: unnestedSql, baseTableSchema: unnestedBaseTableSchema, @@ -305,14 +328,18 @@ export const getResolvedSql = async ({ // Update the SQL to point to the unnested SQL const updatedBaseTableSchema: TableSchema = baseTableSchema; + for (const columnConfig of resolutionConfig.columnConfigs) { + columnConfig.name = getNamespacedKey( + updatedBaseTableSchema.name, + memberKeyToSafeKey(columnConfig.name) + ); + // columnConfig.name = memberKeyToSafeKey(columnConfig.name); + } // Generate resolution schemas for array fields const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ updatedBaseTableSchema, ]); - for (const columnConfig of resolutionConfig.columnConfigs) { - columnConfig.name = memberKeyToSafeKey(columnConfig.name); - } // Generate join paths using existing helper const joinPaths = generateResolutionJoinPaths( updatedBaseTableSchema.name, @@ -322,11 +349,13 @@ export const getResolvedSql = async ({ const tempQuery: Query = { measures: [], - dimensions: baseTableSchema.dimensions.map((d) => d.name), + dimensions: baseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), }; const updatedColumnProjections = columnProjections?.map((cp) => - memberKeyToSafeKey(cp) + getNamespacedKey(updatedBaseTableSchema.name, memberKeyToSafeKey(cp)) ); // Generate resolved dimensions using columnProjections const resolvedDimensions = generateResolvedDimensions( @@ -335,14 +364,6 @@ export const getResolvedSql = async ({ resolutionConfig, updatedColumnProjections ); - - const reconstructingBaseSchema: TableSchema = createBaseTableSchema( - unnestedSql, - [baseTableSchema], - resolutionConfig, - [], - resolvedDimensions - ); // Create query and generate SQL const resolutionQuery: Query = { measures: [], @@ -363,7 +384,8 @@ export const getResolvedSql = async ({ .filter((dim) => { // Exclude columns that need resolution (they'll be replaced by resolved columns) return !resolutionConfig.columnConfigs.some( - (ac) => memberKeyToSafeKey(ac.name) === dim.name + (ac) => + ac.name === getNamespacedKey(updatedBaseTableSchema.name, dim.name) ); }) .map((dim) => dim.name) From dffc096633ea5723e15453e3fcf222d33b9f5101 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 3 Nov 2025 16:43:13 +0530 Subject: [PATCH 12/38] working properly --- meerkat-core/src/resolution/resolution.ts | 88 +- .../cube-to-sql-with-resolution-array.spec.ts | 6334 ++++++++--------- .../cube-to-sql-with-resolution.ts | 68 +- 3 files changed, 3278 insertions(+), 3212 deletions(-) diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 96b5a95d..ba240ca6 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -7,6 +7,7 @@ import { JoinPath, Member, Query } from '../types/cube-types/query'; import { Dimension, Measure, TableSchema } from '../types/cube-types/table'; import { findInDimensionSchemas, + findInSchema, findInSchemas, } from '../utils/find-in-table-schema'; import { @@ -146,6 +147,62 @@ export const generateResolutionSchemas = ( return resolutionSchemas; }; +export const generateResolutionSchemasFromBaseTable = ( + config: ResolutionConfig, + baseTableSchema: TableSchema +) => { + const resolutionSchemas: TableSchema[] = []; + config.columnConfigs.forEach((colConfig) => { + const tableSchema = config.tableSchemas.find( + (ts) => ts.name === colConfig.source + ); + if (!tableSchema) { + throw new Error(`Table schema not found for ${colConfig.source}`); + } + + const baseName = memberKeyToSafeKey(colConfig.name); + const baseAlias = constructAlias({ + name: colConfig.name, + alias: findInSchema(colConfig.name, baseTableSchema)?.alias, + aliasContext: { isTableSchemaAlias: true }, + }); + + // For each column that needs to be resolved, create a copy of the relevant table schema. + // We use the name of the column in the base query as the table schema name + // to avoid conflicts. + const resolutionSchema: TableSchema = { + name: baseName, + sql: tableSchema.sql, + measures: [], + dimensions: colConfig.resolutionColumns.map((col) => { + const dimension = findInDimensionSchemas( + getNamespacedKey(colConfig.source, col), + config.tableSchemas + ); + if (!dimension) { + throw new Error(`Dimension not found: ${col}`); + } + return { + // Need to create a new name due to limitations with how + // CubeToSql handles duplicate dimension names between different sources. + name: memberKeyToSafeKey(getNamespacedKey(colConfig.name, col)), + sql: `${baseName}.${col}`, + type: dimension.type, + alias: `${baseAlias} - ${constructAlias({ + name: col, + alias: dimension.alias, + aliasContext: { isTableSchemaAlias: true }, + })}`, + }; + }), + }; + + resolutionSchemas.push(resolutionSchema); + }); + + return resolutionSchemas; +}; + export const generateResolvedDimensions = ( baseDataSourceName: string, query: Query, @@ -165,19 +222,6 @@ export const generateResolvedDimensions = ( ); if (!columnConfig) { - // TODO: See if this can be optimized - // In a first level join right now, we are just passing the dimension without namespacing it. - // But in a resolution level, we are already adding __unnest_query as the base name and using in - // other places like join paths as it depends on it to work. - // For generating join paths, we expect the table name to be exactly the name of memberToSafeKey(namespace, joinDimensionName) - // Hence, at second level, we are namespacing it everywhere, and we need this condition. - if (dimension.includes('.')) { - return [dimension]; - } else { - return [ - getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), - ]; - } return [ getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), ]; @@ -211,3 +255,21 @@ export const generateResolutionJoinPaths = ( }, ]); }; + +export const generateResolutionJoinPathsFromBaseTable = ( + baseDataSourceName: string, + resolutionConfig: ResolutionConfig, + baseTableSchema: TableSchema +): JoinPath[] => { + return resolutionConfig.columnConfigs.map((config) => [ + { + left: baseDataSourceName, + right: memberKeyToSafeKey(config.name), + on: constructAlias({ + name: config.name, + alias: findInSchema(config.name, baseTableSchema)?.alias, + aliasContext: { isAstIdentifier: false }, + }), + }, + ]); +}; diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts index e2682dbd..7c61162d 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts @@ -1,4 +1,4 @@ -import { Query, ResolutionConfig, TableSchema } from '@devrev/meerkat-core'; +import { TableSchema } from '@devrev/meerkat-core'; import { cubeQueryToSQLWithResolution } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; import { duckdbExec } from '../duckdb-exec'; const CREATE_TEST_TABLE = `CREATE TABLE tickets ( @@ -219,3190 +219,3190 @@ describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { // expect(rowIds.length).toBe(5); // 5 unnested rows total // }); - it('Should handle multiple array fields that need unnesting', async () => { - const query: Query = { - measures: ['tickets.count'], - dimensions: [ - 'tickets.id', - 'tickets.owners', //array - 'tickets.tags', // array - 'tickets.created_by', // scalar - ], - }; + // it('Should handle multiple array fields that need unnesting', async () => { + // const query: Query = { + // measures: ['tickets.count'], + // dimensions: [ + // 'tickets.id', + // 'tickets.owners', //array + // 'tickets.tags', // array + // 'tickets.created_by', // scalar + // ], + // }; - const resolutionConfig: ResolutionConfig = { - columnConfigs: [ - { - name: 'tickets.owners', - isArrayType: true, - source: 'owners_lookup', - joinColumn: 'id', - resolutionColumns: ['display_name'], - }, + // const resolutionConfig: ResolutionConfig = { + // columnConfigs: [ + // { + // name: 'tickets.owners', + // isArrayType: true, + // source: 'owners_lookup', + // joinColumn: 'id', + // resolutionColumns: ['display_name'], + // }, + // { + // name: 'tickets.tags', + // isArrayType: true, + // source: 'tags_lookup', + // joinColumn: 'id', + // resolutionColumns: ['tag_name'], + // }, + // { + // name: 'tickets.created_by', + // isArrayType: false, + // source: 'created_by_lookup', + // joinColumn: 'id', + // resolutionColumns: ['name'], + // }, + // ], + // tableSchemas: [ + // OWNERS_LOOKUP_SCHEMA, + // TAGS_LOOKUP_SCHEMA, + // CREATED_BY_LOOKUP_SCHEMA, + // ], + // }; + + // const columnProjections = [ + // 'tickets.id', + // 'tickets.owners', + // 'tickets.tags', + // 'tickets.created_by', + // 'tickets.count', + // ]; + // const sql = await cubeQueryToSQLWithResolution({ + // query, + // tableSchemas: [TICKETS_TABLE_SCHEMA], + // resolutionConfig, + // columnProjections, + // }); + + // console.log('Phase 1 SQL (multiple arrays):', sql); + + // // Verify row_id is included + // expect(sql).toContain('row_id'); + + // // Both arrays should be unnested + // expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(2); + + // // Execute the SQL to verify it works + // const result = (await duckdbExec(sql)) as any[]; + + // expect(result.length).toBe(7); + + // // Each row should have a row_id + // expect(result[0]).toHaveProperty('__row_id'); + // expect(result[0]).toHaveProperty('tickets__count'); + // expect(result[0]).toHaveProperty('tickets__id'); + // expect(result[0]).toHaveProperty('tickets__owners - display_name'); + // expect(result[0]).toHaveProperty('tickets__tags - tag_name'); + // }); + + it('test an proper query from UI', async () => { + const query = { + dimensions: ['dim_ticket.tags_json'], + filters: [ { - name: 'tickets.tags', - isArrayType: true, - source: 'tags_lookup', - joinColumn: 'id', - resolutionColumns: ['tag_name'], + and: [ + { + and: [ + { + member: 'dim_ticket.created_date', + operator: 'inDateRange', + values: [ + '2025-08-02T09:30:00.000Z', + '2025-10-31T10:29:59.999Z', + ], + }, + { + member: 'dim_ticket.work_type', + operator: 'in', + values: ['ticket'], + }, + { + member: 'dim_ticket.stage_json', + operator: 'in', + values: [ + 'don:core:dvrv-us-1:devo/787:custom_stage/514', + 'don:core:dvrv-us-1:devo/787:custom_stage/512', + 'don:core:dvrv-us-1:devo/787:custom_stage/501', + 'don:core:dvrv-us-1:devo/787:custom_stage/485', + 'don:core:dvrv-us-1:devo/787:custom_stage/440', + 'don:core:dvrv-us-1:devo/787:custom_stage/25', + 'don:core:dvrv-us-1:devo/787:custom_stage/24', + 'don:core:dvrv-us-1:devo/787:custom_stage/22', + 'don:core:dvrv-us-1:devo/787:custom_stage/21', + 'don:core:dvrv-us-1:devo/787:custom_stage/19', + 'don:core:dvrv-us-1:devo/787:custom_stage/13', + 'don:core:dvrv-us-1:devo/787:custom_stage/10', + 'don:core:dvrv-us-1:devo/787:custom_stage/8', + 'don:core:dvrv-us-1:devo/787:custom_stage/7', + 'don:core:dvrv-us-1:devo/787:custom_stage/6', + 'don:core:dvrv-us-1:devo/787:custom_stage/4', + ], + }, + ], + }, + ], }, + ], + limit: 50000, + measures: ['dim_ticket.id_count___function__count'], + } as any; + const tableSchemas = [ + { + dimensions: [ + { + name: 'rev_oid', + sql: 'dim_ticket.rev_oid', + type: 'string', + alias: 'Customer Workspace', + }, + { + name: 'title', + sql: 'dim_ticket.title', + type: 'string', + alias: 'Title', + }, + { + name: 'last_internal_comment_date', + sql: 'dim_ticket.last_internal_comment_date', + type: 'time', + alias: 'Last Internal Comment Date', + }, + { + name: 'work_type', + sql: "'ticket'", + type: 'time', + alias: 'Work Type', + }, + { + name: 'is_spam', + sql: 'dim_ticket.is_spam', + type: 'boolean', + alias: 'Spam', + }, + { + name: 'object_type', + sql: 'dim_ticket.object_type', + type: 'string', + alias: 'Object type', + }, + { + name: 'modified_by_id', + sql: 'dim_ticket.modified_by_id', + type: 'string', + alias: 'Modified by', + }, + { + name: 'source_channel', + sql: 'dim_ticket.source_channel', + type: 'string', + alias: 'Source channel', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'channels', + sql: 'dim_ticket.channels', + type: 'number_array', + alias: 'Channels', + }, + { + name: 'created_by_id', + sql: 'dim_ticket.created_by_id', + type: 'string', + alias: 'Created by', + }, + { + name: 'created_date', + sql: 'dim_ticket.created_date', + type: 'time', + alias: 'Created date', + }, + { + name: 'target_close_date', + sql: 'dim_ticket.target_close_date', + type: 'time', + alias: 'Target close date', + }, + { + name: 'applies_to_part_id', + sql: 'dim_ticket.applies_to_part_id', + type: 'string', + alias: 'Part', + }, + { + name: 'subtype', + sql: 'dim_ticket.subtype', + type: 'string', + alias: 'Subtype', + }, + { + name: 'actual_close_date', + sql: 'dim_ticket.actual_close_date', + type: 'time', + alias: 'Close date', + }, + { + name: 'reported_by_id', + sql: 'dim_ticket.reported_by_id', + type: 'string', + alias: 'Reported by ID', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'reported_by_ids', + sql: 'dim_ticket.reported_by_ids', + type: 'string_array', + alias: 'Reported by', + }, + { + name: 'needs_response', + sql: 'dim_ticket.needs_response', + type: 'boolean', + alias: 'Needs Response', + }, + { + name: 'group', + sql: 'dim_ticket.group', + type: 'string', + alias: 'Group', + }, + { + name: 'modified_date', + sql: 'dim_ticket.modified_date', + type: 'time', + alias: 'Modified date', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'owned_by_ids', + sql: 'dim_ticket.owned_by_ids', + type: 'string_array', + alias: 'Owner', + }, + { + name: 'id', + sql: 'dim_ticket.id', + type: 'string', + alias: 'ID', + }, + { + name: 'sla_tracker_id', + sql: 'dim_ticket.sla_tracker_id', + type: 'string', + alias: 'SLA Tracker', + }, + { + name: 'severity', + sql: 'dim_ticket.severity', + type: 'number', + alias: 'Severity', + }, + { + name: 'stage_json', + sql: "json_extract_string(dim_ticket.stage_json, '$.stage_id')", + type: 'string', + alias: 'Stage', + }, + { + name: 'sla_id', + sql: 'dim_ticket.sla_id', + type: 'string', + alias: 'SLA Name', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'links_json', + sql: "list_distinct(CAST(json_extract_string(dim_ticket.links_json, '$[*].target_object_type') AS VARCHAR[]))", + type: 'string_array', + alias: 'Links', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tags_json', + sql: "CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[])", + type: 'string_array', + alias: 'Tags', + }, + { + name: 'surveys_aggregation_json', + sql: "list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min')", + type: 'number', + alias: 'CSAT Rating', + }, + { + name: 'sla_summary_target_time', + sql: "cast(json_extract_string(dim_ticket.sla_summary, '$.target_time') as timestamp)", + type: 'time', + alias: 'Next SLA Target', + }, + { + name: 'staged_info', + sql: "cast(json_extract_string(dim_ticket.staged_info, '$.is_staged') as boolean)", + type: 'boolean', + alias: 'Changes Need Review', + }, + { + name: 'tnt__account_id', + sql: 'tnt__account_id', + type: 'string', + alias: 'account id', + }, + { + name: 'tnt__actual_effort_spent', + sql: 'tnt__actual_effort_spent', + type: 'number', + alias: 'Actual Effort Spent', + }, + { + name: 'tnt__ai_subtype', + sql: 'tnt__ai_subtype', + type: 'string', + alias: 'AI Subtype', + }, + { + name: 'tnt__asdfad', + sql: 'tnt__asdfad', + type: 'string', + alias: 'asdfad', + }, + { + name: 'tnt__bool_summary', + sql: 'tnt__bool_summary', + type: 'boolean', + alias: 'bool summary', + }, + { + name: 'tnt__boolyesnoresp', + sql: 'tnt__boolyesnoresp', + type: 'boolean', + alias: 'boolyesnoresp', + }, + { + name: 'tnt__capability_part', + sql: 'tnt__capability_part', + type: 'string', + alias: 'Capability part', + }, + { + name: 'tnt__card', + sql: 'tnt__card', + type: 'string', + alias: 'card', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__cust_account', + sql: 'tnt__cust_account', + type: 'string_array', + alias: 'cust_account', + }, + { + name: 'tnt__custom_field_for_sla', + sql: 'tnt__custom_field_for_sla', + type: 'string', + alias: 'custom_field_for_sla', + }, + { + name: 'tnt__custom_id_field', + sql: 'tnt__custom_id_field', + type: 'string', + alias: 'custom id field', + }, + { + name: 'tnt__customer_tier', + sql: 'tnt__customer_tier', + type: 'string', + alias: 'Customer Tier', + }, + { + name: 'tnt__date_field', + sql: 'tnt__date_field', + type: 'time', + alias: 'date_field', + }, + { + name: 'tnt__department_list', + sql: 'tnt__department_list', + type: 'string', + alias: 'Department List', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__email_references', + sql: 'tnt__email_references', + type: 'string_array', + alias: 'tnt__email_references', + }, + { + name: 'tnt__escalated', + sql: 'tnt__escalated', + type: 'boolean', + alias: 'Escalated', + }, + { + name: 'tnt__estimated_effort', + sql: 'tnt__estimated_effort', + type: 'number', + alias: 'Estimated Effort', + }, + { + name: 'tnt__external_source_id', + sql: 'tnt__external_source_id', + type: 'string', + alias: 'tnt__external_source_id', + }, + { + name: 'tnt__feature_affected', + sql: 'tnt__feature_affected', + type: 'string', + alias: 'Feature Affected', + }, + { + name: 'tnt__field1', + sql: 'tnt__field1', + type: 'string', + alias: 'field1', + }, + { + name: 'tnt__foo', + sql: 'tnt__foo', + type: 'string', + alias: 'tnt__foo', + }, + { + name: 'tnt__fruit', + sql: 'tnt__fruit', + type: 'string', + alias: 'Fruit', + }, + { + name: 'tnt__git_template_name', + sql: 'tnt__git_template_name', + type: 'string', + alias: 'Github Template Name', + }, + { + name: 'tnt__id_field', + sql: 'tnt__id_field', + type: 'string', + alias: 'id field', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__id_list_field', + sql: 'tnt__id_list_field', + type: 'string_array', + alias: 'id list field', + }, + { + name: 'tnt__impact', + sql: 'tnt__impact', + type: 'string', + alias: 'Impact', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__include_on_emails', + sql: 'tnt__include_on_emails', + type: 'string_array', + alias: 'Include on emails', + }, + { + name: 'tnt__issue_score', + sql: 'tnt__issue_score', + type: 'number', + alias: 'Issue Score', + }, + { + name: 'tnt__knowledge_gap', + sql: 'tnt__knowledge_gap', + type: 'string', + alias: 'Knowledge Gap', + }, + { + name: 'tnt__linear_assignee', + sql: 'tnt__linear_assignee', + type: 'string', + alias: 'Linear Assignee', + }, + { + name: 'tnt__linear_description', + sql: 'tnt__linear_description', + type: 'string', + alias: 'Linear Description', + }, + { + name: 'tnt__linear_id', + sql: 'tnt__linear_id', + type: 'string', + alias: 'Linear Id', + }, + { + name: 'tnt__linear_identifier', + sql: 'tnt__linear_identifier', + type: 'string', + alias: 'Linear Identifier', + }, + { + name: 'tnt__linear_priority', + sql: 'tnt__linear_priority', + type: 'string', + alias: 'Linear Priority', + }, + { + name: 'tnt__linear_state', + sql: 'tnt__linear_state', + type: 'string', + alias: 'Linear State', + }, + { + name: 'tnt__linear_team', + sql: 'tnt__linear_team', + type: 'string', + alias: 'Linear Team', + }, + { + name: 'tnt__linear_title', + sql: 'tnt__linear_title', + type: 'string', + alias: 'Linear Title', + }, + { + name: 'tnt__linear_url', + sql: 'tnt__linear_url', + type: 'string', + alias: 'Linear Url', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__members', + sql: 'tnt__members', + type: 'string_array', + alias: 'tnt__members', + }, + { + name: 'tnt__micro_service_name', + sql: 'tnt__micro_service_name', + type: 'string', + alias: 'Microservice Name', + }, + { + name: 'tnt__notify_others', + sql: 'tnt__notify_others', + type: 'string', + alias: 'notify others', + }, + { + name: 'tnt__numeric_field', + sql: 'tnt__numeric_field', + type: 'number', + alias: 'numeric field', + }, + { + name: 'tnt__parent_part', + sql: 'tnt__parent_part', + type: 'string', + alias: 'Parent part', + }, + { + name: 'tnt__part_product', + sql: 'tnt__part_product', + type: 'string', + alias: 'part product', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__paytm_customer_name', + sql: 'tnt__paytm_customer_name', + type: 'string_array', + alias: 'paytm customer name', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__platform', + sql: 'tnt__platform', + type: 'string_array', + alias: 'Platform', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__portal_test', + sql: 'tnt__portal_test', + type: 'string_array', + alias: 'portal test', + }, + { + name: 'tnt__product', + sql: 'tnt__product', + type: 'string', + alias: 'Product', + }, + { + name: 'tnt__project', + sql: 'tnt__project', + type: 'string', + alias: 'Project', + }, + { + name: 'tnt__project_stage', + sql: 'tnt__project_stage', + type: 'string', + alias: 'Project Stage', + }, + { + name: 'tnt__rank', + sql: 'tnt__rank', + type: 'number', + alias: 'Rank', + }, + { + name: 'tnt__remaining_effort', + sql: 'tnt__remaining_effort', + type: 'number', + alias: 'Remaining Effort', + }, + { + name: 'tnt__required_text_field', + sql: 'tnt__required_text_field', + type: 'string', + alias: 'required text field', + }, + { + name: 'tnt__resolution_sla_met', + sql: 'tnt__resolution_sla_met', + type: 'string', + alias: 'Resolution SLA Met', + }, + { + name: 'tnt__resolution_sla_target_time', + sql: 'tnt__resolution_sla_target_time', + type: 'string', + alias: 'Resolution SLA Target Time', + }, + { + name: 'tnt__resolution_turnaround_time', + sql: 'tnt__resolution_turnaround_time', + type: 'string', + alias: 'Resolution SLA Turnaround Time', + }, + { + name: 'tnt__response_sla_met', + sql: 'tnt__response_sla_met', + type: 'string', + alias: 'First Response SLA Met', + }, + { + name: 'tnt__response_sla_target_time', + sql: 'tnt__response_sla_target_time', + type: 'string', + alias: 'First Response SLA Target Time', + }, + { + name: 'tnt__response_turnaround_time', + sql: 'tnt__response_turnaround_time', + type: 'string', + alias: 'First Response SLA Turnaround Time', + }, + { + name: 'tnt__root_cause_analysis', + sql: 'tnt__root_cause_analysis', + type: 'string', + alias: 'Root cause analysis', + }, + { + name: 'tnt__search_field_query', + sql: 'tnt__search_field_query', + type: 'string', + alias: 'Search Field Query', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__stakeholders', + sql: 'tnt__stakeholders', + type: 'string_array', + alias: 'stakeholders', + }, + { + name: 'tnt__stray_user', + sql: 'tnt__stray_user', + type: 'string', + alias: 'stray user', + }, + { + name: 'tnt__stray_users', + sql: 'tnt__stray_users', + type: 'string', + alias: 'stray users', + }, + { + name: 'tnt__test', + sql: 'tnt__test', + type: 'number', + alias: 'Test', + }, + { + name: 'tnt__test123', + sql: 'tnt__test123', + type: 'string', + alias: 'test123', + }, + { + name: 'tnt__test_capability_part', + sql: 'tnt__test_capability_part', + type: 'string', + alias: 'Test Capability Part', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__test_field_for_sla', + sql: 'tnt__test_field_for_sla', + type: 'string_array', + alias: 'test field for sla', + }, + { + name: 'tnt__test_filed', + sql: 'tnt__test_filed', + type: 'string', + alias: 'test field', + }, + { + name: 'tnt__test_mikasa', + sql: 'tnt__test_mikasa', + type: 'string', + alias: 'Test mikasa', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__test_multivalue_testrequiredfield', + sql: 'tnt__test_multivalue_testrequiredfield', + type: 'string_array', + alias: 'test multivalue testrequiredfield', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__test_multivalue_textfield', + sql: 'tnt__test_multivalue_textfield', + type: 'string_array', + alias: 'test multivalue textfield', + }, + { + name: 'tnt__test_product_part', + sql: 'tnt__test_product_part', + type: 'string', + alias: 'Test Product part', + }, + { + name: 'tnt__testing_bool_drpdown', + sql: 'tnt__testing_bool_drpdown', + type: 'string', + alias: 'Testing Bool Drpdown', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'tnt__text_list_account', + sql: 'tnt__text_list_account', + type: 'string_array', + alias: 'Text list account', + }, + { + name: 'tnt__ticket_custom_part', + sql: 'tnt__ticket_custom_part', + type: 'string', + alias: 'ticket custom part', + }, + { + name: 'tnt__ticket_dropd', + sql: 'tnt__ticket_dropd', + type: 'string', + alias: 'ticket dropdown', + }, + { + name: 'tnt__ticket_type', + sql: 'tnt__ticket_type', + type: 'string', + alias: 'Ticket type', + }, + { + name: 'tnt__tier', + sql: 'tnt__tier', + type: 'string', + alias: 'Tier', + }, + { + name: 'tnt__urgency', + sql: 'tnt__urgency', + type: 'string', + alias: 'Urgency', + }, + { + name: 'tnt__velocity_test_enum', + sql: 'tnt__velocity_test_enum', + type: 'string', + alias: 'velocity test enum', + }, + { + name: 'tnt__version', + sql: 'tnt__version', + type: 'string', + alias: 'Version', + }, + { + name: 'tnt__work_duration', + sql: 'tnt__work_duration', + type: 'string', + alias: 'Work duration', + }, + { + name: 'tnt__workspace_custom', + sql: 'tnt__workspace_custom', + type: 'string', + alias: 'workspace_custom', + }, + { + name: 'ctype_deal_registration__buying_region', + sql: 'ctype_deal_registration__buying_region', + type: 'string', + alias: 'Theater', + }, + { + name: 'ctype_deal_registration__deal_status', + sql: 'ctype_deal_registration__deal_status', + type: 'string', + alias: 'Deal Status', + }, + { + name: 'ctype_deal_registration__deal_type', + sql: 'ctype_deal_registration__deal_type', + type: 'string', + alias: 'Deal Type', + }, + { + name: 'ctype_deal_registration__employee_count', + sql: 'ctype_deal_registration__employee_count', + type: 'string', + alias: 'Employee Count', + }, + { + name: 'ctype_deal_registration__estimated_deal_value', + sql: 'ctype_deal_registration__estimated_deal_value', + type: 'number', + alias: 'Estimated Deal Value', + }, + { + name: 'ctype_deal_registration__expected_close_date', + sql: 'ctype_deal_registration__expected_close_date', + type: 'time', + alias: 'Expected Close Date', + }, + { + name: 'ctype_deal_registration__partner_type', + sql: 'ctype_deal_registration__partner_type', + type: 'string', + alias: 'Partner Type', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_deal_registration__products_or_services', + sql: 'ctype_deal_registration__products_or_services', + type: 'string_array', + alias: 'Product(s) or Service(s)', + }, + { + name: 'ctype_deal_registration__prospect_company_name', + sql: 'ctype_deal_registration__prospect_company_name', + type: 'string', + alias: 'Prospect Company Name', + }, + { + name: 'ctype_deal_registration__prospect_company_website', + sql: 'ctype_deal_registration__prospect_company_website', + type: 'string', + alias: 'Prospect Company Website', + }, + { + name: 'ctype_deal_registration__prospect_contact_email', + sql: 'ctype_deal_registration__prospect_contact_email', + type: 'string', + alias: 'Prospect Contact Email', + }, + { + name: 'ctype_deal_registration__prospect_contact_name', + sql: 'ctype_deal_registration__prospect_contact_name', + type: 'string', + alias: 'Prospect Contact Name', + }, + { + name: 'ctype_deal_registration__sub_region', + sql: 'ctype_deal_registration__sub_region', + type: 'string', + alias: 'Region ', + }, + { + name: 'ctype_deal_registration__subregion', + sql: 'ctype_deal_registration__subregion', + type: 'string', + alias: 'Subregion', + }, + { + name: 'ctype_Events__campaign_category', + sql: 'ctype_Events__campaign_category', + type: 'string', + alias: 'Campaign Category', + }, + { + name: 'ctype_Events__event_end_date', + sql: 'ctype_Events__event_end_date', + type: 'time', + alias: 'Event End Date', + }, + { + name: 'ctype_Events__event_owner', + sql: 'ctype_Events__event_owner', + type: 'string', + alias: 'Event Owner', + }, + { + name: 'ctype_Events__event_start_date', + sql: 'ctype_Events__event_start_date', + type: 'time', + alias: 'Event Start Date', + }, + { + name: 'ctype_Events__events_test_ref', + sql: 'ctype_Events__events_test_ref', + type: 'string', + alias: 'events test ref', + }, + { + name: 'ctype_Events__external_budget', + sql: 'ctype_Events__external_budget', + type: 'number', + alias: 'External Budget', + }, + { + name: 'ctype_Events__internal_budget', + sql: 'ctype_Events__internal_budget', + type: 'number', + alias: 'Internal Budget', + }, + { + name: 'ctype_Events__is_devrev_event', + sql: 'ctype_Events__is_devrev_event', + type: 'boolean', + alias: 'Is DevRev Event', + }, + { + name: 'ctype_Events__mode', + sql: 'ctype_Events__mode', + type: 'string', + alias: 'Mode', + }, + { + name: 'ctype_Events__organized_by', + sql: 'ctype_Events__organized_by', + type: 'string', + alias: 'Organized by', + }, + { + name: 'ctype_Events__pipeline_generated', + sql: 'ctype_Events__pipeline_generated', + type: 'number', + alias: 'Pipeline Generated', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_Events__representatives', + sql: 'ctype_Events__representatives', + type: 'string_array', + alias: 'Representatives', + }, + { + name: 'ctype_Events__source', + sql: 'ctype_Events__source', + type: 'string', + alias: 'Source', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_Events__sponsors', + sql: 'ctype_Events__sponsors', + type: 'string_array', + alias: 'Sponsors', + }, + { + name: 'ctype_Events__sub_source', + sql: 'ctype_Events__sub_source', + type: 'string', + alias: 'Sub-Source', + }, + { + name: 'ctype_Events__total_budget', + sql: 'ctype_Events__total_budget', + type: 'number', + alias: 'Total Budget', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', + type: 'boolean', + alias: 'allow_attachments', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', + type: 'boolean', + alias: 'allow_channelback', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', + type: 'string_array', + alias: 'Collaborators', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', + type: 'string', + alias: 'External Object Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', + type: 'string', + alias: 'external_id', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', + type: 'string_array', + alias: 'Fields modified during import', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', + type: 'boolean', + alias: 'from_messaging_channel', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', + type: 'boolean', + alias: 'has_incidents', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', + type: 'boolean', + alias: 'is_public', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', + type: 'string_array', + alias: 'Reporters', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', + type: 'string', + alias: 'Customer', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', + type: 'string', + alias: 'raw_subject', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', + type: 'string', + alias: 'recipient', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', + type: 'string', + alias: 'External Source', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', + type: 'string', + alias: 'url', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', + type: 'string', + alias: 'Hold Status', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', + type: 'string', + alias: 'ATI Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', + type: 'string', + alias: 'Event ID', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', + type: 'boolean', + alias: 'Do Not Close', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', + type: 'boolean', + alias: 'Escalated', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', + type: 'string', + alias: 'Partner: Sales Contact', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', + type: 'string', + alias: 'Partner Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', + type: 'string', + alias: 'Env Setup Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', + type: 'string', + alias: 'Additional Comments', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', + type: 'string', + alias: 'Remind me before', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', + type: 'string', + alias: ' zIPS + z3A Licenses', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', + type: 'string', + alias: 'zDefend Licenses', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', + type: 'string', + alias: 'Approval Status', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', + type: 'string', + alias: 'Booking Year', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', + type: 'string', + alias: 'Term', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', + type: 'string', + alias: 'End Customer : Admin Details', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', + type: 'string_array', + alias: 'Root Cause', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', + type: 'string', + alias: 'Request Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', + type: 'string', + alias: 'End User - Full Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', + type: 'string', + alias: 'End Customer: Administrator Email Address', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', + type: 'string', + alias: 'Services Purchased', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', + type: 'string', + alias: 'Support Purchased', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', + type: 'string', + alias: 'Cancellation Reason', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', + type: 'string', + alias: 'Cancellation Date', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', + type: 'string', + alias: 'zConsole URL', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', + type: 'string', + alias: 'Customer Success Representative', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', + type: 'string', + alias: 'VPC Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', + type: 'string', + alias: 'End Customer: Region', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', + type: 'string', + alias: 'Proof of Concept Impact ?', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', + type: 'string', + alias: 'Internal Status', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', + type: 'string', + alias: 'Problem Sub-Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', + type: 'string', + alias: 'Tenant Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', + type: 'string', + alias: 'Customer Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', + type: 'string', + alias: 'Channel', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', + type: 'string', + alias: 'Order Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', + type: 'string', + alias: 'zIPS Licenses', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', + type: 'boolean', + alias: 'z3A License', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', + type: 'string', + alias: 'Sub-Partner Name', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', + type: 'string_array', + alias: 'Ticket Region', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', + type: 'string', + alias: 'zShield Licenses', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', + type: 'string', + alias: 'zScan Apps Licensed', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', + type: 'string', + alias: 'KPE MTD Licenses', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', + type: 'string', + alias: 'Product Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', + type: 'string', + alias: 'whiteCryption agent', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', + type: 'string', + alias: 'zIPS Test Case Types', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', + type: 'string', + alias: 'Product Version', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', + type: 'string', + alias: 'Target Delivery Date', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', + type: 'string', + alias: 'App Version ', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', + type: 'string', + alias: 'End Customer: Company Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', + type: 'string', + alias: 'End Customer: Physical Address', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', + type: 'string', + alias: 'Term Start Date', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', + type: 'string', + alias: 'Term End Date', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', + type: 'string', + alias: 'Order Fulfilled', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', + type: 'string', + alias: 'Requested Delivery Date', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', + type: 'string', + alias: 'Internal Tracking Id', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', + type: 'string', + alias: 'Feature Status', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', + type: 'string', + alias: 'Case Type', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', + type: 'string', + alias: 'Business Impact', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', + type: 'string', + alias: 'Zimperium Console URL', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', + type: 'string', + alias: 'End Customer: Administrator Name', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', + type: 'string', + alias: 'zIPS Only Licenses ', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', + type: 'string', + alias: 'Zimperium Sales Contact: Email Address', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', + type: 'boolean', + alias: 'allow_attachments', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', + type: 'boolean', + alias: 'allow_channelback', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', + type: 'string_array', + alias: 'Collaborators', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', + type: 'string', + alias: 'External Object Type', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', + type: 'string_array', + alias: 'Fields modified during import', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', + type: 'boolean', + alias: 'from_messaging_channel', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', + type: 'boolean', + alias: 'has_incidents', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', + type: 'boolean', + alias: 'is_public', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', + type: 'string_array', + alias: 'Reporters', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', + type: 'string', + alias: 'Customer', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', + type: 'string', + alias: 'raw_subject', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', + type: 'string', + alias: 'recipient', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', + type: 'string', + alias: 'External Source', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', + type: 'string', + alias: 'url', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', + type: 'string', + alias: 'Hold Status', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', + type: 'boolean', + alias: 'Do Not Close', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', + type: 'boolean', + alias: 'Escalated', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', + type: 'string', + alias: 'Partner: Sales Contact', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', + type: 'string', + alias: 'Partner Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', + type: 'string', + alias: 'Env Setup Type', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', + type: 'string', + alias: 'Approval Status', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', + type: 'string', + alias: 'Booking Year', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', + type: 'string_array', + alias: 'Root Cause', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', + type: 'string', + alias: 'Services Purchased', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', + type: 'string', + alias: 'Support Purchased', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', + type: 'string', + alias: 'VPC Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', + type: 'string', + alias: 'End Customer: Region', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', + type: 'string', + alias: 'Proof of Concept Impact ?', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', + type: 'string', + alias: 'Problem Sub-Type', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', + type: 'string', + alias: 'Tenant Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', + type: 'string', + alias: 'Customer Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', + type: 'string', + alias: 'Channel', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', + type: 'string', + alias: 'Order Type', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', + type: 'string', + alias: 'zIPS Licenses', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', + type: 'boolean', + alias: 'z3A License', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', + type: 'string', + alias: 'Sub-Partner Name', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', + type: 'string_array', + alias: 'Ticket Region', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', + type: 'string', + alias: 'Product Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', + type: 'string', + alias: 'whiteCryption agent', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', + type: 'string', + alias: 'Product Version', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', + type: 'string', + alias: 'App Version ', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', + type: 'string', + alias: 'End Customer: Company Name', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', + type: 'string', + alias: 'End Customer: Physical Address', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', + type: 'string', + alias: 'Term Start Date', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', + type: 'string', + alias: 'Term End Date', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', + type: 'string', + alias: 'Order Fulfilled', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', + type: 'string', + alias: 'Internal Tracking Id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', + type: 'string', + alias: 'Feature Status', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', + type: 'string', + alias: 'Case Type', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', + type: 'string', + alias: 'Business Impact', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', + type: 'string', + alias: 'DuplicateField Console URL', + }, + { + name: 'ctype_hell__test', + sql: 'ctype_hell__test', + type: 'string', + alias: 'test', + }, + { + name: 'ctype_hell__text_default_field', + sql: 'ctype_hell__text_default_field', + type: 'string', + alias: 'text default field', + }, + { + name: 'ctype_clone_ticket___clone_field_dropdown', + sql: 'ctype_clone_ticket___clone_field_dropdown', + type: 'string', + alias: 'clone field dropdown', + }, + { + name: 'ctype_test_subtype_sep__new_field_2', + sql: 'ctype_test_subtype_sep__new_field_2', + type: 'string', + alias: 'field_harsh', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_test_subtype_sep__new_field_id_list_3', + sql: 'ctype_test_subtype_sep__new_field_id_list_3', + type: 'string_array', + alias: 'field_id_list_harsh', + }, + { + name: 'ctype_vgtestyesno__booltestyesno', + sql: 'ctype_vgtestyesno__booltestyesno', + type: 'boolean', + alias: 'booltestyesno', + }, + { + name: 'ctype_asd__test_asd', + sql: 'ctype_asd__test_asd', + type: 'string', + alias: 'test asd', + }, + { + name: 'ctype_Microservice__git_template_name', + sql: 'ctype_Microservice__git_template_name', + type: 'string', + alias: 'Github Template Name', + }, + { + name: 'ctype_Microservice__micro_service_name', + sql: 'ctype_Microservice__micro_service_name', + type: 'string', + alias: 'Microservice Name', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_Microservice__platform', + sql: 'ctype_Microservice__platform', + type: 'string_array', + alias: 'Platform', + }, + { + name: 'ctype_Microservice__read_only', + sql: 'ctype_Microservice__read_only', + type: 'string', + alias: 'read only', + }, + { + name: 'ctype_helohelohelohelohelo__dfkdkl', + sql: 'ctype_helohelohelohelohelo__dfkdkl', + type: 'number', + alias: 'dfkdkl', + }, + { + name: 'ctype_helohelohelohelohelo__hello_1122', + sql: 'ctype_helohelohelohelohelo__hello_1122', + type: 'string', + alias: 'hello hello folks', + }, + { + name: 'ctype_helohelohelohelohelo__hello_12', + sql: 'ctype_helohelohelohelohelo__hello_12', + type: 'string', + alias: 'hello_12', + }, + { + name: 'ctype_helohelohelohelohelo__hello_123', + sql: 'ctype_helohelohelohelohelo__hello_123', + type: 'string', + alias: 'hello_123', + }, + { + name: 'ctype_helohelohelohelohelo__hello_12x', + sql: 'ctype_helohelohelohelohelo__hello_12x', + type: 'string', + alias: 'hello_12x', + }, + { + name: 'ctype_helohelohelohelohelo__hello_12y', + sql: 'ctype_helohelohelohelohelo__hello_12y', + type: 'string', + alias: 'hello_12y', + }, + { + name: 'ctype_helohelohelohelohelo__hello_wed', + sql: 'ctype_helohelohelohelohelo__hello_wed', + type: 'string', + alias: 'hello_wed', + }, + { + name: 'ctype_helohelohelohelohelo__hello_world', + sql: 'ctype_helohelohelohelohelo__hello_world', + type: 'string', + alias: 'hello_world', + }, + { + name: 'ctype_dddddddd__dkay_march_23', + sql: 'ctype_dddddddd__dkay_march_23', + type: 'string', + alias: 'dkay_march_23', + }, + { + name: 'ctype_dddddddd__dkay_march_26', + sql: 'ctype_dddddddd__dkay_march_26', + type: 'string', + alias: 'dkay_march_26', + }, + { + name: 'ctype_dddddddd__scope_approval', + sql: 'ctype_dddddddd__scope_approval', + type: 'string', + alias: 'Scope Approval', + }, + { + name: 'ctype_dddddddd__uat_approval', + sql: 'ctype_dddddddd__uat_approval', + type: 'string', + alias: 'UAT approval', + }, + { + name: 'ctype_defaults__customer_uat', + sql: 'ctype_defaults__customer_uat', + type: 'time', + alias: 'customer uat', + }, + { + name: 'ctype_defaults__default', + sql: 'ctype_defaults__default', + type: 'string', + alias: 'default', + }, + { + name: 'ctype_defaults__default_enum', + sql: 'ctype_defaults__default_enum', + type: 'string', + alias: 'default enum', + }, + { + name: 'ctype_defaults__enum_testing', + sql: 'ctype_defaults__enum_testing', + type: 'string', + alias: 'enum testing', + }, + { + name: 'ctype_defaults__uat_date', + sql: 'ctype_defaults__uat_date', + type: 'time', + alias: 'uat date', + }, + { + name: 'ctype_bug__abc', + sql: 'ctype_bug__abc', + type: 'string', + alias: 'abc', + }, + { + name: 'ctype_bug__dependent_test', + sql: 'ctype_bug__dependent_test', + type: 'string', + alias: 'something', + }, + { + name: 'ctype_event_request__approver', + sql: 'ctype_event_request__approver', + type: 'string', + alias: 'Approver', + }, + { + name: 'ctype_event_request__budget', + sql: 'ctype_event_request__budget', + type: 'number', + alias: 'Budget', + }, + { + name: 'ctype_event_request__event_date', + sql: 'ctype_event_request__event_date', + type: 'time', + alias: 'Event Date', + }, + { + name: 'ctype_event_request__event_location', + sql: 'ctype_event_request__event_location', + type: 'string', + alias: 'Event Location', + }, + { + name: 'ctype_event_request__event_name', + sql: 'ctype_event_request__event_name', + type: 'string', + alias: 'Event Name', + }, + { + name: 'ctype_event_request__event_owner', + sql: 'ctype_event_request__event_owner', + type: 'string', + alias: 'Event Owner', + }, + { + name: 'ctype_event_request__event_type', + sql: 'ctype_event_request__event_type', + type: 'string', + alias: 'Event Type', + }, + { + name: 'ctype_event_request__link_to_event', + sql: 'ctype_event_request__link_to_event', + type: 'string', + alias: 'Link To Event', + }, + { + name: 'ctype_event_request__region', + sql: 'ctype_event_request__region', + type: 'string', + alias: 'Region', + }, + { + name: 'ctype_event_request__requested_by', + sql: 'ctype_event_request__requested_by', + type: 'string', + alias: 'Requested by', + }, + { + name: 'ctype_event_request__status', + sql: 'ctype_event_request__status', + type: 'string', + alias: 'Status', + }, + { + name: 'ctype_event_request__target_pipeline', + sql: 'ctype_event_request__target_pipeline', + type: 'number', + alias: 'Target Pipeline', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_event_request__target_segment', + sql: 'ctype_event_request__target_segment', + type: 'string_array', + alias: 'Target Segment', + }, + { + name: 'ctype_mfz_subtype_check__enum_field', + sql: 'ctype_mfz_subtype_check__enum_field', + type: 'string', + alias: 'Custom Enum field', + }, + { + name: 'ctype_mfz_subtype_check__text_field', + sql: 'ctype_mfz_subtype_check__text_field', + type: 'string', + alias: 'Custom text field', + }, + { + name: 'ctype_mfz_subtype_check__user_field', + sql: 'ctype_mfz_subtype_check__user_field', + type: 'string', + alias: 'Custom user field', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_mfz_subtype_check__user_list_field', + sql: 'ctype_mfz_subtype_check__user_list_field', + type: 'string_array', + alias: 'Custom user list field', + }, + { + name: 'ctype_mfz_subtype_check__workspace_field', + sql: 'ctype_mfz_subtype_check__workspace_field', + type: 'string', + alias: 'Custom workspace field', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_mfz_subtype_check__workspace_list_field', + sql: 'ctype_mfz_subtype_check__workspace_list_field', + type: 'string_array', + alias: 'Custom workspace list field', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', + type: 'boolean', + alias: 'allow_attachments', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', + type: 'boolean', + alias: 'allow_channelback', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', + type: 'string_array', + alias: 'Collaborators', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', + type: 'string', + alias: 'External Object Type', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', + type: 'string_array', + alias: 'Fields modified during import', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', + type: 'boolean', + alias: 'from_messaging_channel', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', + type: 'boolean', + alias: 'has_incidents', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', + type: 'boolean', + alias: 'is_public', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', + type: 'string_array', + alias: 'Reporters', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', + type: 'string', + alias: 'Customer', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', + type: 'string', + alias: 'raw_subject', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', + type: 'string', + alias: 'recipient', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', + type: 'string', + alias: 'External Source', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', + type: 'string', + alias: 'url', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', + type: 'string_array', + alias: 'Ticket Region', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', + type: 'string', + alias: 'whiteCryption agent', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', + type: 'string', + alias: 'Feature Status', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', + type: 'string', + alias: 'Case Type', + }, + { + name: 'ctype_mandatory_field_subtype__some_text_field', + sql: 'ctype_mandatory_field_subtype__some_text_field', + type: 'string', + alias: 'Some text field', + }, + { + name: 'ctype_review_subtype__date', + sql: 'ctype_review_subtype__date', + type: 'time', + alias: 'Date', + }, + { + name: 'ctype_review_subtype__play_store_review_url', + sql: 'ctype_review_subtype__play_store_review_url', + type: 'string', + alias: 'Review URL', + }, + { + name: 'ctype_review_subtype__rating', + sql: 'ctype_review_subtype__rating', + type: 'string', + alias: 'Rating', + }, + { + name: 'ctype_review_subtype__reply_date', + sql: 'ctype_review_subtype__reply_date', + type: 'time', + alias: 'Reply date', + }, + { + name: 'ctype_review_subtype__reply_text', + sql: 'ctype_review_subtype__reply_text', + type: 'string', + alias: 'Reply text', + }, + { + name: 'ctype_review_subtype__upvote_count', + sql: 'ctype_review_subtype__upvote_count', + type: 'number', + alias: 'Upvote Count', + }, + { + name: 'ctype_jkjkjkjkjkjk__dgg', + sql: 'ctype_jkjkjkjkjkjk__dgg', + type: 'string', + alias: 'dgg', + }, + { + name: 'ctype_jkjkjkjkjkjk__dggd', + sql: 'ctype_jkjkjkjkjkjk__dggd', + type: 'number', + alias: 'dggd', + }, + { + name: 'ctype_jkjkjkjkjkjk__hello', + sql: 'ctype_jkjkjkjkjkjk__hello', + type: 'string', + alias: 'hello', + }, + { + name: 'ctype_jkjkjkjkjkjk__hello_acl', + sql: 'ctype_jkjkjkjkjkjk__hello_acl', + type: 'string', + alias: 'hellO_acl', + }, + { + name: 'ctype_huryyyyyyyyyyy__all_internal_selected', + sql: 'ctype_huryyyyyyyyyyy__all_internal_selected', + type: 'string', + alias: 'all_external_selected', + }, + { + name: 'ctype_huryyyyyyyyyyy__all_selected', + sql: 'ctype_huryyyyyyyyyyy__all_selected', + type: 'string', + alias: 'all_selected', + }, + { + name: 'ctype_huryyyyyyyyyyy__test_acl_2', + sql: 'ctype_huryyyyyyyyyyy__test_acl_2', + type: 'string', + alias: 'test_acl_2', + }, + { + name: 'ctype_aaa__fkjdsfjkdj', + sql: 'ctype_aaa__fkjdsfjkdj', + type: 'string', + alias: 'fkjdsfjkdj', + }, + { + name: 'ctype_aaa__test_acl_march', + sql: 'ctype_aaa__test_acl_march', + type: 'string', + alias: 'test_acl_march', + }, + { + name: 'ctype_lolololololol__hell_acl_2', + sql: 'ctype_lolololololol__hell_acl_2', + type: 'string', + alias: 'hell_acl_2', + }, + { + name: 'ctype_lolololololol__hello_acl', + sql: 'ctype_lolololololol__hello_acl', + type: 'string', + alias: 'hello_acl', + }, + { + name: 'ctype_lolololololol__hello_acl_3', + sql: 'ctype_lolololololol__hello_acl_3', + type: 'string', + alias: 'hello_acl_3', + }, + { + name: 'ctype_zzz__dds', + sql: 'ctype_zzz__dds', + type: 'string', + alias: 'DDS', + }, + { + name: 'ctype_zzz__hello', + sql: 'ctype_zzz__hello', + type: 'string', + alias: 'hello ', + }, + { + name: 'ctype_zzz__hello_2', + sql: 'ctype_zzz__hello_2', + type: 'string', + alias: 'hello_2', + }, + { + name: 'ctype_zzz__hello_3', + sql: 'ctype_zzz__hello_3', + type: 'string', + alias: 'hello_3', + }, + { + name: 'ctype_zzz__hello_4', + sql: 'ctype_zzz__hello_4', + type: 'string', + alias: 'hello_4', + }, + { + name: 'ctype_zzz__sdkjdsfjk', + sql: 'ctype_zzz__sdkjdsfjk', + type: 'string', + alias: 'SDKJDSFJK', + }, + { + name: 'ctype_ggg__dkay_march_1234', + sql: 'ctype_ggg__dkay_march_1234', + type: 'string', + alias: 'dkay_march_1234', + }, + { + name: 'ctype_ggg__dkay_march_2', + sql: 'ctype_ggg__dkay_march_2', + type: 'string', + alias: 'dkay_march_2', + }, + { + name: 'ctype_ggg__dkay_test', + sql: 'ctype_ggg__dkay_test', + type: 'string', + alias: 'dkay_test', + }, + { + name: 'ctype_ooooo__dkay_march_3', + sql: 'ctype_ooooo__dkay_march_3', + type: 'string', + alias: 'dkay_march_3', + }, + { + name: 'ctype_ooooo__dkay_march_4', + sql: 'ctype_ooooo__dkay_march_4', + type: 'string', + alias: 'dkay_march_4', + }, + { + name: 'ctype_ooooo__dkay_march_5', + sql: 'ctype_ooooo__dkay_march_5', + type: 'string', + alias: 'dkay_march_5', + }, + { + name: 'ctype_ooooo__dkay_march_7', + sql: 'ctype_ooooo__dkay_march_7', + type: 'string', + alias: 'dkay_march_7', + }, + { + name: 'ctype_sync_engine_test__field1', + sql: 'ctype_sync_engine_test__field1', + type: 'string', + alias: 'field1', + }, + { + name: 'ctype_sync_engine_test__field2_bool', + sql: 'ctype_sync_engine_test__field2_bool', + type: 'boolean', + alias: 'field2_bool', + }, + { + name: 'ctype_priviliges_testing__read_only_field', + sql: 'ctype_priviliges_testing__read_only_field', + type: 'string', + alias: 'read only field', + }, + { + name: 'ctype_priviliges_testing__read_write_field', + sql: 'ctype_priviliges_testing__read_write_field', + type: 'string', + alias: 'read/write field', + }, + { + name: 'ctype_dashboard_test__pulkit', + sql: 'ctype_dashboard_test__pulkit', + type: 'boolean', + alias: 'pulkit', + }, + { + name: 'ctype_tttt__dropdown', + sql: 'ctype_tttt__dropdown', + type: 'string', + alias: 'dropdown', + }, + ], + joins: [ + { + sql: 'dim_ticket.sla_tracker_id = dim_sla_tracker.id', + }, + { + sql: 'dim_ticket.id = dim_survey_response.object', + }, + { + sql: 'dim_ticket.id = dim_link_issue_target.source_id', + }, + { + sql: 'dim_ticket.id = dim_link_conversation_source.target_id', + }, + { + sql: 'dim_ticket.rev_oid = dim_revo.id', + }, + { + sql: 'dim_ticket.applies_to_part_id = dim_part.id', + }, + ], + measures: [ + { + name: 'id_count', + function: { + type: 'count', + }, + sql: 'count(dim_ticket.id)', + type: 'string', + alias: 'Ticket Id', + }, + { + name: 'created_date_max', + function: { + type: 'max', + }, + sql: 'max(dim_ticket.created_date)', + type: 'time', + alias: 'Created date', + }, + { + name: 'actual_close_date_max', + function: { + type: 'max', + }, + sql: 'max(dim_ticket.actual_close_date)', + type: 'time', + alias: 'Closed date', + }, + { + name: 'sla_tracker_id_count', + function: { + type: 'count', + }, + sql: 'count(dim_ticket.sla_tracker_id)', + type: 'string', + alias: 'Sla Tracker Id', + }, + { + name: 'resolution_time', + sql: "case WHEN actual_close_date > created_date THEN date_diff('minutes', created_date, actual_close_date) ELSE null END ", + type: 'number', + alias: 'Resolution Time', + }, + { + name: 'surveys_aggregation_json_measure', + function: { + type: 'median', + }, + sql: "median(list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min'))", + type: 'number', + alias: 'CSAT Rating', + }, + { + name: 'tnt__account_id', + sql: 'tnt__account_id', + type: 'string', + alias: 'account id', + }, + { + name: 'tnt__actual_effort_spent', + sql: 'tnt__actual_effort_spent', + type: 'number', + alias: 'Actual Effort Spent', + }, + { + name: 'tnt__capability_part', + sql: 'tnt__capability_part', + type: 'string', + alias: 'Capability part', + }, + { + name: 'tnt__custom_id_field', + sql: 'tnt__custom_id_field', + type: 'string', + alias: 'custom id field', + }, + { + name: 'tnt__date_field', + sql: 'tnt__date_field', + type: 'time', + alias: 'date_field', + }, + { + name: 'tnt__estimated_effort', + sql: 'tnt__estimated_effort', + type: 'number', + alias: 'Estimated Effort', + }, + { + name: 'tnt__fruit', + sql: 'tnt__fruit', + type: 'string', + alias: 'Fruit', + }, + { + name: 'tnt__id_field', + sql: 'tnt__id_field', + type: 'string', + alias: 'id field', + }, + { + name: 'tnt__issue_score', + sql: 'tnt__issue_score', + type: 'number', + alias: 'Issue Score', + }, + { + name: 'tnt__numeric_field', + sql: 'tnt__numeric_field', + type: 'number', + alias: 'numeric field', + }, + { + name: 'tnt__parent_part', + sql: 'tnt__parent_part', + type: 'string', + alias: 'Parent part', + }, + { + name: 'tnt__part_product', + sql: 'tnt__part_product', + type: 'string', + alias: 'part product', + }, + { + name: 'tnt__rank', + sql: 'tnt__rank', + type: 'number', + alias: 'Rank', + }, + { + name: 'tnt__remaining_effort', + sql: 'tnt__remaining_effort', + type: 'number', + alias: 'Remaining Effort', + }, + { + name: 'tnt__stray_user', + sql: 'tnt__stray_user', + type: 'string', + alias: 'stray user', + }, + { + name: 'tnt__stray_users', + sql: 'tnt__stray_users', + type: 'string', + alias: 'stray users', + }, + { + name: 'tnt__test', + sql: 'tnt__test', + type: 'number', + alias: 'Test', + }, + { + name: 'tnt__test_capability_part', + sql: 'tnt__test_capability_part', + type: 'string', + alias: 'Test Capability Part', + }, + { + name: 'tnt__test_product_part', + sql: 'tnt__test_product_part', + type: 'string', + alias: 'Test Product part', + }, + { + name: 'tnt__ticket_custom_part', + sql: 'tnt__ticket_custom_part', + type: 'string', + alias: 'ticket custom part', + }, + { + name: 'tnt__workspace_custom', + sql: 'tnt__workspace_custom', + type: 'string', + alias: 'workspace_custom', + }, + { + name: 'ctype_deal_registration__estimated_deal_value', + sql: 'ctype_deal_registration__estimated_deal_value', + type: 'number', + alias: 'Estimated Deal Value', + }, + { + name: 'ctype_deal_registration__expected_close_date', + sql: 'ctype_deal_registration__expected_close_date', + type: 'time', + alias: 'Expected Close Date', + }, + { + name: 'ctype_Events__event_end_date', + sql: 'ctype_Events__event_end_date', + type: 'time', + alias: 'Event End Date', + }, + { + name: 'ctype_Events__event_owner', + sql: 'ctype_Events__event_owner', + type: 'string', + alias: 'Event Owner', + }, + { + name: 'ctype_Events__event_start_date', + sql: 'ctype_Events__event_start_date', + type: 'time', + alias: 'Event Start Date', + }, + { + name: 'ctype_Events__events_test_ref', + sql: 'ctype_Events__events_test_ref', + type: 'string', + alias: 'events test ref', + }, + { + name: 'ctype_Events__external_budget', + sql: 'ctype_Events__external_budget', + type: 'number', + alias: 'External Budget', + }, + { + name: 'ctype_Events__internal_budget', + sql: 'ctype_Events__internal_budget', + type: 'number', + alias: 'Internal Budget', + }, + { + name: 'ctype_Events__pipeline_generated', + sql: 'ctype_Events__pipeline_generated', + type: 'number', + alias: 'Pipeline Generated', + }, + { + name: 'ctype_Events__total_budget', + sql: 'ctype_Events__total_budget', + type: 'number', + alias: 'Total Budget', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_helohelohelohelohelo__dfkdkl', + sql: 'ctype_helohelohelohelohelo__dfkdkl', + type: 'number', + alias: 'dfkdkl', + }, + { + name: 'ctype_defaults__customer_uat', + sql: 'ctype_defaults__customer_uat', + type: 'time', + alias: 'customer uat', + }, + { + name: 'ctype_defaults__uat_date', + sql: 'ctype_defaults__uat_date', + type: 'time', + alias: 'uat date', + }, + { + name: 'ctype_event_request__approver', + sql: 'ctype_event_request__approver', + type: 'string', + alias: 'Approver', + }, + { + name: 'ctype_event_request__budget', + sql: 'ctype_event_request__budget', + type: 'number', + alias: 'Budget', + }, + { + name: 'ctype_event_request__event_date', + sql: 'ctype_event_request__event_date', + type: 'time', + alias: 'Event Date', + }, + { + name: 'ctype_event_request__event_owner', + sql: 'ctype_event_request__event_owner', + type: 'string', + alias: 'Event Owner', + }, + { + name: 'ctype_event_request__requested_by', + sql: 'ctype_event_request__requested_by', + type: 'string', + alias: 'Requested by', + }, + { + name: 'ctype_event_request__target_pipeline', + sql: 'ctype_event_request__target_pipeline', + type: 'number', + alias: 'Target Pipeline', + }, + { + name: 'ctype_mfz_subtype_check__user_field', + sql: 'ctype_mfz_subtype_check__user_field', + type: 'string', + alias: 'Custom user field', + }, + { + name: 'ctype_mfz_subtype_check__workspace_field', + sql: 'ctype_mfz_subtype_check__workspace_field', + type: 'string', + alias: 'Custom workspace field', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', + type: 'number', + alias: 'brand_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', + type: 'time', + alias: 'created_at', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', + type: 'time', + alias: 'generated_timestamp', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', + type: 'number', + alias: 'group_id', + }, + { + name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', + type: 'number', + alias: 'ticket_form_id', + }, + { + name: 'ctype_review_subtype__date', + sql: 'ctype_review_subtype__date', + type: 'time', + alias: 'Date', + }, + { + name: 'ctype_review_subtype__reply_date', + sql: 'ctype_review_subtype__reply_date', + type: 'time', + alias: 'Reply date', + }, + { + name: 'ctype_review_subtype__upvote_count', + sql: 'ctype_review_subtype__upvote_count', + type: 'number', + alias: 'Upvote Count', + }, + { + name: 'ctype_jkjkjkjkjkjk__dggd', + sql: 'ctype_jkjkjkjkjkjk__dggd', + type: 'number', + alias: 'dggd', + }, + { + name: 'id_count___function__count', + function: { + type: 'count', + }, + sql: 'count(dim_ticket.id)', + type: 'string', + alias: 'Ticket Id', + }, + ], + name: 'dim_ticket', + sql: 'select * from devrev.dim_ticket', + }, + ] as any; + const resolutionConfig = { + columnConfigs: [ { - name: 'tickets.created_by', - isArrayType: false, - source: 'created_by_lookup', joinColumn: 'id', + name: 'dim_ticket.tags_json', + isArrayType: true, resolutionColumns: ['name'], + source: 'dim_tag', }, ], tableSchemas: [ - OWNERS_LOOKUP_SCHEMA, - TAGS_LOOKUP_SCHEMA, - CREATED_BY_LOOKUP_SCHEMA, + { + dimensions: [ + { + name: 'created_by_id', + sql: 'created_by_id', + type: 'string', + alias: 'Created by', + }, + { + name: 'dev_oid', + sql: 'dev_oid', + type: 'string', + alias: 'Dev organization ID', + }, + { + name: 'modified_date', + sql: 'modified_date', + type: 'time', + alias: 'Modified date', + }, + { + name: 'object_type', + sql: 'object_type', + type: 'string', + alias: 'Object type', + }, + { + name: 'tag_type', + sql: 'tag_type', + type: 'number', + alias: 'Values enabled', + }, + { + modifier: { + shouldUnnestGroupBy: false, + }, + name: 'allowed_values', + sql: 'allowed_values', + type: 'string_array', + alias: 'Allowed values', + }, + { + name: 'is_deleted', + sql: 'is_deleted', + type: 'boolean', + alias: 'Is deleted', + }, + { + name: 'modified_by_id', + sql: 'modified_by_id', + type: 'string', + alias: 'Modified by', + }, + { + name: 'access_level', + sql: 'access_level', + type: 'number', + alias: 'Access level', + }, + { + name: 'display_id', + sql: 'display_id', + type: 'string', + alias: 'Display ID', + }, + { + name: 'id', + sql: 'id', + type: 'string', + alias: 'ID', + }, + { + name: 'name', + sql: 'name', + type: 'string', + alias: 'Tag name', + }, + { + name: 'object_version', + sql: 'object_version', + type: 'number', + alias: 'Object Version', + }, + { + name: 'created_date', + sql: 'created_date', + type: 'time', + alias: 'Created date', + }, + ], + measures: [], + name: 'dim_tag', + sql: 'SELECT * FROM devrev.dim_tag', + }, ], - }; - + } as any; const columnProjections = [ - 'tickets.id', - 'tickets.owners', - 'tickets.tags', - 'tickets.created_by', - 'tickets.count', + 'dim_ticket.tags_json', + 'dim_ticket.id_count___function__count', ]; - const sql = await cubeQueryToSQLWithResolution({ + + const resultQuery = await cubeQueryToSQLWithResolution({ query, - tableSchemas: [TICKETS_TABLE_SCHEMA], + tableSchemas, resolutionConfig, columnProjections, }); - console.log('Phase 1 SQL (multiple arrays):', sql); - - // Verify row_id is included - expect(sql).toContain('row_id'); - - // Both arrays should be unnested - expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(2); - - // Execute the SQL to verify it works - const result = (await duckdbExec(sql)) as any[]; - - expect(result.length).toBe(7); - - // Each row should have a row_id - expect(result[0]).toHaveProperty('__row_id'); - expect(result[0]).toHaveProperty('tickets__count'); - expect(result[0]).toHaveProperty('tickets__id'); - expect(result[0]).toHaveProperty('tickets__owners - display_name'); - expect(result[0]).toHaveProperty('tickets__tags - tag_name'); + const expectedQuery = `select * exclude(__row_id) from (SELECT MAX(__resolved_query."Ticket Id") AS "Ticket Id" , ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name" , "__row_id" FROM (SELECT __resolved_query."__row_id" AS "__row_id", * FROM (SELECT "Tags - Tag name", "Ticket Id", "__row_id" FROM (SELECT __unnested_base_query."Ticket Id" AS "Ticket Id", __unnested_base_query."__row_id" AS "__row_id", * FROM (SELECT "Ticket Id", "Tags", "__row_id" FROM (SELECT __base_query."Ticket Id" AS "Ticket Id", unnest(__base_query."Tags") AS "Tags", row_number() OVER () AS "__row_id", * FROM (SELECT count("ID") AS "Ticket Id" , "Tags" FROM (SELECT dim_ticket.created_date AS "Created date", 'ticket' AS "Work Type", json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[]) AS "Tags", dim_ticket.id AS "ID", * FROM (select * from devrev.dim_ticket) AS dim_ticket) AS dim_ticket WHERE (((("Created date" >= '2025-08-02T09:30:00.000Z') AND ("Created date" <= '2025-10-31T10:29:59.999Z')) AND ("Work Type" IN ('ticket')) AND (Stage IN ('don:core:dvrv-us-1:devo/787:custom_stage/514', 'don:core:dvrv-us-1:devo/787:custom_stage/512', 'don:core:dvrv-us-1:devo/787:custom_stage/501', 'don:core:dvrv-us-1:devo/787:custom_stage/485', 'don:core:dvrv-us-1:devo/787:custom_stage/440', 'don:core:dvrv-us-1:devo/787:custom_stage/25', 'don:core:dvrv-us-1:devo/787:custom_stage/24', 'don:core:dvrv-us-1:devo/787:custom_stage/22', 'don:core:dvrv-us-1:devo/787:custom_stage/21', 'don:core:dvrv-us-1:devo/787:custom_stage/19', 'don:core:dvrv-us-1:devo/787:custom_stage/13', 'don:core:dvrv-us-1:devo/787:custom_stage/10', 'don:core:dvrv-us-1:devo/787:custom_stage/8', 'don:core:dvrv-us-1:devo/787:custom_stage/7', 'don:core:dvrv-us-1:devo/787:custom_stage/6', 'don:core:dvrv-us-1:devo/787:custom_stage/4')))) GROUP BY Tags LIMIT 50000) AS __base_query) AS __base_query) AS __unnested_base_query LEFT JOIN (SELECT __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", * FROM (SELECT * FROM devrev.dim_tag) AS __unnested_base_query__dim_ticket__tags_json) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags"=__unnested_base_query__dim_ticket__tags_json.id) AS MEERKAT_GENERATED_TABLE) AS __resolved_query) AS __resolved_query GROUP BY __row_id)`; + expect(resultQuery).toBe(expectedQuery); }); - - // it('test an proper query from UI', async () => { - // const query = { - // dimensions: ['dim_ticket.tags_json'], - // filters: [ - // { - // and: [ - // { - // and: [ - // { - // member: 'dim_ticket.created_date', - // operator: 'inDateRange', - // values: [ - // '2025-08-02T09:30:00.000Z', - // '2025-10-31T10:29:59.999Z', - // ], - // }, - // { - // member: 'dim_ticket.work_type', - // operator: 'in', - // values: ['ticket'], - // }, - // { - // member: 'dim_ticket.stage_json', - // operator: 'in', - // values: [ - // 'don:core:dvrv-us-1:devo/787:custom_stage/514', - // 'don:core:dvrv-us-1:devo/787:custom_stage/512', - // 'don:core:dvrv-us-1:devo/787:custom_stage/501', - // 'don:core:dvrv-us-1:devo/787:custom_stage/485', - // 'don:core:dvrv-us-1:devo/787:custom_stage/440', - // 'don:core:dvrv-us-1:devo/787:custom_stage/25', - // 'don:core:dvrv-us-1:devo/787:custom_stage/24', - // 'don:core:dvrv-us-1:devo/787:custom_stage/22', - // 'don:core:dvrv-us-1:devo/787:custom_stage/21', - // 'don:core:dvrv-us-1:devo/787:custom_stage/19', - // 'don:core:dvrv-us-1:devo/787:custom_stage/13', - // 'don:core:dvrv-us-1:devo/787:custom_stage/10', - // 'don:core:dvrv-us-1:devo/787:custom_stage/8', - // 'don:core:dvrv-us-1:devo/787:custom_stage/7', - // 'don:core:dvrv-us-1:devo/787:custom_stage/6', - // 'don:core:dvrv-us-1:devo/787:custom_stage/4', - // ], - // }, - // ], - // }, - // ], - // }, - // ], - // limit: 50000, - // measures: ['dim_ticket.id_count___function__count'], - // } as any; - // const tableSchemas = [ - // { - // dimensions: [ - // { - // name: 'rev_oid', - // sql: 'dim_ticket.rev_oid', - // type: 'string', - // alias: 'Customer Workspace', - // }, - // { - // name: 'title', - // sql: 'dim_ticket.title', - // type: 'string', - // alias: 'Title', - // }, - // { - // name: 'last_internal_comment_date', - // sql: 'dim_ticket.last_internal_comment_date', - // type: 'time', - // alias: 'Last Internal Comment Date', - // }, - // { - // name: 'work_type', - // sql: "'ticket'", - // type: 'time', - // alias: 'Work Type', - // }, - // { - // name: 'is_spam', - // sql: 'dim_ticket.is_spam', - // type: 'boolean', - // alias: 'Spam', - // }, - // { - // name: 'object_type', - // sql: 'dim_ticket.object_type', - // type: 'string', - // alias: 'Object type', - // }, - // { - // name: 'modified_by_id', - // sql: 'dim_ticket.modified_by_id', - // type: 'string', - // alias: 'Modified by', - // }, - // { - // name: 'source_channel', - // sql: 'dim_ticket.source_channel', - // type: 'string', - // alias: 'Source channel', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'channels', - // sql: 'dim_ticket.channels', - // type: 'number_array', - // alias: 'Channels', - // }, - // { - // name: 'created_by_id', - // sql: 'dim_ticket.created_by_id', - // type: 'string', - // alias: 'Created by', - // }, - // { - // name: 'created_date', - // sql: 'dim_ticket.created_date', - // type: 'time', - // alias: 'Created date', - // }, - // { - // name: 'target_close_date', - // sql: 'dim_ticket.target_close_date', - // type: 'time', - // alias: 'Target close date', - // }, - // { - // name: 'applies_to_part_id', - // sql: 'dim_ticket.applies_to_part_id', - // type: 'string', - // alias: 'Part', - // }, - // { - // name: 'subtype', - // sql: 'dim_ticket.subtype', - // type: 'string', - // alias: 'Subtype', - // }, - // { - // name: 'actual_close_date', - // sql: 'dim_ticket.actual_close_date', - // type: 'time', - // alias: 'Close date', - // }, - // { - // name: 'reported_by_id', - // sql: 'dim_ticket.reported_by_id', - // type: 'string', - // alias: 'Reported by ID', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'reported_by_ids', - // sql: 'dim_ticket.reported_by_ids', - // type: 'string_array', - // alias: 'Reported by', - // }, - // { - // name: 'needs_response', - // sql: 'dim_ticket.needs_response', - // type: 'boolean', - // alias: 'Needs Response', - // }, - // { - // name: 'group', - // sql: 'dim_ticket.group', - // type: 'string', - // alias: 'Group', - // }, - // { - // name: 'modified_date', - // sql: 'dim_ticket.modified_date', - // type: 'time', - // alias: 'Modified date', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'owned_by_ids', - // sql: 'dim_ticket.owned_by_ids', - // type: 'string_array', - // alias: 'Owner', - // }, - // { - // name: 'id', - // sql: 'dim_ticket.id', - // type: 'string', - // alias: 'ID', - // }, - // { - // name: 'sla_tracker_id', - // sql: 'dim_ticket.sla_tracker_id', - // type: 'string', - // alias: 'SLA Tracker', - // }, - // { - // name: 'severity', - // sql: 'dim_ticket.severity', - // type: 'number', - // alias: 'Severity', - // }, - // { - // name: 'stage_json', - // sql: "json_extract_string(dim_ticket.stage_json, '$.stage_id')", - // type: 'string', - // alias: 'Stage', - // }, - // { - // name: 'sla_id', - // sql: 'dim_ticket.sla_id', - // type: 'string', - // alias: 'SLA Name', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'links_json', - // sql: "list_distinct(CAST(json_extract_string(dim_ticket.links_json, '$[*].target_object_type') AS VARCHAR[]))", - // type: 'string_array', - // alias: 'Links', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tags_json', - // sql: "CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[])", - // type: 'string_array', - // alias: 'Tags', - // }, - // { - // name: 'surveys_aggregation_json', - // sql: "list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min')", - // type: 'number', - // alias: 'CSAT Rating', - // }, - // { - // name: 'sla_summary_target_time', - // sql: "cast(json_extract_string(dim_ticket.sla_summary, '$.target_time') as timestamp)", - // type: 'time', - // alias: 'Next SLA Target', - // }, - // { - // name: 'staged_info', - // sql: "cast(json_extract_string(dim_ticket.staged_info, '$.is_staged') as boolean)", - // type: 'boolean', - // alias: 'Changes Need Review', - // }, - // { - // name: 'tnt__account_id', - // sql: 'tnt__account_id', - // type: 'string', - // alias: 'account id', - // }, - // { - // name: 'tnt__actual_effort_spent', - // sql: 'tnt__actual_effort_spent', - // type: 'number', - // alias: 'Actual Effort Spent', - // }, - // { - // name: 'tnt__ai_subtype', - // sql: 'tnt__ai_subtype', - // type: 'string', - // alias: 'AI Subtype', - // }, - // { - // name: 'tnt__asdfad', - // sql: 'tnt__asdfad', - // type: 'string', - // alias: 'asdfad', - // }, - // { - // name: 'tnt__bool_summary', - // sql: 'tnt__bool_summary', - // type: 'boolean', - // alias: 'bool summary', - // }, - // { - // name: 'tnt__boolyesnoresp', - // sql: 'tnt__boolyesnoresp', - // type: 'boolean', - // alias: 'boolyesnoresp', - // }, - // { - // name: 'tnt__capability_part', - // sql: 'tnt__capability_part', - // type: 'string', - // alias: 'Capability part', - // }, - // { - // name: 'tnt__card', - // sql: 'tnt__card', - // type: 'string', - // alias: 'card', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__cust_account', - // sql: 'tnt__cust_account', - // type: 'string_array', - // alias: 'cust_account', - // }, - // { - // name: 'tnt__custom_field_for_sla', - // sql: 'tnt__custom_field_for_sla', - // type: 'string', - // alias: 'custom_field_for_sla', - // }, - // { - // name: 'tnt__custom_id_field', - // sql: 'tnt__custom_id_field', - // type: 'string', - // alias: 'custom id field', - // }, - // { - // name: 'tnt__customer_tier', - // sql: 'tnt__customer_tier', - // type: 'string', - // alias: 'Customer Tier', - // }, - // { - // name: 'tnt__date_field', - // sql: 'tnt__date_field', - // type: 'time', - // alias: 'date_field', - // }, - // { - // name: 'tnt__department_list', - // sql: 'tnt__department_list', - // type: 'string', - // alias: 'Department List', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__email_references', - // sql: 'tnt__email_references', - // type: 'string_array', - // alias: 'tnt__email_references', - // }, - // { - // name: 'tnt__escalated', - // sql: 'tnt__escalated', - // type: 'boolean', - // alias: 'Escalated', - // }, - // { - // name: 'tnt__estimated_effort', - // sql: 'tnt__estimated_effort', - // type: 'number', - // alias: 'Estimated Effort', - // }, - // { - // name: 'tnt__external_source_id', - // sql: 'tnt__external_source_id', - // type: 'string', - // alias: 'tnt__external_source_id', - // }, - // { - // name: 'tnt__feature_affected', - // sql: 'tnt__feature_affected', - // type: 'string', - // alias: 'Feature Affected', - // }, - // { - // name: 'tnt__field1', - // sql: 'tnt__field1', - // type: 'string', - // alias: 'field1', - // }, - // { - // name: 'tnt__foo', - // sql: 'tnt__foo', - // type: 'string', - // alias: 'tnt__foo', - // }, - // { - // name: 'tnt__fruit', - // sql: 'tnt__fruit', - // type: 'string', - // alias: 'Fruit', - // }, - // { - // name: 'tnt__git_template_name', - // sql: 'tnt__git_template_name', - // type: 'string', - // alias: 'Github Template Name', - // }, - // { - // name: 'tnt__id_field', - // sql: 'tnt__id_field', - // type: 'string', - // alias: 'id field', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__id_list_field', - // sql: 'tnt__id_list_field', - // type: 'string_array', - // alias: 'id list field', - // }, - // { - // name: 'tnt__impact', - // sql: 'tnt__impact', - // type: 'string', - // alias: 'Impact', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__include_on_emails', - // sql: 'tnt__include_on_emails', - // type: 'string_array', - // alias: 'Include on emails', - // }, - // { - // name: 'tnt__issue_score', - // sql: 'tnt__issue_score', - // type: 'number', - // alias: 'Issue Score', - // }, - // { - // name: 'tnt__knowledge_gap', - // sql: 'tnt__knowledge_gap', - // type: 'string', - // alias: 'Knowledge Gap', - // }, - // { - // name: 'tnt__linear_assignee', - // sql: 'tnt__linear_assignee', - // type: 'string', - // alias: 'Linear Assignee', - // }, - // { - // name: 'tnt__linear_description', - // sql: 'tnt__linear_description', - // type: 'string', - // alias: 'Linear Description', - // }, - // { - // name: 'tnt__linear_id', - // sql: 'tnt__linear_id', - // type: 'string', - // alias: 'Linear Id', - // }, - // { - // name: 'tnt__linear_identifier', - // sql: 'tnt__linear_identifier', - // type: 'string', - // alias: 'Linear Identifier', - // }, - // { - // name: 'tnt__linear_priority', - // sql: 'tnt__linear_priority', - // type: 'string', - // alias: 'Linear Priority', - // }, - // { - // name: 'tnt__linear_state', - // sql: 'tnt__linear_state', - // type: 'string', - // alias: 'Linear State', - // }, - // { - // name: 'tnt__linear_team', - // sql: 'tnt__linear_team', - // type: 'string', - // alias: 'Linear Team', - // }, - // { - // name: 'tnt__linear_title', - // sql: 'tnt__linear_title', - // type: 'string', - // alias: 'Linear Title', - // }, - // { - // name: 'tnt__linear_url', - // sql: 'tnt__linear_url', - // type: 'string', - // alias: 'Linear Url', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__members', - // sql: 'tnt__members', - // type: 'string_array', - // alias: 'tnt__members', - // }, - // { - // name: 'tnt__micro_service_name', - // sql: 'tnt__micro_service_name', - // type: 'string', - // alias: 'Microservice Name', - // }, - // { - // name: 'tnt__notify_others', - // sql: 'tnt__notify_others', - // type: 'string', - // alias: 'notify others', - // }, - // { - // name: 'tnt__numeric_field', - // sql: 'tnt__numeric_field', - // type: 'number', - // alias: 'numeric field', - // }, - // { - // name: 'tnt__parent_part', - // sql: 'tnt__parent_part', - // type: 'string', - // alias: 'Parent part', - // }, - // { - // name: 'tnt__part_product', - // sql: 'tnt__part_product', - // type: 'string', - // alias: 'part product', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__paytm_customer_name', - // sql: 'tnt__paytm_customer_name', - // type: 'string_array', - // alias: 'paytm customer name', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__platform', - // sql: 'tnt__platform', - // type: 'string_array', - // alias: 'Platform', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__portal_test', - // sql: 'tnt__portal_test', - // type: 'string_array', - // alias: 'portal test', - // }, - // { - // name: 'tnt__product', - // sql: 'tnt__product', - // type: 'string', - // alias: 'Product', - // }, - // { - // name: 'tnt__project', - // sql: 'tnt__project', - // type: 'string', - // alias: 'Project', - // }, - // { - // name: 'tnt__project_stage', - // sql: 'tnt__project_stage', - // type: 'string', - // alias: 'Project Stage', - // }, - // { - // name: 'tnt__rank', - // sql: 'tnt__rank', - // type: 'number', - // alias: 'Rank', - // }, - // { - // name: 'tnt__remaining_effort', - // sql: 'tnt__remaining_effort', - // type: 'number', - // alias: 'Remaining Effort', - // }, - // { - // name: 'tnt__required_text_field', - // sql: 'tnt__required_text_field', - // type: 'string', - // alias: 'required text field', - // }, - // { - // name: 'tnt__resolution_sla_met', - // sql: 'tnt__resolution_sla_met', - // type: 'string', - // alias: 'Resolution SLA Met', - // }, - // { - // name: 'tnt__resolution_sla_target_time', - // sql: 'tnt__resolution_sla_target_time', - // type: 'string', - // alias: 'Resolution SLA Target Time', - // }, - // { - // name: 'tnt__resolution_turnaround_time', - // sql: 'tnt__resolution_turnaround_time', - // type: 'string', - // alias: 'Resolution SLA Turnaround Time', - // }, - // { - // name: 'tnt__response_sla_met', - // sql: 'tnt__response_sla_met', - // type: 'string', - // alias: 'First Response SLA Met', - // }, - // { - // name: 'tnt__response_sla_target_time', - // sql: 'tnt__response_sla_target_time', - // type: 'string', - // alias: 'First Response SLA Target Time', - // }, - // { - // name: 'tnt__response_turnaround_time', - // sql: 'tnt__response_turnaround_time', - // type: 'string', - // alias: 'First Response SLA Turnaround Time', - // }, - // { - // name: 'tnt__root_cause_analysis', - // sql: 'tnt__root_cause_analysis', - // type: 'string', - // alias: 'Root cause analysis', - // }, - // { - // name: 'tnt__search_field_query', - // sql: 'tnt__search_field_query', - // type: 'string', - // alias: 'Search Field Query', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__stakeholders', - // sql: 'tnt__stakeholders', - // type: 'string_array', - // alias: 'stakeholders', - // }, - // { - // name: 'tnt__stray_user', - // sql: 'tnt__stray_user', - // type: 'string', - // alias: 'stray user', - // }, - // { - // name: 'tnt__stray_users', - // sql: 'tnt__stray_users', - // type: 'string', - // alias: 'stray users', - // }, - // { - // name: 'tnt__test', - // sql: 'tnt__test', - // type: 'number', - // alias: 'Test', - // }, - // { - // name: 'tnt__test123', - // sql: 'tnt__test123', - // type: 'string', - // alias: 'test123', - // }, - // { - // name: 'tnt__test_capability_part', - // sql: 'tnt__test_capability_part', - // type: 'string', - // alias: 'Test Capability Part', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__test_field_for_sla', - // sql: 'tnt__test_field_for_sla', - // type: 'string_array', - // alias: 'test field for sla', - // }, - // { - // name: 'tnt__test_filed', - // sql: 'tnt__test_filed', - // type: 'string', - // alias: 'test field', - // }, - // { - // name: 'tnt__test_mikasa', - // sql: 'tnt__test_mikasa', - // type: 'string', - // alias: 'Test mikasa', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__test_multivalue_testrequiredfield', - // sql: 'tnt__test_multivalue_testrequiredfield', - // type: 'string_array', - // alias: 'test multivalue testrequiredfield', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__test_multivalue_textfield', - // sql: 'tnt__test_multivalue_textfield', - // type: 'string_array', - // alias: 'test multivalue textfield', - // }, - // { - // name: 'tnt__test_product_part', - // sql: 'tnt__test_product_part', - // type: 'string', - // alias: 'Test Product part', - // }, - // { - // name: 'tnt__testing_bool_drpdown', - // sql: 'tnt__testing_bool_drpdown', - // type: 'string', - // alias: 'Testing Bool Drpdown', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'tnt__text_list_account', - // sql: 'tnt__text_list_account', - // type: 'string_array', - // alias: 'Text list account', - // }, - // { - // name: 'tnt__ticket_custom_part', - // sql: 'tnt__ticket_custom_part', - // type: 'string', - // alias: 'ticket custom part', - // }, - // { - // name: 'tnt__ticket_dropd', - // sql: 'tnt__ticket_dropd', - // type: 'string', - // alias: 'ticket dropdown', - // }, - // { - // name: 'tnt__ticket_type', - // sql: 'tnt__ticket_type', - // type: 'string', - // alias: 'Ticket type', - // }, - // { - // name: 'tnt__tier', - // sql: 'tnt__tier', - // type: 'string', - // alias: 'Tier', - // }, - // { - // name: 'tnt__urgency', - // sql: 'tnt__urgency', - // type: 'string', - // alias: 'Urgency', - // }, - // { - // name: 'tnt__velocity_test_enum', - // sql: 'tnt__velocity_test_enum', - // type: 'string', - // alias: 'velocity test enum', - // }, - // { - // name: 'tnt__version', - // sql: 'tnt__version', - // type: 'string', - // alias: 'Version', - // }, - // { - // name: 'tnt__work_duration', - // sql: 'tnt__work_duration', - // type: 'string', - // alias: 'Work duration', - // }, - // { - // name: 'tnt__workspace_custom', - // sql: 'tnt__workspace_custom', - // type: 'string', - // alias: 'workspace_custom', - // }, - // { - // name: 'ctype_deal_registration__buying_region', - // sql: 'ctype_deal_registration__buying_region', - // type: 'string', - // alias: 'Theater', - // }, - // { - // name: 'ctype_deal_registration__deal_status', - // sql: 'ctype_deal_registration__deal_status', - // type: 'string', - // alias: 'Deal Status', - // }, - // { - // name: 'ctype_deal_registration__deal_type', - // sql: 'ctype_deal_registration__deal_type', - // type: 'string', - // alias: 'Deal Type', - // }, - // { - // name: 'ctype_deal_registration__employee_count', - // sql: 'ctype_deal_registration__employee_count', - // type: 'string', - // alias: 'Employee Count', - // }, - // { - // name: 'ctype_deal_registration__estimated_deal_value', - // sql: 'ctype_deal_registration__estimated_deal_value', - // type: 'number', - // alias: 'Estimated Deal Value', - // }, - // { - // name: 'ctype_deal_registration__expected_close_date', - // sql: 'ctype_deal_registration__expected_close_date', - // type: 'time', - // alias: 'Expected Close Date', - // }, - // { - // name: 'ctype_deal_registration__partner_type', - // sql: 'ctype_deal_registration__partner_type', - // type: 'string', - // alias: 'Partner Type', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_deal_registration__products_or_services', - // sql: 'ctype_deal_registration__products_or_services', - // type: 'string_array', - // alias: 'Product(s) or Service(s)', - // }, - // { - // name: 'ctype_deal_registration__prospect_company_name', - // sql: 'ctype_deal_registration__prospect_company_name', - // type: 'string', - // alias: 'Prospect Company Name', - // }, - // { - // name: 'ctype_deal_registration__prospect_company_website', - // sql: 'ctype_deal_registration__prospect_company_website', - // type: 'string', - // alias: 'Prospect Company Website', - // }, - // { - // name: 'ctype_deal_registration__prospect_contact_email', - // sql: 'ctype_deal_registration__prospect_contact_email', - // type: 'string', - // alias: 'Prospect Contact Email', - // }, - // { - // name: 'ctype_deal_registration__prospect_contact_name', - // sql: 'ctype_deal_registration__prospect_contact_name', - // type: 'string', - // alias: 'Prospect Contact Name', - // }, - // { - // name: 'ctype_deal_registration__sub_region', - // sql: 'ctype_deal_registration__sub_region', - // type: 'string', - // alias: 'Region ', - // }, - // { - // name: 'ctype_deal_registration__subregion', - // sql: 'ctype_deal_registration__subregion', - // type: 'string', - // alias: 'Subregion', - // }, - // { - // name: 'ctype_Events__campaign_category', - // sql: 'ctype_Events__campaign_category', - // type: 'string', - // alias: 'Campaign Category', - // }, - // { - // name: 'ctype_Events__event_end_date', - // sql: 'ctype_Events__event_end_date', - // type: 'time', - // alias: 'Event End Date', - // }, - // { - // name: 'ctype_Events__event_owner', - // sql: 'ctype_Events__event_owner', - // type: 'string', - // alias: 'Event Owner', - // }, - // { - // name: 'ctype_Events__event_start_date', - // sql: 'ctype_Events__event_start_date', - // type: 'time', - // alias: 'Event Start Date', - // }, - // { - // name: 'ctype_Events__events_test_ref', - // sql: 'ctype_Events__events_test_ref', - // type: 'string', - // alias: 'events test ref', - // }, - // { - // name: 'ctype_Events__external_budget', - // sql: 'ctype_Events__external_budget', - // type: 'number', - // alias: 'External Budget', - // }, - // { - // name: 'ctype_Events__internal_budget', - // sql: 'ctype_Events__internal_budget', - // type: 'number', - // alias: 'Internal Budget', - // }, - // { - // name: 'ctype_Events__is_devrev_event', - // sql: 'ctype_Events__is_devrev_event', - // type: 'boolean', - // alias: 'Is DevRev Event', - // }, - // { - // name: 'ctype_Events__mode', - // sql: 'ctype_Events__mode', - // type: 'string', - // alias: 'Mode', - // }, - // { - // name: 'ctype_Events__organized_by', - // sql: 'ctype_Events__organized_by', - // type: 'string', - // alias: 'Organized by', - // }, - // { - // name: 'ctype_Events__pipeline_generated', - // sql: 'ctype_Events__pipeline_generated', - // type: 'number', - // alias: 'Pipeline Generated', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_Events__representatives', - // sql: 'ctype_Events__representatives', - // type: 'string_array', - // alias: 'Representatives', - // }, - // { - // name: 'ctype_Events__source', - // sql: 'ctype_Events__source', - // type: 'string', - // alias: 'Source', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_Events__sponsors', - // sql: 'ctype_Events__sponsors', - // type: 'string_array', - // alias: 'Sponsors', - // }, - // { - // name: 'ctype_Events__sub_source', - // sql: 'ctype_Events__sub_source', - // type: 'string', - // alias: 'Sub-Source', - // }, - // { - // name: 'ctype_Events__total_budget', - // sql: 'ctype_Events__total_budget', - // type: 'number', - // alias: 'Total Budget', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', - // type: 'boolean', - // alias: 'allow_attachments', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', - // type: 'boolean', - // alias: 'allow_channelback', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', - // type: 'string_array', - // alias: 'Collaborators', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', - // type: 'string', - // alias: 'External Object Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', - // type: 'string', - // alias: 'external_id', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', - // type: 'string_array', - // alias: 'Fields modified during import', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', - // type: 'boolean', - // alias: 'from_messaging_channel', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', - // type: 'boolean', - // alias: 'has_incidents', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', - // type: 'boolean', - // alias: 'is_public', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', - // type: 'string_array', - // alias: 'Reporters', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', - // type: 'string', - // alias: 'Customer', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', - // type: 'string', - // alias: 'raw_subject', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', - // type: 'string', - // alias: 'recipient', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', - // type: 'string', - // alias: 'External Source', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', - // type: 'string', - // alias: 'url', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', - // type: 'string', - // alias: 'Hold Status', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', - // type: 'string', - // alias: 'ATI Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', - // type: 'string', - // alias: 'Event ID', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', - // type: 'boolean', - // alias: 'Do Not Close', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', - // type: 'boolean', - // alias: 'Escalated', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', - // type: 'string', - // alias: 'Partner: Sales Contact', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', - // type: 'string', - // alias: 'Partner Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', - // type: 'string', - // alias: 'Env Setup Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', - // type: 'string', - // alias: 'Additional Comments', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', - // type: 'string', - // alias: 'Remind me before', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', - // type: 'string', - // alias: ' zIPS + z3A Licenses', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', - // type: 'string', - // alias: 'zDefend Licenses', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', - // type: 'string', - // alias: 'Approval Status', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', - // type: 'string', - // alias: 'Booking Year', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', - // type: 'string', - // alias: 'Term', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', - // type: 'string', - // alias: 'End Customer : Admin Details', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', - // type: 'string_array', - // alias: 'Root Cause', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', - // type: 'string', - // alias: 'Request Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', - // type: 'string', - // alias: 'End User - Full Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', - // type: 'string', - // alias: 'End Customer: Administrator Email Address', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', - // type: 'string', - // alias: 'Services Purchased', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', - // type: 'string', - // alias: 'Support Purchased', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', - // type: 'string', - // alias: 'Cancellation Reason', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', - // type: 'string', - // alias: 'Cancellation Date', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', - // type: 'string', - // alias: 'zConsole URL', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', - // type: 'string', - // alias: 'Customer Success Representative', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', - // type: 'string', - // alias: 'VPC Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', - // type: 'string', - // alias: 'End Customer: Region', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', - // type: 'string', - // alias: 'Proof of Concept Impact ?', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', - // type: 'string', - // alias: 'Internal Status', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', - // type: 'string', - // alias: 'Problem Sub-Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', - // type: 'string', - // alias: 'Tenant Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', - // type: 'string', - // alias: 'Customer Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', - // type: 'string', - // alias: 'Channel', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', - // type: 'string', - // alias: 'Order Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', - // type: 'string', - // alias: 'zIPS Licenses', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', - // type: 'boolean', - // alias: 'z3A License', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', - // type: 'string', - // alias: 'Sub-Partner Name', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', - // type: 'string_array', - // alias: 'Ticket Region', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', - // type: 'string', - // alias: 'zShield Licenses', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', - // type: 'string', - // alias: 'zScan Apps Licensed', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', - // type: 'string', - // alias: 'KPE MTD Licenses', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', - // type: 'string', - // alias: 'Product Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', - // type: 'string', - // alias: 'whiteCryption agent', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', - // type: 'string', - // alias: 'zIPS Test Case Types', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', - // type: 'string', - // alias: 'Product Version', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', - // type: 'string', - // alias: 'Target Delivery Date', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', - // type: 'string', - // alias: 'App Version ', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', - // type: 'string', - // alias: 'End Customer: Company Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', - // type: 'string', - // alias: 'End Customer: Physical Address', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', - // type: 'string', - // alias: 'Term Start Date', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', - // type: 'string', - // alias: 'Term End Date', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', - // type: 'string', - // alias: 'Order Fulfilled', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', - // type: 'string', - // alias: 'Requested Delivery Date', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', - // type: 'string', - // alias: 'Internal Tracking Id', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', - // type: 'string', - // alias: 'Feature Status', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', - // type: 'string', - // alias: 'Case Type', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', - // type: 'string', - // alias: 'Business Impact', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', - // type: 'string', - // alias: 'Zimperium Console URL', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', - // type: 'string', - // alias: 'End Customer: Administrator Name', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', - // type: 'string', - // alias: 'zIPS Only Licenses ', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', - // type: 'string', - // alias: 'Zimperium Sales Contact: Email Address', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', - // type: 'boolean', - // alias: 'allow_attachments', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', - // type: 'boolean', - // alias: 'allow_channelback', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', - // type: 'string_array', - // alias: 'Collaborators', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', - // type: 'string', - // alias: 'External Object Type', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', - // type: 'string_array', - // alias: 'Fields modified during import', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', - // type: 'boolean', - // alias: 'from_messaging_channel', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', - // type: 'boolean', - // alias: 'has_incidents', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', - // type: 'boolean', - // alias: 'is_public', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', - // type: 'string_array', - // alias: 'Reporters', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', - // type: 'string', - // alias: 'Customer', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', - // type: 'string', - // alias: 'raw_subject', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', - // type: 'string', - // alias: 'recipient', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', - // type: 'string', - // alias: 'External Source', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', - // type: 'string', - // alias: 'url', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', - // type: 'string', - // alias: 'Hold Status', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', - // type: 'boolean', - // alias: 'Do Not Close', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', - // type: 'boolean', - // alias: 'Escalated', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', - // type: 'string', - // alias: 'Partner: Sales Contact', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', - // type: 'string', - // alias: 'Partner Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', - // type: 'string', - // alias: 'Env Setup Type', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', - // type: 'string', - // alias: 'Approval Status', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', - // type: 'string', - // alias: 'Booking Year', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', - // type: 'string_array', - // alias: 'Root Cause', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', - // type: 'string', - // alias: 'Services Purchased', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', - // type: 'string', - // alias: 'Support Purchased', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', - // type: 'string', - // alias: 'VPC Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', - // type: 'string', - // alias: 'End Customer: Region', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', - // type: 'string', - // alias: 'Proof of Concept Impact ?', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', - // type: 'string', - // alias: 'Problem Sub-Type', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', - // type: 'string', - // alias: 'Tenant Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', - // type: 'string', - // alias: 'Customer Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', - // type: 'string', - // alias: 'Channel', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', - // type: 'string', - // alias: 'Order Type', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', - // type: 'string', - // alias: 'zIPS Licenses', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', - // type: 'boolean', - // alias: 'z3A License', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', - // type: 'string', - // alias: 'Sub-Partner Name', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', - // type: 'string_array', - // alias: 'Ticket Region', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', - // type: 'string', - // alias: 'Product Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', - // type: 'string', - // alias: 'whiteCryption agent', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', - // type: 'string', - // alias: 'Product Version', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', - // type: 'string', - // alias: 'App Version ', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', - // type: 'string', - // alias: 'End Customer: Company Name', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', - // type: 'string', - // alias: 'End Customer: Physical Address', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', - // type: 'string', - // alias: 'Term Start Date', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', - // type: 'string', - // alias: 'Term End Date', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', - // type: 'string', - // alias: 'Order Fulfilled', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', - // type: 'string', - // alias: 'Internal Tracking Id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', - // type: 'string', - // alias: 'Feature Status', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', - // type: 'string', - // alias: 'Case Type', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', - // type: 'string', - // alias: 'Business Impact', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', - // type: 'string', - // alias: 'DuplicateField Console URL', - // }, - // { - // name: 'ctype_hell__test', - // sql: 'ctype_hell__test', - // type: 'string', - // alias: 'test', - // }, - // { - // name: 'ctype_hell__text_default_field', - // sql: 'ctype_hell__text_default_field', - // type: 'string', - // alias: 'text default field', - // }, - // { - // name: 'ctype_clone_ticket___clone_field_dropdown', - // sql: 'ctype_clone_ticket___clone_field_dropdown', - // type: 'string', - // alias: 'clone field dropdown', - // }, - // { - // name: 'ctype_test_subtype_sep__new_field_2', - // sql: 'ctype_test_subtype_sep__new_field_2', - // type: 'string', - // alias: 'field_harsh', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_test_subtype_sep__new_field_id_list_3', - // sql: 'ctype_test_subtype_sep__new_field_id_list_3', - // type: 'string_array', - // alias: 'field_id_list_harsh', - // }, - // { - // name: 'ctype_vgtestyesno__booltestyesno', - // sql: 'ctype_vgtestyesno__booltestyesno', - // type: 'boolean', - // alias: 'booltestyesno', - // }, - // { - // name: 'ctype_asd__test_asd', - // sql: 'ctype_asd__test_asd', - // type: 'string', - // alias: 'test asd', - // }, - // { - // name: 'ctype_Microservice__git_template_name', - // sql: 'ctype_Microservice__git_template_name', - // type: 'string', - // alias: 'Github Template Name', - // }, - // { - // name: 'ctype_Microservice__micro_service_name', - // sql: 'ctype_Microservice__micro_service_name', - // type: 'string', - // alias: 'Microservice Name', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_Microservice__platform', - // sql: 'ctype_Microservice__platform', - // type: 'string_array', - // alias: 'Platform', - // }, - // { - // name: 'ctype_Microservice__read_only', - // sql: 'ctype_Microservice__read_only', - // type: 'string', - // alias: 'read only', - // }, - // { - // name: 'ctype_helohelohelohelohelo__dfkdkl', - // sql: 'ctype_helohelohelohelohelo__dfkdkl', - // type: 'number', - // alias: 'dfkdkl', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_1122', - // sql: 'ctype_helohelohelohelohelo__hello_1122', - // type: 'string', - // alias: 'hello hello folks', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_12', - // sql: 'ctype_helohelohelohelohelo__hello_12', - // type: 'string', - // alias: 'hello_12', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_123', - // sql: 'ctype_helohelohelohelohelo__hello_123', - // type: 'string', - // alias: 'hello_123', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_12x', - // sql: 'ctype_helohelohelohelohelo__hello_12x', - // type: 'string', - // alias: 'hello_12x', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_12y', - // sql: 'ctype_helohelohelohelohelo__hello_12y', - // type: 'string', - // alias: 'hello_12y', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_wed', - // sql: 'ctype_helohelohelohelohelo__hello_wed', - // type: 'string', - // alias: 'hello_wed', - // }, - // { - // name: 'ctype_helohelohelohelohelo__hello_world', - // sql: 'ctype_helohelohelohelohelo__hello_world', - // type: 'string', - // alias: 'hello_world', - // }, - // { - // name: 'ctype_dddddddd__dkay_march_23', - // sql: 'ctype_dddddddd__dkay_march_23', - // type: 'string', - // alias: 'dkay_march_23', - // }, - // { - // name: 'ctype_dddddddd__dkay_march_26', - // sql: 'ctype_dddddddd__dkay_march_26', - // type: 'string', - // alias: 'dkay_march_26', - // }, - // { - // name: 'ctype_dddddddd__scope_approval', - // sql: 'ctype_dddddddd__scope_approval', - // type: 'string', - // alias: 'Scope Approval', - // }, - // { - // name: 'ctype_dddddddd__uat_approval', - // sql: 'ctype_dddddddd__uat_approval', - // type: 'string', - // alias: 'UAT approval', - // }, - // { - // name: 'ctype_defaults__customer_uat', - // sql: 'ctype_defaults__customer_uat', - // type: 'time', - // alias: 'customer uat', - // }, - // { - // name: 'ctype_defaults__default', - // sql: 'ctype_defaults__default', - // type: 'string', - // alias: 'default', - // }, - // { - // name: 'ctype_defaults__default_enum', - // sql: 'ctype_defaults__default_enum', - // type: 'string', - // alias: 'default enum', - // }, - // { - // name: 'ctype_defaults__enum_testing', - // sql: 'ctype_defaults__enum_testing', - // type: 'string', - // alias: 'enum testing', - // }, - // { - // name: 'ctype_defaults__uat_date', - // sql: 'ctype_defaults__uat_date', - // type: 'time', - // alias: 'uat date', - // }, - // { - // name: 'ctype_bug__abc', - // sql: 'ctype_bug__abc', - // type: 'string', - // alias: 'abc', - // }, - // { - // name: 'ctype_bug__dependent_test', - // sql: 'ctype_bug__dependent_test', - // type: 'string', - // alias: 'something', - // }, - // { - // name: 'ctype_event_request__approver', - // sql: 'ctype_event_request__approver', - // type: 'string', - // alias: 'Approver', - // }, - // { - // name: 'ctype_event_request__budget', - // sql: 'ctype_event_request__budget', - // type: 'number', - // alias: 'Budget', - // }, - // { - // name: 'ctype_event_request__event_date', - // sql: 'ctype_event_request__event_date', - // type: 'time', - // alias: 'Event Date', - // }, - // { - // name: 'ctype_event_request__event_location', - // sql: 'ctype_event_request__event_location', - // type: 'string', - // alias: 'Event Location', - // }, - // { - // name: 'ctype_event_request__event_name', - // sql: 'ctype_event_request__event_name', - // type: 'string', - // alias: 'Event Name', - // }, - // { - // name: 'ctype_event_request__event_owner', - // sql: 'ctype_event_request__event_owner', - // type: 'string', - // alias: 'Event Owner', - // }, - // { - // name: 'ctype_event_request__event_type', - // sql: 'ctype_event_request__event_type', - // type: 'string', - // alias: 'Event Type', - // }, - // { - // name: 'ctype_event_request__link_to_event', - // sql: 'ctype_event_request__link_to_event', - // type: 'string', - // alias: 'Link To Event', - // }, - // { - // name: 'ctype_event_request__region', - // sql: 'ctype_event_request__region', - // type: 'string', - // alias: 'Region', - // }, - // { - // name: 'ctype_event_request__requested_by', - // sql: 'ctype_event_request__requested_by', - // type: 'string', - // alias: 'Requested by', - // }, - // { - // name: 'ctype_event_request__status', - // sql: 'ctype_event_request__status', - // type: 'string', - // alias: 'Status', - // }, - // { - // name: 'ctype_event_request__target_pipeline', - // sql: 'ctype_event_request__target_pipeline', - // type: 'number', - // alias: 'Target Pipeline', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_event_request__target_segment', - // sql: 'ctype_event_request__target_segment', - // type: 'string_array', - // alias: 'Target Segment', - // }, - // { - // name: 'ctype_mfz_subtype_check__enum_field', - // sql: 'ctype_mfz_subtype_check__enum_field', - // type: 'string', - // alias: 'Custom Enum field', - // }, - // { - // name: 'ctype_mfz_subtype_check__text_field', - // sql: 'ctype_mfz_subtype_check__text_field', - // type: 'string', - // alias: 'Custom text field', - // }, - // { - // name: 'ctype_mfz_subtype_check__user_field', - // sql: 'ctype_mfz_subtype_check__user_field', - // type: 'string', - // alias: 'Custom user field', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_mfz_subtype_check__user_list_field', - // sql: 'ctype_mfz_subtype_check__user_list_field', - // type: 'string_array', - // alias: 'Custom user list field', - // }, - // { - // name: 'ctype_mfz_subtype_check__workspace_field', - // sql: 'ctype_mfz_subtype_check__workspace_field', - // type: 'string', - // alias: 'Custom workspace field', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_mfz_subtype_check__workspace_list_field', - // sql: 'ctype_mfz_subtype_check__workspace_list_field', - // type: 'string_array', - // alias: 'Custom workspace list field', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', - // type: 'boolean', - // alias: 'allow_attachments', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', - // type: 'boolean', - // alias: 'allow_channelback', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', - // type: 'string_array', - // alias: 'Collaborators', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', - // type: 'string', - // alias: 'External Object Type', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', - // type: 'string_array', - // alias: 'Fields modified during import', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', - // type: 'boolean', - // alias: 'from_messaging_channel', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', - // type: 'boolean', - // alias: 'has_incidents', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', - // type: 'boolean', - // alias: 'is_public', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', - // type: 'string_array', - // alias: 'Reporters', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', - // type: 'string', - // alias: 'Customer', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', - // type: 'string', - // alias: 'raw_subject', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', - // type: 'string', - // alias: 'recipient', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', - // type: 'string', - // alias: 'External Source', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', - // type: 'string', - // alias: 'url', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', - // type: 'string_array', - // alias: 'Ticket Region', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', - // type: 'string', - // alias: 'whiteCryption agent', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', - // type: 'string', - // alias: 'Feature Status', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', - // type: 'string', - // alias: 'Case Type', - // }, - // { - // name: 'ctype_mandatory_field_subtype__some_text_field', - // sql: 'ctype_mandatory_field_subtype__some_text_field', - // type: 'string', - // alias: 'Some text field', - // }, - // { - // name: 'ctype_review_subtype__date', - // sql: 'ctype_review_subtype__date', - // type: 'time', - // alias: 'Date', - // }, - // { - // name: 'ctype_review_subtype__play_store_review_url', - // sql: 'ctype_review_subtype__play_store_review_url', - // type: 'string', - // alias: 'Review URL', - // }, - // { - // name: 'ctype_review_subtype__rating', - // sql: 'ctype_review_subtype__rating', - // type: 'string', - // alias: 'Rating', - // }, - // { - // name: 'ctype_review_subtype__reply_date', - // sql: 'ctype_review_subtype__reply_date', - // type: 'time', - // alias: 'Reply date', - // }, - // { - // name: 'ctype_review_subtype__reply_text', - // sql: 'ctype_review_subtype__reply_text', - // type: 'string', - // alias: 'Reply text', - // }, - // { - // name: 'ctype_review_subtype__upvote_count', - // sql: 'ctype_review_subtype__upvote_count', - // type: 'number', - // alias: 'Upvote Count', - // }, - // { - // name: 'ctype_jkjkjkjkjkjk__dgg', - // sql: 'ctype_jkjkjkjkjkjk__dgg', - // type: 'string', - // alias: 'dgg', - // }, - // { - // name: 'ctype_jkjkjkjkjkjk__dggd', - // sql: 'ctype_jkjkjkjkjkjk__dggd', - // type: 'number', - // alias: 'dggd', - // }, - // { - // name: 'ctype_jkjkjkjkjkjk__hello', - // sql: 'ctype_jkjkjkjkjkjk__hello', - // type: 'string', - // alias: 'hello', - // }, - // { - // name: 'ctype_jkjkjkjkjkjk__hello_acl', - // sql: 'ctype_jkjkjkjkjkjk__hello_acl', - // type: 'string', - // alias: 'hellO_acl', - // }, - // { - // name: 'ctype_huryyyyyyyyyyy__all_internal_selected', - // sql: 'ctype_huryyyyyyyyyyy__all_internal_selected', - // type: 'string', - // alias: 'all_external_selected', - // }, - // { - // name: 'ctype_huryyyyyyyyyyy__all_selected', - // sql: 'ctype_huryyyyyyyyyyy__all_selected', - // type: 'string', - // alias: 'all_selected', - // }, - // { - // name: 'ctype_huryyyyyyyyyyy__test_acl_2', - // sql: 'ctype_huryyyyyyyyyyy__test_acl_2', - // type: 'string', - // alias: 'test_acl_2', - // }, - // { - // name: 'ctype_aaa__fkjdsfjkdj', - // sql: 'ctype_aaa__fkjdsfjkdj', - // type: 'string', - // alias: 'fkjdsfjkdj', - // }, - // { - // name: 'ctype_aaa__test_acl_march', - // sql: 'ctype_aaa__test_acl_march', - // type: 'string', - // alias: 'test_acl_march', - // }, - // { - // name: 'ctype_lolololololol__hell_acl_2', - // sql: 'ctype_lolololololol__hell_acl_2', - // type: 'string', - // alias: 'hell_acl_2', - // }, - // { - // name: 'ctype_lolololololol__hello_acl', - // sql: 'ctype_lolololololol__hello_acl', - // type: 'string', - // alias: 'hello_acl', - // }, - // { - // name: 'ctype_lolololololol__hello_acl_3', - // sql: 'ctype_lolololololol__hello_acl_3', - // type: 'string', - // alias: 'hello_acl_3', - // }, - // { - // name: 'ctype_zzz__dds', - // sql: 'ctype_zzz__dds', - // type: 'string', - // alias: 'DDS', - // }, - // { - // name: 'ctype_zzz__hello', - // sql: 'ctype_zzz__hello', - // type: 'string', - // alias: 'hello ', - // }, - // { - // name: 'ctype_zzz__hello_2', - // sql: 'ctype_zzz__hello_2', - // type: 'string', - // alias: 'hello_2', - // }, - // { - // name: 'ctype_zzz__hello_3', - // sql: 'ctype_zzz__hello_3', - // type: 'string', - // alias: 'hello_3', - // }, - // { - // name: 'ctype_zzz__hello_4', - // sql: 'ctype_zzz__hello_4', - // type: 'string', - // alias: 'hello_4', - // }, - // { - // name: 'ctype_zzz__sdkjdsfjk', - // sql: 'ctype_zzz__sdkjdsfjk', - // type: 'string', - // alias: 'SDKJDSFJK', - // }, - // { - // name: 'ctype_ggg__dkay_march_1234', - // sql: 'ctype_ggg__dkay_march_1234', - // type: 'string', - // alias: 'dkay_march_1234', - // }, - // { - // name: 'ctype_ggg__dkay_march_2', - // sql: 'ctype_ggg__dkay_march_2', - // type: 'string', - // alias: 'dkay_march_2', - // }, - // { - // name: 'ctype_ggg__dkay_test', - // sql: 'ctype_ggg__dkay_test', - // type: 'string', - // alias: 'dkay_test', - // }, - // { - // name: 'ctype_ooooo__dkay_march_3', - // sql: 'ctype_ooooo__dkay_march_3', - // type: 'string', - // alias: 'dkay_march_3', - // }, - // { - // name: 'ctype_ooooo__dkay_march_4', - // sql: 'ctype_ooooo__dkay_march_4', - // type: 'string', - // alias: 'dkay_march_4', - // }, - // { - // name: 'ctype_ooooo__dkay_march_5', - // sql: 'ctype_ooooo__dkay_march_5', - // type: 'string', - // alias: 'dkay_march_5', - // }, - // { - // name: 'ctype_ooooo__dkay_march_7', - // sql: 'ctype_ooooo__dkay_march_7', - // type: 'string', - // alias: 'dkay_march_7', - // }, - // { - // name: 'ctype_sync_engine_test__field1', - // sql: 'ctype_sync_engine_test__field1', - // type: 'string', - // alias: 'field1', - // }, - // { - // name: 'ctype_sync_engine_test__field2_bool', - // sql: 'ctype_sync_engine_test__field2_bool', - // type: 'boolean', - // alias: 'field2_bool', - // }, - // { - // name: 'ctype_priviliges_testing__read_only_field', - // sql: 'ctype_priviliges_testing__read_only_field', - // type: 'string', - // alias: 'read only field', - // }, - // { - // name: 'ctype_priviliges_testing__read_write_field', - // sql: 'ctype_priviliges_testing__read_write_field', - // type: 'string', - // alias: 'read/write field', - // }, - // { - // name: 'ctype_dashboard_test__pulkit', - // sql: 'ctype_dashboard_test__pulkit', - // type: 'boolean', - // alias: 'pulkit', - // }, - // { - // name: 'ctype_tttt__dropdown', - // sql: 'ctype_tttt__dropdown', - // type: 'string', - // alias: 'dropdown', - // }, - // ], - // joins: [ - // { - // sql: 'dim_ticket.sla_tracker_id = dim_sla_tracker.id', - // }, - // { - // sql: 'dim_ticket.id = dim_survey_response.object', - // }, - // { - // sql: 'dim_ticket.id = dim_link_issue_target.source_id', - // }, - // { - // sql: 'dim_ticket.id = dim_link_conversation_source.target_id', - // }, - // { - // sql: 'dim_ticket.rev_oid = dim_revo.id', - // }, - // { - // sql: 'dim_ticket.applies_to_part_id = dim_part.id', - // }, - // ], - // measures: [ - // { - // name: 'id_count', - // function: { - // type: 'count', - // }, - // sql: 'count(dim_ticket.id)', - // type: 'string', - // alias: 'Ticket Id', - // }, - // { - // name: 'created_date_max', - // function: { - // type: 'max', - // }, - // sql: 'max(dim_ticket.created_date)', - // type: 'time', - // alias: 'Created date', - // }, - // { - // name: 'actual_close_date_max', - // function: { - // type: 'max', - // }, - // sql: 'max(dim_ticket.actual_close_date)', - // type: 'time', - // alias: 'Closed date', - // }, - // { - // name: 'sla_tracker_id_count', - // function: { - // type: 'count', - // }, - // sql: 'count(dim_ticket.sla_tracker_id)', - // type: 'string', - // alias: 'Sla Tracker Id', - // }, - // { - // name: 'resolution_time', - // sql: "case WHEN actual_close_date > created_date THEN date_diff('minutes', created_date, actual_close_date) ELSE null END ", - // type: 'number', - // alias: 'Resolution Time', - // }, - // { - // name: 'surveys_aggregation_json_measure', - // function: { - // type: 'median', - // }, - // sql: "median(list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min'))", - // type: 'number', - // alias: 'CSAT Rating', - // }, - // { - // name: 'tnt__account_id', - // sql: 'tnt__account_id', - // type: 'string', - // alias: 'account id', - // }, - // { - // name: 'tnt__actual_effort_spent', - // sql: 'tnt__actual_effort_spent', - // type: 'number', - // alias: 'Actual Effort Spent', - // }, - // { - // name: 'tnt__capability_part', - // sql: 'tnt__capability_part', - // type: 'string', - // alias: 'Capability part', - // }, - // { - // name: 'tnt__custom_id_field', - // sql: 'tnt__custom_id_field', - // type: 'string', - // alias: 'custom id field', - // }, - // { - // name: 'tnt__date_field', - // sql: 'tnt__date_field', - // type: 'time', - // alias: 'date_field', - // }, - // { - // name: 'tnt__estimated_effort', - // sql: 'tnt__estimated_effort', - // type: 'number', - // alias: 'Estimated Effort', - // }, - // { - // name: 'tnt__fruit', - // sql: 'tnt__fruit', - // type: 'string', - // alias: 'Fruit', - // }, - // { - // name: 'tnt__id_field', - // sql: 'tnt__id_field', - // type: 'string', - // alias: 'id field', - // }, - // { - // name: 'tnt__issue_score', - // sql: 'tnt__issue_score', - // type: 'number', - // alias: 'Issue Score', - // }, - // { - // name: 'tnt__numeric_field', - // sql: 'tnt__numeric_field', - // type: 'number', - // alias: 'numeric field', - // }, - // { - // name: 'tnt__parent_part', - // sql: 'tnt__parent_part', - // type: 'string', - // alias: 'Parent part', - // }, - // { - // name: 'tnt__part_product', - // sql: 'tnt__part_product', - // type: 'string', - // alias: 'part product', - // }, - // { - // name: 'tnt__rank', - // sql: 'tnt__rank', - // type: 'number', - // alias: 'Rank', - // }, - // { - // name: 'tnt__remaining_effort', - // sql: 'tnt__remaining_effort', - // type: 'number', - // alias: 'Remaining Effort', - // }, - // { - // name: 'tnt__stray_user', - // sql: 'tnt__stray_user', - // type: 'string', - // alias: 'stray user', - // }, - // { - // name: 'tnt__stray_users', - // sql: 'tnt__stray_users', - // type: 'string', - // alias: 'stray users', - // }, - // { - // name: 'tnt__test', - // sql: 'tnt__test', - // type: 'number', - // alias: 'Test', - // }, - // { - // name: 'tnt__test_capability_part', - // sql: 'tnt__test_capability_part', - // type: 'string', - // alias: 'Test Capability Part', - // }, - // { - // name: 'tnt__test_product_part', - // sql: 'tnt__test_product_part', - // type: 'string', - // alias: 'Test Product part', - // }, - // { - // name: 'tnt__ticket_custom_part', - // sql: 'tnt__ticket_custom_part', - // type: 'string', - // alias: 'ticket custom part', - // }, - // { - // name: 'tnt__workspace_custom', - // sql: 'tnt__workspace_custom', - // type: 'string', - // alias: 'workspace_custom', - // }, - // { - // name: 'ctype_deal_registration__estimated_deal_value', - // sql: 'ctype_deal_registration__estimated_deal_value', - // type: 'number', - // alias: 'Estimated Deal Value', - // }, - // { - // name: 'ctype_deal_registration__expected_close_date', - // sql: 'ctype_deal_registration__expected_close_date', - // type: 'time', - // alias: 'Expected Close Date', - // }, - // { - // name: 'ctype_Events__event_end_date', - // sql: 'ctype_Events__event_end_date', - // type: 'time', - // alias: 'Event End Date', - // }, - // { - // name: 'ctype_Events__event_owner', - // sql: 'ctype_Events__event_owner', - // type: 'string', - // alias: 'Event Owner', - // }, - // { - // name: 'ctype_Events__event_start_date', - // sql: 'ctype_Events__event_start_date', - // type: 'time', - // alias: 'Event Start Date', - // }, - // { - // name: 'ctype_Events__events_test_ref', - // sql: 'ctype_Events__events_test_ref', - // type: 'string', - // alias: 'events test ref', - // }, - // { - // name: 'ctype_Events__external_budget', - // sql: 'ctype_Events__external_budget', - // type: 'number', - // alias: 'External Budget', - // }, - // { - // name: 'ctype_Events__internal_budget', - // sql: 'ctype_Events__internal_budget', - // type: 'number', - // alias: 'Internal Budget', - // }, - // { - // name: 'ctype_Events__pipeline_generated', - // sql: 'ctype_Events__pipeline_generated', - // type: 'number', - // alias: 'Pipeline Generated', - // }, - // { - // name: 'ctype_Events__total_budget', - // sql: 'ctype_Events__total_budget', - // type: 'number', - // alias: 'Total Budget', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - // sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_helohelohelohelohelo__dfkdkl', - // sql: 'ctype_helohelohelohelohelo__dfkdkl', - // type: 'number', - // alias: 'dfkdkl', - // }, - // { - // name: 'ctype_defaults__customer_uat', - // sql: 'ctype_defaults__customer_uat', - // type: 'time', - // alias: 'customer uat', - // }, - // { - // name: 'ctype_defaults__uat_date', - // sql: 'ctype_defaults__uat_date', - // type: 'time', - // alias: 'uat date', - // }, - // { - // name: 'ctype_event_request__approver', - // sql: 'ctype_event_request__approver', - // type: 'string', - // alias: 'Approver', - // }, - // { - // name: 'ctype_event_request__budget', - // sql: 'ctype_event_request__budget', - // type: 'number', - // alias: 'Budget', - // }, - // { - // name: 'ctype_event_request__event_date', - // sql: 'ctype_event_request__event_date', - // type: 'time', - // alias: 'Event Date', - // }, - // { - // name: 'ctype_event_request__event_owner', - // sql: 'ctype_event_request__event_owner', - // type: 'string', - // alias: 'Event Owner', - // }, - // { - // name: 'ctype_event_request__requested_by', - // sql: 'ctype_event_request__requested_by', - // type: 'string', - // alias: 'Requested by', - // }, - // { - // name: 'ctype_event_request__target_pipeline', - // sql: 'ctype_event_request__target_pipeline', - // type: 'number', - // alias: 'Target Pipeline', - // }, - // { - // name: 'ctype_mfz_subtype_check__user_field', - // sql: 'ctype_mfz_subtype_check__user_field', - // type: 'string', - // alias: 'Custom user field', - // }, - // { - // name: 'ctype_mfz_subtype_check__workspace_field', - // sql: 'ctype_mfz_subtype_check__workspace_field', - // type: 'string', - // alias: 'Custom workspace field', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - // type: 'number', - // alias: 'brand_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - // type: 'time', - // alias: 'created_at', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - // type: 'time', - // alias: 'generated_timestamp', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - // type: 'number', - // alias: 'group_id', - // }, - // { - // name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - // sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - // type: 'number', - // alias: 'ticket_form_id', - // }, - // { - // name: 'ctype_review_subtype__date', - // sql: 'ctype_review_subtype__date', - // type: 'time', - // alias: 'Date', - // }, - // { - // name: 'ctype_review_subtype__reply_date', - // sql: 'ctype_review_subtype__reply_date', - // type: 'time', - // alias: 'Reply date', - // }, - // { - // name: 'ctype_review_subtype__upvote_count', - // sql: 'ctype_review_subtype__upvote_count', - // type: 'number', - // alias: 'Upvote Count', - // }, - // { - // name: 'ctype_jkjkjkjkjkjk__dggd', - // sql: 'ctype_jkjkjkjkjkjk__dggd', - // type: 'number', - // alias: 'dggd', - // }, - // { - // name: 'id_count___function__count', - // function: { - // type: 'count', - // }, - // sql: 'count(dim_ticket.id)', - // type: 'string', - // alias: 'Ticket Id', - // }, - // ], - // name: 'dim_ticket', - // sql: 'select * from devrev.dim_ticket', - // }, - // ] as any; - // const resolutionConfig = { - // columnConfigs: [ - // { - // joinColumn: 'id', - // name: 'dim_ticket.tags_json', - // isArrayType: true, - // resolutionColumns: ['name'], - // source: 'dim_tag', - // }, - // ], - // tableSchemas: [ - // { - // dimensions: [ - // { - // name: 'created_by_id', - // sql: 'created_by_id', - // type: 'string', - // alias: 'Created by', - // }, - // { - // name: 'dev_oid', - // sql: 'dev_oid', - // type: 'string', - // alias: 'Dev organization ID', - // }, - // { - // name: 'modified_date', - // sql: 'modified_date', - // type: 'time', - // alias: 'Modified date', - // }, - // { - // name: 'object_type', - // sql: 'object_type', - // type: 'string', - // alias: 'Object type', - // }, - // { - // name: 'tag_type', - // sql: 'tag_type', - // type: 'number', - // alias: 'Values enabled', - // }, - // { - // modifier: { - // shouldUnnestGroupBy: false, - // }, - // name: 'allowed_values', - // sql: 'allowed_values', - // type: 'string_array', - // alias: 'Allowed values', - // }, - // { - // name: 'is_deleted', - // sql: 'is_deleted', - // type: 'boolean', - // alias: 'Is deleted', - // }, - // { - // name: 'modified_by_id', - // sql: 'modified_by_id', - // type: 'string', - // alias: 'Modified by', - // }, - // { - // name: 'access_level', - // sql: 'access_level', - // type: 'number', - // alias: 'Access level', - // }, - // { - // name: 'display_id', - // sql: 'display_id', - // type: 'string', - // alias: 'Display ID', - // }, - // { - // name: 'id', - // sql: 'id', - // type: 'string', - // alias: 'ID', - // }, - // { - // name: 'name', - // sql: 'name', - // type: 'string', - // alias: 'Tag name', - // }, - // { - // name: 'object_version', - // sql: 'object_version', - // type: 'number', - // alias: 'Object Version', - // }, - // { - // name: 'created_date', - // sql: 'created_date', - // type: 'time', - // alias: 'Created date', - // }, - // ], - // measures: [], - // name: 'dim_tag', - // sql: 'SELECT * FROM devrev.dim_tag', - // }, - // ], - // } as any; - // const columnProjections = [ - // 'dim_ticket.tags_json', - // 'dim_ticket.id_count___function__count', - // ]; - - // const resultQuery = await cubeQueryToSQLWithResolution({ - // query, - // tableSchemas, - // resolutionConfig, - // columnProjections, - // }); - - // const expectedQuery = `select * exclude(__row_id) from (SELECT MAX(__resolved_query."Ticket Id") AS "Ticket Id" , ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name" , "__row_id" FROM (SELECT __resolved_query."__row_id" AS "__row_id", * FROM (SELECT "Tags - Tag name", "Ticket Id", "__row_id" FROM (SELECT __unnested_base_query."Ticket Id" AS "Ticket Id", __unnested_base_query."__row_id" AS "__row_id", * FROM (SELECT "Ticket Id", "Tags", "__row_id" FROM (SELECT __base_query."Ticket Id" AS "Ticket Id", unnest(__base_query."Tags") AS "Tags", row_number() OVER () AS "__row_id", * FROM (SELECT count("ID") AS "Ticket Id" , "Tags" FROM (SELECT dim_ticket.created_date AS "Created date", 'ticket' AS "Work Type", json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[]) AS "Tags", dim_ticket.id AS "ID", * FROM (select * from devrev.dim_ticket) AS dim_ticket) AS dim_ticket WHERE (((("Created date" >= '2025-08-02T09:30:00.000Z') AND ("Created date" <= '2025-10-31T10:29:59.999Z')) AND ("Work Type" IN ('ticket')) AND (Stage IN ('don:core:dvrv-us-1:devo/787:custom_stage/514', 'don:core:dvrv-us-1:devo/787:custom_stage/512', 'don:core:dvrv-us-1:devo/787:custom_stage/501', 'don:core:dvrv-us-1:devo/787:custom_stage/485', 'don:core:dvrv-us-1:devo/787:custom_stage/440', 'don:core:dvrv-us-1:devo/787:custom_stage/25', 'don:core:dvrv-us-1:devo/787:custom_stage/24', 'don:core:dvrv-us-1:devo/787:custom_stage/22', 'don:core:dvrv-us-1:devo/787:custom_stage/21', 'don:core:dvrv-us-1:devo/787:custom_stage/19', 'don:core:dvrv-us-1:devo/787:custom_stage/13', 'don:core:dvrv-us-1:devo/787:custom_stage/10', 'don:core:dvrv-us-1:devo/787:custom_stage/8', 'don:core:dvrv-us-1:devo/787:custom_stage/7', 'don:core:dvrv-us-1:devo/787:custom_stage/6', 'don:core:dvrv-us-1:devo/787:custom_stage/4')))) GROUP BY Tags LIMIT 50000) AS __base_query) AS __base_query) AS __unnested_base_query LEFT JOIN (SELECT __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", * FROM (SELECT * FROM devrev.dim_tag) AS __unnested_base_query__dim_ticket__tags_json) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags"=__unnested_base_query__dim_ticket__tags_json.id) AS MEERKAT_GENERATED_TABLE) AS __resolved_query) AS __resolved_query GROUP BY __row_id)`; - // expect(resultQuery).toBe(expectedQuery); - // }); // it('Should handle only scalar field resolution without unnesting', async () => { // const query: Query = { // measures: ['tickets.count'], diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 581b162c..25fc911c 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -4,7 +4,9 @@ import { createBaseTableSchema, Dimension, generateResolutionJoinPaths, + generateResolutionJoinPathsFromBaseTable, generateResolutionSchemas, + generateResolutionSchemasFromBaseTable, generateResolvedDimensions, getNamespacedKey, memberKeyToSafeKey, @@ -245,11 +247,11 @@ export const getUnnestBaseSql = async ({ // const projectedColumns = [...dimensions, 'row_id', ...measures]; const unnestedBaseTableSchema: TableSchema = { - name: '__unnested_base_query', + name: '__base_query', sql: unnestedSql, dimensions: schemaWithRowId.dimensions.map((d) => ({ name: d.name, - sql: `__unnested_base_query."${d.alias || d.name}"`, + sql: `__base_query."${d.alias || d.name}"`, type: d.type, alias: d.alias, })), @@ -273,21 +275,24 @@ export const getUnnestBaseSql = async ({ const rightJoin = join.sql.split('=')[1]; const rightJoinNamespace = rightJoin.split('.')[0].trim(); const rightJoinField = rightJoin.split('.')[1].trim(); - join.sql = join.sql.replace( - leftJoin, - `${unnestedBaseTableSchema.name}.${ - dimensionToJoinOn[0].alias || dimensionToJoinOn[0].name - }` - ); - // TODO: Confirm if name also needs "" like this. - join.sql = `${unnestedBaseTableSchema.name}.${ - dimensionToJoinOn[0].alias - ? `"${dimensionToJoinOn[0].alias}"` - : dimensionToJoinOn[0].name - }=${memberKeyToSafeKey( - getNamespacedKey(unnestedBaseTableSchema.name, rightJoinNamespace) - )}.${rightJoinField}`; + // join.sql = join.sql.replace( + // leftJoin, + // `${unnestedBaseTableSchema.name}.${ + // dimensionToJoinOn[0].alias || dimensionToJoinOn[0].name + // }` + // ); + // // TODO: Confirm if name also needs "" like this. + // join.sql = `${unnestedBaseTableSchema.name}.${ + // dimensionToJoinOn[0].alias + // ? `"${dimensionToJoinOn[0].alias}"` + // : dimensionToJoinOn[0].name + // }=${memberKeyToSafeKey( + // getNamespacedKey(unnestedBaseTableSchema.name, rightJoinNamespace) + // )}.${rightJoinField}`; } + resolutionConfig.columnConfigs.forEach((config) => { + config.name = memberKeyToSafeKey(config.name); + }); return { sql: unnestedSql, baseTableSchema: unnestedBaseTableSchema, @@ -328,23 +333,24 @@ export const getResolvedSql = async ({ // Update the SQL to point to the unnested SQL const updatedBaseTableSchema: TableSchema = baseTableSchema; - for (const columnConfig of resolutionConfig.columnConfigs) { - columnConfig.name = getNamespacedKey( - updatedBaseTableSchema.name, - memberKeyToSafeKey(columnConfig.name) - ); - // columnConfig.name = memberKeyToSafeKey(columnConfig.name); - } + // for (const columnConfig of resolutionConfig.columnConfigs) { + // columnConfig.name = getNamespacedKey( + // updatedBaseTableSchema.name, + // memberKeyToSafeKey(columnConfig.name) + // ); + // // columnConfig.name = memberKeyToSafeKey(columnConfig.name); + // } // Generate resolution schemas for array fields - const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ - updatedBaseTableSchema, - ]); + const resolutionSchemas = generateResolutionSchemasFromBaseTable( + resolutionConfig, + updatedBaseTableSchema + ); // Generate join paths using existing helper - const joinPaths = generateResolutionJoinPaths( + const joinPaths = generateResolutionJoinPathsFromBaseTable( updatedBaseTableSchema.name, resolutionConfig, - [updatedBaseTableSchema] + updatedBaseTableSchema ); const tempQuery: Query = { @@ -355,7 +361,7 @@ export const getResolvedSql = async ({ }; const updatedColumnProjections = columnProjections?.map((cp) => - getNamespacedKey(updatedBaseTableSchema.name, memberKeyToSafeKey(cp)) + memberKeyToSafeKey(cp) ); // Generate resolved dimensions using columnProjections const resolvedDimensions = generateResolvedDimensions( @@ -377,15 +383,13 @@ export const getResolvedSql = async ({ contextParams, }); - // Build list of dimension names that should be in output // Use the baseTableSchema which already has all the column info const baseDimensionNames = new Set( baseTableSchema.dimensions .filter((dim) => { // Exclude columns that need resolution (they'll be replaced by resolved columns) return !resolutionConfig.columnConfigs.some( - (ac) => - ac.name === getNamespacedKey(updatedBaseTableSchema.name, dim.name) + (ac) => ac.name === dim.name ); }) .map((dim) => dim.name) From 4f063ff076ec460092a11944f4812b80fce15a9f Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 3 Nov 2025 18:00:48 +0530 Subject: [PATCH 13/38] working again --- .../src/member-formatters/constants.ts | 1 + meerkat-core/src/resolution/resolution.ts | 47 +++ .../cube-to-sql-with-resolution.ts | 319 +++++------------- 3 files changed, 139 insertions(+), 228 deletions(-) diff --git a/meerkat-core/src/member-formatters/constants.ts b/meerkat-core/src/member-formatters/constants.ts index 60413962..108e656e 100644 --- a/meerkat-core/src/member-formatters/constants.ts +++ b/meerkat-core/src/member-formatters/constants.ts @@ -1,2 +1,3 @@ export const COLUMN_NAME_DELIMITER = '.'; export const MEERKAT_OUTPUT_DELIMITER = '__'; +export const ROW_ID_DIMENSION_NAME = 'row_id'; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index ba240ca6..02c11b5d 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -91,6 +91,53 @@ export const createBaseTableSchema = ( }; }; +export const createWrapperTableSchema = ( + sql: string, + baseTableSchema: TableSchema +) => { + return { + name: BASE_DATA_SOURCE_NAME, + sql: sql, + dimensions: baseTableSchema.dimensions.map((d) => ({ + name: d.name, + sql: `${BASE_DATA_SOURCE_NAME}."${d.alias || d.name}"`, + type: d.type, + alias: d.alias, + })), + measures: baseTableSchema.measures.map((m) => ({ + name: m.name, + sql: `${BASE_DATA_SOURCE_NAME}."${m.alias || m.name}"`, + type: m.type, + alias: m.alias, + })), + joins: baseTableSchema.joins, + }; +}; + +export const updateArrayFlattenModifierUsingResolutionConfig = ( + baseTableSchema: TableSchema, + resolutionConfig: ResolutionConfig +) => { + const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); + for (const dimension of baseTableSchema.dimensions) { + if ( + arrayColumns.some( + (ac: ResolutionColumnConfig) => ac.name === dimension.name + ) + ) { + dimension.modifier = { shouldFlattenArray: true }; + } + } +}; + +export const getArrayTypeResolutionColumnConfigs = ( + resolutionConfig: ResolutionConfig +) => { + return resolutionConfig.columnConfigs.filter( + (config) => config.isArrayType === true + ); +}; + export const generateResolutionSchemas = ( config: ResolutionConfig, baseTableSchemas: TableSchema[] diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 25fc911c..25e3ef4b 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -2,18 +2,23 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, + createWrapperTableSchema, Dimension, generateResolutionJoinPaths, generateResolutionJoinPathsFromBaseTable, generateResolutionSchemas, generateResolutionSchemasFromBaseTable, generateResolvedDimensions, + getArrayTypeResolutionColumnConfigs, getNamespacedKey, + Measure, memberKeyToSafeKey, Query, ResolutionConfig, TableSchema, + updateArrayFlattenModifierUsingResolutionConfig, } from '@devrev/meerkat-core'; +import { ROW_ID_DIMENSION_NAME } from 'meerkat-core/src/member-formatters/constants'; import { cubeQueryToSQL, CubeQueryToSQLParams, @@ -96,15 +101,6 @@ export const cubeQueryToSQLWithResolution = async ({ return sql; }; -/** - * Helper function to get array-type columns from resolution config - */ -const getArrayTypeColumns = (resolutionConfig: ResolutionConfig) => { - return resolutionConfig.columnConfigs.filter( - (config) => config.isArrayType === true - ); -}; - export interface CubeQueryToSQLWithResolutionWithArrayParams { baseSql: string; measures: string[]; @@ -124,22 +120,38 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ columnProjections, contextParams, }: CubeQueryToSQLWithResolutionWithArrayParams) => { - // Phase 1: Generate SQL with row_id and unnested arrays - const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ + // Step 1: Create schema for the base SQL + const baseSchema: TableSchema = createBaseTableSchema( baseSql, - measures, - dimensions, tableSchemas, resolutionConfig, + measures, + dimensions + ); + + baseSchema.dimensions.push({ + name: 'row_id', + sql: 'row_number() OVER ()', + type: 'number', + alias: '__row_id', + } as Dimension); + columnProjections?.push(ROW_ID_DIMENSION_NAME); + + // Doing this because we need to use the original name of the column in the base table schema. + resolutionConfig.columnConfigs.forEach((config) => { + config.name = memberKeyToSafeKey(config.name); + }); + + // Phase 1: Generate SQL with row_id and unnested arrays + const unnestTableSchema = await getUnnestTableSchema({ + baseTableSchema: baseSchema, + resolutionConfig, contextParams, }); - columnProjections?.push(`${tableSchemas[0].name}.row_id`); // Phase 2: Apply resolution (join with lookup tables) - const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ - unnestedSql: unnestBaseSql, - baseTableSchema, - tableSchemas, + const resolvedTableSchema = await getResolvedSql({ + baseTableSchema: unnestTableSchema, resolutionConfig, contextParams, columnProjections, @@ -147,7 +159,6 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ // Phase 3: Re-aggregate to reverse the unnest const aggregatedSql = await getAggregatedSql({ - resolvedSql, resolvedTableSchema, resolutionConfig, contextParams, @@ -157,146 +168,46 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ }; /** - * Phase 1: Add row_id and apply unnesting + * Phase 1: Apply unnesting * - * This function performs 3 steps: - * 1. Wrap base SQL and add row_id dimension - * 2. Create schema with unnest modifiers for array columns - * 3. Generate final unnested SQL - * - * @param baseSql - Base SQL generated from the original query (no modifications) - * @param measures - Original measures from the query - * @param dimensions - Original dimensions from the query - * @returns SQL with row_id and unnested arrays, the schema, and list of projected columns + * This function performs 1 step: + * 1. Create schema with unnest modifiers for array columns + * 2. Generate final unnested SQL + * @returns Table schema with unnest modifiers for array columns */ -export const getUnnestBaseSql = async ({ - baseSql, - measures, - dimensions, - tableSchemas, +export const getUnnestTableSchema = async ({ + baseTableSchema, resolutionConfig, contextParams, }: { - baseSql: string; - measures: string[]; - dimensions: string[]; - tableSchemas: TableSchema[]; + baseTableSchema: TableSchema; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; -}): Promise<{ - sql: string; - baseTableSchema: TableSchema; -}> => { - // Step 1: Create schema for the base SQL - const baseTableSchema: TableSchema = createBaseTableSchema( - baseSql, - tableSchemas, - resolutionConfig, - measures, - dimensions +}): Promise => { + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig ); - const arrayColumns = getArrayTypeColumns(resolutionConfig); - for (const dimension of baseTableSchema.dimensions) { - const arrayResolvedFields = arrayColumns.map((ac) => - memberKeyToSafeKey(ac.name) - ); - if ( - arrayResolvedFields.some( - (resolvedField) => resolvedField === dimension.name - ) - ) { - dimension.modifier = { shouldFlattenArray: true }; - } - } - debugger; - - // Step 2: Add row_id dimension to schema and generate SQL with row_id - const schemaWithRowId: TableSchema = { - ...baseTableSchema, - dimensions: [ - ...baseTableSchema.dimensions, - { - name: memberKeyToSafeKey(`${tableSchemas[0].name}.row_id`), - sql: 'row_number() OVER ()', - type: 'number', - alias: '__row_id', - } as Dimension, - ], - joins: baseTableSchema.joins, - }; - - // Query that projects all original columns plus row_id - const queryWithRowId: Query = { - measures: [], - dimensions: [ - ...schemaWithRowId.dimensions.map((d) => - getNamespacedKey(schemaWithRowId.name, d.name) - ), - ], - }; - const unnestedSql = await cubeQueryToSQL({ - query: queryWithRowId, - tableSchemas: [schemaWithRowId], + query: { + measures: [], + dimensions: [ + ...baseTableSchema.dimensions.map((d) => + getNamespacedKey(baseTableSchema.name, d.name) + ), + ], + }, + tableSchemas: [baseTableSchema], contextParams, }); - // Build list of projected columns (in original format, not namespaced) - // This is what was actually projected in the unnested SQL - // const projectedColumns = [...dimensions, 'row_id', ...measures]; - - const unnestedBaseTableSchema: TableSchema = { - name: '__base_query', - sql: unnestedSql, - dimensions: schemaWithRowId.dimensions.map((d) => ({ - name: d.name, - sql: `__base_query."${d.alias || d.name}"`, - type: d.type, - alias: d.alias, - })), - measures: [], - joins: schemaWithRowId.joins, - }; - for (const join of unnestedBaseTableSchema.joins || []) { - const leftJoin = join.sql.split('=')[0]; - const namespace = leftJoin.split('.')[0]; - const name = leftJoin.split('.')[1]; - const toFind = `${namespace}.${name}`.trim(); - const dimensionToJoinOn = schemaWithRowId.dimensions.filter( - (d) => d.sql.trim() === toFind - ); - if (dimensionToJoinOn.length === 0) { - throw new Error(`Dimension not found: ${namespace}.${name}`); - } - if (dimensionToJoinOn.length > 1) { - throw new Error(`Multiple dimensions found: ${namespace}.${name}`); - } - const rightJoin = join.sql.split('=')[1]; - const rightJoinNamespace = rightJoin.split('.')[0].trim(); - const rightJoinField = rightJoin.split('.')[1].trim(); - // join.sql = join.sql.replace( - // leftJoin, - // `${unnestedBaseTableSchema.name}.${ - // dimensionToJoinOn[0].alias || dimensionToJoinOn[0].name - // }` - // ); - // // TODO: Confirm if name also needs "" like this. - // join.sql = `${unnestedBaseTableSchema.name}.${ - // dimensionToJoinOn[0].alias - // ? `"${dimensionToJoinOn[0].alias}"` - // : dimensionToJoinOn[0].name - // }=${memberKeyToSafeKey( - // getNamespacedKey(unnestedBaseTableSchema.name, rightJoinNamespace) - // )}.${rightJoinField}`; - } - resolutionConfig.columnConfigs.forEach((config) => { - config.name = memberKeyToSafeKey(config.name); - }); - return { - sql: unnestedSql, - baseTableSchema: unnestedBaseTableSchema, - }; + const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( + unnestedSql, + baseTableSchema + ); + + return unnestedBaseTableSchema; }; /** @@ -306,47 +217,27 @@ export const getUnnestBaseSql = async ({ * 1. Uses the base table schema from Phase 1 (source of truth) * 2. Generates resolution schemas for array fields * 3. Sets up join paths between unnested data and resolution tables - * 4. Generates SQL with resolved values - * - * @param unnestedSql - SQL output from Phase 1 (with row_id and unnested arrays) - * @param baseTableSchema - Schema from Phase 1 that describes the unnested SQL - * @returns SQL with row_id, unnested arrays, and resolved values from lookup tables + * @returns Table schema with resolved values from lookup tables */ export const getResolvedSql = async ({ - unnestedSql, baseTableSchema, - tableSchemas, resolutionConfig, contextParams, columnProjections, }: { - unnestedSql: string; baseTableSchema: TableSchema; - tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; columnProjections?: string[]; -}): Promise<{ - sql: string; - resolvedTableSchema: TableSchema; -}> => { - // Update the SQL to point to the unnested SQL +}): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; - // for (const columnConfig of resolutionConfig.columnConfigs) { - // columnConfig.name = getNamespacedKey( - // updatedBaseTableSchema.name, - // memberKeyToSafeKey(columnConfig.name) - // ); - // // columnConfig.name = memberKeyToSafeKey(columnConfig.name); - // } // Generate resolution schemas for array fields const resolutionSchemas = generateResolutionSchemasFromBaseTable( resolutionConfig, updatedBaseTableSchema ); - // Generate join paths using existing helper const joinPaths = generateResolutionJoinPathsFromBaseTable( updatedBaseTableSchema.name, resolutionConfig, @@ -370,6 +261,7 @@ export const getResolvedSql = async ({ resolutionConfig, updatedColumnProjections ); + // Create query and generate SQL const resolutionQuery: Query = { measures: [], @@ -395,36 +287,24 @@ export const getResolvedSql = async ({ .map((dim) => dim.name) ); - const resolvedTableSchema: TableSchema = { - name: '__resolved_query', - sql: resolvedSql, - measures: [], - dimensions: [ - // Dimensions from base table that were queried (row_id, measures, non-resolved dimensions) - ...updatedBaseTableSchema.dimensions - .filter((dim) => baseDimensionNames.has(dim.name)) - .map((dim) => ({ - name: dim.name, - sql: `__resolved_query."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })), - // All dimensions from resolution schemas (resolved columns from JOINs) - ...resolutionSchemas.flatMap((resSchema) => - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `__resolved_query."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ), - ], - }; - - return { - sql: resolvedSql, - resolvedTableSchema, - }; + const resolvedTableSchema: TableSchema = createWrapperTableSchema( + resolvedSql, + updatedBaseTableSchema + ); + resolvedTableSchema.dimensions = resolvedTableSchema.dimensions.filter( + (dim) => baseDimensionNames.has(dim.name) + ); + resolvedTableSchema.dimensions.push( + ...resolutionSchemas.flatMap((resSchema) => + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ) + ); + return resolvedTableSchema; }; /** @@ -443,48 +323,34 @@ export const getResolvedSql = async ({ * @returns Final SQL with arrays containing resolved values */ export const getAggregatedSql = async ({ - resolvedSql, resolvedTableSchema, resolutionConfig, contextParams, }: { - resolvedSql: string; resolvedTableSchema: TableSchema; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; }): Promise => { - // Step 1: Use the resolved table schema from Phase 2 as source of truth - // Update the SQL to point to the resolved SQL - const aggregationBaseTableSchema: TableSchema = { - ...resolvedTableSchema, - sql: resolvedSql, - }; + const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; // Step 2: Identify which columns need ARRAY_AGG vs MAX - const arrayColumns = getArrayTypeColumns(resolutionConfig); + const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); const baseTableName = aggregationBaseTableSchema.name; - // Helper to check if a dimension is a resolved array column const isResolvedArrayColumn = (dimName: string) => { - // Check if the dimension name matches a resolved column pattern - // e.g., "tickets__owners__display_name" return arrayColumns.some((arrayCol) => { - const arrayColPrefix = memberKeyToSafeKey(arrayCol.name); - return ( - dimName.includes(`${arrayColPrefix}__`) && - !dimName.startsWith('__base_query__') - ); + return dimName.includes(`${arrayCol.name}__`); }); }; // Step 3: Create aggregation measures with proper aggregation functions // Get row_id dimension for GROUP BY const rowIdDimension = aggregationBaseTableSchema.dimensions.find( - (d) => d.name === 'row_id' || d.name.endsWith('__row_id') + (d) => d.name === ROW_ID_DIMENSION_NAME ); // Create measures with MAX or ARRAY_AGG based on column type - const aggregationMeasures: TableSchema['measures'] = []; + const aggregationMeasures: Measure[] = []; aggregationBaseTableSchema.dimensions .filter((dim) => dim.name !== rowIdDimension?.name) @@ -516,19 +382,16 @@ export const getAggregatedSql = async ({ dimensions: rowIdDimension ? [rowIdDimension] : [], }; - // Step 4: Create the aggregation query - const aggregationQuery: Query = { - measures: aggregationMeasures.map((m) => - getNamespacedKey(baseTableName, m.name) - ), - dimensions: rowIdDimension - ? [getNamespacedKey(baseTableName, rowIdDimension.name)] - : [], - }; - // Step 5: Generate the final SQL const aggregatedSql = await cubeQueryToSQL({ - query: aggregationQuery, + query: { + measures: aggregationMeasures.map((m) => + getNamespacedKey(baseTableName, m.name) + ), + dimensions: rowIdDimension + ? [getNamespacedKey(baseTableName, rowIdDimension.name)] + : [], + }, tableSchemas: [schemaWithAggregation], contextParams, }); From 94a1e10005f799f9b2cbda3190c457e69036e86b Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 3 Nov 2025 23:10:44 +0530 Subject: [PATCH 14/38] moving everything to browser too --- .../browser-cube-to-sql-with-resolution.ts | 400 ++++++++---------- meerkat-core/src/joins/joins.ts | 2 - .../src/member-formatters/constants.d.ts | 3 + .../src/member-formatters/constants.ts | 2 +- meerkat-core/src/member-formatters/index.ts | 2 +- meerkat-core/src/resolution/resolution.ts | 75 ---- .../src/utils/find-in-table-schema.ts | 9 + .../cube-to-sql-with-resolution.ts | 26 +- 8 files changed, 203 insertions(+), 316 deletions(-) create mode 100644 meerkat-core/src/member-formatters/constants.d.ts diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 2d021536..80a8d06d 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -2,13 +2,20 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, + createWrapperTableSchema, Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + getArrayTypeResolutionColumnConfigs, + getNamespacedKey, + Measure, + memberKeyToSafeKey, Query, ResolutionConfig, + ROW_ID_DIMENSION_NAME, TableSchema, + updateArrayFlattenModifierUsingResolutionConfig, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; import { @@ -42,10 +49,21 @@ export const cubeQueryToSQLWithResolution = async ({ columnProjections, contextParams, }: CubeQueryToSQLWithResolutionParams) => { + const duplicateTableSchemas = tableSchemas.map((schema) => ({ + name: schema.name, + sql: schema.sql, + dimensions: schema.dimensions.map((dim) => ({ + ...dim, + modifier: dim.modifier ? { ...dim.modifier } : undefined, + alias: undefined, + })), + measures: schema.measures.map((measure) => ({ ...measure })), + joins: schema.joins?.map((join) => ({ ...join })), + })); const baseSql = await cubeQueryToSQL({ connection, query, - tableSchemas, + tableSchemas: duplicateTableSchemas, contextParams, }); @@ -54,6 +72,21 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } + // Step 2: Check if array-type resolution is needed + if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { + // Delegate to array handler, passing baseSql instead of query + return cubeQueryToSQLWithResolutionWithArray({ + connection, + baseSql, + measures: query.measures, + dimensions: query.dimensions || [], + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, + }); + } + // Create a table schema for the base query. const baseTable: TableSchema = createBaseTableSchema( baseSql, @@ -69,7 +102,7 @@ export const cubeQueryToSQLWithResolution = async ({ ); const resolveParams: CubeQueryToSQLParams = { - connection: connection, + connection, query: { measures: [], dimensions: generateResolvedDimensions( @@ -91,30 +124,60 @@ export const cubeQueryToSQLWithResolution = async ({ return sql; }; +export interface CubeQueryToSQLWithResolutionWithArrayParams { + connection: AsyncDuckDBConnection; + baseSql: string; + measures: string[]; + dimensions: string[]; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + columnProjections?: string[]; + contextParams?: ContextParams; +} + export const cubeQueryToSQLWithResolutionWithArray = async ({ connection, - query, + baseSql, + measures, + dimensions, tableSchemas, resolutionConfig, columnProjections, contextParams, -}: CubeQueryToSQLWithResolutionParams) => { +}: CubeQueryToSQLWithResolutionWithArrayParams) => { + const baseSchema: TableSchema = createBaseTableSchema( + baseSql, + tableSchemas, + resolutionConfig, + measures, + dimensions + ); + + baseSchema.dimensions.push({ + name: ROW_ID_DIMENSION_NAME, + sql: 'row_number() OVER ()', + type: 'number', + alias: ROW_ID_DIMENSION_NAME, + } as Dimension); + columnProjections?.push(ROW_ID_DIMENSION_NAME); + + // Doing this because we need to use the original name of the column in the base table schema. + resolutionConfig.columnConfigs.forEach((config) => { + config.name = memberKeyToSafeKey(config.name); + }); + // Phase 1: Generate SQL with row_id and unnested arrays - const { sql: unnestBaseSql, baseTableSchema } = await getUnnestBaseSql({ + const unnestTableSchema = await getUnnestTableSchema({ connection, - query, - tableSchemas, + baseTableSchema: baseSchema, resolutionConfig, contextParams, }); - // // // Phase 2: Apply resolution (join with lookup tables) - const { sql: resolvedSql, resolvedTableSchema } = await getResolvedSql({ + // Phase 2: Apply resolution (join with lookup tables) + const resolvedTableSchema = await getResolvedSql({ connection, - unnestedSql: unnestBaseSql, - baseTableSchema, - query, - tableSchemas, + baseTableSchema: unnestTableSchema, resolutionConfig, contextParams, columnProjections, @@ -123,115 +186,57 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ // Phase 3: Re-aggregate to reverse the unnest const aggregatedSql = await getAggregatedSql({ connection, - resolvedSql, resolvedTableSchema, - query, resolutionConfig, contextParams, }); return aggregatedSql; }; - -export const getUnnestBaseSql = async ({ +/** + * Phase 1: Apply unnesting + * + * This function performs 1 step: + * 1. Create schema with unnest modifiers for array columns + * 2. Generate final unnested SQL + * @returns Table schema with unnest modifiers for array columns + */ +export const getUnnestTableSchema = async ({ connection, - query, - tableSchemas, + baseTableSchema, resolutionConfig, contextParams, -}: CubeQueryToSQLWithResolutionParams): Promise<{ - sql: string; +}: { + connection: AsyncDuckDBConnection; baseTableSchema: TableSchema; -}> => { - // Step 1: Add row_id to the first table schema and generate base SQL (without unnesting) - const modifiedTableSchemasWithRowId = tableSchemas.map((schema, index) => { - // Add row_id to the first table only - if (index !== 0) { - return schema; - } - - // Add row_id dimension (no unnest modifier yet) - // TODO: Will this cause a problem of adding row_id to the first schema ? - const newDimensions = [ - { - name: 'row_id', - sql: 'row_number() OVER ()', - type: 'number' as const, - alias: '__row_id', - }, - ...schema.dimensions, - ]; - - return { - ...schema, - dimensions: newDimensions, - }; - }); - - // Use the first table for row_id reference - const firstTable = tableSchemas[0]; - - const queryWithRowId: Query = { - measures: query.measures, - dimensions: [`${firstTable.name}.row_id`, ...(query.dimensions || [])], - joinPaths: query.joinPaths, - filters: query.filters, - order: query.order, - limit: query.limit, - offset: query.offset, - }; - - // Generate base SQL with row_id - const baseSqlWithRowId = await cubeQueryToSQL({ - connection, - query: queryWithRowId, - tableSchemas: modifiedTableSchemasWithRowId, - contextParams, - }); - - // Step 2: Create a new table schema from the base SQL with row_id - // This will be used to apply unnesting - const baseTableName = '__base_query'; // Use standard name to work with helpers - - const baseTableSchema: TableSchema = createBaseTableSchema( - baseSqlWithRowId, - tableSchemas, - resolutionConfig, - query.measures, - query.dimensions + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig ); - baseTableSchema.dimensions.push({ - name: 'row_id', - sql: '__row_id', - type: 'number', - alias: '__row_id', - } as Dimension); - - // Step 3: Create query with unnest modifiers applied - const unnestQuery: Query = { - measures: [], - dimensions: [ - `${baseTableName}.row_id`, - ...query.measures.map((m) => `${baseTableName}.${m.replace('.', '__')}`), - ...(query.dimensions || []).map( - (d) => `${baseTableName}.${d.replace('.', '__')}` - ), - ], - }; - - // Generate the final SQL with unnesting applied - const unnestedBaseSql = await cubeQueryToSQL({ + const unnestedSql = await cubeQueryToSQL({ connection, - query: unnestQuery, + query: { + measures: [], + dimensions: [ + ...baseTableSchema.dimensions.map((d) => + getNamespacedKey(baseTableSchema.name, d.name) + ), + ], + }, tableSchemas: [baseTableSchema], contextParams, }); - return { - sql: unnestedBaseSql, - baseTableSchema, - }; + const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( + unnestedSql, + baseTableSchema + ); + + return unnestedBaseTableSchema; }; /** @@ -241,68 +246,53 @@ export const getUnnestBaseSql = async ({ * 1. Uses the base table schema from Phase 1 (source of truth) * 2. Generates resolution schemas for array fields * 3. Sets up join paths between unnested data and resolution tables - * 4. Generates SQL with resolved values - * - * @param unnestedSql - SQL output from Phase 1 (with row_id and unnested arrays) - * @param baseTableSchema - Schema from Phase 1 that describes the unnested SQL - * @returns SQL with row_id, unnested arrays, and resolved values from lookup tables + * @returns Table schema with resolved values from lookup tables */ export const getResolvedSql = async ({ connection, - unnestedSql, baseTableSchema, - query, - tableSchemas, resolutionConfig, contextParams, columnProjections, }: { connection: AsyncDuckDBConnection; - unnestedSql: string; baseTableSchema: TableSchema; - query: Query; - tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; columnProjections?: string[]; -}): Promise<{ - sql: string; - resolvedTableSchema: TableSchema; -}> => { - // Step 1: Use the base table schema from Phase 1 as source of truth - // Update the SQL to point to the unnested SQL - const updatedBaseTableSchema: TableSchema = { - ...baseTableSchema, - sql: unnestedSql, - }; - - debugger; - // Step 2: Generate resolution schemas for array fields only - // Use the existing generateResolutionSchemas helper +}): Promise => { + const updatedBaseTableSchema: TableSchema = baseTableSchema; - const resolutionSchemas = generateResolutionSchemas( - resolutionConfig, - tableSchemas - ); + // Generate resolution schemas for array fields + const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ + updatedBaseTableSchema, + ]); - // Step 3: Generate join paths using existing helper - // Note: Pass the base table schema (from Phase 1) to generate correct join paths const joinPaths = generateResolutionJoinPaths( - BASE_DATA_SOURCE_NAME, + updatedBaseTableSchema.name, resolutionConfig, - tableSchemas + [updatedBaseTableSchema] ); - query.dimensions?.push('row_id'); - // Step 4: Generate resolved dimensions using existing helper - // should not be using column projections here, it doesn't have row_id. + const tempQuery: Query = { + measures: [], + dimensions: baseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), + }; + + const updatedColumnProjections = columnProjections?.map((cp) => + memberKeyToSafeKey(cp) + ); + // Generate resolved dimensions using columnProjections const resolvedDimensions = generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig + updatedBaseTableSchema.name, + tempQuery, + resolutionConfig, + updatedColumnProjections ); - // Step 5: Create query and generate SQL + // Create query and generate SQL const resolutionQuery: Query = { measures: [], dimensions: resolvedDimensions, @@ -316,53 +306,36 @@ export const getResolvedSql = async ({ contextParams, }); - // Create a schema describing Phase 2's output by combining: - // 1. Dimensions from updatedBaseTableSchema that were actually queried - // 2. Dimensions from resolutionSchemas (resolved columns) - // The key insight: cubeQueryToSQL outputs columns using their alias field - - // Build list of dimension names that should be in output - const baseDimensionNames = new Set([ - 'row_id', - ...query.measures.map((m) => m.replace('.', '__')), - ...(query.dimensions || []) - .filter( - (d) => !resolutionConfig.columnConfigs.some((ac) => ac.name === d) - ) - .map((d) => d.replace('.', '__')), - ]); - - debugger; - const resolvedTableSchema: TableSchema = { - name: '__resolved_query', - sql: resolvedSql, - measures: [], - dimensions: [ - // Dimensions from base table that were queried (row_id, measures, non-resolved dimensions) - ...updatedBaseTableSchema.dimensions - .filter((dim) => baseDimensionNames.has(dim.name)) - .map((dim) => ({ - name: dim.name, - sql: `__resolved_query."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })), - // All dimensions from resolution schemas (resolved columns from JOINs) - ...resolutionSchemas.flatMap((resSchema) => - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `__resolved_query."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ), - ], - }; + // Use the baseTableSchema which already has all the column info + const baseDimensionNames = new Set( + baseTableSchema.dimensions + .filter((dim) => { + // Exclude columns that need resolution (they'll be replaced by resolved columns) + return !resolutionConfig.columnConfigs.some( + (ac) => ac.name === dim.name + ); + }) + .map((dim) => dim.name) + ); - return { - sql: resolvedSql, - resolvedTableSchema, - }; + const resolvedTableSchema: TableSchema = createWrapperTableSchema( + resolvedSql, + updatedBaseTableSchema + ); + resolvedTableSchema.dimensions = resolvedTableSchema.dimensions.filter( + (dim) => baseDimensionNames.has(dim.name) + ); + resolvedTableSchema.dimensions.push( + ...resolutionSchemas.flatMap((resSchema) => + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ) + ); + return resolvedTableSchema; }; /** @@ -375,58 +348,42 @@ export const getResolvedSql = async ({ * 4. Uses ARRAY_AGG for resolved array columns * * @param resolvedSql - SQL output from Phase 2 (with resolved values) - * @param query - Original query + * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) * @param resolutionConfig - Resolution configuration * @param contextParams - Optional context parameters * @returns Final SQL with arrays containing resolved values */ export const getAggregatedSql = async ({ connection, - resolvedSql, resolvedTableSchema, - query, resolutionConfig, contextParams, }: { connection: AsyncDuckDBConnection; - resolvedSql: string; resolvedTableSchema: TableSchema; - query: Query; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; }): Promise => { - // Step 1: Use the resolved table schema from Phase 2 as source of truth - // Update the SQL to point to the resolved SQL - const aggregationBaseTableSchema: TableSchema = { - ...resolvedTableSchema, - sql: resolvedSql, - }; + const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; // Step 2: Identify which columns need ARRAY_AGG vs MAX - const arrayColumns = getArrayTypeColumns(resolutionConfig); + const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); const baseTableName = aggregationBaseTableSchema.name; - // Helper to check if a dimension is a resolved array column const isResolvedArrayColumn = (dimName: string) => { - // Check if the dimension name matches a resolved column pattern - // e.g., "tickets__owners__display_name" return arrayColumns.some((arrayCol) => { - const arrayColPrefix = arrayCol.name.replace('.', '__'); - return ( - dimName.includes(`${arrayColPrefix}__`) && - !dimName.startsWith('__base_query__') - ); + return dimName.includes(`${arrayCol.name}__`); }); }; // Step 3: Create aggregation measures with proper aggregation functions // Get row_id dimension for GROUP BY const rowIdDimension = aggregationBaseTableSchema.dimensions.find( - (d) => d.name === 'row_id' || d.name.endsWith('__row_id') + (d) => d.name === ROW_ID_DIMENSION_NAME ); // Create measures with MAX or ARRAY_AGG based on column type - const aggregationMeasures: TableSchema['measures'] = []; + const aggregationMeasures: Measure[] = []; aggregationBaseTableSchema.dimensions .filter((dim) => dim.name !== rowIdDimension?.name) @@ -458,22 +415,21 @@ export const getAggregatedSql = async ({ dimensions: rowIdDimension ? [rowIdDimension] : [], }; - // Step 4: Create the aggregation query - const aggregationQuery: Query = { - measures: aggregationMeasures.map((m) => `${baseTableName}.${m.name}`), - dimensions: rowIdDimension - ? [`${baseTableName}.${rowIdDimension.name}`] - : [], - }; - // Step 5: Generate the final SQL const aggregatedSql = await cubeQueryToSQL({ connection, - query: aggregationQuery, + query: { + measures: aggregationMeasures.map((m) => + getNamespacedKey(baseTableName, m.name) + ), + dimensions: rowIdDimension + ? [getNamespacedKey(baseTableName, rowIdDimension.name)] + : [], + }, tableSchemas: [schemaWithAggregation], contextParams, }); - const rowIdExcludedSql = `select * exclude(__row_id) from (${aggregatedSql})`; + const rowIdExcludedSql = `select * exclude(${ROW_ID_DIMENSION_NAME}) from (${aggregatedSql})`; return rowIdExcludedSql; }; diff --git a/meerkat-core/src/joins/joins.ts b/meerkat-core/src/joins/joins.ts index 81468cea..46bdb7fe 100644 --- a/meerkat-core/src/joins/joins.ts +++ b/meerkat-core/src/joins/joins.ts @@ -56,7 +56,6 @@ export function generateSqlQuery( // If visitedFrom is undefined, this is the first visit to the node visitedNodes.set(currentEdge.right, currentEdge); - debugger; query += ` LEFT JOIN (${tableSchemaSqlMap[currentEdge.right]}) AS ${ currentEdge.right @@ -73,7 +72,6 @@ export const createDirectedGraph = ( tableSchema: TableSchema[], tableSchemaSqlMap: { [key: string]: string } ) => { - debugger; const directedGraph: { [key: string]: { [key: string]: { [key: string]: string } }; } = {}; diff --git a/meerkat-core/src/member-formatters/constants.d.ts b/meerkat-core/src/member-formatters/constants.d.ts new file mode 100644 index 00000000..5997a512 --- /dev/null +++ b/meerkat-core/src/member-formatters/constants.d.ts @@ -0,0 +1,3 @@ +export declare const COLUMN_NAME_DELIMITER = "."; +export declare const MEERKAT_OUTPUT_DELIMITER = "__"; +export declare const ROW_ID_DIMENSION_NAME = "__row_id"; diff --git a/meerkat-core/src/member-formatters/constants.ts b/meerkat-core/src/member-formatters/constants.ts index 108e656e..0a92bbfd 100644 --- a/meerkat-core/src/member-formatters/constants.ts +++ b/meerkat-core/src/member-formatters/constants.ts @@ -1,3 +1,3 @@ export const COLUMN_NAME_DELIMITER = '.'; export const MEERKAT_OUTPUT_DELIMITER = '__'; -export const ROW_ID_DIMENSION_NAME = 'row_id'; +export const ROW_ID_DIMENSION_NAME = '__row_id'; diff --git a/meerkat-core/src/member-formatters/index.ts b/meerkat-core/src/member-formatters/index.ts index f9e67e98..2acd05ce 100644 --- a/meerkat-core/src/member-formatters/index.ts +++ b/meerkat-core/src/member-formatters/index.ts @@ -1,4 +1,4 @@ -export { COLUMN_NAME_DELIMITER } from './constants'; +export { COLUMN_NAME_DELIMITER, ROW_ID_DIMENSION_NAME } from './constants'; export { constructAlias, getAliasFromSchema } from './get-alias'; export { getNamespacedKey } from './get-namespaced-key'; export { memberKeyToSafeKey } from './member-key-to-safe-key'; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 02c11b5d..689dae46 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -7,7 +7,6 @@ import { JoinPath, Member, Query } from '../types/cube-types/query'; import { Dimension, Measure, TableSchema } from '../types/cube-types/table'; import { findInDimensionSchemas, - findInSchema, findInSchemas, } from '../utils/find-in-table-schema'; import { @@ -194,62 +193,6 @@ export const generateResolutionSchemas = ( return resolutionSchemas; }; -export const generateResolutionSchemasFromBaseTable = ( - config: ResolutionConfig, - baseTableSchema: TableSchema -) => { - const resolutionSchemas: TableSchema[] = []; - config.columnConfigs.forEach((colConfig) => { - const tableSchema = config.tableSchemas.find( - (ts) => ts.name === colConfig.source - ); - if (!tableSchema) { - throw new Error(`Table schema not found for ${colConfig.source}`); - } - - const baseName = memberKeyToSafeKey(colConfig.name); - const baseAlias = constructAlias({ - name: colConfig.name, - alias: findInSchema(colConfig.name, baseTableSchema)?.alias, - aliasContext: { isTableSchemaAlias: true }, - }); - - // For each column that needs to be resolved, create a copy of the relevant table schema. - // We use the name of the column in the base query as the table schema name - // to avoid conflicts. - const resolutionSchema: TableSchema = { - name: baseName, - sql: tableSchema.sql, - measures: [], - dimensions: colConfig.resolutionColumns.map((col) => { - const dimension = findInDimensionSchemas( - getNamespacedKey(colConfig.source, col), - config.tableSchemas - ); - if (!dimension) { - throw new Error(`Dimension not found: ${col}`); - } - return { - // Need to create a new name due to limitations with how - // CubeToSql handles duplicate dimension names between different sources. - name: memberKeyToSafeKey(getNamespacedKey(colConfig.name, col)), - sql: `${baseName}.${col}`, - type: dimension.type, - alias: `${baseAlias} - ${constructAlias({ - name: col, - alias: dimension.alias, - aliasContext: { isTableSchemaAlias: true }, - })}`, - }; - }), - }; - - resolutionSchemas.push(resolutionSchema); - }); - - return resolutionSchemas; -}; - export const generateResolvedDimensions = ( baseDataSourceName: string, query: Query, @@ -302,21 +245,3 @@ export const generateResolutionJoinPaths = ( }, ]); }; - -export const generateResolutionJoinPathsFromBaseTable = ( - baseDataSourceName: string, - resolutionConfig: ResolutionConfig, - baseTableSchema: TableSchema -): JoinPath[] => { - return resolutionConfig.columnConfigs.map((config) => [ - { - left: baseDataSourceName, - right: memberKeyToSafeKey(config.name), - on: constructAlias({ - name: config.name, - alias: findInSchema(config.name, baseTableSchema)?.alias, - aliasContext: { isAstIdentifier: false }, - }), - }, - ]); -}; diff --git a/meerkat-core/src/utils/find-in-table-schema.ts b/meerkat-core/src/utils/find-in-table-schema.ts index c58e25ea..08f42ddf 100644 --- a/meerkat-core/src/utils/find-in-table-schema.ts +++ b/meerkat-core/src/utils/find-in-table-schema.ts @@ -52,6 +52,15 @@ export const findInSchemas = (name: string, tableSchemas: TableSchema[]) => { ** Finds the dimension or measure in the provided table schemas. ** Assumes the provided name is namespaced as `tableName.columnName`. */ + if (!name.includes('.')) { + if (tableSchemas.length > 1) { + throw new Error( + `Multiple table schemas found for ${name} and field doesn't have a table name` + ); + } + return findInSchema(name, tableSchemas[0]); + } + const [tableName, columnName] = splitIntoDataSourceAndFields(name); const tableSchema = tableSchemas.find((table) => table.name === tableName); if (!tableSchema) { diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 25e3ef4b..22e4d735 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -5,9 +5,7 @@ import { createWrapperTableSchema, Dimension, generateResolutionJoinPaths, - generateResolutionJoinPathsFromBaseTable, generateResolutionSchemas, - generateResolutionSchemasFromBaseTable, generateResolvedDimensions, getArrayTypeResolutionColumnConfigs, getNamespacedKey, @@ -15,10 +13,10 @@ import { memberKeyToSafeKey, Query, ResolutionConfig, + ROW_ID_DIMENSION_NAME, TableSchema, updateArrayFlattenModifierUsingResolutionConfig, } from '@devrev/meerkat-core'; -import { ROW_ID_DIMENSION_NAME } from 'meerkat-core/src/member-formatters/constants'; import { cubeQueryToSQL, CubeQueryToSQLParams, @@ -52,7 +50,6 @@ export const cubeQueryToSQLWithResolution = async ({ // Step 2: Check if array-type resolution is needed if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - debugger; // Delegate to array handler, passing baseSql instead of query return cubeQueryToSQLWithResolutionWithArray({ baseSql, @@ -76,7 +73,7 @@ export const cubeQueryToSQLWithResolution = async ({ const resolutionSchemas: TableSchema[] = generateResolutionSchemas( resolutionConfig, - tableSchemas + [baseTable] ); const resolveParams: CubeQueryToSQLParams = { @@ -91,7 +88,7 @@ export const cubeQueryToSQLWithResolution = async ({ joinPaths: generateResolutionJoinPaths( BASE_DATA_SOURCE_NAME, resolutionConfig, - tableSchemas + [baseTable] ), }, tableSchemas: [baseTable, ...resolutionSchemas], @@ -130,10 +127,10 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ ); baseSchema.dimensions.push({ - name: 'row_id', + name: ROW_ID_DIMENSION_NAME, sql: 'row_number() OVER ()', type: 'number', - alias: '__row_id', + alias: ROW_ID_DIMENSION_NAME, } as Dimension); columnProjections?.push(ROW_ID_DIMENSION_NAME); @@ -233,15 +230,14 @@ export const getResolvedSql = async ({ const updatedBaseTableSchema: TableSchema = baseTableSchema; // Generate resolution schemas for array fields - const resolutionSchemas = generateResolutionSchemasFromBaseTable( - resolutionConfig, - updatedBaseTableSchema - ); + const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ + updatedBaseTableSchema, + ]); - const joinPaths = generateResolutionJoinPathsFromBaseTable( + const joinPaths = generateResolutionJoinPaths( updatedBaseTableSchema.name, resolutionConfig, - updatedBaseTableSchema + [updatedBaseTableSchema] ); const tempQuery: Query = { @@ -396,6 +392,6 @@ export const getAggregatedSql = async ({ contextParams, }); - const rowIdExcludedSql = `select * exclude(__row_id) from (${aggregatedSql})`; + const rowIdExcludedSql = `select * exclude(${ROW_ID_DIMENSION_NAME}) from (${aggregatedSql})`; return rowIdExcludedSql; }; From 2f077b14fc4fb82c402a04823efd690372d37723 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 10 Nov 2025 00:08:13 +0530 Subject: [PATCH 15/38] mionr refactoring working --- .../browser-cube-to-sql-with-resolution.ts | 194 +++++++++--------- meerkat-core/src/resolution/resolution.ts | 20 +- 2 files changed, 97 insertions(+), 117 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 80a8d06d..c8da175e 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -49,21 +49,10 @@ export const cubeQueryToSQLWithResolution = async ({ columnProjections, contextParams, }: CubeQueryToSQLWithResolutionParams) => { - const duplicateTableSchemas = tableSchemas.map((schema) => ({ - name: schema.name, - sql: schema.sql, - dimensions: schema.dimensions.map((dim) => ({ - ...dim, - modifier: dim.modifier ? { ...dim.modifier } : undefined, - alias: undefined, - })), - measures: schema.measures.map((measure) => ({ ...measure })), - joins: schema.joins?.map((join) => ({ ...join })), - })); const baseSql = await cubeQueryToSQL({ connection, query, - tableSchemas: duplicateTableSchemas, + tableSchemas, contextParams, }); @@ -72,85 +61,82 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } - // Step 2: Check if array-type resolution is needed if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - // Delegate to array handler, passing baseSql instead of query + // This is to ensure that, only the column projection columns + // are being resolved and other definitions are ignored. + resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( + (config) => { + return columnProjections?.includes(config.name); + } + ); return cubeQueryToSQLWithResolutionWithArray({ connection, baseSql, - measures: query.measures, - dimensions: query.dimensions || [], tableSchemas, resolutionConfig, columnProjections, contextParams, }); - } - - // Create a table schema for the base query. - const baseTable: TableSchema = createBaseTableSchema( - baseSql, - tableSchemas, - resolutionConfig, - query.measures, - query.dimensions - ); + } else { + // Create a table schema for the base query. + const baseTable: TableSchema = createBaseTableSchema( + baseSql, + tableSchemas, + resolutionConfig, + query.measures, + query.dimensions + ); - const resolutionSchemas: TableSchema[] = generateResolutionSchemas( - resolutionConfig, - tableSchemas - ); + const resolutionSchemas: TableSchema[] = generateResolutionSchemas( + resolutionConfig, + tableSchemas + ); - const resolveParams: CubeQueryToSQLParams = { - connection, - query: { - measures: [], - dimensions: generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), - joinPaths: generateResolutionJoinPaths( - BASE_DATA_SOURCE_NAME, - resolutionConfig, - tableSchemas - ), - }, - tableSchemas: [baseTable, ...resolutionSchemas], - }; - const sql = await cubeQueryToSQL(resolveParams); + const resolveParams: CubeQueryToSQLParams = { + connection, + query: { + measures: [], + dimensions: generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, + query, + resolutionConfig, + columnProjections + ), + joinPaths: generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + tableSchemas + ), + }, + tableSchemas: [baseTable, ...resolutionSchemas], + }; + const sql = await cubeQueryToSQL(resolveParams); - return sql; + return sql; + } }; -export interface CubeQueryToSQLWithResolutionWithArrayParams { - connection: AsyncDuckDBConnection; - baseSql: string; - measures: string[]; - dimensions: string[]; - tableSchemas: TableSchema[]; - resolutionConfig: ResolutionConfig; - columnProjections?: string[]; - contextParams?: ContextParams; -} - export const cubeQueryToSQLWithResolutionWithArray = async ({ connection, baseSql, - measures, - dimensions, tableSchemas, resolutionConfig, columnProjections, contextParams, -}: CubeQueryToSQLWithResolutionWithArrayParams) => { +}: { + connection: AsyncDuckDBConnection; + baseSql: string; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + columnProjections?: string[]; + contextParams?: ContextParams; +}): Promise => { const baseSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, resolutionConfig, - measures, - dimensions + [], + columnProjections ); baseSchema.dimensions.push({ @@ -166,7 +152,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ config.name = memberKeyToSafeKey(config.name); }); - // Phase 1: Generate SQL with row_id and unnested arrays + // Generate SQL with row_id and unnested arrays const unnestTableSchema = await getUnnestTableSchema({ connection, baseTableSchema: baseSchema, @@ -174,8 +160,8 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ contextParams, }); - // Phase 2: Apply resolution (join with lookup tables) - const resolvedTableSchema = await getResolvedSql({ + // Apply resolution (join with lookup tables) + const resolvedTableSchema = await getResolvedTableSchema({ connection, baseTableSchema: unnestTableSchema, resolutionConfig, @@ -183,7 +169,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ columnProjections, }); - // Phase 3: Re-aggregate to reverse the unnest + // Re-aggregate to reverse the unnest const aggregatedSql = await getAggregatedSql({ connection, resolvedTableSchema, @@ -194,7 +180,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ return aggregatedSql; }; /** - * Phase 1: Apply unnesting + * Apply unnesting * * This function performs 1 step: * 1. Create schema with unnest modifiers for array columns @@ -240,7 +226,7 @@ export const getUnnestTableSchema = async ({ }; /** - * Phase 2: Apply resolution (join with lookup tables) + * Apply resolution (join with lookup tables) * * This function: * 1. Uses the base table schema from Phase 1 (source of truth) @@ -248,7 +234,7 @@ export const getUnnestTableSchema = async ({ * 3. Sets up join paths between unnested data and resolution tables * @returns Table schema with resolved values from lookup tables */ -export const getResolvedSql = async ({ +export const getResolvedTableSchema = async ({ connection, baseTableSchema, resolutionConfig, @@ -307,39 +293,48 @@ export const getResolvedSql = async ({ }); // Use the baseTableSchema which already has all the column info - const baseDimensionNames = new Set( - baseTableSchema.dimensions - .filter((dim) => { - // Exclude columns that need resolution (they'll be replaced by resolved columns) - return !resolutionConfig.columnConfigs.some( - (ac) => ac.name === dim.name - ); - }) - .map((dim) => dim.name) - ); - const resolvedTableSchema: TableSchema = createWrapperTableSchema( resolvedSql, updatedBaseTableSchema ); - resolvedTableSchema.dimensions = resolvedTableSchema.dimensions.filter( - (dim) => baseDimensionNames.has(dim.name) - ); - resolvedTableSchema.dimensions.push( - ...resolutionSchemas.flatMap((resSchema) => - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ) - ); + + // Create a map of resolution schema dimensions by original column name + const resolutionDimensionsByColumnName = new Map(); + resolutionConfig.columnConfigs.forEach((config) => { + const resSchema = resolutionSchemas.find((rs) => + rs.dimensions.some((dim) => dim.name.startsWith(config.name)) + ); + if (resSchema) { + resolutionDimensionsByColumnName.set( + config.name, + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ); + } + }); + + // Maintain the same order as baseTableSchema.dimensions + // Replace dimensions that need resolution with their resolved counterparts + resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { + const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); + if (resolvedDims) { + // Replace with resolved dimensions + return resolvedDims; + } else { + // Keep the original dimension with correct SQL reference + return [dim]; + } + }); + return resolvedTableSchema; }; /** - * Phase 3: Re-aggregate to reverse the unnest + * Re-aggregate to reverse the unnest * * This function: * 1. Wraps Phase 2 SQL as a new base table @@ -396,8 +391,9 @@ export const getAggregatedSql = async ({ dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; // Use ARRAY_AGG for resolved array columns, MAX for others + // Filter out null values for ARRAY_AGG using FILTER clause const aggregationFn = isArrayColumn - ? `ARRAY_AGG(DISTINCT ${columnRef})` + ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` : `MAX(${columnRef})`; aggregationMeasures.push({ diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 689dae46..55423935 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -15,11 +15,7 @@ import { ResolutionConfig, } from './types'; -const constructBaseDimension = ( - name: string, - schema: Measure | Dimension, - resolutionColumnConfigs: ResolutionColumnConfig[] -) => { +const constructBaseDimension = (name: string, schema: Measure | Dimension) => { const dimension: Dimension = { name: memberKeyToSafeKey(name), sql: `${BASE_DATA_SOURCE_NAME}.${constructAlias({ @@ -35,14 +31,6 @@ const constructBaseDimension = ( aliasContext: { isTableSchemaAlias: true }, }), }; - // const shouldFlattenField = resolutionColumnConfigs.some( - // (config) => config.name == name && config.isArrayType - // ); - // if (shouldFlattenField) { - // dimension.modifier = { - // shouldFlattenArray: true, - // }; - // } return dimension; }; @@ -71,11 +59,7 @@ export const createBaseTableSchema = ( dimensions: [...measures, ...(dimensions || [])].map((member) => { const schema = schemaByName[member]; if (schema) { - return constructBaseDimension( - member, - schema, - resolutionConfig.columnConfigs - ); + return constructBaseDimension(member, schema); } else { throw new Error(`Not found: ${member}`); } From c273540b695d6f427cf369f9406f17609693e624 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 10 Nov 2025 01:08:46 +0530 Subject: [PATCH 16/38] final changes after testing and copy pasting same code from browser into node --- .../sql-expression-modifiers.ts | 5 +- .../cube-to-sql-with-resolution.ts | 181 +++++++++--------- 2 files changed, 97 insertions(+), 89 deletions(-) diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts index 195c4b88..16b49cf9 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts @@ -17,7 +17,10 @@ export const arrayFieldUnNestModifier = ({ export const arrayFlattenModifier = ({ sqlExpression, }: DimensionModifier): string => { - return `unnest(${sqlExpression})`; + // Ensure NULL or empty arrays produce at least one row with NULL value + // This prevents rows from being dropped when arrays are NULL or empty + // COALESCE handles NULL, and len() = 0 check handles empty arrays [] + return `unnest(CASE WHEN ${sqlExpression} IS NULL OR len(COALESCE(${sqlExpression}, [])) = 0 THEN [NULL] ELSE ${sqlExpression} END)`; }; export const shouldUnnest = ({ diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 22e4d735..2fa22d71 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -48,82 +48,78 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } - // Step 2: Check if array-type resolution is needed if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - // Delegate to array handler, passing baseSql instead of query + // This is to ensure that, only the column projection columns + // are being resolved and other definitions are ignored. + resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( + (config) => { + return columnProjections?.includes(config.name); + } + ); return cubeQueryToSQLWithResolutionWithArray({ baseSql, - measures: query.measures, - dimensions: query.dimensions || [], tableSchemas, resolutionConfig, columnProjections, contextParams, }); - } - - // Create a table schema for the base query. - const baseTable: TableSchema = createBaseTableSchema( - baseSql, - tableSchemas, - resolutionConfig, - query.measures, - query.dimensions - ); - - const resolutionSchemas: TableSchema[] = generateResolutionSchemas( - resolutionConfig, - [baseTable] - ); + } else { + // Create a table schema for the base query. + const baseTable: TableSchema = createBaseTableSchema( + baseSql, + tableSchemas, + resolutionConfig, + query.measures, + query.dimensions + ); - const resolveParams: CubeQueryToSQLParams = { - query: { - measures: [], - dimensions: generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), - joinPaths: generateResolutionJoinPaths( - BASE_DATA_SOURCE_NAME, - resolutionConfig, - [baseTable] - ), - }, - tableSchemas: [baseTable, ...resolutionSchemas], - }; - const sql = await cubeQueryToSQL(resolveParams); + const resolutionSchemas: TableSchema[] = generateResolutionSchemas( + resolutionConfig, + tableSchemas + ); + + const resolveParams: CubeQueryToSQLParams = { + query: { + measures: [], + dimensions: generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, + query, + resolutionConfig, + columnProjections + ), + joinPaths: generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + tableSchemas + ), + }, + tableSchemas: [baseTable, ...resolutionSchemas], + }; + const sql = await cubeQueryToSQL(resolveParams); - return sql; + return sql; + } }; -export interface CubeQueryToSQLWithResolutionWithArrayParams { - baseSql: string; - measures: string[]; - dimensions: string[]; - tableSchemas: TableSchema[]; - resolutionConfig: ResolutionConfig; - columnProjections?: string[]; - contextParams?: ContextParams; -} - export const cubeQueryToSQLWithResolutionWithArray = async ({ baseSql, - measures, - dimensions, tableSchemas, resolutionConfig, columnProjections, contextParams, -}: CubeQueryToSQLWithResolutionWithArrayParams) => { - // Step 1: Create schema for the base SQL +}: { + baseSql: string; + tableSchemas: TableSchema[]; + resolutionConfig: ResolutionConfig; + columnProjections?: string[]; + contextParams?: ContextParams; +}): Promise => { const baseSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, resolutionConfig, - measures, - dimensions + [], + columnProjections ); baseSchema.dimensions.push({ @@ -139,22 +135,22 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ config.name = memberKeyToSafeKey(config.name); }); - // Phase 1: Generate SQL with row_id and unnested arrays + // Generate SQL with row_id and unnested arrays const unnestTableSchema = await getUnnestTableSchema({ baseTableSchema: baseSchema, resolutionConfig, contextParams, }); - // Phase 2: Apply resolution (join with lookup tables) - const resolvedTableSchema = await getResolvedSql({ + // Apply resolution (join with lookup tables) + const resolvedTableSchema = await getResolvedTableSchema({ baseTableSchema: unnestTableSchema, resolutionConfig, contextParams, columnProjections, }); - // Phase 3: Re-aggregate to reverse the unnest + // Re-aggregate to reverse the unnest const aggregatedSql = await getAggregatedSql({ resolvedTableSchema, resolutionConfig, @@ -163,9 +159,8 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ return aggregatedSql; }; - /** - * Phase 1: Apply unnesting + * Apply unnesting * * This function performs 1 step: * 1. Create schema with unnest modifiers for array columns @@ -208,7 +203,7 @@ export const getUnnestTableSchema = async ({ }; /** - * Phase 2: Apply resolution (join with lookup tables) + * Apply resolution (join with lookup tables) * * This function: * 1. Uses the base table schema from Phase 1 (source of truth) @@ -216,7 +211,7 @@ export const getUnnestTableSchema = async ({ * 3. Sets up join paths between unnested data and resolution tables * @returns Table schema with resolved values from lookup tables */ -export const getResolvedSql = async ({ +export const getResolvedTableSchema = async ({ baseTableSchema, resolutionConfig, contextParams, @@ -272,39 +267,48 @@ export const getResolvedSql = async ({ }); // Use the baseTableSchema which already has all the column info - const baseDimensionNames = new Set( - baseTableSchema.dimensions - .filter((dim) => { - // Exclude columns that need resolution (they'll be replaced by resolved columns) - return !resolutionConfig.columnConfigs.some( - (ac) => ac.name === dim.name - ); - }) - .map((dim) => dim.name) - ); - const resolvedTableSchema: TableSchema = createWrapperTableSchema( resolvedSql, updatedBaseTableSchema ); - resolvedTableSchema.dimensions = resolvedTableSchema.dimensions.filter( - (dim) => baseDimensionNames.has(dim.name) - ); - resolvedTableSchema.dimensions.push( - ...resolutionSchemas.flatMap((resSchema) => - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ) - ); + + // Create a map of resolution schema dimensions by original column name + const resolutionDimensionsByColumnName = new Map(); + resolutionConfig.columnConfigs.forEach((config) => { + const resSchema = resolutionSchemas.find((rs) => + rs.dimensions.some((dim) => dim.name.startsWith(config.name)) + ); + if (resSchema) { + resolutionDimensionsByColumnName.set( + config.name, + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ); + } + }); + + // Maintain the same order as baseTableSchema.dimensions + // Replace dimensions that need resolution with their resolved counterparts + resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { + const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); + if (resolvedDims) { + // Replace with resolved dimensions + return resolvedDims; + } else { + // Keep the original dimension with correct SQL reference + return [dim]; + } + }); + return resolvedTableSchema; }; /** - * Phase 3: Re-aggregate to reverse the unnest + * Re-aggregate to reverse the unnest * * This function: * 1. Wraps Phase 2 SQL as a new base table @@ -359,8 +363,9 @@ export const getAggregatedSql = async ({ dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; // Use ARRAY_AGG for resolved array columns, MAX for others + // Filter out null values for ARRAY_AGG using FILTER clause const aggregationFn = isArrayColumn - ? `ARRAY_AGG(DISTINCT ${columnRef})` + ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` : `MAX(${columnRef})`; aggregationMeasures.push({ From 06662f53034670e623ed0925ffb482bd520b6291 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Mon, 10 Nov 2025 01:19:27 +0530 Subject: [PATCH 17/38] minor refactoring --- .vscode/launch.json | 44 ------------------- .../browser-cube-to-sql-with-resolution.ts | 25 +++-------- .../src/member-formatters/constants.d.ts | 3 -- meerkat-core/src/resolution/resolution.ts | 3 +- .../src/utils/find-in-table-schema.ts | 2 +- .../cube-to-sql-with-resolution.ts | 16 +++---- 6 files changed, 16 insertions(+), 77 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 meerkat-core/src/member-formatters/constants.d.ts diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 64487837..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Launch Program", - "skipFiles": [ - "/**" - ], - "program": "${workspaceFolder}/meerkat-core/src/resolution/resolution.spec.ts", - "outFiles": [ - "${workspaceFolder}/**/*.js" - ] - }, - { - "type": "node", - "request": "launch", - "name": "Debug Jest Tests", - "runtimeExecutable": "node", - "runtimeArgs": [ - "--inspect-brk", - "${workspaceFolder}/node_modules/.bin/jest", - "--runInBand", - "--no-cache", - "--testPathPattern=meerkat-core/src/resolution/resolution.spec.ts" - ], - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen", - "cwd": "${workspaceFolder}", - "env": { - "NODE_ENV": "test" - }, - "sourceMaps": true, - "resolveSourceMapLocations": [ - "${workspaceFolder}/**", - "!**/node_modules/**" - ] - }, - ] -} \ No newline at end of file diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index c8da175e..60ddca69 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -32,15 +32,6 @@ export interface CubeQueryToSQLWithResolutionParams { contextParams?: ContextParams; } -/** - * Helper function to get array-type columns from resolution config - */ -const getArrayTypeColumns = (resolutionConfig: ResolutionConfig) => { - return resolutionConfig.columnConfigs.filter( - (config) => config.isArrayType === true - ); -}; - export const cubeQueryToSQLWithResolution = async ({ connection, query, @@ -249,7 +240,7 @@ export const getResolvedTableSchema = async ({ }): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; - // Generate resolution schemas for array fields + // Generate resolution schemas for fields that need resolution const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ updatedBaseTableSchema, ]); @@ -337,12 +328,10 @@ export const getResolvedTableSchema = async ({ * Re-aggregate to reverse the unnest * * This function: - * 1. Wraps Phase 2 SQL as a new base table - * 2. Groups by row_id - * 3. Uses MAX for non-array columns (they're duplicated) - * 4. Uses ARRAY_AGG for resolved array columns + * 1. Groups by row_id + * 2. Uses MAX for non-array columns (they're duplicated) + * 3. Uses ARRAY_AGG for resolved array columns * - * @param resolvedSql - SQL output from Phase 2 (with resolved values) * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) * @param resolutionConfig - Resolution configuration * @param contextParams - Optional context parameters @@ -361,7 +350,7 @@ export const getAggregatedSql = async ({ }): Promise => { const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; - // Step 2: Identify which columns need ARRAY_AGG vs MAX + // Identify which columns need ARRAY_AGG vs MAX const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); const baseTableName = aggregationBaseTableSchema.name; @@ -371,7 +360,7 @@ export const getAggregatedSql = async ({ }); }; - // Step 3: Create aggregation measures with proper aggregation functions + // Create aggregation measures with proper aggregation functions // Get row_id dimension for GROUP BY const rowIdDimension = aggregationBaseTableSchema.dimensions.find( (d) => d.name === ROW_ID_DIMENSION_NAME @@ -411,7 +400,7 @@ export const getAggregatedSql = async ({ dimensions: rowIdDimension ? [rowIdDimension] : [], }; - // Step 5: Generate the final SQL + // Generate the final SQL const aggregatedSql = await cubeQueryToSQL({ connection, query: { diff --git a/meerkat-core/src/member-formatters/constants.d.ts b/meerkat-core/src/member-formatters/constants.d.ts deleted file mode 100644 index 5997a512..00000000 --- a/meerkat-core/src/member-formatters/constants.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -export declare const COLUMN_NAME_DELIMITER = "."; -export declare const MEERKAT_OUTPUT_DELIMITER = "__"; -export declare const ROW_ID_DIMENSION_NAME = "__row_id"; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 55423935..45066252 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -16,7 +16,7 @@ import { } from './types'; const constructBaseDimension = (name: string, schema: Measure | Dimension) => { - const dimension: Dimension = { + return { name: memberKeyToSafeKey(name), sql: `${BASE_DATA_SOURCE_NAME}.${constructAlias({ name, @@ -31,7 +31,6 @@ const constructBaseDimension = (name: string, schema: Measure | Dimension) => { aliasContext: { isTableSchemaAlias: true }, }), }; - return dimension; }; export const createBaseTableSchema = ( diff --git a/meerkat-core/src/utils/find-in-table-schema.ts b/meerkat-core/src/utils/find-in-table-schema.ts index 08f42ddf..8eaa5e5f 100644 --- a/meerkat-core/src/utils/find-in-table-schema.ts +++ b/meerkat-core/src/utils/find-in-table-schema.ts @@ -50,7 +50,7 @@ export const findInDimensionSchemas = ( export const findInSchemas = (name: string, tableSchemas: TableSchema[]) => { /* ** Finds the dimension or measure in the provided table schemas. - ** Assumes the provided name is namespaced as `tableName.columnName`. + ** Handles both namespaced (`tableName.columnName`) and non-namespaced names. */ if (!name.includes('.')) { if (tableSchemas.length > 1) { diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 2fa22d71..f564fc3e 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -224,7 +224,7 @@ export const getResolvedTableSchema = async ({ }): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; - // Generate resolution schemas for array fields + // Generate resolution schemas for fields that need resolution const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ updatedBaseTableSchema, ]); @@ -311,12 +311,10 @@ export const getResolvedTableSchema = async ({ * Re-aggregate to reverse the unnest * * This function: - * 1. Wraps Phase 2 SQL as a new base table - * 2. Groups by row_id - * 3. Uses MAX for non-array columns (they're duplicated) - * 4. Uses ARRAY_AGG for resolved array columns + * 1. Groups by row_id + * 2. Uses MAX for non-array columns (they're duplicated) + * 3. Uses ARRAY_AGG for resolved array columns * - * @param resolvedSql - SQL output from Phase 2 (with resolved values) * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) * @param resolutionConfig - Resolution configuration * @param contextParams - Optional context parameters @@ -333,7 +331,7 @@ export const getAggregatedSql = async ({ }): Promise => { const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; - // Step 2: Identify which columns need ARRAY_AGG vs MAX + // Identify which columns need ARRAY_AGG vs MAX const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); const baseTableName = aggregationBaseTableSchema.name; @@ -343,7 +341,7 @@ export const getAggregatedSql = async ({ }); }; - // Step 3: Create aggregation measures with proper aggregation functions + // Create aggregation measures with proper aggregation functions // Get row_id dimension for GROUP BY const rowIdDimension = aggregationBaseTableSchema.dimensions.find( (d) => d.name === ROW_ID_DIMENSION_NAME @@ -383,7 +381,7 @@ export const getAggregatedSql = async ({ dimensions: rowIdDimension ? [rowIdDimension] : [], }; - // Step 5: Generate the final SQL + // Generate the final SQL const aggregatedSql = await cubeQueryToSQL({ query: { measures: aggregationMeasures.map((m) => From 9bb01ade8d91af34c00322562b01f037c5a16a3e Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 00:08:32 +0530 Subject: [PATCH 18/38] udpated tests for resolution.ts --- .../src/resolution/resolution.spec.ts | 432 ++++++++++++++++-- meerkat-core/src/resolution/types.ts | 2 +- 2 files changed, 406 insertions(+), 28 deletions(-) diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index 45170199..a5fde3cc 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -1,10 +1,13 @@ import { createBaseTableSchema, + createWrapperTableSchema, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + getArrayTypeResolutionColumnConfigs, + updateArrayFlattenModifierUsingResolutionConfig, } from './resolution'; -import { ResolutionConfig } from './types'; +import { BASE_DATA_SOURCE_NAME, ResolutionConfig } from './types'; describe('Create base table schema', () => { it('dimensions and measures are converted to dimensions', () => { @@ -18,19 +21,19 @@ describe('Create base table schema', () => { { name: 'count', sql: 'COUNT(*)', - type: 'number', + type: 'number' as const, }, ], dimensions: [ { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, }, { name: 'column2', sql: 'base_table.column2', - type: 'string', + type: 'string' as const, }, ], }, @@ -89,12 +92,12 @@ describe('Create base table schema', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, }, { name: 'column2', sql: 'base_table.column2', - type: 'string', + type: 'string' as const, }, ], }, @@ -104,6 +107,7 @@ describe('Create base table schema', () => { { name: 'base_table.column1', source: 'resolution_table', + isArrayType: false, joinColumn: 'id', resolutionColumns: ['display_id'], }, @@ -166,7 +170,7 @@ describe('Create base table schema', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, }, ], }, @@ -199,13 +203,13 @@ describe('Create base table schema', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, alias: 'Column 1', }, { name: 'column2', sql: 'base_table.column2', - type: 'string', + type: 'string' as const, alias: 'Column 2', }, ], @@ -215,12 +219,14 @@ describe('Create base table schema', () => { columnConfigs: [ { name: 'base_table.column1', + isArrayType: false, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + isArrayType: false, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_name'], @@ -279,12 +285,12 @@ describe('Generate resolution schemas', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, }, { name: 'column2', sql: 'base_table.column2', - type: 'string', + type: 'string' as const, }, ], }, @@ -314,17 +320,17 @@ describe('Generate resolution schemas', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, }, { name: 'display_name', sql: 'resolution_table.display_name', - type: 'string', + type: 'string' as const, }, ], }, @@ -397,17 +403,17 @@ describe('Generate resolution schemas', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, }, { name: 'display_name', sql: 'resolution_table.display_name', - type: 'string', + type: 'string' as const, }, ], }, @@ -437,14 +443,14 @@ describe('Generate resolution schemas', () => { { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, }, ], dimensions: [ { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, }, ], }, @@ -475,12 +481,12 @@ describe('Generate resolution schemas', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, }, ], }, @@ -515,7 +521,7 @@ describe('Generate resolution schemas', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, alias: 'Column 1', }, ], @@ -540,13 +546,13 @@ describe('Generate resolution schemas', () => { { name: 'id', sql: 'resolution_table.id', - type: 'string', + type: 'string' as const, alias: 'ID', }, { name: 'display_id', sql: 'resolution_table.display_id', - type: 'string', + type: 'string' as const, alias: 'Display ID', }, ], @@ -601,6 +607,7 @@ describe('Generate resolved dimensions', () => { }; const resolvedDimensions = generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig ); @@ -629,6 +636,7 @@ describe('Generate resolved dimensions', () => { }; const resolvedDimensions = generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig ); @@ -668,6 +676,7 @@ describe('Generate resolved dimensions', () => { ]; const resolvedDimensions = generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, query, resolutionConfig, projections @@ -702,7 +711,11 @@ describe('Generate resolution join paths', () => { tableSchemas: [], }; - const joinPaths = generateResolutionJoinPaths(resolutionConfig, []); + const joinPaths = generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, + resolutionConfig, + [] + ); expect(joinPaths).toEqual([ [ @@ -732,13 +745,13 @@ describe('Generate resolution join paths', () => { { name: 'column1', sql: 'base_table.column1', - type: 'string', + type: 'string' as const, alias: 'Column 1', }, { name: 'column2', sql: 'base_table.column2', - type: 'string', + type: 'string' as const, alias: 'Column 2', }, ], @@ -758,6 +771,7 @@ describe('Generate resolution join paths', () => { }; const joinPaths = generateResolutionJoinPaths( + BASE_DATA_SOURCE_NAME, resolutionConfig, baseTableSchemas ); @@ -772,3 +786,367 @@ describe('Generate resolution join paths', () => { ]); }); }); + +describe('createWrapperTableSchema', () => { + it('should create wrapper schema with correct structure', () => { + const sql = 'SELECT * FROM base_table'; + const baseTableSchema = { + name: 'original_table', + sql: 'original sql', + dimensions: [ + { + name: 'column1', + sql: 'original_table.column1', + type: 'string' as const, + alias: 'Column 1', + }, + { + name: 'column2', + sql: 'original_table.column2', + type: 'number' as const, + alias: 'Column 2', + }, + ], + measures: [ + { + name: 'count', + sql: 'COUNT(*)', + type: 'number' as const, + alias: 'Count', + }, + ], + joins: [ + { + sql: 'some_join_condition', + }, + ], + } as any; + + const result = createWrapperTableSchema(sql, baseTableSchema); + + expect(result).toEqual({ + name: '__base_query', + sql: 'SELECT * FROM base_table', + dimensions: [ + { + name: 'column1', + sql: '__base_query."Column 1"', + type: 'string', + alias: 'Column 1', + }, + { + name: 'column2', + sql: '__base_query."Column 2"', + type: 'number', + alias: 'Column 2', + }, + ], + measures: [ + { + name: 'count', + sql: '__base_query."Count"', + type: 'number', + alias: 'Count', + }, + ], + joins: [ + { + sql: 'some_join_condition', + }, + ], + }); + }); + + it('should handle dimensions without aliases', () => { + const sql = 'SELECT column1 FROM base_table'; + const baseTableSchema = { + name: 'original_table', + sql: 'original sql', + dimensions: [ + { + name: 'column1', + sql: 'original_table.column1', + type: 'string' as const, + }, + ], + measures: [], + joins: [], + } as any; + + const result = createWrapperTableSchema(sql, baseTableSchema); + + expect(result.dimensions[0].sql).toBe('__base_query."column1"'); + }); + + it('should handle empty dimensions and measures', () => { + const sql = 'SELECT * FROM base_table'; + const baseTableSchema = { + name: 'original_table', + sql: 'original sql', + dimensions: [], + measures: [], + joins: [], + } as any; + + const result = createWrapperTableSchema(sql, baseTableSchema); + + expect(result).toEqual({ + name: '__base_query', + sql: 'SELECT * FROM base_table', + dimensions: [], + measures: [], + joins: [], + }); + }); +}); + +describe('getArrayTypeResolutionColumnConfigs', () => { + it('should filter and return only array type column configs', () => { + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'table.array_column', + isArrayType: true, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + { + name: 'table.scalar_column', + isArrayType: false, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + { + name: 'table.another_array', + isArrayType: true, + source: 'lookup_table2', + joinColumn: 'id', + resolutionColumns: ['value'], + }, + ], + tableSchemas: [], + }; + + const result = getArrayTypeResolutionColumnConfigs(resolutionConfig); + + expect(result).toHaveLength(2); + expect(result[0].name).toBe('table.array_column'); + expect(result[1].name).toBe('table.another_array'); + expect(result.every((config) => config.isArrayType === true)).toBe(true); + }); + + it('should return empty array when no array type configs exist', () => { + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'table.scalar_column1', + isArrayType: false, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + { + name: 'table.scalar_column2', + isArrayType: false, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [], + }; + + const result = getArrayTypeResolutionColumnConfigs(resolutionConfig); + + expect(result).toEqual([]); + }); + + it('should return empty array when columnConfigs is empty', () => { + const resolutionConfig: ResolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + + const result = getArrayTypeResolutionColumnConfigs(resolutionConfig); + + expect(result).toEqual([]); + }); +}); + +describe('updateArrayFlattenModifierUsingResolutionConfig', () => { + it('should add shouldFlattenArray modifier to array columns', () => { + const baseTableSchema = { + name: 'base_table', + sql: 'SELECT * FROM base_table', + dimensions: [ + { + name: 'array_column', + sql: 'base_table.array_column', + type: 'string_array' as const, + }, + { + name: 'scalar_column', + sql: 'base_table.scalar_column', + type: 'string' as const, + }, + ], + measures: [], + } as any; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'array_column', + isArrayType: true, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [], + }; + + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig + ); + + expect(baseTableSchema.dimensions[0].modifier).toEqual({ + shouldFlattenArray: true, + }); + expect(baseTableSchema.dimensions[1].modifier).toBeUndefined(); + }); + + it('should handle multiple array columns', () => { + const baseTableSchema = { + name: 'base_table', + sql: 'SELECT * FROM base_table', + dimensions: [ + { + name: 'array_column1', + sql: 'base_table.array_column1', + type: 'string_array' as const, + }, + { + name: 'array_column2', + sql: 'base_table.array_column2', + type: 'string_array' as const, + }, + { + name: 'scalar_column', + sql: 'base_table.scalar_column', + type: 'string' as const, + }, + ], + measures: [], + } as any; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'array_column1', + isArrayType: true, + source: 'lookup_table1', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + { + name: 'array_column2', + isArrayType: true, + source: 'lookup_table2', + joinColumn: 'id', + resolutionColumns: ['value'], + }, + ], + tableSchemas: [], + }; + + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig + ); + + expect(baseTableSchema.dimensions[0].modifier).toEqual({ + shouldFlattenArray: true, + }); + expect(baseTableSchema.dimensions[1].modifier).toEqual({ + shouldFlattenArray: true, + }); + expect(baseTableSchema.dimensions[2].modifier).toBeUndefined(); + }); + + it('should not modify dimensions when no array columns in config', () => { + const baseTableSchema = { + name: 'base_table', + sql: 'SELECT * FROM base_table', + dimensions: [ + { + name: 'column1', + sql: 'base_table.column1', + type: 'string', + }, + { + name: 'column2', + sql: 'base_table.column2', + type: 'number', + }, + ], + measures: [], + } as any; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'column1', + isArrayType: false, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [], + }; + + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig + ); + + expect(baseTableSchema.dimensions[0].modifier).toBeUndefined(); + expect(baseTableSchema.dimensions[1].modifier).toBeUndefined(); + }); + + it('should handle empty dimensions array', () => { + const baseTableSchema = { + name: 'base_table', + sql: 'SELECT * FROM base_table', + dimensions: [], + measures: [], + } as any; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'array_column', + isArrayType: true, + source: 'lookup_table', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [], + }; + + // Should not throw error + expect(() => { + updateArrayFlattenModifierUsingResolutionConfig( + baseTableSchema, + resolutionConfig + ); + }).not.toThrow(); + + expect(baseTableSchema.dimensions).toEqual([]); + }); +}); diff --git a/meerkat-core/src/resolution/types.ts b/meerkat-core/src/resolution/types.ts index 4f1581e0..921ff797 100644 --- a/meerkat-core/src/resolution/types.ts +++ b/meerkat-core/src/resolution/types.ts @@ -5,7 +5,7 @@ export interface ResolutionColumnConfig { // Should match a measure or dimension in the query. name: string; // is array type - isArrayType: boolean; + isArrayType?: boolean; // Name of the data source to use for resolution. source: string; // Name of the column in the data source to join on. From 2b7a570f2e62e305ab37264695a96d383448ff0c Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 00:44:01 +0530 Subject: [PATCH 19/38] adding a test --- .../cube-to-sql-with-resolution-array.spec.ts | 3624 ----------------- .../cube-to-sql-with-resolution.spec.ts | 454 +++ 2 files changed, 454 insertions(+), 3624 deletions(-) delete mode 100644 meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts create mode 100644 meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts deleted file mode 100644 index 7c61162d..00000000 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution-array.spec.ts +++ /dev/null @@ -1,3624 +0,0 @@ -import { TableSchema } from '@devrev/meerkat-core'; -import { cubeQueryToSQLWithResolution } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; -import { duckdbExec } from '../duckdb-exec'; -const CREATE_TEST_TABLE = `CREATE TABLE tickets ( - id INTEGER, - owners VARCHAR[], - tags VARCHAR[], - created_by VARCHAR, - subscribers_count INTEGER -)`; - -const INPUT_DATA_QUERY = `INSERT INTO tickets VALUES -(1, ['owner1', 'owner2'], ['tag1'], 'user1', 30), -(2, ['owner2', 'owner3'], ['tag2', 'tag3'], 'user2', 10), -(3, ['owner4'], ['tag1', 'tag4', 'tag3'], 'user3', 80)`; - -const CREATE_RESOLUTION_TABLE = `CREATE TABLE owners_lookup ( - id VARCHAR, - display_name VARCHAR, - email VARCHAR -)`; - -const RESOLUTION_DATA_QUERY = `INSERT INTO owners_lookup VALUES -('owner1', 'Alice Smith', 'alice@example.com'), -('owner2', 'Bob Jones', 'bob@example.com'), -('owner3', 'Charlie Brown', 'charlie@example.com'), -('owner4', 'Diana Prince', 'diana@example.com')`; - -const CREATE_TAGS_LOOKUP_TABLE = `CREATE TABLE tags_lookup ( - id VARCHAR, - tag_name VARCHAR -)`; -const CREATE_CREATED_BY_LOOKUP_TABLE = `CREATE TABLE created_by_lookup ( - id VARCHAR, - name VARCHAR -)`; -const CREATED_BY_LOOKUP_DATA_QUERY = `INSERT INTO created_by_lookup VALUES -('user1', 'User 1'), -('user2', 'User 2'), -('user3', 'User 3')`; -const TAGS_LOOKUP_DATA_QUERY = `INSERT INTO tags_lookup VALUES -('tag1', 'Tag 1'), -('tag2', 'Tag 2'), -('tag3', 'Tag 3'), -('tag4', 'Tag 4')`; - -const TICKETS_TABLE_SCHEMA: TableSchema = { - name: 'tickets', - sql: 'select * from tickets', - measures: [ - { - name: 'count', - sql: 'COUNT(*)', - type: 'number', - }, - ], - dimensions: [ - { - alias: 'ID', - name: 'id', - sql: 'id', - type: 'number', - }, - { - alias: 'Created By', - name: 'created_by', - sql: 'created_by', - type: 'string', - }, - { - alias: 'Owners', - name: 'owners', - sql: 'owners', - type: 'string_array', - }, - { - alias: 'Tags', - name: 'tags', - sql: 'tags', - type: 'string_array', - }, - ], -}; - -const OWNERS_LOOKUP_SCHEMA: TableSchema = { - name: 'owners_lookup', - sql: 'select * from owners_lookup', - measures: [], - dimensions: [ - { - alias: 'ID', - name: 'id', - sql: 'id', - type: 'string', - }, - { - alias: 'Display Name', - name: 'display_name', - sql: 'display_name', - type: 'string', - }, - { - alias: 'Email', - name: 'email', - sql: 'email', - type: 'string', - }, - ], -}; - -const TAGS_LOOKUP_SCHEMA: TableSchema = { - name: 'tags_lookup', - sql: 'select * from tags_lookup', - measures: [], - dimensions: [ - { - alias: 'ID', - name: 'id', - sql: 'id', - type: 'string', - }, - { - alias: 'Tag Name', - name: 'tag_name', - sql: 'tag_name', - type: 'string', - }, - ], -}; - -const CREATED_BY_LOOKUP_SCHEMA: TableSchema = { - name: 'created_by_lookup', - sql: 'select * from created_by_lookup', - measures: [], - dimensions: [ - { - alias: 'ID', - name: 'id', - sql: 'id', - type: 'string', - }, - { - alias: 'Name', - name: 'name', - sql: 'name', - type: 'string', - }, - ], -}; -describe('cubeQueryToSQLWithResolutionWithArray - Phase 1: Unnest', () => { - jest.setTimeout(1000000); - beforeAll(async () => { - // Create test tables - await duckdbExec(CREATE_TEST_TABLE); - await duckdbExec(INPUT_DATA_QUERY); - await duckdbExec(CREATE_RESOLUTION_TABLE); - await duckdbExec(RESOLUTION_DATA_QUERY); - await duckdbExec(CREATE_TAGS_LOOKUP_TABLE); - await duckdbExec(TAGS_LOOKUP_DATA_QUERY); - await duckdbExec(CREATE_CREATED_BY_LOOKUP_TABLE); - await duckdbExec(CREATED_BY_LOOKUP_DATA_QUERY); - }); - - // it('Should add row_id and unnest array fields that need resolution', async () => { - // const query: Query = { - // measures: ['tickets.count'], - // dimensions: ['tickets.id', 'tickets.owners'], - // }; - - // const resolutionConfig: ResolutionConfig = { - // columnConfigs: [ - // { - // name: 'tickets.owners', - // isArrayType: true, - // source: 'owners_lookup', - // joinColumn: 'id', - // resolutionColumns: ['display_name', 'email'], - // }, - // ], - // tableSchemas: [OWNERS_LOOKUP_SCHEMA], - // }; - - // debugger; - // const sql = await cubeQueryToSQLWithResolutionWithArray({ - // query, - // tableSchemas: [TICKETS_TABLE_SCHEMA], - // resolutionConfig, - // }); - - // console.log('Phase 1 SQL (with row_id and unnest):', sql); - - // // Verify the SQL includes row_id - // expect(sql).toContain('row_id'); - - // // Verify the SQL unnests the owners array - // expect(sql).toContain('unnest'); - - // // Verify it includes the owners dimension - // expect(sql).toContain('owners'); - - // // Execute the SQL to verify it works - // const result = (await duckdbExec(sql)) as any[]; - // // console.log('Phase 1 Result:', JSON.stringify(result, null, 2)); - - // // The result should have unnested rows (more rows than the original 3) - // // Original: 3 rows, but ticket 1 has 2 owners, ticket 2 has 2 owners, ticket 3 has 1 owner - // // Expected: 5 unnested rows - // expect(result.length).toBe(5); - - // // Each row should have a row_id - // expect(result[0]).toHaveProperty('__row_id'); - // expect(result[0]).toHaveProperty('tickets__count'); - // expect(result[0]).toHaveProperty('tickets__id'); - // expect(result[0]).toHaveProperty('tickets__owners'); - - // debugger; - // // Verify row_ids are preserved (rows with same original row should have same row_id) - // const rowIds = result.map((r) => r.row_id); - // expect(rowIds.length).toBe(5); // 5 unnested rows total - // }); - - // it('Should handle multiple array fields that need unnesting', async () => { - // const query: Query = { - // measures: ['tickets.count'], - // dimensions: [ - // 'tickets.id', - // 'tickets.owners', //array - // 'tickets.tags', // array - // 'tickets.created_by', // scalar - // ], - // }; - - // const resolutionConfig: ResolutionConfig = { - // columnConfigs: [ - // { - // name: 'tickets.owners', - // isArrayType: true, - // source: 'owners_lookup', - // joinColumn: 'id', - // resolutionColumns: ['display_name'], - // }, - // { - // name: 'tickets.tags', - // isArrayType: true, - // source: 'tags_lookup', - // joinColumn: 'id', - // resolutionColumns: ['tag_name'], - // }, - // { - // name: 'tickets.created_by', - // isArrayType: false, - // source: 'created_by_lookup', - // joinColumn: 'id', - // resolutionColumns: ['name'], - // }, - // ], - // tableSchemas: [ - // OWNERS_LOOKUP_SCHEMA, - // TAGS_LOOKUP_SCHEMA, - // CREATED_BY_LOOKUP_SCHEMA, - // ], - // }; - - // const columnProjections = [ - // 'tickets.id', - // 'tickets.owners', - // 'tickets.tags', - // 'tickets.created_by', - // 'tickets.count', - // ]; - // const sql = await cubeQueryToSQLWithResolution({ - // query, - // tableSchemas: [TICKETS_TABLE_SCHEMA], - // resolutionConfig, - // columnProjections, - // }); - - // console.log('Phase 1 SQL (multiple arrays):', sql); - - // // Verify row_id is included - // expect(sql).toContain('row_id'); - - // // Both arrays should be unnested - // expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(2); - - // // Execute the SQL to verify it works - // const result = (await duckdbExec(sql)) as any[]; - - // expect(result.length).toBe(7); - - // // Each row should have a row_id - // expect(result[0]).toHaveProperty('__row_id'); - // expect(result[0]).toHaveProperty('tickets__count'); - // expect(result[0]).toHaveProperty('tickets__id'); - // expect(result[0]).toHaveProperty('tickets__owners - display_name'); - // expect(result[0]).toHaveProperty('tickets__tags - tag_name'); - // }); - - it('test an proper query from UI', async () => { - const query = { - dimensions: ['dim_ticket.tags_json'], - filters: [ - { - and: [ - { - and: [ - { - member: 'dim_ticket.created_date', - operator: 'inDateRange', - values: [ - '2025-08-02T09:30:00.000Z', - '2025-10-31T10:29:59.999Z', - ], - }, - { - member: 'dim_ticket.work_type', - operator: 'in', - values: ['ticket'], - }, - { - member: 'dim_ticket.stage_json', - operator: 'in', - values: [ - 'don:core:dvrv-us-1:devo/787:custom_stage/514', - 'don:core:dvrv-us-1:devo/787:custom_stage/512', - 'don:core:dvrv-us-1:devo/787:custom_stage/501', - 'don:core:dvrv-us-1:devo/787:custom_stage/485', - 'don:core:dvrv-us-1:devo/787:custom_stage/440', - 'don:core:dvrv-us-1:devo/787:custom_stage/25', - 'don:core:dvrv-us-1:devo/787:custom_stage/24', - 'don:core:dvrv-us-1:devo/787:custom_stage/22', - 'don:core:dvrv-us-1:devo/787:custom_stage/21', - 'don:core:dvrv-us-1:devo/787:custom_stage/19', - 'don:core:dvrv-us-1:devo/787:custom_stage/13', - 'don:core:dvrv-us-1:devo/787:custom_stage/10', - 'don:core:dvrv-us-1:devo/787:custom_stage/8', - 'don:core:dvrv-us-1:devo/787:custom_stage/7', - 'don:core:dvrv-us-1:devo/787:custom_stage/6', - 'don:core:dvrv-us-1:devo/787:custom_stage/4', - ], - }, - ], - }, - ], - }, - ], - limit: 50000, - measures: ['dim_ticket.id_count___function__count'], - } as any; - const tableSchemas = [ - { - dimensions: [ - { - name: 'rev_oid', - sql: 'dim_ticket.rev_oid', - type: 'string', - alias: 'Customer Workspace', - }, - { - name: 'title', - sql: 'dim_ticket.title', - type: 'string', - alias: 'Title', - }, - { - name: 'last_internal_comment_date', - sql: 'dim_ticket.last_internal_comment_date', - type: 'time', - alias: 'Last Internal Comment Date', - }, - { - name: 'work_type', - sql: "'ticket'", - type: 'time', - alias: 'Work Type', - }, - { - name: 'is_spam', - sql: 'dim_ticket.is_spam', - type: 'boolean', - alias: 'Spam', - }, - { - name: 'object_type', - sql: 'dim_ticket.object_type', - type: 'string', - alias: 'Object type', - }, - { - name: 'modified_by_id', - sql: 'dim_ticket.modified_by_id', - type: 'string', - alias: 'Modified by', - }, - { - name: 'source_channel', - sql: 'dim_ticket.source_channel', - type: 'string', - alias: 'Source channel', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'channels', - sql: 'dim_ticket.channels', - type: 'number_array', - alias: 'Channels', - }, - { - name: 'created_by_id', - sql: 'dim_ticket.created_by_id', - type: 'string', - alias: 'Created by', - }, - { - name: 'created_date', - sql: 'dim_ticket.created_date', - type: 'time', - alias: 'Created date', - }, - { - name: 'target_close_date', - sql: 'dim_ticket.target_close_date', - type: 'time', - alias: 'Target close date', - }, - { - name: 'applies_to_part_id', - sql: 'dim_ticket.applies_to_part_id', - type: 'string', - alias: 'Part', - }, - { - name: 'subtype', - sql: 'dim_ticket.subtype', - type: 'string', - alias: 'Subtype', - }, - { - name: 'actual_close_date', - sql: 'dim_ticket.actual_close_date', - type: 'time', - alias: 'Close date', - }, - { - name: 'reported_by_id', - sql: 'dim_ticket.reported_by_id', - type: 'string', - alias: 'Reported by ID', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'reported_by_ids', - sql: 'dim_ticket.reported_by_ids', - type: 'string_array', - alias: 'Reported by', - }, - { - name: 'needs_response', - sql: 'dim_ticket.needs_response', - type: 'boolean', - alias: 'Needs Response', - }, - { - name: 'group', - sql: 'dim_ticket.group', - type: 'string', - alias: 'Group', - }, - { - name: 'modified_date', - sql: 'dim_ticket.modified_date', - type: 'time', - alias: 'Modified date', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'owned_by_ids', - sql: 'dim_ticket.owned_by_ids', - type: 'string_array', - alias: 'Owner', - }, - { - name: 'id', - sql: 'dim_ticket.id', - type: 'string', - alias: 'ID', - }, - { - name: 'sla_tracker_id', - sql: 'dim_ticket.sla_tracker_id', - type: 'string', - alias: 'SLA Tracker', - }, - { - name: 'severity', - sql: 'dim_ticket.severity', - type: 'number', - alias: 'Severity', - }, - { - name: 'stage_json', - sql: "json_extract_string(dim_ticket.stage_json, '$.stage_id')", - type: 'string', - alias: 'Stage', - }, - { - name: 'sla_id', - sql: 'dim_ticket.sla_id', - type: 'string', - alias: 'SLA Name', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'links_json', - sql: "list_distinct(CAST(json_extract_string(dim_ticket.links_json, '$[*].target_object_type') AS VARCHAR[]))", - type: 'string_array', - alias: 'Links', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tags_json', - sql: "CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[])", - type: 'string_array', - alias: 'Tags', - }, - { - name: 'surveys_aggregation_json', - sql: "list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min')", - type: 'number', - alias: 'CSAT Rating', - }, - { - name: 'sla_summary_target_time', - sql: "cast(json_extract_string(dim_ticket.sla_summary, '$.target_time') as timestamp)", - type: 'time', - alias: 'Next SLA Target', - }, - { - name: 'staged_info', - sql: "cast(json_extract_string(dim_ticket.staged_info, '$.is_staged') as boolean)", - type: 'boolean', - alias: 'Changes Need Review', - }, - { - name: 'tnt__account_id', - sql: 'tnt__account_id', - type: 'string', - alias: 'account id', - }, - { - name: 'tnt__actual_effort_spent', - sql: 'tnt__actual_effort_spent', - type: 'number', - alias: 'Actual Effort Spent', - }, - { - name: 'tnt__ai_subtype', - sql: 'tnt__ai_subtype', - type: 'string', - alias: 'AI Subtype', - }, - { - name: 'tnt__asdfad', - sql: 'tnt__asdfad', - type: 'string', - alias: 'asdfad', - }, - { - name: 'tnt__bool_summary', - sql: 'tnt__bool_summary', - type: 'boolean', - alias: 'bool summary', - }, - { - name: 'tnt__boolyesnoresp', - sql: 'tnt__boolyesnoresp', - type: 'boolean', - alias: 'boolyesnoresp', - }, - { - name: 'tnt__capability_part', - sql: 'tnt__capability_part', - type: 'string', - alias: 'Capability part', - }, - { - name: 'tnt__card', - sql: 'tnt__card', - type: 'string', - alias: 'card', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__cust_account', - sql: 'tnt__cust_account', - type: 'string_array', - alias: 'cust_account', - }, - { - name: 'tnt__custom_field_for_sla', - sql: 'tnt__custom_field_for_sla', - type: 'string', - alias: 'custom_field_for_sla', - }, - { - name: 'tnt__custom_id_field', - sql: 'tnt__custom_id_field', - type: 'string', - alias: 'custom id field', - }, - { - name: 'tnt__customer_tier', - sql: 'tnt__customer_tier', - type: 'string', - alias: 'Customer Tier', - }, - { - name: 'tnt__date_field', - sql: 'tnt__date_field', - type: 'time', - alias: 'date_field', - }, - { - name: 'tnt__department_list', - sql: 'tnt__department_list', - type: 'string', - alias: 'Department List', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__email_references', - sql: 'tnt__email_references', - type: 'string_array', - alias: 'tnt__email_references', - }, - { - name: 'tnt__escalated', - sql: 'tnt__escalated', - type: 'boolean', - alias: 'Escalated', - }, - { - name: 'tnt__estimated_effort', - sql: 'tnt__estimated_effort', - type: 'number', - alias: 'Estimated Effort', - }, - { - name: 'tnt__external_source_id', - sql: 'tnt__external_source_id', - type: 'string', - alias: 'tnt__external_source_id', - }, - { - name: 'tnt__feature_affected', - sql: 'tnt__feature_affected', - type: 'string', - alias: 'Feature Affected', - }, - { - name: 'tnt__field1', - sql: 'tnt__field1', - type: 'string', - alias: 'field1', - }, - { - name: 'tnt__foo', - sql: 'tnt__foo', - type: 'string', - alias: 'tnt__foo', - }, - { - name: 'tnt__fruit', - sql: 'tnt__fruit', - type: 'string', - alias: 'Fruit', - }, - { - name: 'tnt__git_template_name', - sql: 'tnt__git_template_name', - type: 'string', - alias: 'Github Template Name', - }, - { - name: 'tnt__id_field', - sql: 'tnt__id_field', - type: 'string', - alias: 'id field', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__id_list_field', - sql: 'tnt__id_list_field', - type: 'string_array', - alias: 'id list field', - }, - { - name: 'tnt__impact', - sql: 'tnt__impact', - type: 'string', - alias: 'Impact', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__include_on_emails', - sql: 'tnt__include_on_emails', - type: 'string_array', - alias: 'Include on emails', - }, - { - name: 'tnt__issue_score', - sql: 'tnt__issue_score', - type: 'number', - alias: 'Issue Score', - }, - { - name: 'tnt__knowledge_gap', - sql: 'tnt__knowledge_gap', - type: 'string', - alias: 'Knowledge Gap', - }, - { - name: 'tnt__linear_assignee', - sql: 'tnt__linear_assignee', - type: 'string', - alias: 'Linear Assignee', - }, - { - name: 'tnt__linear_description', - sql: 'tnt__linear_description', - type: 'string', - alias: 'Linear Description', - }, - { - name: 'tnt__linear_id', - sql: 'tnt__linear_id', - type: 'string', - alias: 'Linear Id', - }, - { - name: 'tnt__linear_identifier', - sql: 'tnt__linear_identifier', - type: 'string', - alias: 'Linear Identifier', - }, - { - name: 'tnt__linear_priority', - sql: 'tnt__linear_priority', - type: 'string', - alias: 'Linear Priority', - }, - { - name: 'tnt__linear_state', - sql: 'tnt__linear_state', - type: 'string', - alias: 'Linear State', - }, - { - name: 'tnt__linear_team', - sql: 'tnt__linear_team', - type: 'string', - alias: 'Linear Team', - }, - { - name: 'tnt__linear_title', - sql: 'tnt__linear_title', - type: 'string', - alias: 'Linear Title', - }, - { - name: 'tnt__linear_url', - sql: 'tnt__linear_url', - type: 'string', - alias: 'Linear Url', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__members', - sql: 'tnt__members', - type: 'string_array', - alias: 'tnt__members', - }, - { - name: 'tnt__micro_service_name', - sql: 'tnt__micro_service_name', - type: 'string', - alias: 'Microservice Name', - }, - { - name: 'tnt__notify_others', - sql: 'tnt__notify_others', - type: 'string', - alias: 'notify others', - }, - { - name: 'tnt__numeric_field', - sql: 'tnt__numeric_field', - type: 'number', - alias: 'numeric field', - }, - { - name: 'tnt__parent_part', - sql: 'tnt__parent_part', - type: 'string', - alias: 'Parent part', - }, - { - name: 'tnt__part_product', - sql: 'tnt__part_product', - type: 'string', - alias: 'part product', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__paytm_customer_name', - sql: 'tnt__paytm_customer_name', - type: 'string_array', - alias: 'paytm customer name', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__platform', - sql: 'tnt__platform', - type: 'string_array', - alias: 'Platform', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__portal_test', - sql: 'tnt__portal_test', - type: 'string_array', - alias: 'portal test', - }, - { - name: 'tnt__product', - sql: 'tnt__product', - type: 'string', - alias: 'Product', - }, - { - name: 'tnt__project', - sql: 'tnt__project', - type: 'string', - alias: 'Project', - }, - { - name: 'tnt__project_stage', - sql: 'tnt__project_stage', - type: 'string', - alias: 'Project Stage', - }, - { - name: 'tnt__rank', - sql: 'tnt__rank', - type: 'number', - alias: 'Rank', - }, - { - name: 'tnt__remaining_effort', - sql: 'tnt__remaining_effort', - type: 'number', - alias: 'Remaining Effort', - }, - { - name: 'tnt__required_text_field', - sql: 'tnt__required_text_field', - type: 'string', - alias: 'required text field', - }, - { - name: 'tnt__resolution_sla_met', - sql: 'tnt__resolution_sla_met', - type: 'string', - alias: 'Resolution SLA Met', - }, - { - name: 'tnt__resolution_sla_target_time', - sql: 'tnt__resolution_sla_target_time', - type: 'string', - alias: 'Resolution SLA Target Time', - }, - { - name: 'tnt__resolution_turnaround_time', - sql: 'tnt__resolution_turnaround_time', - type: 'string', - alias: 'Resolution SLA Turnaround Time', - }, - { - name: 'tnt__response_sla_met', - sql: 'tnt__response_sla_met', - type: 'string', - alias: 'First Response SLA Met', - }, - { - name: 'tnt__response_sla_target_time', - sql: 'tnt__response_sla_target_time', - type: 'string', - alias: 'First Response SLA Target Time', - }, - { - name: 'tnt__response_turnaround_time', - sql: 'tnt__response_turnaround_time', - type: 'string', - alias: 'First Response SLA Turnaround Time', - }, - { - name: 'tnt__root_cause_analysis', - sql: 'tnt__root_cause_analysis', - type: 'string', - alias: 'Root cause analysis', - }, - { - name: 'tnt__search_field_query', - sql: 'tnt__search_field_query', - type: 'string', - alias: 'Search Field Query', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__stakeholders', - sql: 'tnt__stakeholders', - type: 'string_array', - alias: 'stakeholders', - }, - { - name: 'tnt__stray_user', - sql: 'tnt__stray_user', - type: 'string', - alias: 'stray user', - }, - { - name: 'tnt__stray_users', - sql: 'tnt__stray_users', - type: 'string', - alias: 'stray users', - }, - { - name: 'tnt__test', - sql: 'tnt__test', - type: 'number', - alias: 'Test', - }, - { - name: 'tnt__test123', - sql: 'tnt__test123', - type: 'string', - alias: 'test123', - }, - { - name: 'tnt__test_capability_part', - sql: 'tnt__test_capability_part', - type: 'string', - alias: 'Test Capability Part', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__test_field_for_sla', - sql: 'tnt__test_field_for_sla', - type: 'string_array', - alias: 'test field for sla', - }, - { - name: 'tnt__test_filed', - sql: 'tnt__test_filed', - type: 'string', - alias: 'test field', - }, - { - name: 'tnt__test_mikasa', - sql: 'tnt__test_mikasa', - type: 'string', - alias: 'Test mikasa', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__test_multivalue_testrequiredfield', - sql: 'tnt__test_multivalue_testrequiredfield', - type: 'string_array', - alias: 'test multivalue testrequiredfield', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__test_multivalue_textfield', - sql: 'tnt__test_multivalue_textfield', - type: 'string_array', - alias: 'test multivalue textfield', - }, - { - name: 'tnt__test_product_part', - sql: 'tnt__test_product_part', - type: 'string', - alias: 'Test Product part', - }, - { - name: 'tnt__testing_bool_drpdown', - sql: 'tnt__testing_bool_drpdown', - type: 'string', - alias: 'Testing Bool Drpdown', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'tnt__text_list_account', - sql: 'tnt__text_list_account', - type: 'string_array', - alias: 'Text list account', - }, - { - name: 'tnt__ticket_custom_part', - sql: 'tnt__ticket_custom_part', - type: 'string', - alias: 'ticket custom part', - }, - { - name: 'tnt__ticket_dropd', - sql: 'tnt__ticket_dropd', - type: 'string', - alias: 'ticket dropdown', - }, - { - name: 'tnt__ticket_type', - sql: 'tnt__ticket_type', - type: 'string', - alias: 'Ticket type', - }, - { - name: 'tnt__tier', - sql: 'tnt__tier', - type: 'string', - alias: 'Tier', - }, - { - name: 'tnt__urgency', - sql: 'tnt__urgency', - type: 'string', - alias: 'Urgency', - }, - { - name: 'tnt__velocity_test_enum', - sql: 'tnt__velocity_test_enum', - type: 'string', - alias: 'velocity test enum', - }, - { - name: 'tnt__version', - sql: 'tnt__version', - type: 'string', - alias: 'Version', - }, - { - name: 'tnt__work_duration', - sql: 'tnt__work_duration', - type: 'string', - alias: 'Work duration', - }, - { - name: 'tnt__workspace_custom', - sql: 'tnt__workspace_custom', - type: 'string', - alias: 'workspace_custom', - }, - { - name: 'ctype_deal_registration__buying_region', - sql: 'ctype_deal_registration__buying_region', - type: 'string', - alias: 'Theater', - }, - { - name: 'ctype_deal_registration__deal_status', - sql: 'ctype_deal_registration__deal_status', - type: 'string', - alias: 'Deal Status', - }, - { - name: 'ctype_deal_registration__deal_type', - sql: 'ctype_deal_registration__deal_type', - type: 'string', - alias: 'Deal Type', - }, - { - name: 'ctype_deal_registration__employee_count', - sql: 'ctype_deal_registration__employee_count', - type: 'string', - alias: 'Employee Count', - }, - { - name: 'ctype_deal_registration__estimated_deal_value', - sql: 'ctype_deal_registration__estimated_deal_value', - type: 'number', - alias: 'Estimated Deal Value', - }, - { - name: 'ctype_deal_registration__expected_close_date', - sql: 'ctype_deal_registration__expected_close_date', - type: 'time', - alias: 'Expected Close Date', - }, - { - name: 'ctype_deal_registration__partner_type', - sql: 'ctype_deal_registration__partner_type', - type: 'string', - alias: 'Partner Type', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_deal_registration__products_or_services', - sql: 'ctype_deal_registration__products_or_services', - type: 'string_array', - alias: 'Product(s) or Service(s)', - }, - { - name: 'ctype_deal_registration__prospect_company_name', - sql: 'ctype_deal_registration__prospect_company_name', - type: 'string', - alias: 'Prospect Company Name', - }, - { - name: 'ctype_deal_registration__prospect_company_website', - sql: 'ctype_deal_registration__prospect_company_website', - type: 'string', - alias: 'Prospect Company Website', - }, - { - name: 'ctype_deal_registration__prospect_contact_email', - sql: 'ctype_deal_registration__prospect_contact_email', - type: 'string', - alias: 'Prospect Contact Email', - }, - { - name: 'ctype_deal_registration__prospect_contact_name', - sql: 'ctype_deal_registration__prospect_contact_name', - type: 'string', - alias: 'Prospect Contact Name', - }, - { - name: 'ctype_deal_registration__sub_region', - sql: 'ctype_deal_registration__sub_region', - type: 'string', - alias: 'Region ', - }, - { - name: 'ctype_deal_registration__subregion', - sql: 'ctype_deal_registration__subregion', - type: 'string', - alias: 'Subregion', - }, - { - name: 'ctype_Events__campaign_category', - sql: 'ctype_Events__campaign_category', - type: 'string', - alias: 'Campaign Category', - }, - { - name: 'ctype_Events__event_end_date', - sql: 'ctype_Events__event_end_date', - type: 'time', - alias: 'Event End Date', - }, - { - name: 'ctype_Events__event_owner', - sql: 'ctype_Events__event_owner', - type: 'string', - alias: 'Event Owner', - }, - { - name: 'ctype_Events__event_start_date', - sql: 'ctype_Events__event_start_date', - type: 'time', - alias: 'Event Start Date', - }, - { - name: 'ctype_Events__events_test_ref', - sql: 'ctype_Events__events_test_ref', - type: 'string', - alias: 'events test ref', - }, - { - name: 'ctype_Events__external_budget', - sql: 'ctype_Events__external_budget', - type: 'number', - alias: 'External Budget', - }, - { - name: 'ctype_Events__internal_budget', - sql: 'ctype_Events__internal_budget', - type: 'number', - alias: 'Internal Budget', - }, - { - name: 'ctype_Events__is_devrev_event', - sql: 'ctype_Events__is_devrev_event', - type: 'boolean', - alias: 'Is DevRev Event', - }, - { - name: 'ctype_Events__mode', - sql: 'ctype_Events__mode', - type: 'string', - alias: 'Mode', - }, - { - name: 'ctype_Events__organized_by', - sql: 'ctype_Events__organized_by', - type: 'string', - alias: 'Organized by', - }, - { - name: 'ctype_Events__pipeline_generated', - sql: 'ctype_Events__pipeline_generated', - type: 'number', - alias: 'Pipeline Generated', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_Events__representatives', - sql: 'ctype_Events__representatives', - type: 'string_array', - alias: 'Representatives', - }, - { - name: 'ctype_Events__source', - sql: 'ctype_Events__source', - type: 'string', - alias: 'Source', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_Events__sponsors', - sql: 'ctype_Events__sponsors', - type: 'string_array', - alias: 'Sponsors', - }, - { - name: 'ctype_Events__sub_source', - sql: 'ctype_Events__sub_source', - type: 'string', - alias: 'Sub-Source', - }, - { - name: 'ctype_Events__total_budget', - sql: 'ctype_Events__total_budget', - type: 'number', - alias: 'Total Budget', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_attachments_cfid', - type: 'boolean', - alias: 'allow_attachments', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__allow_channelback_cfid', - type: 'boolean', - alias: 'allow_channelback', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__collaborator_ids', - type: 'string_array', - alias: 'Collaborators', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ext_object_type', - type: 'string', - alias: 'External Object Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__external_id_cfid', - type: 'string', - alias: 'external_id', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__fields_modified_during_import', - type: 'string_array', - alias: 'Fields modified during import', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__from_messaging_channel_cfid', - type: 'boolean', - alias: 'from_messaging_channel', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__has_incidents_cfid', - type: 'boolean', - alias: 'has_incidents', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__is_public_cfid', - type: 'boolean', - alias: 'is_public', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_reporters', - type: 'string_array', - alias: 'Reporters', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__original_rev_org', - type: 'string', - alias: 'Customer', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__raw_subject_cfid', - type: 'string', - alias: 'raw_subject', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__recipient_cfid', - type: 'string', - alias: 'recipient', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ro_source_item', - type: 'string', - alias: 'External Source', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__url_cfid', - type: 'string', - alias: 'url', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x12840984514707_cfid', - type: 'string', - alias: 'Hold Status', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007715801_cfid', - type: 'string', - alias: 'ATI Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x1500007831982_cfid', - type: 'string', - alias: 'Event ID', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x16770809925651_cfid', - type: 'boolean', - alias: 'Do Not Close', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x19445665382931_cfid', - type: 'boolean', - alias: 'Escalated', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000203467_cfid', - type: 'string', - alias: 'Partner: Sales Contact', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000212308_cfid', - type: 'string', - alias: 'Partner Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000220288_cfid', - type: 'string', - alias: 'Env Setup Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000222327_cfid', - type: 'string', - alias: 'Additional Comments', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000229528_cfid', - type: 'string', - alias: 'Remind me before', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245507_cfid', - type: 'string', - alias: ' zIPS + z3A Licenses', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360000245527_cfid', - type: 'string', - alias: 'zDefend Licenses', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360002877873_cfid', - type: 'string', - alias: 'Approval Status', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003033353_cfid', - type: 'string', - alias: 'Booking Year', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003070794_cfid', - type: 'string', - alias: 'Term', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360003637633_cfid', - type: 'string', - alias: 'End Customer : Admin Details', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004164014_cfid', - type: 'string_array', - alias: 'Root Cause', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360004926333_cfid', - type: 'string', - alias: 'Request Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403033_cfid', - type: 'string', - alias: 'End User - Full Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360005403153_cfid', - type: 'string', - alias: 'End Customer: Administrator Email Address', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006394633_cfid', - type: 'string', - alias: 'Services Purchased', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360006484754_cfid', - type: 'string', - alias: 'Support Purchased', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007214193_cfid', - type: 'string', - alias: 'Cancellation Reason', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360007291334_cfid', - type: 'string', - alias: 'Cancellation Date', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928013_cfid', - type: 'string', - alias: 'zConsole URL', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009928193_cfid', - type: 'string', - alias: 'Customer Success Representative', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360009978873_cfid', - type: 'string', - alias: 'VPC Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360010017554_cfid', - type: 'string', - alias: 'End Customer: Region', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011850853_cfid', - type: 'string', - alias: 'Proof of Concept Impact ?', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011929414_cfid', - type: 'string', - alias: 'Internal Status', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360011934634_cfid', - type: 'string', - alias: 'Problem Sub-Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360012820673_cfid', - type: 'string', - alias: 'Tenant Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360019252873_cfid', - type: 'string', - alias: 'Customer Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277473_cfid', - type: 'string', - alias: 'Channel', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021277493_cfid', - type: 'string', - alias: 'Order Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289714_cfid', - type: 'string', - alias: 'zIPS Licenses', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021289734_cfid', - type: 'boolean', - alias: 'z3A License', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360021522734_cfid', - type: 'string', - alias: 'Sub-Partner Name', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360033488674_cfid', - type: 'string_array', - alias: 'Ticket Region', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036191933_cfid', - type: 'string', - alias: 'zShield Licenses', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360036262514_cfid', - type: 'string', - alias: 'zScan Apps Licensed', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x360046636694_cfid', - type: 'string', - alias: 'KPE MTD Licenses', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417555758995_cfid', - type: 'string', - alias: 'Product Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417589587987_cfid', - type: 'string', - alias: 'whiteCryption agent', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4417734769043_cfid', - type: 'string', - alias: 'zIPS Test Case Types', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x4619649774611_cfid', - type: 'string', - alias: 'Product Version', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80938687_cfid', - type: 'string', - alias: 'Target Delivery Date', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80942287_cfid', - type: 'string', - alias: 'App Version ', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949067_cfid', - type: 'string', - alias: 'End Customer: Company Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949087_cfid', - type: 'string', - alias: 'End Customer: Physical Address', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949527_cfid', - type: 'string', - alias: 'Term Start Date', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949547_cfid', - type: 'string', - alias: 'Term End Date', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80949587_cfid', - type: 'string', - alias: 'Order Fulfilled', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80953107_cfid', - type: 'string', - alias: 'Requested Delivery Date', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80966647_cfid', - type: 'string', - alias: 'Internal Tracking Id', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x80971487_cfid', - type: 'string', - alias: 'Feature Status', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81275168_cfid', - type: 'string', - alias: 'Case Type', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81315948_cfid', - type: 'string', - alias: 'Business Impact', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81335888_cfid', - type: 'string', - alias: 'Zimperium Console URL', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342748_cfid', - type: 'string', - alias: 'End Customer: Administrator Name', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81342948_cfid', - type: 'string', - alias: 'zIPS Only Licenses ', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__x81343048_cfid', - type: 'string', - alias: 'Zimperium Sales Contact: Email Address', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_attachments_cfid', - type: 'boolean', - alias: 'allow_attachments', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__allow_channelback_cfid', - type: 'boolean', - alias: 'allow_channelback', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__collaborator_ids', - type: 'string_array', - alias: 'Collaborators', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ext_object_type', - type: 'string', - alias: 'External Object Type', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__fields_modified_during_import', - type: 'string_array', - alias: 'Fields modified during import', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__from_messaging_channel_cfid', - type: 'boolean', - alias: 'from_messaging_channel', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__has_incidents_cfid', - type: 'boolean', - alias: 'has_incidents', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__is_public_cfid', - type: 'boolean', - alias: 'is_public', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_reporters', - type: 'string_array', - alias: 'Reporters', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__original_rev_org', - type: 'string', - alias: 'Customer', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__raw_subject_cfid', - type: 'string', - alias: 'raw_subject', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__recipient_cfid', - type: 'string', - alias: 'recipient', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ro_source_item', - type: 'string', - alias: 'External Source', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__url_cfid', - type: 'string', - alias: 'url', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x12840984514707_cfid', - type: 'string', - alias: 'Hold Status', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x16770809925651_cfid', - type: 'boolean', - alias: 'Do Not Close', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x19445665382931_cfid', - type: 'boolean', - alias: 'Escalated', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000203467_cfid', - type: 'string', - alias: 'Partner: Sales Contact', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000212308_cfid', - type: 'string', - alias: 'Partner Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360000220288_cfid', - type: 'string', - alias: 'Env Setup Type', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360002877873_cfid', - type: 'string', - alias: 'Approval Status', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360003033353_cfid', - type: 'string', - alias: 'Booking Year', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360004164014_cfid', - type: 'string_array', - alias: 'Root Cause', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006394633_cfid', - type: 'string', - alias: 'Services Purchased', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360006484754_cfid', - type: 'string', - alias: 'Support Purchased', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360009978873_cfid', - type: 'string', - alias: 'VPC Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360010017554_cfid', - type: 'string', - alias: 'End Customer: Region', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011850853_cfid', - type: 'string', - alias: 'Proof of Concept Impact ?', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360011934634_cfid', - type: 'string', - alias: 'Problem Sub-Type', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360012820673_cfid', - type: 'string', - alias: 'Tenant Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360019252873_cfid', - type: 'string', - alias: 'Customer Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277473_cfid', - type: 'string', - alias: 'Channel', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021277493_cfid', - type: 'string', - alias: 'Order Type', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289714_cfid', - type: 'string', - alias: 'zIPS Licenses', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021289734_cfid', - type: 'boolean', - alias: 'z3A License', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360021522734_cfid', - type: 'string', - alias: 'Sub-Partner Name', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x360033488674_cfid', - type: 'string_array', - alias: 'Ticket Region', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417555758995_cfid', - type: 'string', - alias: 'Product Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4417589587987_cfid', - type: 'string', - alias: 'whiteCryption agent', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x4619649774611_cfid', - type: 'string', - alias: 'Product Version', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80942287_cfid', - type: 'string', - alias: 'App Version ', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949067_cfid', - type: 'string', - alias: 'End Customer: Company Name', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949087_cfid', - type: 'string', - alias: 'End Customer: Physical Address', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949527_cfid', - type: 'string', - alias: 'Term Start Date', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949547_cfid', - type: 'string', - alias: 'Term End Date', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80949587_cfid', - type: 'string', - alias: 'Order Fulfilled', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80966647_cfid', - type: 'string', - alias: 'Internal Tracking Id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x80971487_cfid', - type: 'string', - alias: 'Feature Status', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81275168_cfid', - type: 'string', - alias: 'Case Type', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81315948_cfid', - type: 'string', - alias: 'Business Impact', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__x81335888_cfid', - type: 'string', - alias: 'DuplicateField Console URL', - }, - { - name: 'ctype_hell__test', - sql: 'ctype_hell__test', - type: 'string', - alias: 'test', - }, - { - name: 'ctype_hell__text_default_field', - sql: 'ctype_hell__text_default_field', - type: 'string', - alias: 'text default field', - }, - { - name: 'ctype_clone_ticket___clone_field_dropdown', - sql: 'ctype_clone_ticket___clone_field_dropdown', - type: 'string', - alias: 'clone field dropdown', - }, - { - name: 'ctype_test_subtype_sep__new_field_2', - sql: 'ctype_test_subtype_sep__new_field_2', - type: 'string', - alias: 'field_harsh', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_test_subtype_sep__new_field_id_list_3', - sql: 'ctype_test_subtype_sep__new_field_id_list_3', - type: 'string_array', - alias: 'field_id_list_harsh', - }, - { - name: 'ctype_vgtestyesno__booltestyesno', - sql: 'ctype_vgtestyesno__booltestyesno', - type: 'boolean', - alias: 'booltestyesno', - }, - { - name: 'ctype_asd__test_asd', - sql: 'ctype_asd__test_asd', - type: 'string', - alias: 'test asd', - }, - { - name: 'ctype_Microservice__git_template_name', - sql: 'ctype_Microservice__git_template_name', - type: 'string', - alias: 'Github Template Name', - }, - { - name: 'ctype_Microservice__micro_service_name', - sql: 'ctype_Microservice__micro_service_name', - type: 'string', - alias: 'Microservice Name', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_Microservice__platform', - sql: 'ctype_Microservice__platform', - type: 'string_array', - alias: 'Platform', - }, - { - name: 'ctype_Microservice__read_only', - sql: 'ctype_Microservice__read_only', - type: 'string', - alias: 'read only', - }, - { - name: 'ctype_helohelohelohelohelo__dfkdkl', - sql: 'ctype_helohelohelohelohelo__dfkdkl', - type: 'number', - alias: 'dfkdkl', - }, - { - name: 'ctype_helohelohelohelohelo__hello_1122', - sql: 'ctype_helohelohelohelohelo__hello_1122', - type: 'string', - alias: 'hello hello folks', - }, - { - name: 'ctype_helohelohelohelohelo__hello_12', - sql: 'ctype_helohelohelohelohelo__hello_12', - type: 'string', - alias: 'hello_12', - }, - { - name: 'ctype_helohelohelohelohelo__hello_123', - sql: 'ctype_helohelohelohelohelo__hello_123', - type: 'string', - alias: 'hello_123', - }, - { - name: 'ctype_helohelohelohelohelo__hello_12x', - sql: 'ctype_helohelohelohelohelo__hello_12x', - type: 'string', - alias: 'hello_12x', - }, - { - name: 'ctype_helohelohelohelohelo__hello_12y', - sql: 'ctype_helohelohelohelohelo__hello_12y', - type: 'string', - alias: 'hello_12y', - }, - { - name: 'ctype_helohelohelohelohelo__hello_wed', - sql: 'ctype_helohelohelohelohelo__hello_wed', - type: 'string', - alias: 'hello_wed', - }, - { - name: 'ctype_helohelohelohelohelo__hello_world', - sql: 'ctype_helohelohelohelohelo__hello_world', - type: 'string', - alias: 'hello_world', - }, - { - name: 'ctype_dddddddd__dkay_march_23', - sql: 'ctype_dddddddd__dkay_march_23', - type: 'string', - alias: 'dkay_march_23', - }, - { - name: 'ctype_dddddddd__dkay_march_26', - sql: 'ctype_dddddddd__dkay_march_26', - type: 'string', - alias: 'dkay_march_26', - }, - { - name: 'ctype_dddddddd__scope_approval', - sql: 'ctype_dddddddd__scope_approval', - type: 'string', - alias: 'Scope Approval', - }, - { - name: 'ctype_dddddddd__uat_approval', - sql: 'ctype_dddddddd__uat_approval', - type: 'string', - alias: 'UAT approval', - }, - { - name: 'ctype_defaults__customer_uat', - sql: 'ctype_defaults__customer_uat', - type: 'time', - alias: 'customer uat', - }, - { - name: 'ctype_defaults__default', - sql: 'ctype_defaults__default', - type: 'string', - alias: 'default', - }, - { - name: 'ctype_defaults__default_enum', - sql: 'ctype_defaults__default_enum', - type: 'string', - alias: 'default enum', - }, - { - name: 'ctype_defaults__enum_testing', - sql: 'ctype_defaults__enum_testing', - type: 'string', - alias: 'enum testing', - }, - { - name: 'ctype_defaults__uat_date', - sql: 'ctype_defaults__uat_date', - type: 'time', - alias: 'uat date', - }, - { - name: 'ctype_bug__abc', - sql: 'ctype_bug__abc', - type: 'string', - alias: 'abc', - }, - { - name: 'ctype_bug__dependent_test', - sql: 'ctype_bug__dependent_test', - type: 'string', - alias: 'something', - }, - { - name: 'ctype_event_request__approver', - sql: 'ctype_event_request__approver', - type: 'string', - alias: 'Approver', - }, - { - name: 'ctype_event_request__budget', - sql: 'ctype_event_request__budget', - type: 'number', - alias: 'Budget', - }, - { - name: 'ctype_event_request__event_date', - sql: 'ctype_event_request__event_date', - type: 'time', - alias: 'Event Date', - }, - { - name: 'ctype_event_request__event_location', - sql: 'ctype_event_request__event_location', - type: 'string', - alias: 'Event Location', - }, - { - name: 'ctype_event_request__event_name', - sql: 'ctype_event_request__event_name', - type: 'string', - alias: 'Event Name', - }, - { - name: 'ctype_event_request__event_owner', - sql: 'ctype_event_request__event_owner', - type: 'string', - alias: 'Event Owner', - }, - { - name: 'ctype_event_request__event_type', - sql: 'ctype_event_request__event_type', - type: 'string', - alias: 'Event Type', - }, - { - name: 'ctype_event_request__link_to_event', - sql: 'ctype_event_request__link_to_event', - type: 'string', - alias: 'Link To Event', - }, - { - name: 'ctype_event_request__region', - sql: 'ctype_event_request__region', - type: 'string', - alias: 'Region', - }, - { - name: 'ctype_event_request__requested_by', - sql: 'ctype_event_request__requested_by', - type: 'string', - alias: 'Requested by', - }, - { - name: 'ctype_event_request__status', - sql: 'ctype_event_request__status', - type: 'string', - alias: 'Status', - }, - { - name: 'ctype_event_request__target_pipeline', - sql: 'ctype_event_request__target_pipeline', - type: 'number', - alias: 'Target Pipeline', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_event_request__target_segment', - sql: 'ctype_event_request__target_segment', - type: 'string_array', - alias: 'Target Segment', - }, - { - name: 'ctype_mfz_subtype_check__enum_field', - sql: 'ctype_mfz_subtype_check__enum_field', - type: 'string', - alias: 'Custom Enum field', - }, - { - name: 'ctype_mfz_subtype_check__text_field', - sql: 'ctype_mfz_subtype_check__text_field', - type: 'string', - alias: 'Custom text field', - }, - { - name: 'ctype_mfz_subtype_check__user_field', - sql: 'ctype_mfz_subtype_check__user_field', - type: 'string', - alias: 'Custom user field', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_mfz_subtype_check__user_list_field', - sql: 'ctype_mfz_subtype_check__user_list_field', - type: 'string_array', - alias: 'Custom user list field', - }, - { - name: 'ctype_mfz_subtype_check__workspace_field', - sql: 'ctype_mfz_subtype_check__workspace_field', - type: 'string', - alias: 'Custom workspace field', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_mfz_subtype_check__workspace_list_field', - sql: 'ctype_mfz_subtype_check__workspace_list_field', - type: 'string_array', - alias: 'Custom workspace list field', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_attachments_cfid', - type: 'boolean', - alias: 'allow_attachments', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__allow_channelback_cfid', - type: 'boolean', - alias: 'allow_channelback', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__collaborator_ids', - type: 'string_array', - alias: 'Collaborators', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ext_object_type', - type: 'string', - alias: 'External Object Type', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__fields_modified_during_import', - type: 'string_array', - alias: 'Fields modified during import', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__from_messaging_channel_cfid', - type: 'boolean', - alias: 'from_messaging_channel', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__has_incidents_cfid', - type: 'boolean', - alias: 'has_incidents', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__is_public_cfid', - type: 'boolean', - alias: 'is_public', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_reporters', - type: 'string_array', - alias: 'Reporters', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__original_rev_org', - type: 'string', - alias: 'Customer', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__raw_subject_cfid', - type: 'string', - alias: 'raw_subject', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__recipient_cfid', - type: 'string', - alias: 'recipient', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ro_source_item', - type: 'string', - alias: 'External Source', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__url_cfid', - type: 'string', - alias: 'url', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x360033488674_cfid', - type: 'string_array', - alias: 'Ticket Region', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x4417589587987_cfid', - type: 'string', - alias: 'whiteCryption agent', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x80971487_cfid', - type: 'string', - alias: 'Feature Status', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__x81275168_cfid', - type: 'string', - alias: 'Case Type', - }, - { - name: 'ctype_mandatory_field_subtype__some_text_field', - sql: 'ctype_mandatory_field_subtype__some_text_field', - type: 'string', - alias: 'Some text field', - }, - { - name: 'ctype_review_subtype__date', - sql: 'ctype_review_subtype__date', - type: 'time', - alias: 'Date', - }, - { - name: 'ctype_review_subtype__play_store_review_url', - sql: 'ctype_review_subtype__play_store_review_url', - type: 'string', - alias: 'Review URL', - }, - { - name: 'ctype_review_subtype__rating', - sql: 'ctype_review_subtype__rating', - type: 'string', - alias: 'Rating', - }, - { - name: 'ctype_review_subtype__reply_date', - sql: 'ctype_review_subtype__reply_date', - type: 'time', - alias: 'Reply date', - }, - { - name: 'ctype_review_subtype__reply_text', - sql: 'ctype_review_subtype__reply_text', - type: 'string', - alias: 'Reply text', - }, - { - name: 'ctype_review_subtype__upvote_count', - sql: 'ctype_review_subtype__upvote_count', - type: 'number', - alias: 'Upvote Count', - }, - { - name: 'ctype_jkjkjkjkjkjk__dgg', - sql: 'ctype_jkjkjkjkjkjk__dgg', - type: 'string', - alias: 'dgg', - }, - { - name: 'ctype_jkjkjkjkjkjk__dggd', - sql: 'ctype_jkjkjkjkjkjk__dggd', - type: 'number', - alias: 'dggd', - }, - { - name: 'ctype_jkjkjkjkjkjk__hello', - sql: 'ctype_jkjkjkjkjkjk__hello', - type: 'string', - alias: 'hello', - }, - { - name: 'ctype_jkjkjkjkjkjk__hello_acl', - sql: 'ctype_jkjkjkjkjkjk__hello_acl', - type: 'string', - alias: 'hellO_acl', - }, - { - name: 'ctype_huryyyyyyyyyyy__all_internal_selected', - sql: 'ctype_huryyyyyyyyyyy__all_internal_selected', - type: 'string', - alias: 'all_external_selected', - }, - { - name: 'ctype_huryyyyyyyyyyy__all_selected', - sql: 'ctype_huryyyyyyyyyyy__all_selected', - type: 'string', - alias: 'all_selected', - }, - { - name: 'ctype_huryyyyyyyyyyy__test_acl_2', - sql: 'ctype_huryyyyyyyyyyy__test_acl_2', - type: 'string', - alias: 'test_acl_2', - }, - { - name: 'ctype_aaa__fkjdsfjkdj', - sql: 'ctype_aaa__fkjdsfjkdj', - type: 'string', - alias: 'fkjdsfjkdj', - }, - { - name: 'ctype_aaa__test_acl_march', - sql: 'ctype_aaa__test_acl_march', - type: 'string', - alias: 'test_acl_march', - }, - { - name: 'ctype_lolololololol__hell_acl_2', - sql: 'ctype_lolololololol__hell_acl_2', - type: 'string', - alias: 'hell_acl_2', - }, - { - name: 'ctype_lolololololol__hello_acl', - sql: 'ctype_lolololololol__hello_acl', - type: 'string', - alias: 'hello_acl', - }, - { - name: 'ctype_lolololololol__hello_acl_3', - sql: 'ctype_lolololololol__hello_acl_3', - type: 'string', - alias: 'hello_acl_3', - }, - { - name: 'ctype_zzz__dds', - sql: 'ctype_zzz__dds', - type: 'string', - alias: 'DDS', - }, - { - name: 'ctype_zzz__hello', - sql: 'ctype_zzz__hello', - type: 'string', - alias: 'hello ', - }, - { - name: 'ctype_zzz__hello_2', - sql: 'ctype_zzz__hello_2', - type: 'string', - alias: 'hello_2', - }, - { - name: 'ctype_zzz__hello_3', - sql: 'ctype_zzz__hello_3', - type: 'string', - alias: 'hello_3', - }, - { - name: 'ctype_zzz__hello_4', - sql: 'ctype_zzz__hello_4', - type: 'string', - alias: 'hello_4', - }, - { - name: 'ctype_zzz__sdkjdsfjk', - sql: 'ctype_zzz__sdkjdsfjk', - type: 'string', - alias: 'SDKJDSFJK', - }, - { - name: 'ctype_ggg__dkay_march_1234', - sql: 'ctype_ggg__dkay_march_1234', - type: 'string', - alias: 'dkay_march_1234', - }, - { - name: 'ctype_ggg__dkay_march_2', - sql: 'ctype_ggg__dkay_march_2', - type: 'string', - alias: 'dkay_march_2', - }, - { - name: 'ctype_ggg__dkay_test', - sql: 'ctype_ggg__dkay_test', - type: 'string', - alias: 'dkay_test', - }, - { - name: 'ctype_ooooo__dkay_march_3', - sql: 'ctype_ooooo__dkay_march_3', - type: 'string', - alias: 'dkay_march_3', - }, - { - name: 'ctype_ooooo__dkay_march_4', - sql: 'ctype_ooooo__dkay_march_4', - type: 'string', - alias: 'dkay_march_4', - }, - { - name: 'ctype_ooooo__dkay_march_5', - sql: 'ctype_ooooo__dkay_march_5', - type: 'string', - alias: 'dkay_march_5', - }, - { - name: 'ctype_ooooo__dkay_march_7', - sql: 'ctype_ooooo__dkay_march_7', - type: 'string', - alias: 'dkay_march_7', - }, - { - name: 'ctype_sync_engine_test__field1', - sql: 'ctype_sync_engine_test__field1', - type: 'string', - alias: 'field1', - }, - { - name: 'ctype_sync_engine_test__field2_bool', - sql: 'ctype_sync_engine_test__field2_bool', - type: 'boolean', - alias: 'field2_bool', - }, - { - name: 'ctype_priviliges_testing__read_only_field', - sql: 'ctype_priviliges_testing__read_only_field', - type: 'string', - alias: 'read only field', - }, - { - name: 'ctype_priviliges_testing__read_write_field', - sql: 'ctype_priviliges_testing__read_write_field', - type: 'string', - alias: 'read/write field', - }, - { - name: 'ctype_dashboard_test__pulkit', - sql: 'ctype_dashboard_test__pulkit', - type: 'boolean', - alias: 'pulkit', - }, - { - name: 'ctype_tttt__dropdown', - sql: 'ctype_tttt__dropdown', - type: 'string', - alias: 'dropdown', - }, - ], - joins: [ - { - sql: 'dim_ticket.sla_tracker_id = dim_sla_tracker.id', - }, - { - sql: 'dim_ticket.id = dim_survey_response.object', - }, - { - sql: 'dim_ticket.id = dim_link_issue_target.source_id', - }, - { - sql: 'dim_ticket.id = dim_link_conversation_source.target_id', - }, - { - sql: 'dim_ticket.rev_oid = dim_revo.id', - }, - { - sql: 'dim_ticket.applies_to_part_id = dim_part.id', - }, - ], - measures: [ - { - name: 'id_count', - function: { - type: 'count', - }, - sql: 'count(dim_ticket.id)', - type: 'string', - alias: 'Ticket Id', - }, - { - name: 'created_date_max', - function: { - type: 'max', - }, - sql: 'max(dim_ticket.created_date)', - type: 'time', - alias: 'Created date', - }, - { - name: 'actual_close_date_max', - function: { - type: 'max', - }, - sql: 'max(dim_ticket.actual_close_date)', - type: 'time', - alias: 'Closed date', - }, - { - name: 'sla_tracker_id_count', - function: { - type: 'count', - }, - sql: 'count(dim_ticket.sla_tracker_id)', - type: 'string', - alias: 'Sla Tracker Id', - }, - { - name: 'resolution_time', - sql: "case WHEN actual_close_date > created_date THEN date_diff('minutes', created_date, actual_close_date) ELSE null END ", - type: 'number', - alias: 'Resolution Time', - }, - { - name: 'surveys_aggregation_json_measure', - function: { - type: 'median', - }, - sql: "median(list_aggregate(CAST(json_extract_string(dim_ticket.surveys_aggregation_json, '$[*].minimum') AS integer[]), 'min'))", - type: 'number', - alias: 'CSAT Rating', - }, - { - name: 'tnt__account_id', - sql: 'tnt__account_id', - type: 'string', - alias: 'account id', - }, - { - name: 'tnt__actual_effort_spent', - sql: 'tnt__actual_effort_spent', - type: 'number', - alias: 'Actual Effort Spent', - }, - { - name: 'tnt__capability_part', - sql: 'tnt__capability_part', - type: 'string', - alias: 'Capability part', - }, - { - name: 'tnt__custom_id_field', - sql: 'tnt__custom_id_field', - type: 'string', - alias: 'custom id field', - }, - { - name: 'tnt__date_field', - sql: 'tnt__date_field', - type: 'time', - alias: 'date_field', - }, - { - name: 'tnt__estimated_effort', - sql: 'tnt__estimated_effort', - type: 'number', - alias: 'Estimated Effort', - }, - { - name: 'tnt__fruit', - sql: 'tnt__fruit', - type: 'string', - alias: 'Fruit', - }, - { - name: 'tnt__id_field', - sql: 'tnt__id_field', - type: 'string', - alias: 'id field', - }, - { - name: 'tnt__issue_score', - sql: 'tnt__issue_score', - type: 'number', - alias: 'Issue Score', - }, - { - name: 'tnt__numeric_field', - sql: 'tnt__numeric_field', - type: 'number', - alias: 'numeric field', - }, - { - name: 'tnt__parent_part', - sql: 'tnt__parent_part', - type: 'string', - alias: 'Parent part', - }, - { - name: 'tnt__part_product', - sql: 'tnt__part_product', - type: 'string', - alias: 'part product', - }, - { - name: 'tnt__rank', - sql: 'tnt__rank', - type: 'number', - alias: 'Rank', - }, - { - name: 'tnt__remaining_effort', - sql: 'tnt__remaining_effort', - type: 'number', - alias: 'Remaining Effort', - }, - { - name: 'tnt__stray_user', - sql: 'tnt__stray_user', - type: 'string', - alias: 'stray user', - }, - { - name: 'tnt__stray_users', - sql: 'tnt__stray_users', - type: 'string', - alias: 'stray users', - }, - { - name: 'tnt__test', - sql: 'tnt__test', - type: 'number', - alias: 'Test', - }, - { - name: 'tnt__test_capability_part', - sql: 'tnt__test_capability_part', - type: 'string', - alias: 'Test Capability Part', - }, - { - name: 'tnt__test_product_part', - sql: 'tnt__test_product_part', - type: 'string', - alias: 'Test Product part', - }, - { - name: 'tnt__ticket_custom_part', - sql: 'tnt__ticket_custom_part', - type: 'string', - alias: 'ticket custom part', - }, - { - name: 'tnt__workspace_custom', - sql: 'tnt__workspace_custom', - type: 'string', - alias: 'workspace_custom', - }, - { - name: 'ctype_deal_registration__estimated_deal_value', - sql: 'ctype_deal_registration__estimated_deal_value', - type: 'number', - alias: 'Estimated Deal Value', - }, - { - name: 'ctype_deal_registration__expected_close_date', - sql: 'ctype_deal_registration__expected_close_date', - type: 'time', - alias: 'Expected Close Date', - }, - { - name: 'ctype_Events__event_end_date', - sql: 'ctype_Events__event_end_date', - type: 'time', - alias: 'Event End Date', - }, - { - name: 'ctype_Events__event_owner', - sql: 'ctype_Events__event_owner', - type: 'string', - alias: 'Event Owner', - }, - { - name: 'ctype_Events__event_start_date', - sql: 'ctype_Events__event_start_date', - type: 'time', - alias: 'Event Start Date', - }, - { - name: 'ctype_Events__events_test_ref', - sql: 'ctype_Events__events_test_ref', - type: 'string', - alias: 'events test ref', - }, - { - name: 'ctype_Events__external_budget', - sql: 'ctype_Events__external_budget', - type: 'number', - alias: 'External Budget', - }, - { - name: 'ctype_Events__internal_budget', - sql: 'ctype_Events__internal_budget', - type: 'number', - alias: 'Internal Budget', - }, - { - name: 'ctype_Events__pipeline_generated', - sql: 'ctype_Events__pipeline_generated', - type: 'number', - alias: 'Pipeline Generated', - }, - { - name: 'ctype_Events__total_budget', - sql: 'ctype_Events__total_budget', - type: 'number', - alias: 'Total Budget', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - sql: 'ctype_svxsdhkjcbdsgcbvsdjgchgdshckweds__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwhe33cnrsy2__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_helohelohelohelohelo__dfkdkl', - sql: 'ctype_helohelohelohelohelo__dfkdkl', - type: 'number', - alias: 'dfkdkl', - }, - { - name: 'ctype_defaults__customer_uat', - sql: 'ctype_defaults__customer_uat', - type: 'time', - alias: 'customer uat', - }, - { - name: 'ctype_defaults__uat_date', - sql: 'ctype_defaults__uat_date', - type: 'time', - alias: 'uat date', - }, - { - name: 'ctype_event_request__approver', - sql: 'ctype_event_request__approver', - type: 'string', - alias: 'Approver', - }, - { - name: 'ctype_event_request__budget', - sql: 'ctype_event_request__budget', - type: 'number', - alias: 'Budget', - }, - { - name: 'ctype_event_request__event_date', - sql: 'ctype_event_request__event_date', - type: 'time', - alias: 'Event Date', - }, - { - name: 'ctype_event_request__event_owner', - sql: 'ctype_event_request__event_owner', - type: 'string', - alias: 'Event Owner', - }, - { - name: 'ctype_event_request__requested_by', - sql: 'ctype_event_request__requested_by', - type: 'string', - alias: 'Requested by', - }, - { - name: 'ctype_event_request__target_pipeline', - sql: 'ctype_event_request__target_pipeline', - type: 'number', - alias: 'Target Pipeline', - }, - { - name: 'ctype_mfz_subtype_check__user_field', - sql: 'ctype_mfz_subtype_check__user_field', - type: 'string', - alias: 'Custom user field', - }, - { - name: 'ctype_mfz_subtype_check__workspace_field', - sql: 'ctype_mfz_subtype_check__workspace_field', - type: 'string', - alias: 'Custom workspace field', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__brand_id_cfid', - type: 'number', - alias: 'brand_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__created_at_cfid', - type: 'time', - alias: 'created_at', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__generated_timestamp_cfid', - type: 'time', - alias: 'generated_timestamp', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__group_id_cfid', - type: 'number', - alias: 'group_id', - }, - { - name: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - sql: 'ctype_pjsy4zdfonxx66tjnxwgk4tjoxyx65djmnxyk5dtfzwvkzltoruy63q__ticket_form_id_cfid', - type: 'number', - alias: 'ticket_form_id', - }, - { - name: 'ctype_review_subtype__date', - sql: 'ctype_review_subtype__date', - type: 'time', - alias: 'Date', - }, - { - name: 'ctype_review_subtype__reply_date', - sql: 'ctype_review_subtype__reply_date', - type: 'time', - alias: 'Reply date', - }, - { - name: 'ctype_review_subtype__upvote_count', - sql: 'ctype_review_subtype__upvote_count', - type: 'number', - alias: 'Upvote Count', - }, - { - name: 'ctype_jkjkjkjkjkjk__dggd', - sql: 'ctype_jkjkjkjkjkjk__dggd', - type: 'number', - alias: 'dggd', - }, - { - name: 'id_count___function__count', - function: { - type: 'count', - }, - sql: 'count(dim_ticket.id)', - type: 'string', - alias: 'Ticket Id', - }, - ], - name: 'dim_ticket', - sql: 'select * from devrev.dim_ticket', - }, - ] as any; - const resolutionConfig = { - columnConfigs: [ - { - joinColumn: 'id', - name: 'dim_ticket.tags_json', - isArrayType: true, - resolutionColumns: ['name'], - source: 'dim_tag', - }, - ], - tableSchemas: [ - { - dimensions: [ - { - name: 'created_by_id', - sql: 'created_by_id', - type: 'string', - alias: 'Created by', - }, - { - name: 'dev_oid', - sql: 'dev_oid', - type: 'string', - alias: 'Dev organization ID', - }, - { - name: 'modified_date', - sql: 'modified_date', - type: 'time', - alias: 'Modified date', - }, - { - name: 'object_type', - sql: 'object_type', - type: 'string', - alias: 'Object type', - }, - { - name: 'tag_type', - sql: 'tag_type', - type: 'number', - alias: 'Values enabled', - }, - { - modifier: { - shouldUnnestGroupBy: false, - }, - name: 'allowed_values', - sql: 'allowed_values', - type: 'string_array', - alias: 'Allowed values', - }, - { - name: 'is_deleted', - sql: 'is_deleted', - type: 'boolean', - alias: 'Is deleted', - }, - { - name: 'modified_by_id', - sql: 'modified_by_id', - type: 'string', - alias: 'Modified by', - }, - { - name: 'access_level', - sql: 'access_level', - type: 'number', - alias: 'Access level', - }, - { - name: 'display_id', - sql: 'display_id', - type: 'string', - alias: 'Display ID', - }, - { - name: 'id', - sql: 'id', - type: 'string', - alias: 'ID', - }, - { - name: 'name', - sql: 'name', - type: 'string', - alias: 'Tag name', - }, - { - name: 'object_version', - sql: 'object_version', - type: 'number', - alias: 'Object Version', - }, - { - name: 'created_date', - sql: 'created_date', - type: 'time', - alias: 'Created date', - }, - ], - measures: [], - name: 'dim_tag', - sql: 'SELECT * FROM devrev.dim_tag', - }, - ], - } as any; - const columnProjections = [ - 'dim_ticket.tags_json', - 'dim_ticket.id_count___function__count', - ]; - - const resultQuery = await cubeQueryToSQLWithResolution({ - query, - tableSchemas, - resolutionConfig, - columnProjections, - }); - - const expectedQuery = `select * exclude(__row_id) from (SELECT MAX(__resolved_query."Ticket Id") AS "Ticket Id" , ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name" , "__row_id" FROM (SELECT __resolved_query."__row_id" AS "__row_id", * FROM (SELECT "Tags - Tag name", "Ticket Id", "__row_id" FROM (SELECT __unnested_base_query."Ticket Id" AS "Ticket Id", __unnested_base_query."__row_id" AS "__row_id", * FROM (SELECT "Ticket Id", "Tags", "__row_id" FROM (SELECT __base_query."Ticket Id" AS "Ticket Id", unnest(__base_query."Tags") AS "Tags", row_number() OVER () AS "__row_id", * FROM (SELECT count("ID") AS "Ticket Id" , "Tags" FROM (SELECT dim_ticket.created_date AS "Created date", 'ticket' AS "Work Type", json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", CAST(json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[]) AS "Tags", dim_ticket.id AS "ID", * FROM (select * from devrev.dim_ticket) AS dim_ticket) AS dim_ticket WHERE (((("Created date" >= '2025-08-02T09:30:00.000Z') AND ("Created date" <= '2025-10-31T10:29:59.999Z')) AND ("Work Type" IN ('ticket')) AND (Stage IN ('don:core:dvrv-us-1:devo/787:custom_stage/514', 'don:core:dvrv-us-1:devo/787:custom_stage/512', 'don:core:dvrv-us-1:devo/787:custom_stage/501', 'don:core:dvrv-us-1:devo/787:custom_stage/485', 'don:core:dvrv-us-1:devo/787:custom_stage/440', 'don:core:dvrv-us-1:devo/787:custom_stage/25', 'don:core:dvrv-us-1:devo/787:custom_stage/24', 'don:core:dvrv-us-1:devo/787:custom_stage/22', 'don:core:dvrv-us-1:devo/787:custom_stage/21', 'don:core:dvrv-us-1:devo/787:custom_stage/19', 'don:core:dvrv-us-1:devo/787:custom_stage/13', 'don:core:dvrv-us-1:devo/787:custom_stage/10', 'don:core:dvrv-us-1:devo/787:custom_stage/8', 'don:core:dvrv-us-1:devo/787:custom_stage/7', 'don:core:dvrv-us-1:devo/787:custom_stage/6', 'don:core:dvrv-us-1:devo/787:custom_stage/4')))) GROUP BY Tags LIMIT 50000) AS __base_query) AS __base_query) AS __unnested_base_query LEFT JOIN (SELECT __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", * FROM (SELECT * FROM devrev.dim_tag) AS __unnested_base_query__dim_ticket__tags_json) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags"=__unnested_base_query__dim_ticket__tags_json.id) AS MEERKAT_GENERATED_TABLE) AS __resolved_query) AS __resolved_query GROUP BY __row_id)`; - expect(resultQuery).toBe(expectedQuery); - }); - // it('Should handle only scalar field resolution without unnesting', async () => { - // const query: Query = { - // measures: ['tickets.count'], - // dimensions: [ - // 'tickets.id', - // 'tickets.owners', - // 'tickets.tags', - // 'tickets.created_by', - // ], - // }; - - // const resolutionConfig: ResolutionConfig = { - // columnConfigs: [ - // { - // name: 'tickets.created_by', - // isArrayType: false, - // source: 'created_by_lookup', - // joinColumn: 'id', - // resolutionColumns: ['name'], - // }, - // ], - // tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], - // }; - - // const sql = await cubeQueryToSQLWithResolutionWithArray({ - // query, - // tableSchemas: [TICKETS_TABLE_SCHEMA], - // resolutionConfig, - // }); - - // console.log('Phase 1 SQL (multiple arrays):', sql); - - // // Verify row_id is included - // expect(sql).toContain('row_id'); - - // // Both arrays should be unnested - // expect(sql.match(/unnest/g)?.length).toBeGreaterThanOrEqual(0); - - // // Execute the SQL to verify it works - // const result = (await duckdbExec(sql)) as any[]; - - // expect(result.length).toBe(7); - - // // Each row should have a row_id - // expect(result[0]).toHaveProperty('__row_id'); - // expect(result[0]).toHaveProperty('tickets__count'); - // expect(result[0]).toHaveProperty('tickets__id'); - // expect(result[0]).toHaveProperty('tickets__owners'); - // expect(result[0]).toHaveProperty('tickets__tags'); - // expect(result[0]).toHaveProperty('tickets__created_by - name'); - // }); - - // it('Should return regular SQL when no array fields need resolution', async () => { - // const query: Query = { - // measures: ['tickets.count'], - // dimensions: ['tickets.id', 'tickets.created_by'], - // }; - - // const resolutionConfig: ResolutionConfig = { - // columnConfigs: [], - // tableSchemas: [], - // }; - - // const sql = await cubeQueryToSQLWithResolutionWithArray({ - // query, - // tableSchemas: [TICKETS_TABLE_SCHEMA], - // resolutionConfig, - // }); - - // console.log('SQL without resolution:', sql); - - // // Should not have row_id or unnest when no array resolution is needed - // expect(sql).not.toContain('row_id'); - // expect(sql).not.toContain('unnest'); - // }); -}); - -// describe('cubeQueryToSQLWithResolutionWithArray - Phase 2: Resolution', () => { -// jest.setTimeout(1000000); - -// beforeAll(async () => { -// // Tables are already created in Phase 1 tests -// }); - -// it('Should resolve array values by joining with lookup tables', async () => { -// const query: Query = { -// measures: ['tickets.count'], -// dimensions: ['tickets.id', 'tickets.owners'], -// }; - -// const resolutionConfig: ResolutionConfig = { -// columnConfigs: [ -// { -// name: 'tickets.owners', -// isArrayType: true, -// source: 'owners_lookup', -// joinColumn: 'id', -// resolutionColumns: ['display_name', 'email'], -// }, -// ], -// tableSchemas: [OWNERS_LOOKUP_SCHEMA], -// }; - -// const sql = await cubeQueryToSQLWithResolutionWithArray({ -// query, -// tableSchemas: [TICKETS_TABLE_SCHEMA], -// resolutionConfig, -// }); - -// console.log('Phase 2 SQL (with resolution):', sql); - -// // Verify the SQL includes row_id -// expect(sql).toContain('row_id'); - -// // Verify the SQL has joins to resolution tables -// expect(sql).toContain('owners_lookup'); - -// // Verify it includes resolution columns -// expect(sql).toContain('display_name'); -// expect(sql).toContain('email'); - -// // Execute the SQL to verify it works -// const result = (await duckdbExec(sql)) as any[]; -// console.log( -// 'Phase 2 Result (first 3 rows):', -// JSON.stringify(result.slice(0, 3), null, 2) -// ); - -// // The result should have 5 unnested + resolved rows -// expect(result.length).toBe(5); - -// // Each row should have resolution columns -// expect(result[0]).toHaveProperty('__unnested_base_query__row_id'); -// expect(result[0]).toHaveProperty('tickets__owners__display_name'); -// expect(result[0]).toHaveProperty('tickets__owners__email'); - -// // Verify actual resolved values -// const owner1Rows = result.filter( -// (r: any) => r['__unnested_base_query__tickets__owners'] === 'owner1' -// ); -// expect(owner1Rows[0]['tickets__owners__display_name']).toBe('Alice Smith'); -// expect(owner1Rows[0]['tickets__owners__email']).toBe('alice@example.com'); -// }); -// }); - -// describe('cubeQueryToSQLWithResolutionWithArray - Phase 3: Re-aggregation', () => { -// jest.setTimeout(1000000); - -// beforeAll(async () => { -// // Tables are already created in Phase 1 tests -// }); - -// it('Should re-aggregate unnested rows back to original count with resolved arrays', async () => { -// const query: Query = { -// measures: ['tickets.count'], -// dimensions: ['tickets.id', 'tickets.owners'], -// }; - -// const resolutionConfig: ResolutionConfig = { -// columnConfigs: [ -// { -// name: 'tickets.owners', -// isArrayType: true, -// source: 'owners_lookup', -// joinColumn: 'id', -// resolutionColumns: ['display_name', 'email'], -// }, -// ], -// tableSchemas: [OWNERS_LOOKUP_SCHEMA], -// }; - -// const sql = await cubeQueryToSQLWithResolutionWithArray({ -// query, -// tableSchemas: [TICKETS_TABLE_SCHEMA], -// resolutionConfig, -// }); - -// console.log('Phase 3 SQL (re-aggregated):', sql); - -// // Verify the SQL includes aggregation functions -// expect(sql).toContain('GROUP BY'); -// expect(sql).toContain('row_id'); - -// // Execute the SQL to verify it works -// const result = (await duckdbExec(sql)) as any[]; -// console.log('Phase 3 Result:', JSON.stringify(result, null, 2)); - -// // Should have 3 rows (back to original count) -// expect(result.length).toBe(3); - -// // Each row should have aggregated data -// expect(result[0]).toHaveProperty('__aggregation_base__row_id'); -// expect(result[0]).toHaveProperty('__aggregation_base__tickets__count'); -// expect(result[0]).toHaveProperty('__aggregation_base__tickets__id'); - -// // Should have arrays with resolved values -// expect(result[0]).toHaveProperty( -// '__aggregation_base__tickets__owners__display_name' -// ); -// expect(result[0]).toHaveProperty( -// '__aggregation_base__tickets__owners__email' -// ); - -// // Verify the resolved arrays contain the correct values -// // Ticket 1 has owners: owner1, owner2 -// const ticket1 = result.find( -// (r: any) => r['__aggregation_base__tickets__id'] === 1 -// ); -// expect(ticket1).toBeDefined(); - -// // The display names should be aggregated into an array -// const displayNames = -// ticket1['__aggregation_base__tickets__owners__display_name']; -// console.log('Ticket 1 display names:', displayNames); - -// // Note: The actual format depends on how cubeQueryToSQL handles ARRAY_AGG -// // It should be an array containing the resolved values -// }); -// }); diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts new file mode 100644 index 00000000..3c3ef8b4 --- /dev/null +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -0,0 +1,454 @@ +import { Query, ResolutionConfig, TableSchema } from '@devrev/meerkat-core'; +import { cubeQueryToSQLWithResolution } from '../cube-to-sql-with-resolution/cube-to-sql-with-resolution'; +import { duckdbExec } from '../duckdb-exec'; +const CREATE_TEST_TABLE = `CREATE TABLE tickets ( + id INTEGER, + owners VARCHAR[], + tags VARCHAR[], + created_by VARCHAR, + subscribers_count INTEGER +)`; + +const INPUT_DATA_QUERY = `INSERT INTO tickets VALUES +(1, ['owner1', 'owner2'], ['tag1'], 'user1', 30), +(2, ['owner2', 'owner3'], ['tag2', 'tag3'], 'user2', 10), +(3, ['owner4'], ['tag1', 'tag4', 'tag3'], 'user3', 80)`; + +const CREATE_RESOLUTION_TABLE = `CREATE TABLE owners_lookup ( + id VARCHAR, + display_name VARCHAR, + email VARCHAR +)`; + +const RESOLUTION_DATA_QUERY = `INSERT INTO owners_lookup VALUES +('owner1', 'Alice Smith', 'alice@example.com'), +('owner2', 'Bob Jones', 'bob@example.com'), +('owner3', 'Charlie Brown', 'charlie@example.com'), +('owner4', 'Diana Prince', 'diana@example.com')`; + +const CREATE_TAGS_LOOKUP_TABLE = `CREATE TABLE tags_lookup ( + id VARCHAR, + tag_name VARCHAR +)`; +const CREATE_CREATED_BY_LOOKUP_TABLE = `CREATE TABLE created_by_lookup ( + id VARCHAR, + name VARCHAR +)`; +const CREATED_BY_LOOKUP_DATA_QUERY = `INSERT INTO created_by_lookup VALUES +('user1', 'User 1'), +('user2', 'User 2'), +('user3', 'User 3')`; +const TAGS_LOOKUP_DATA_QUERY = `INSERT INTO tags_lookup VALUES +('tag1', 'Tag 1'), +('tag2', 'Tag 2'), +('tag3', 'Tag 3'), +('tag4', 'Tag 4')`; + +const TICKETS_TABLE_SCHEMA: TableSchema = { + name: 'tickets', + sql: 'select * from tickets', + measures: [ + { + name: 'count', + sql: 'COUNT(*)', + type: 'number', + }, + ], + dimensions: [ + { + alias: 'ID', + name: 'id', + sql: 'id', + type: 'number', + }, + { + alias: 'Created By', + name: 'created_by', + sql: 'created_by', + type: 'string', + }, + { + alias: 'Owners', + name: 'owners', + sql: 'owners', + type: 'string_array', + }, + { + alias: 'Tags', + name: 'tags', + sql: 'tags', + type: 'string_array', + }, + ], +}; + +const OWNERS_LOOKUP_SCHEMA: TableSchema = { + name: 'owners_lookup', + sql: 'select * from owners_lookup', + measures: [], + dimensions: [ + { + alias: 'ID', + name: 'id', + sql: 'id', + type: 'string', + }, + { + alias: 'Display Name', + name: 'display_name', + sql: 'display_name', + type: 'string', + }, + { + alias: 'Email', + name: 'email', + sql: 'email', + type: 'string', + }, + ], +}; + +const TAGS_LOOKUP_SCHEMA: TableSchema = { + name: 'tags_lookup', + sql: 'select * from tags_lookup', + measures: [], + dimensions: [ + { + alias: 'ID', + name: 'id', + sql: 'id', + type: 'string', + }, + { + alias: 'Tag Name', + name: 'tag_name', + sql: 'tag_name', + type: 'string', + }, + ], +}; + +const CREATED_BY_LOOKUP_SCHEMA: TableSchema = { + name: 'created_by_lookup', + sql: 'select * from created_by_lookup', + measures: [], + dimensions: [ + { + alias: 'ID', + name: 'id', + sql: 'id', + type: 'string', + }, + { + alias: 'Name', + name: 'name', + sql: 'name', + type: 'string', + }, + ], +}; +describe('cubeQueryToSQLWithResolution - Array field resolution', () => { + jest.setTimeout(1000000); + beforeAll(async () => { + // Create test tables + await duckdbExec(CREATE_TEST_TABLE); + await duckdbExec(INPUT_DATA_QUERY); + await duckdbExec(CREATE_RESOLUTION_TABLE); + await duckdbExec(RESOLUTION_DATA_QUERY); + await duckdbExec(CREATE_TAGS_LOOKUP_TABLE); + await duckdbExec(TAGS_LOOKUP_DATA_QUERY); + await duckdbExec(CREATE_CREATED_BY_LOOKUP_TABLE); + await duckdbExec(CREATED_BY_LOOKUP_DATA_QUERY); + }); + + it('Should resolve array fields with lookup tables', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: ['tickets.id', 'tickets.owners'], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.owners', + isArrayType: true, + source: 'owners_lookup', + joinColumn: 'id', + resolutionColumns: ['display_name', 'email'], + }, + ], + tableSchemas: [OWNERS_LOOKUP_SCHEMA], + }; + + const sql = await cubeQueryToSQLWithResolution({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + columnProjections: ['tickets.owners', 'tickets.count', 'tickets.id'], + }); + + console.log('SQL with resolution:', sql); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + console.log('Result:', result); + + // Without array unnesting, should have 3 rows (original count) + expect(result.length).toBe(3); + + // Each row should have the expected properties + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('ID'); + + // The owners field should be resolved with display_name and email + expect(result[0]).toHaveProperty('Owners - Display Name'); + expect(result[0]).toHaveProperty('Owners - Email'); + + const id1Record = result.find((r: any) => r.ID === 1); + // Note: Array order may not be preserved without index tracking in UNNEST/ARRAY_AGG + expect(id1Record['Owners - Display Name']).toEqual( + expect.arrayContaining(['Alice Smith', 'Bob Jones']) + ); + expect(id1Record['Owners - Email']).toEqual( + expect.arrayContaining(['alice@example.com', 'bob@example.com']) + ); + + const id2Record = result.find((r: any) => r.ID === 2); + expect(id2Record['Owners - Display Name']).toEqual( + expect.arrayContaining(['Bob Jones', 'Charlie Brown']) + ); + expect(id2Record['Owners - Email']).toEqual( + expect.arrayContaining(['bob@example.com', 'charlie@example.com']) + ); + }); + + it('Should handle multiple array fields that need unnesting', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: [ + 'tickets.id', + 'tickets.owners', //array + 'tickets.tags', // array + 'tickets.created_by', // scalar + ], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.owners', + isArrayType: true, + source: 'owners_lookup', + joinColumn: 'id', + resolutionColumns: ['display_name'], + }, + { + name: 'tickets.tags', + isArrayType: true, + source: 'tags_lookup', + joinColumn: 'id', + resolutionColumns: ['tag_name'], + }, + { + name: 'tickets.created_by', + isArrayType: false, + source: 'created_by_lookup', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [ + OWNERS_LOOKUP_SCHEMA, + TAGS_LOOKUP_SCHEMA, + CREATED_BY_LOOKUP_SCHEMA, + ], + }; + + const columnProjections = [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + 'tickets.count', + ]; + const sql = await cubeQueryToSQLWithResolution({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + columnProjections, + }); + + console.log('SQL (multiple arrays):', sql); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + console.log('Result:', result); + + // Should have 3 rows (original ticket count) + expect(result.length).toBe(3); + + // Each row should have the expected properties + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('ID'); + expect(result[0]).toHaveProperty('Owners - Display Name'); + expect(result[0]).toHaveProperty('Tags - Tag Name'); + expect(result[0]).toHaveProperty('Created By - Name'); + + // Verify ticket 1: 2 owners, 1 tag + const ticket1 = result.find((r: any) => r.ID === 1); + expect(ticket1['Owners - Display Name']).toEqual( + expect.arrayContaining(['Alice Smith', 'Bob Jones']) + ); + expect(ticket1['Owners - Display Name'].length).toBe(2); + expect(ticket1['Tags - Tag Name']).toEqual( + expect.arrayContaining(['Tag 1']) + ); + expect(ticket1['Tags - Tag Name'].length).toBe(1); + expect(ticket1['Created By - Name']).toBe('User 1'); + + // Verify ticket 2: 2 owners, 2 tags + const ticket2 = result.find((r: any) => r.ID === 2); + expect(ticket2['Owners - Display Name']).toEqual( + expect.arrayContaining(['Bob Jones', 'Charlie Brown']) + ); + expect(ticket2['Owners - Display Name'].length).toBe(2); + expect(ticket2['Tags - Tag Name']).toEqual( + expect.arrayContaining(['Tag 2', 'Tag 3']) + ); + expect(ticket2['Tags - Tag Name'].length).toBe(2); + expect(ticket2['Created By - Name']).toBe('User 2'); + + // Verify ticket 3: 1 owner, 3 tags + const ticket3 = result.find((r: any) => r.ID === 3); + expect(ticket3['Owners - Display Name']).toEqual( + expect.arrayContaining(['Diana Prince']) + ); + expect(ticket3['Owners - Display Name'].length).toBe(1); + expect(ticket3['Tags - Tag Name']).toEqual( + expect.arrayContaining(['Tag 1', 'Tag 3', 'Tag 4']) + ); + expect(ticket3['Tags - Tag Name'].length).toBe(3); + expect(ticket3['Created By - Name']).toBe('User 3'); + }); + + it('Should handle only scalar field resolution without unnesting', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + ], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.created_by', + isArrayType: false, + source: 'created_by_lookup', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], + }; + + const columnProjections = [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + 'tickets.count', + ]; + + const sql = await cubeQueryToSQLWithResolution({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + columnProjections, + }); + + console.log('SQL (scalar resolution only):', sql); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + console.log('Result:', result); + + // Should have 3 rows (no array unnesting, only scalar resolution) + expect(result.length).toBe(3); + + // Each row should have the expected properties + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('ID'); + expect(result[0]).toHaveProperty('Owners'); // Original array, not resolved + expect(result[0]).toHaveProperty('Tags'); // Original array, not resolved + expect(result[0]).toHaveProperty('Created By - Name'); // Resolved scalar field + + // Verify scalar resolution worked correctly + const ticket1 = result.find((r: any) => r.ID === 1); + expect(ticket1['Created By - Name']).toBe('User 1'); + expect(Array.isArray(ticket1['Owners'])).toBe(true); + expect(ticket1['Owners']).toEqual(['owner1', 'owner2']); + expect(Array.isArray(ticket1['Tags'])).toBe(true); + expect(ticket1['Tags']).toEqual(['tag1']); + + const ticket2 = result.find((r: any) => r.ID === 2); + expect(ticket2['Created By - Name']).toBe('User 2'); + expect(ticket2['Owners']).toEqual(['owner2', 'owner3']); + expect(ticket2['Tags']).toEqual(['tag2', 'tag3']); + + const ticket3 = result.find((r: any) => r.ID === 3); + expect(ticket3['Created By - Name']).toBe('User 3'); + expect(ticket3['Owners']).toEqual(['owner4']); + expect(ticket3['Tags']).toEqual(['tag1', 'tag4', 'tag3']); + }); + + it('Should return regular SQL when no resolution is configured', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: ['tickets.id', 'tickets.created_by'], + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [], + tableSchemas: [], + }; + + const sql = await cubeQueryToSQLWithResolution({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + }); + + console.log('SQL without resolution:', sql); + + // Should not have resolution-specific features when no resolution is configured + expect(sql).not.toContain('__row_id'); + expect(sql).not.toContain('unnest'); + expect(sql).not.toContain('ARRAY_AGG'); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + console.log('Result:', result); + + // Should have 3 rows (original ticket count) + expect(result.length).toBe(3); + + // Each row should have basic properties (no resolution, so original column names) + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('ID'); + expect(result[0]).toHaveProperty('Created By'); + + // Verify data is correct (original scalar values, not resolved) + const ticket1 = result.find((r: any) => r.ID === 1); + expect(ticket1['Created By']).toBe('user1'); + + const ticket2 = result.find((r: any) => r.ID === 2); + expect(ticket2['Created By']).toBe('user2'); + + const ticket3 = result.find((r: any) => r.ID === 3); + expect(ticket3['Created By']).toBe('user3'); + }); +}); From 95beab12b6ee9587835c0834538f905d1402ccd3 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 01:29:44 +0530 Subject: [PATCH 20/38] ensuring we are having the same order by using row_id --- .../browser-cube-to-sql-with-resolution.ts | 35 ++++++++++++++----- meerkat-core/src/resolution/resolution.ts | 15 ++++++++ .../cube-to-sql-with-resolution.ts | 35 ++++++++++++++----- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 60ddca69..504a3066 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -16,6 +16,7 @@ import { ROW_ID_DIMENSION_NAME, TableSchema, updateArrayFlattenModifierUsingResolutionConfig, + wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; import { @@ -78,6 +79,14 @@ export const cubeQueryToSQLWithResolution = async ({ query.dimensions ); + // Add row_id dimension to preserve ordering from base SQL + baseTable.dimensions.push({ + name: ROW_ID_DIMENSION_NAME, + sql: 'row_number() OVER ()', + type: 'number', + alias: ROW_ID_DIMENSION_NAME, + } as Dimension); + const resolutionSchemas: TableSchema[] = generateResolutionSchemas( resolutionConfig, tableSchemas @@ -87,12 +96,16 @@ export const cubeQueryToSQLWithResolution = async ({ connection, query: { measures: [], - dimensions: generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), + dimensions: [ + ...generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, + query, + resolutionConfig, + columnProjections + ), + // Include row_id in dimensions to preserve it through the query + getNamespacedKey(BASE_DATA_SOURCE_NAME, ROW_ID_DIMENSION_NAME), + ], joinPaths: generateResolutionJoinPaths( BASE_DATA_SOURCE_NAME, resolutionConfig, @@ -103,7 +116,8 @@ export const cubeQueryToSQLWithResolution = async ({ }; const sql = await cubeQueryToSQL(resolveParams); - return sql; + // Order by row_id to maintain base SQL ordering, then exclude it + return wrapWithRowIdOrderingAndExclusion(sql, ROW_ID_DIMENSION_NAME); } }; @@ -415,6 +429,9 @@ export const getAggregatedSql = async ({ contextParams, }); - const rowIdExcludedSql = `select * exclude(${ROW_ID_DIMENSION_NAME}) from (${aggregatedSql})`; - return rowIdExcludedSql; + // Order by row_id to maintain consistent ordering before excluding it + return wrapWithRowIdOrderingAndExclusion( + aggregatedSql, + ROW_ID_DIMENSION_NAME + ); }; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 45066252..90fa674f 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -228,3 +228,18 @@ export const generateResolutionJoinPaths = ( }, ]); }; + +/** + * Wraps SQL to order by row_id and then exclude it from results. + * This maintains the ordering from the base query while removing the internal row_id column. + * + * @param sql - The SQL query that includes a __row_id column + * @param rowIdColumnName - The name of the row_id column (defaults to '__row_id') + * @returns SQL query ordered by row_id with the row_id column excluded + */ +export const wrapWithRowIdOrderingAndExclusion = ( + sql: string, + rowIdColumnName: string +): string => { + return `select * exclude(${rowIdColumnName}) from (${sql}) order by ${rowIdColumnName}`; +}; diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index f564fc3e..c21315a0 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -16,6 +16,7 @@ import { ROW_ID_DIMENSION_NAME, TableSchema, updateArrayFlattenModifierUsingResolutionConfig, + wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { cubeQueryToSQL, @@ -73,6 +74,14 @@ export const cubeQueryToSQLWithResolution = async ({ query.dimensions ); + // Add row_id dimension to preserve ordering from base SQL + baseTable.dimensions.push({ + name: ROW_ID_DIMENSION_NAME, + sql: 'row_number() OVER ()', + type: 'number', + alias: ROW_ID_DIMENSION_NAME, + } as Dimension); + const resolutionSchemas: TableSchema[] = generateResolutionSchemas( resolutionConfig, tableSchemas @@ -81,12 +90,16 @@ export const cubeQueryToSQLWithResolution = async ({ const resolveParams: CubeQueryToSQLParams = { query: { measures: [], - dimensions: generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), + dimensions: [ + ...generateResolvedDimensions( + BASE_DATA_SOURCE_NAME, + query, + resolutionConfig, + columnProjections + ), + // Include row_id in dimensions to preserve it through the query + getNamespacedKey(BASE_DATA_SOURCE_NAME, ROW_ID_DIMENSION_NAME), + ], joinPaths: generateResolutionJoinPaths( BASE_DATA_SOURCE_NAME, resolutionConfig, @@ -97,7 +110,8 @@ export const cubeQueryToSQLWithResolution = async ({ }; const sql = await cubeQueryToSQL(resolveParams); - return sql; + // Order by row_id to maintain base SQL ordering, then exclude it + return wrapWithRowIdOrderingAndExclusion(sql, ROW_ID_DIMENSION_NAME); } }; @@ -395,6 +409,9 @@ export const getAggregatedSql = async ({ contextParams, }); - const rowIdExcludedSql = `select * exclude(${ROW_ID_DIMENSION_NAME}) from (${aggregatedSql})`; - return rowIdExcludedSql; + // Order by row_id to maintain consistent ordering before excluding it + return wrapWithRowIdOrderingAndExclusion( + aggregatedSql, + ROW_ID_DIMENSION_NAME + ); }; From 4ae0e4e7ee551680500de7fb0797f49885bcb719 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 02:50:58 +0530 Subject: [PATCH 21/38] final ordering changes for row number --- .../browser-cube-to-sql-with-resolution.ts | 21 ++- .../src/resolution/resolution.spec.ts | 177 ++++++++++++++++++ meerkat-core/src/resolution/resolution.ts | 35 ++++ .../cube-to-sql-with-resolution.spec.ts | 120 +++++++++++- .../cube-to-sql-with-resolution.ts | 16 +- 5 files changed, 351 insertions(+), 18 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 504a3066..b77bfbba 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -7,6 +7,7 @@ import { generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + generateRowNumberSql, getArrayTypeResolutionColumnConfigs, getNamespacedKey, Measure, @@ -48,11 +49,6 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0) { - // If no resolution is needed, return the base SQL. - return baseSql; - } - if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { // This is to ensure that, only the column projection columns // are being resolved and other definitions are ignored. @@ -64,6 +60,7 @@ export const cubeQueryToSQLWithResolution = async ({ return cubeQueryToSQLWithResolutionWithArray({ connection, baseSql, + query, tableSchemas, resolutionConfig, columnProjections, @@ -82,7 +79,11 @@ export const cubeQueryToSQLWithResolution = async ({ // Add row_id dimension to preserve ordering from base SQL baseTable.dimensions.push({ name: ROW_ID_DIMENSION_NAME, - sql: 'row_number() OVER ()', + sql: generateRowNumberSql( + query, + baseTable.dimensions, + BASE_DATA_SOURCE_NAME + ), type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); @@ -124,6 +125,7 @@ export const cubeQueryToSQLWithResolution = async ({ export const cubeQueryToSQLWithResolutionWithArray = async ({ connection, baseSql, + query, tableSchemas, resolutionConfig, columnProjections, @@ -131,6 +133,7 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ }: { connection: AsyncDuckDBConnection; baseSql: string; + query: Query; tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; columnProjections?: string[]; @@ -146,7 +149,11 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ baseSchema.dimensions.push({ name: ROW_ID_DIMENSION_NAME, - sql: 'row_number() OVER ()', + sql: generateRowNumberSql( + query, + baseSchema.dimensions, + BASE_DATA_SOURCE_NAME + ), type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index a5fde3cc..e872835c 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -4,6 +4,7 @@ import { generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + generateRowNumberSql, getArrayTypeResolutionColumnConfigs, updateArrayFlattenModifierUsingResolutionConfig, } from './resolution'; @@ -1150,3 +1151,179 @@ describe('updateArrayFlattenModifierUsingResolutionConfig', () => { expect(baseTableSchema.dimensions).toEqual([]); }); }); + +describe('generateRowNumberSql', () => { + it('should generate row_number with ORDER BY for single column', () => { + const query = { + order: { 'table.id': 'asc' }, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + { + name: 'table__name', + alias: 'Name', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe('row_number() OVER (ORDER BY __base_query."ID" ASC)'); + }); + + it('should generate row_number with ORDER BY for multiple columns', () => { + const query = { + order: { 'table.id': 'asc', 'table.name': 'desc' }, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + { + name: 'table__name', + alias: 'Name', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe( + 'row_number() OVER (ORDER BY __base_query."ID" ASC, __base_query."Name" DESC)' + ); + }); + + it('should generate row_number without ORDER BY when query has no order', () => { + const query = {}; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe('row_number() OVER ()'); + }); + + it('should generate row_number without ORDER BY when order is empty', () => { + const query = { + order: {}, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe('row_number() OVER ()'); + }); + + it('should use dimension name when alias is not present', () => { + const query = { + order: { 'table.id': 'asc' }, + }; + const dimensions = [ + { + name: 'table__id', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe( + 'row_number() OVER (ORDER BY __base_query."table__id" ASC)' + ); + }); + + it('should handle dimension not found by using safe member name', () => { + const query = { + order: { 'table.unknown_column': 'desc' }, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe( + 'row_number() OVER (ORDER BY __base_query."table__unknown_column" DESC)' + ); + }); + + it('should handle mixed case order directions', () => { + const query = { + order: { 'table.id': 'asc', 'table.name': 'desc' }, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + { + name: 'table__name', + alias: 'Name', + }, + ]; + + const result = generateRowNumberSql( + query, + dimensions, + BASE_DATA_SOURCE_NAME + ); + + expect(result).toBe( + 'row_number() OVER (ORDER BY __base_query."ID" ASC, __base_query."Name" DESC)' + ); + }); + + it('should use custom base table name', () => { + const query = { + order: { 'table.id': 'asc' }, + }; + const dimensions = [ + { + name: 'table__id', + alias: 'ID', + }, + ]; + const customBaseTableName = 'custom_table'; + + const result = generateRowNumberSql(query, dimensions, customBaseTableName); + + expect(result).toBe('row_number() OVER (ORDER BY custom_table."ID" ASC)'); + }); +}); diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 90fa674f..3192f804 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -229,6 +229,41 @@ export const generateResolutionJoinPaths = ( ]); }; +/** + * Generates row_number() OVER (ORDER BY ...) SQL based on query order. + * This is used to preserve the original query ordering through resolution operations. + * + * @param query - The query object that may contain an order clause + * @param dimensions - The dimensions array from the base table schema + * @param baseTableName - The base table name to use in column references + * @returns SQL expression for row_number() OVER (ORDER BY ...) + */ +export const generateRowNumberSql = ( + query: { order?: Record }, + dimensions: { name: string; alias?: string }[], + baseTableName: string +): string => { + let rowNumberSql = 'row_number() OVER ('; + if (query.order && Object.keys(query.order).length > 0) { + const orderClauses = Object.entries(query.order).map( + ([member, direction]) => { + // Find the actual column name/alias in the base table dimensions + const safeMember = memberKeyToSafeKey(member); + const dimension = dimensions.find( + (d) => d.name === safeMember || d.alias === safeMember + ); + const columnName = dimension + ? dimension.alias || dimension.name + : safeMember; + return `${baseTableName}."${columnName}" ${direction.toUpperCase()}`; + } + ); + rowNumberSql += `ORDER BY ${orderClauses.join(', ')}`; + } + rowNumberSql += ')'; + return rowNumberSql; +}; + /** * Wraps SQL to order by row_id and then exclude it from results. * This maintains the ordering from the base query while removing the internal row_id column. diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts index 3c3ef8b4..c9f7a66b 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -10,8 +10,8 @@ const CREATE_TEST_TABLE = `CREATE TABLE tickets ( )`; const INPUT_DATA_QUERY = `INSERT INTO tickets VALUES -(1, ['owner1', 'owner2'], ['tag1'], 'user1', 30), (2, ['owner2', 'owner3'], ['tag2', 'tag3'], 'user2', 10), +(1, ['owner1', 'owner2'], ['tag1'], 'user1', 30), (3, ['owner4'], ['tag1', 'tag4', 'tag3'], 'user3', 80)`; const CREATE_RESOLUTION_TABLE = `CREATE TABLE owners_lookup ( @@ -165,6 +165,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { const query: Query = { measures: ['tickets.count'], dimensions: ['tickets.id', 'tickets.owners'], + order: { 'tickets.id': 'asc' }, }; const resolutionConfig: ResolutionConfig = { @@ -196,6 +197,11 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { // Without array unnesting, should have 3 rows (original count) expect(result.length).toBe(3); + // Verify ordering is maintained (ORDER BY tickets.id ASC) + expect(result[0].ID).toBe(1); + expect(result[1].ID).toBe(2); + expect(result[2].ID).toBe(3); + // Each row should have the expected properties expect(result[0]).toHaveProperty('tickets__count'); expect(result[0]).toHaveProperty('ID'); @@ -204,7 +210,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(result[0]).toHaveProperty('Owners - Display Name'); expect(result[0]).toHaveProperty('Owners - Email'); - const id1Record = result.find((r: any) => r.ID === 1); + const id1Record = result[0]; // Note: Array order may not be preserved without index tracking in UNNEST/ARRAY_AGG expect(id1Record['Owners - Display Name']).toEqual( expect.arrayContaining(['Alice Smith', 'Bob Jones']) @@ -213,7 +219,8 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect.arrayContaining(['alice@example.com', 'bob@example.com']) ); - const id2Record = result.find((r: any) => r.ID === 2); + const id2Record = result[1]; + expect(id2Record.ID).toBe(2); expect(id2Record['Owners - Display Name']).toEqual( expect.arrayContaining(['Bob Jones', 'Charlie Brown']) ); @@ -231,6 +238,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { 'tickets.tags', // array 'tickets.created_by', // scalar ], + order: { 'tickets.id': 'asc' }, }; const resolutionConfig: ResolutionConfig = { @@ -287,6 +295,11 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { // Should have 3 rows (original ticket count) expect(result.length).toBe(3); + // Verify ordering is maintained (ORDER BY tickets.id ASC) + expect(result[0].ID).toBe(1); + expect(result[1].ID).toBe(2); + expect(result[2].ID).toBe(3); + // Each row should have the expected properties expect(result[0]).toHaveProperty('tickets__count'); expect(result[0]).toHaveProperty('ID'); @@ -295,7 +308,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(result[0]).toHaveProperty('Created By - Name'); // Verify ticket 1: 2 owners, 1 tag - const ticket1 = result.find((r: any) => r.ID === 1); + const ticket1 = result[0]; expect(ticket1['Owners - Display Name']).toEqual( expect.arrayContaining(['Alice Smith', 'Bob Jones']) ); @@ -307,7 +320,8 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(ticket1['Created By - Name']).toBe('User 1'); // Verify ticket 2: 2 owners, 2 tags - const ticket2 = result.find((r: any) => r.ID === 2); + const ticket2 = result[1]; + expect(ticket2.ID).toBe(2); expect(ticket2['Owners - Display Name']).toEqual( expect.arrayContaining(['Bob Jones', 'Charlie Brown']) ); @@ -319,7 +333,8 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(ticket2['Created By - Name']).toBe('User 2'); // Verify ticket 3: 1 owner, 3 tags - const ticket3 = result.find((r: any) => r.ID === 3); + const ticket3 = result[2]; + expect(ticket3.ID).toBe(3); expect(ticket3['Owners - Display Name']).toEqual( expect.arrayContaining(['Diana Prince']) ); @@ -340,6 +355,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { 'tickets.tags', 'tickets.created_by', ], + order: { 'tickets.id': 'asc' }, }; const resolutionConfig: ResolutionConfig = { @@ -379,6 +395,11 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { // Should have 3 rows (no array unnesting, only scalar resolution) expect(result.length).toBe(3); + // Verify ordering is maintained (ORDER BY tickets.id ASC) + expect(result[0].ID).toBe(1); + expect(result[1].ID).toBe(2); + expect(result[2].ID).toBe(3); + // Each row should have the expected properties expect(result[0]).toHaveProperty('tickets__count'); expect(result[0]).toHaveProperty('ID'); @@ -387,19 +408,22 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(result[0]).toHaveProperty('Created By - Name'); // Resolved scalar field // Verify scalar resolution worked correctly - const ticket1 = result.find((r: any) => r.ID === 1); + const ticket1 = result[0]; + expect(ticket1.ID).toBe(1); expect(ticket1['Created By - Name']).toBe('User 1'); expect(Array.isArray(ticket1['Owners'])).toBe(true); expect(ticket1['Owners']).toEqual(['owner1', 'owner2']); expect(Array.isArray(ticket1['Tags'])).toBe(true); expect(ticket1['Tags']).toEqual(['tag1']); - const ticket2 = result.find((r: any) => r.ID === 2); + const ticket2 = result[1]; + expect(ticket2.ID).toBe(2); expect(ticket2['Created By - Name']).toBe('User 2'); expect(ticket2['Owners']).toEqual(['owner2', 'owner3']); expect(ticket2['Tags']).toEqual(['tag2', 'tag3']); - const ticket3 = result.find((r: any) => r.ID === 3); + const ticket3 = result[2]; + expect(ticket3.ID).toBe(3); expect(ticket3['Created By - Name']).toBe('User 3'); expect(ticket3['Owners']).toEqual(['owner4']); expect(ticket3['Tags']).toEqual(['tag1', 'tag4', 'tag3']); @@ -451,4 +475,82 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { const ticket3 = result.find((r: any) => r.ID === 3); expect(ticket3['Created By']).toBe('user3'); }); + + it('Should handle resolution without ORDER BY clause', async () => { + const query: Query = { + measures: ['tickets.count'], + dimensions: [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + ], + // NOTE: No order clause specified + }; + + const resolutionConfig: ResolutionConfig = { + columnConfigs: [ + { + name: 'tickets.created_by', + source: 'created_by_lookup', + joinColumn: 'id', + resolutionColumns: ['name'], + }, + ], + tableSchemas: [CREATED_BY_LOOKUP_SCHEMA], + }; + + const columnProjections = [ + 'tickets.id', + 'tickets.owners', + 'tickets.tags', + 'tickets.created_by', + 'tickets.count', + ]; + + const sql = await cubeQueryToSQLWithResolution({ + query, + tableSchemas: [TICKETS_TABLE_SCHEMA], + resolutionConfig, + columnProjections, + }); + + console.log('SQL (no ORDER BY):', sql); + + // Should contain row_id even without ORDER BY (for consistency) + expect(sql).toContain('__row_id'); + // Should contain row_number() OVER () without ORDER BY inside + expect(sql).toContain('row_number() OVER ()'); + // Should still order by row_id at the end + expect(sql).toContain('order by __row_id'); + + // Execute the SQL to verify it works + const result = (await duckdbExec(sql)) as any[]; + console.log('Result (no ORDER BY):', result); + + // Should have 3 rows (no array unnesting, only scalar resolution) + expect(result.length).toBe(3); + + // Each row should have the expected properties + expect(result[0]).toHaveProperty('tickets__count'); + expect(result[0]).toHaveProperty('ID'); + expect(result[0]).toHaveProperty('Owners'); // Original array, not resolved + expect(result[0]).toHaveProperty('Tags'); // Original array, not resolved + expect(result[0]).toHaveProperty('Created By - Name'); // Resolved scalar field + + // Verify scalar resolution worked correctly + // Order might vary without ORDER BY, so we find by ID + const ticket1 = result.find((r: any) => r.ID === 1); + expect(ticket1['Created By - Name']).toBe('User 1'); + expect(Array.isArray(ticket1['Owners'])).toBe(true); + expect(ticket1['Owners']).toEqual(['owner1', 'owner2']); + + const ticket2 = result.find((r: any) => r.ID === 2); + expect(ticket2['Created By - Name']).toBe('User 2'); + expect(ticket2['Owners']).toEqual(['owner2', 'owner3']); + + const ticket3 = result.find((r: any) => r.ID === 3); + expect(ticket3['Created By - Name']).toBe('User 3'); + expect(ticket3['Owners']).toEqual(['owner4']); + }); }); diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index c21315a0..d0dd9375 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -7,6 +7,7 @@ import { generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + generateRowNumberSql, getArrayTypeResolutionColumnConfigs, getNamespacedKey, Measure, @@ -59,6 +60,7 @@ export const cubeQueryToSQLWithResolution = async ({ ); return cubeQueryToSQLWithResolutionWithArray({ baseSql, + query, tableSchemas, resolutionConfig, columnProjections, @@ -77,7 +79,11 @@ export const cubeQueryToSQLWithResolution = async ({ // Add row_id dimension to preserve ordering from base SQL baseTable.dimensions.push({ name: ROW_ID_DIMENSION_NAME, - sql: 'row_number() OVER ()', + sql: generateRowNumberSql( + query, + baseTable.dimensions, + BASE_DATA_SOURCE_NAME + ), type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); @@ -117,12 +123,14 @@ export const cubeQueryToSQLWithResolution = async ({ export const cubeQueryToSQLWithResolutionWithArray = async ({ baseSql, + query, tableSchemas, resolutionConfig, columnProjections, contextParams, }: { baseSql: string; + query: Query; tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; columnProjections?: string[]; @@ -138,7 +146,11 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ baseSchema.dimensions.push({ name: ROW_ID_DIMENSION_NAME, - sql: 'row_number() OVER ()', + sql: generateRowNumberSql( + query, + baseSchema.dimensions, + BASE_DATA_SOURCE_NAME + ), type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); From 086736b42f30c823987c10312227f6ef46f2e78b Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 03:05:37 +0530 Subject: [PATCH 22/38] minor --- .../browser-cube-to-sql-with-resolution.ts | 4 ++++ .../cube-to-sql-with-resolution.ts | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index b77bfbba..ff4128bf 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -49,6 +49,10 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); + if (resolutionConfig.columnConfigs.length === 0 && !query.order) { + return baseSql; + } + if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { // This is to ensure that, only the column projection columns // are being resolved and other definitions are ignored. diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index d0dd9375..b4c9bcd8 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -45,8 +45,7 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0) { - // If no resolution is needed, return the base SQL. + if (resolutionConfig.columnConfigs.length === 0 || !query.order) { return baseSql; } From 2f271839f7ce84efd22c3725184860d4b7273b12 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 03:12:45 +0530 Subject: [PATCH 23/38] final tests --- .../src/__tests__/cube-to-sql-with-resolution.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts index c9f7a66b..025b039f 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -518,11 +518,11 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { console.log('SQL (no ORDER BY):', sql); // Should contain row_id even without ORDER BY (for consistency) - expect(sql).toContain('__row_id'); + expect(sql).not.toContain('__row_id'); // Should contain row_number() OVER () without ORDER BY inside - expect(sql).toContain('row_number() OVER ()'); + expect(sql).not.toContain('row_number() OVER ('); // Should still order by row_id at the end - expect(sql).toContain('order by __row_id'); + expect(sql).not.toContain('order by __row_id'); // Execute the SQL to verify it works const result = (await duckdbExec(sql)) as any[]; From 6f27e7430ec586b386a325541fde3db9610868aa Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 03:16:56 +0530 Subject: [PATCH 24/38] fixed final tests --- .../cube-to-sql-with-resolution.spec.ts | 6 +- meerkat-node/src/__tests__/working_query.txt | 117 ++++++++++++++++++ .../cube-to-sql-with-resolution.ts | 2 +- 3 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 meerkat-node/src/__tests__/working_query.txt diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts index 025b039f..c9f7a66b 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -518,11 +518,11 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { console.log('SQL (no ORDER BY):', sql); // Should contain row_id even without ORDER BY (for consistency) - expect(sql).not.toContain('__row_id'); + expect(sql).toContain('__row_id'); // Should contain row_number() OVER () without ORDER BY inside - expect(sql).not.toContain('row_number() OVER ('); + expect(sql).toContain('row_number() OVER ()'); // Should still order by row_id at the end - expect(sql).not.toContain('order by __row_id'); + expect(sql).toContain('order by __row_id'); // Execute the SQL to verify it works const result = (await duckdbExec(sql)) as any[]; diff --git a/meerkat-node/src/__tests__/working_query.txt b/meerkat-node/src/__tests__/working_query.txt new file mode 100644 index 00000000..02298503 --- /dev/null +++ b/meerkat-node/src/__tests__/working_query.txt @@ -0,0 +1,117 @@ +SELECT + * exclude (__row_id) +FROM + ( + SELECT + MAX(__resolved_query."Ticket Id") AS "Ticket Id", + ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name", + "__row_id" + FROM + ( + SELECT + __resolved_query."__row_id" AS "__row_id", + * + FROM + ( + SELECT + "Tags - Tag name", + "Ticket Id", + "__row_id" + FROM + ( + SELECT + __unnested_base_query."Ticket Id" AS "Ticket Id", + __unnested_base_query."__row_id" AS "__row_id", + * + FROM + ( + SELECT + "Ticket Id", + "Tags", + "__row_id" + FROM + ( + SELECT + __base_query."Ticket Id" AS "Ticket Id", + unnest(__base_query."Tags") AS "Tags", + row_number() OVER () AS "__row_id", + * + FROM + ( + SELECT + count("ID") AS "Ticket Id", + "Tags" + FROM + ( + SELECT + dim_ticket.created_date AS "Created date", + 'ticket' AS "Work Type", + json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", + CAST( + json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[] + ) AS "Tags", + dim_ticket.id AS "ID", + * + FROM + ( + SELECT + * + FROM + devrev.dim_ticket + ) AS dim_ticket + ) AS dim_ticket + WHERE + ( + ( + ( + ("Created date" >= '2025-08-02T09:30:00.000Z') + AND ("Created date" <= '2025-10-31T10:29:59.999Z') + ) + AND ("Work Type" IN ('ticket')) + AND ( + Stage IN ( + 'don:core:dvrv-us-1:devo/787:custom_stage/514', + 'don:core:dvrv-us-1:devo/787:custom_stage/512', + 'don:core:dvrv-us-1:devo/787:custom_stage/501', + 'don:core:dvrv-us-1:devo/787:custom_stage/485', + 'don:core:dvrv-us-1:devo/787:custom_stage/440', + 'don:core:dvrv-us-1:devo/787:custom_stage/25', + 'don:core:dvrv-us-1:devo/787:custom_stage/24', + 'don:core:dvrv-us-1:devo/787:custom_stage/22', + 'don:core:dvrv-us-1:devo/787:custom_stage/21', + 'don:core:dvrv-us-1:devo/787:custom_stage/19', + 'don:core:dvrv-us-1:devo/787:custom_stage/13', + 'don:core:dvrv-us-1:devo/787:custom_stage/10', + 'don:core:dvrv-us-1:devo/787:custom_stage/8', + 'don:core:dvrv-us-1:devo/787:custom_stage/7', + 'don:core:dvrv-us-1:devo/787:custom_stage/6', + 'don:core:dvrv-us-1:devo/787:custom_stage/4' + ) + ) + ) + ) + GROUP BY + Tags + LIMIT + 50000 + ) AS __base_query + ) AS __base_query + ) AS __unnested_base_query + LEFT JOIN ( + SELECT + __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", + * + FROM + ( + SELECT + * + FROM + devrev.dim_tag + ) AS __unnested_base_query__dim_ticket__tags_json + ) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags" = __unnested_base_query__dim_ticket__tags_json.id + ) AS MEERKAT_GENERATED_TABLE + ) AS __resolved_query + ) AS __resolved_query + GROUP BY + __row_id + ) \ No newline at end of file diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index b4c9bcd8..3bb9387a 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -45,7 +45,7 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0 || !query.order) { + if (resolutionConfig.columnConfigs.length === 0 && !query.order) { return baseSql; } From 804383c2e656e11b69763ea091bdeece45731d9c Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 03:27:53 +0530 Subject: [PATCH 25/38] fixing test --- meerkat-node/src/__tests__/resolution.spec.ts | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/meerkat-node/src/__tests__/resolution.spec.ts b/meerkat-node/src/__tests__/resolution.spec.ts index 90bde754..f336a591 100644 --- a/meerkat-node/src/__tests__/resolution.spec.ts +++ b/meerkat-node/src/__tests__/resolution.spec.ts @@ -242,21 +242,22 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - SELECT + select * exclude(__row_id) from (SELECT "base_table__part_id_1 - display_id", "base_table__random_column", "base_table__work_id - display_id", "base_table__work_id - title", - "base_table__part_id_2 - display_id" + "base_table__part_id_2 - display_id", + "__row_id" FROM - (SELECT __base_query.base_table__random_column AS "base_table__random_column", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query + (SELECT __base_query.base_table__random_column AS "base_table__random_column", row_number() OVER () AS "__row_id", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query.base_table__part_id_1 = base_table__part_id_1.id LEFT JOIN (SELECT base_table__work_id.display_id AS "base_table__work_id - display_id", base_table__work_id.title AS "base_table__work_id - title", * FROM (select id, display_id, title from system.dim_issue) AS base_table__work_id) AS base_table__work_id ON __base_query.base_table__work_id = base_table__work_id.id LEFT JOIN (SELECT base_table__part_id_2.display_id AS "base_table__part_id_2 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 ON __base_query.base_table__part_id_2 = base_table__part_id_2.id) - AS MEERKAT_GENERATED_TABLE + AS MEERKAT_GENERATED_TABLE) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -285,14 +286,15 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - SELECT + select * exclude(__row_id) from (SELECT "base_table__count", - "base_table__part_id_1 - display_id" + "base_table__part_id_1 - display_id", + "__row_id" FROM - (SELECT __base_query.base_table__count AS "base_table__count", * FROM (SELECT count(*) AS base_table__count , base_table__part_id_1 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, * FROM (select * from base_table) AS base_table) AS base_table GROUP BY base_table__part_id_1) AS __base_query + (SELECT __base_query.base_table__count AS "base_table__count", row_number() OVER () AS "__row_id", * FROM (SELECT count(*) AS base_table__count , base_table__part_id_1 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, * FROM (select * from base_table) AS base_table) AS base_table GROUP BY base_table__part_id_1) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query.base_table__part_id_1 = base_table__part_id_1.id) - AS MEERKAT_GENERATED_TABLE + AS MEERKAT_GENERATED_TABLE) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -332,17 +334,18 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - SELECT + select * exclude(__row_id) from (SELECT "Part ID 1 - Display ID", "Random Column", - "Part ID 2 - Display ID" + "Part ID 2 - Display ID", + "__row_id" FROM - (SELECT __base_query."Random Column" AS "Random Column", * FROM (SELECT "Part ID 1", "Random Column", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query + (SELECT __base_query."Random Column" AS "Random Column", row_number() OVER () AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query."Part ID 1" = base_table__part_id_1.id LEFT JOIN (SELECT base_table__part_id_2.display_id AS "Part ID 2 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 ON __base_query."Part ID 2" = base_table__part_id_2.id) - AS MEERKAT_GENERATED_TABLE + AS MEERKAT_GENERATED_TABLE) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -378,14 +381,15 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - SELECT + select * exclude(__row_id) from (SELECT "base_table__random_column", - "base_table__part_id_1 - display_id" + "base_table__part_id_1 - display_id", + "__row_id" FROM - (SELECT __base_query.base_table__random_column AS "base_table__random_column", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query + (SELECT __base_query.base_table__random_column AS "base_table__random_column", row_number() OVER () AS "__row_id", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query.base_table__part_id_1 = base_table__part_id_1.id) - AS MEERKAT_GENERATED_TABLE + AS MEERKAT_GENERATED_TABLE) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() From 61d6bdec7dac311d0cfd400900699c954641bd06 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 13:56:17 +0530 Subject: [PATCH 26/38] cr comments --- .../browser-cube-to-sql-with-resolution.ts | 24 ++++-- meerkat-core/src/constants/index.ts | 1 + .../modifiers/array-flatten-modifier.ts | 25 ++++++ .../modifiers/array-unnest-modifier.ts | 23 ++++++ .../sql-expression-modifiers.ts | 82 ++++++------------- .../types.ts | 15 ++++ meerkat-core/src/index.ts | 1 + .../src/member-formatters/constants.ts | 2 - meerkat-core/src/member-formatters/index.ts | 2 +- .../src/resolution/resolution.spec.ts | 45 +++++----- meerkat-core/src/resolution/resolution.ts | 29 ++++--- meerkat-core/src/resolution/types.ts | 3 +- .../cube-to-sql-with-resolution.ts | 25 ++++-- 13 files changed, 160 insertions(+), 117 deletions(-) create mode 100644 meerkat-core/src/constants/index.ts create mode 100644 meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-flatten-modifier.ts create mode 100644 meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-unnest-modifier.ts create mode 100644 meerkat-core/src/get-wrapped-base-query-with-projections/types.ts diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index ff4128bf..f2153f5d 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -11,12 +11,13 @@ import { getArrayTypeResolutionColumnConfigs, getNamespacedKey, Measure, + MEERKAT_OUTPUT_DELIMITER, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - updateArrayFlattenModifierUsingResolutionConfig, + withArrayFlattenModifier, wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; @@ -53,6 +54,7 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } + // Check if any columns to be resolved are array types if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { // This is to ensure that, only the column projection columns // are being resolved and other definitions are ignored. @@ -81,7 +83,7 @@ export const cubeQueryToSQLWithResolution = async ({ ); // Add row_id dimension to preserve ordering from base SQL - baseTable.dimensions.push({ + const rowIdDimension: Dimension = { name: ROW_ID_DIMENSION_NAME, sql: generateRowNumberSql( query, @@ -90,7 +92,8 @@ export const cubeQueryToSQLWithResolution = async ({ ), type: 'number', alias: ROW_ID_DIMENSION_NAME, - } as Dimension); + }; + baseTable.dimensions.push(rowIdDimension); const resolutionSchemas: TableSchema[] = generateResolutionSchemas( resolutionConfig, @@ -214,7 +217,7 @@ export const getUnnestTableSchema = async ({ resolutionConfig: ResolutionConfig; contextParams?: ContextParams; }): Promise => { - updateArrayFlattenModifierUsingResolutionConfig( + const updatedBaseTableSchema = withArrayFlattenModifier( baseTableSchema, resolutionConfig ); @@ -224,12 +227,12 @@ export const getUnnestTableSchema = async ({ query: { measures: [], dimensions: [ - ...baseTableSchema.dimensions.map((d) => - getNamespacedKey(baseTableSchema.name, d.name) + ...updatedBaseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) ), ], }, - tableSchemas: [baseTableSchema], + tableSchemas: [updatedBaseTableSchema], contextParams, }); @@ -381,7 +384,7 @@ export const getAggregatedSql = async ({ const isResolvedArrayColumn = (dimName: string) => { return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}__`); + return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); }); }; @@ -391,6 +394,9 @@ export const getAggregatedSql = async ({ (d) => d.name === ROW_ID_DIMENSION_NAME ); + if (!rowIdDimension) { + throw new Error('Row id dimension not found'); + } // Create measures with MAX or ARRAY_AGG based on column type const aggregationMeasures: Measure[] = []; @@ -422,7 +428,7 @@ export const getAggregatedSql = async ({ const schemaWithAggregation: TableSchema = { ...aggregationBaseTableSchema, measures: aggregationMeasures, - dimensions: rowIdDimension ? [rowIdDimension] : [], + dimensions: [rowIdDimension], }; // Generate the final SQL diff --git a/meerkat-core/src/constants/index.ts b/meerkat-core/src/constants/index.ts new file mode 100644 index 00000000..c6a7041c --- /dev/null +++ b/meerkat-core/src/constants/index.ts @@ -0,0 +1 @@ +export const ROW_ID_DIMENSION_NAME = '__row_id'; diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-flatten-modifier.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-flatten-modifier.ts new file mode 100644 index 00000000..5b3b2573 --- /dev/null +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-flatten-modifier.ts @@ -0,0 +1,25 @@ +import { isArrayTypeMember } from '../../utils/is-array-member-type'; +import { DimensionModifier, Modifier } from '../types'; + +export const arrayFlattenModifier = ({ + sqlExpression, +}: DimensionModifier): string => { + // Ensure NULL or empty arrays produce at least one row with NULL value + // This prevents rows from being dropped when arrays are NULL or empty + // COALESCE handles NULL, and len() = 0 check handles empty arrays [] + return `unnest(CASE WHEN ${sqlExpression} IS NULL OR len(COALESCE(${sqlExpression}, [])) = 0 THEN [NULL] ELSE ${sqlExpression} END)`; +}; + +export const shouldFlattenArray = ({ + dimension, +}: DimensionModifier): boolean => { + const isArrayType = isArrayTypeMember(dimension.type); + const shouldFlattenArray = dimension.modifier?.shouldFlattenArray; + return !!(isArrayType && shouldFlattenArray); +}; + +export const arrayFlattenModifierConfig: Modifier = { + name: 'shouldFlattenArray', + matcher: shouldFlattenArray, + modifier: arrayFlattenModifier, +}; diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-unnest-modifier.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-unnest-modifier.ts new file mode 100644 index 00000000..697173ed --- /dev/null +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-unnest-modifier.ts @@ -0,0 +1,23 @@ +import { isArrayTypeMember } from '../../utils/is-array-member-type'; +import { DimensionModifier, Modifier } from '../types'; + +export const arrayFieldUnNestModifier = ({ + sqlExpression, +}: DimensionModifier): string => { + return `array[unnest(${sqlExpression})]`; +}; + +export const shouldUnnest = ({ + dimension, + query, +}: DimensionModifier): boolean => { + const isArrayType = isArrayTypeMember(dimension.type); + const hasUnNestedGroupBy = dimension.modifier?.shouldUnnestGroupBy; + return !!(isArrayType && hasUnNestedGroupBy && query.measures.length > 0); +}; + +export const arrayUnnestModifier: Modifier = { + name: 'shouldUnnestGroupBy', + matcher: shouldUnnest, + modifier: arrayFieldUnNestModifier, +}; diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts index 16b49cf9..9d89a18b 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts @@ -1,64 +1,28 @@ -import { Dimension, Query } from '../types/cube-types'; -import { isArrayTypeMember } from '../utils/is-array-member-type'; - -export interface DimensionModifier { - sqlExpression: string; - dimension: Dimension; - key: string; - query: Query; -} - -export const arrayFieldUnNestModifier = ({ - sqlExpression, -}: DimensionModifier): string => { - return `array[unnest(${sqlExpression})]`; -}; - -export const arrayFlattenModifier = ({ - sqlExpression, -}: DimensionModifier): string => { - // Ensure NULL or empty arrays produce at least one row with NULL value - // This prevents rows from being dropped when arrays are NULL or empty - // COALESCE handles NULL, and len() = 0 check handles empty arrays [] - return `unnest(CASE WHEN ${sqlExpression} IS NULL OR len(COALESCE(${sqlExpression}, [])) = 0 THEN [NULL] ELSE ${sqlExpression} END)`; -}; - -export const shouldUnnest = ({ - dimension, - query, -}: DimensionModifier): boolean => { - const isArrayType = isArrayTypeMember(dimension.type); - const hasUnNestedGroupBy = dimension.modifier?.shouldUnnestGroupBy; - return !!(isArrayType && hasUnNestedGroupBy && query.measures.length > 0); -}; - -export const shouldFlattenArray = ({ - dimension, -}: DimensionModifier): boolean => { - const isArrayType = isArrayTypeMember(dimension.type); - const hasUnNestedGroupBy = dimension.modifier?.shouldFlattenArray; - return !!(isArrayType && hasUnNestedGroupBy); -}; - -export type Modifier = { - name: string; - matcher: (modifier: DimensionModifier) => boolean; - modifier: (modifier: DimensionModifier) => string; +import { + arrayFlattenModifier, + arrayFlattenModifierConfig, + shouldFlattenArray, +} from './modifiers/array-flatten-modifier'; +import { + arrayFieldUnNestModifier, + arrayUnnestModifier, + shouldUnnest, +} from './modifiers/array-unnest-modifier'; +import { DimensionModifier, Modifier } from './types'; + +// Re-export types for backward compatibility +export type { DimensionModifier, Modifier }; + +export const MODIFIERS = [arrayUnnestModifier, arrayFlattenModifierConfig]; + +// Export individual modifier functions for backward compatibility +export { + arrayFieldUnNestModifier, + arrayFlattenModifier, + shouldFlattenArray, + shouldUnnest, }; -export const MODIFIERS: Modifier[] = [ - { - name: 'shouldUnnestGroupBy', - matcher: shouldUnnest, - modifier: arrayFieldUnNestModifier, - }, - { - name: 'shouldFlattenArray', - matcher: shouldFlattenArray, - modifier: arrayFlattenModifier, - }, -]; - export const getModifiedSqlExpression = ({ sqlExpression, dimension, diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts new file mode 100644 index 00000000..3d66e914 --- /dev/null +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts @@ -0,0 +1,15 @@ +import { Dimension, Query } from '../types/cube-types'; + +export interface DimensionModifier { + sqlExpression: string; + dimension: Dimension; + key: string; + query: Query; +} + +export type Modifier = { + name: string; + matcher: (modifier: DimensionModifier) => boolean; + modifier: (modifier: DimensionModifier) => string; +}; + diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index 2b6d03c7..9b0541cd 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -2,6 +2,7 @@ export * from './ast-builder/ast-builder'; export * from './ast-deserializer/ast-deserializer'; export * from './ast-serializer/ast-serializer'; export * from './ast-validator'; +export * from './constants'; export { detectApplyContextParamsToBaseSQL } from './context-params/context-params-ast'; export * from './cube-measure-transformer/cube-measure-transformer'; export * from './cube-to-duckdb/cube-filter-to-duckdb'; diff --git a/meerkat-core/src/member-formatters/constants.ts b/meerkat-core/src/member-formatters/constants.ts index 41e3b7a6..8bb6f9a2 100644 --- a/meerkat-core/src/member-formatters/constants.ts +++ b/meerkat-core/src/member-formatters/constants.ts @@ -1,7 +1,5 @@ export const COLUMN_NAME_DELIMITER = '.'; export const MEERKAT_OUTPUT_DELIMITER = '__'; -export const ROW_ID_DIMENSION_NAME = '__row_id'; - // Multi-character delimiter using three different uncommon characters // to minimize the chance of collision with real data export const STRING_ARRAY_DELIMITER = '§‡¶'; diff --git a/meerkat-core/src/member-formatters/index.ts b/meerkat-core/src/member-formatters/index.ts index 2acd05ce..b668014e 100644 --- a/meerkat-core/src/member-formatters/index.ts +++ b/meerkat-core/src/member-formatters/index.ts @@ -1,4 +1,4 @@ -export { COLUMN_NAME_DELIMITER, ROW_ID_DIMENSION_NAME } from './constants'; +export { COLUMN_NAME_DELIMITER, MEERKAT_OUTPUT_DELIMITER } from './constants'; export { constructAlias, getAliasFromSchema } from './get-alias'; export { getNamespacedKey } from './get-namespaced-key'; export { memberKeyToSafeKey } from './member-key-to-safe-key'; diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index e872835c..0ed7deef 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -6,7 +6,7 @@ import { generateResolvedDimensions, generateRowNumberSql, getArrayTypeResolutionColumnConfigs, - updateArrayFlattenModifierUsingResolutionConfig, + withArrayFlattenModifier, } from './resolution'; import { BASE_DATA_SOURCE_NAME, ResolutionConfig } from './types'; @@ -976,7 +976,7 @@ describe('getArrayTypeResolutionColumnConfigs', () => { }); }); -describe('updateArrayFlattenModifierUsingResolutionConfig', () => { +describe('withArrayFlattenModifier', () => { it('should add shouldFlattenArray modifier to array columns', () => { const baseTableSchema = { name: 'base_table', @@ -1009,15 +1009,14 @@ describe('updateArrayFlattenModifierUsingResolutionConfig', () => { tableSchemas: [], }; - updateArrayFlattenModifierUsingResolutionConfig( - baseTableSchema, - resolutionConfig - ); + const result = withArrayFlattenModifier(baseTableSchema, resolutionConfig); - expect(baseTableSchema.dimensions[0].modifier).toEqual({ + expect(result.dimensions[0].modifier).toEqual({ shouldFlattenArray: true, }); - expect(baseTableSchema.dimensions[1].modifier).toBeUndefined(); + expect(result.dimensions[1].modifier).toBeUndefined(); + // Verify immutability + expect(baseTableSchema.dimensions[0].modifier).toBeUndefined(); }); it('should handle multiple array columns', () => { @@ -1064,18 +1063,17 @@ describe('updateArrayFlattenModifierUsingResolutionConfig', () => { tableSchemas: [], }; - updateArrayFlattenModifierUsingResolutionConfig( - baseTableSchema, - resolutionConfig - ); + const result = withArrayFlattenModifier(baseTableSchema, resolutionConfig); - expect(baseTableSchema.dimensions[0].modifier).toEqual({ + expect(result.dimensions[0].modifier).toEqual({ shouldFlattenArray: true, }); - expect(baseTableSchema.dimensions[1].modifier).toEqual({ + expect(result.dimensions[1].modifier).toEqual({ shouldFlattenArray: true, }); - expect(baseTableSchema.dimensions[2].modifier).toBeUndefined(); + expect(result.dimensions[2].modifier).toBeUndefined(); + // Verify immutability + expect(baseTableSchema.dimensions[0].modifier).toBeUndefined(); }); it('should not modify dimensions when no array columns in config', () => { @@ -1110,13 +1108,10 @@ describe('updateArrayFlattenModifierUsingResolutionConfig', () => { tableSchemas: [], }; - updateArrayFlattenModifierUsingResolutionConfig( - baseTableSchema, - resolutionConfig - ); + const result = withArrayFlattenModifier(baseTableSchema, resolutionConfig); - expect(baseTableSchema.dimensions[0].modifier).toBeUndefined(); - expect(baseTableSchema.dimensions[1].modifier).toBeUndefined(); + expect(result.dimensions[0].modifier).toBeUndefined(); + expect(result.dimensions[1].modifier).toBeUndefined(); }); it('should handle empty dimensions array', () => { @@ -1142,13 +1137,11 @@ describe('updateArrayFlattenModifierUsingResolutionConfig', () => { // Should not throw error expect(() => { - updateArrayFlattenModifierUsingResolutionConfig( - baseTableSchema, - resolutionConfig - ); + withArrayFlattenModifier(baseTableSchema, resolutionConfig); }).not.toThrow(); - expect(baseTableSchema.dimensions).toEqual([]); + const result = withArrayFlattenModifier(baseTableSchema, resolutionConfig); + expect(result.dimensions).toEqual([]); }); }); diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 3192f804..3e3c8a64 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -96,20 +96,29 @@ export const createWrapperTableSchema = ( }; }; -export const updateArrayFlattenModifierUsingResolutionConfig = ( +export const withArrayFlattenModifier = ( baseTableSchema: TableSchema, resolutionConfig: ResolutionConfig -) => { +): TableSchema => { const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); - for (const dimension of baseTableSchema.dimensions) { - if ( - arrayColumns.some( + + return { + ...baseTableSchema, + dimensions: baseTableSchema.dimensions.map((dimension) => { + const shouldFlatten = arrayColumns.some( (ac: ResolutionColumnConfig) => ac.name === dimension.name - ) - ) { - dimension.modifier = { shouldFlattenArray: true }; - } - } + ); + + if (shouldFlatten) { + return { + ...dimension, + modifier: { shouldFlattenArray: true }, + }; + } + + return dimension; + }), + }; }; export const getArrayTypeResolutionColumnConfigs = ( diff --git a/meerkat-core/src/resolution/types.ts b/meerkat-core/src/resolution/types.ts index 921ff797..31cfb58d 100644 --- a/meerkat-core/src/resolution/types.ts +++ b/meerkat-core/src/resolution/types.ts @@ -4,7 +4,8 @@ export interface ResolutionColumnConfig { // Name of the column that needs resolution. // Should match a measure or dimension in the query. name: string; - // is array type + // Indicates if this column is an array type that needs special handling (UNNEST/ARRAY_AGG). + // This is essential metadata for the multi-phase resolution process. isArrayType?: boolean; // Name of the data source to use for resolution. source: string; diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 3bb9387a..6fc152fc 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -11,12 +11,13 @@ import { getArrayTypeResolutionColumnConfigs, getNamespacedKey, Measure, + MEERKAT_OUTPUT_DELIMITER, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - updateArrayFlattenModifierUsingResolutionConfig, + withArrayFlattenModifier, wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { @@ -49,6 +50,7 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } + // Check if any columns to be resolved are array types if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { // This is to ensure that, only the column projection columns // are being resolved and other definitions are ignored. @@ -76,7 +78,7 @@ export const cubeQueryToSQLWithResolution = async ({ ); // Add row_id dimension to preserve ordering from base SQL - baseTable.dimensions.push({ + const rowIdDimension: Dimension = { name: ROW_ID_DIMENSION_NAME, sql: generateRowNumberSql( query, @@ -85,7 +87,8 @@ export const cubeQueryToSQLWithResolution = async ({ ), type: 'number', alias: ROW_ID_DIMENSION_NAME, - } as Dimension); + }; + baseTable.dimensions.push(rowIdDimension); const resolutionSchemas: TableSchema[] = generateResolutionSchemas( resolutionConfig, @@ -201,7 +204,7 @@ export const getUnnestTableSchema = async ({ resolutionConfig: ResolutionConfig; contextParams?: ContextParams; }): Promise => { - updateArrayFlattenModifierUsingResolutionConfig( + const updatedBaseTableSchema = withArrayFlattenModifier( baseTableSchema, resolutionConfig ); @@ -210,12 +213,12 @@ export const getUnnestTableSchema = async ({ query: { measures: [], dimensions: [ - ...baseTableSchema.dimensions.map((d) => - getNamespacedKey(baseTableSchema.name, d.name) + ...updatedBaseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) ), ], }, - tableSchemas: [baseTableSchema], + tableSchemas: [updatedBaseTableSchema], contextParams, }); @@ -362,7 +365,7 @@ export const getAggregatedSql = async ({ const isResolvedArrayColumn = (dimName: string) => { return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}__`); + return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); }); }; @@ -372,6 +375,10 @@ export const getAggregatedSql = async ({ (d) => d.name === ROW_ID_DIMENSION_NAME ); + if (!rowIdDimension) { + throw new Error('Row id dimension not found'); + } + // Create measures with MAX or ARRAY_AGG based on column type const aggregationMeasures: Measure[] = []; @@ -403,7 +410,7 @@ export const getAggregatedSql = async ({ const schemaWithAggregation: TableSchema = { ...aggregationBaseTableSchema, measures: aggregationMeasures, - dimensions: rowIdDimension ? [rowIdDimension] : [], + dimensions: [rowIdDimension], }; // Generate the final SQL From 34ae940162983eb2cffe9272b87db830bd718e97 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 19:44:26 +0530 Subject: [PATCH 27/38] moving code into dependent files for better readability --- meerkat-browser/package.json | 2 +- .../browser-cube-to-sql-with-resolution.ts | 264 +----------------- .../steps/aggregation-step.ts | 114 ++++++++ .../steps/resolution-step.ts | 123 ++++++++ .../steps/unnest-step.ts | 58 ++++ .../types.ts | 1 - meerkat-node/package.json | 2 +- .../cube-to-sql-with-resolution.ts | 256 +---------------- .../steps/aggregation-step.ts | 111 ++++++++ .../steps/resolution-step.ts | 118 ++++++++ .../steps/unnest-step.ts | 52 ++++ 11 files changed, 586 insertions(+), 515 deletions(-) create mode 100644 meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts create mode 100644 meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts create mode 100644 meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts create mode 100644 meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts create mode 100644 meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts create mode 100644 meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index 24bf7dec..d158b1d6 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.105", + "version": "0.0.106", "dependencies": { "tslib": "^2.3.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index f2153f5d..1ea82670 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -2,22 +2,17 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, - createWrapperTableSchema, Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, generateRowNumberSql, - getArrayTypeResolutionColumnConfigs, getNamespacedKey, - Measure, - MEERKAT_OUTPUT_DELIMITER, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - withArrayFlattenModifier, wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; @@ -25,6 +20,9 @@ import { cubeQueryToSQL, CubeQueryToSQLParams, } from '../browser-cube-to-sql/browser-cube-to-sql'; +import { getAggregatedSql } from './steps/aggregation-step'; +import { getResolvedTableSchema } from './steps/resolution-step'; +import { getUnnestTableSchema } from './steps/unnest-step'; export interface CubeQueryToSQLWithResolutionParams { connection: AsyncDuckDBConnection; @@ -50,7 +48,7 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0 && !query.order) { + if (resolutionConfig.columnConfigs.length === 0) { return baseSql; } @@ -198,257 +196,3 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ return aggregatedSql; }; -/** - * Apply unnesting - * - * This function performs 1 step: - * 1. Create schema with unnest modifiers for array columns - * 2. Generate final unnested SQL - * @returns Table schema with unnest modifiers for array columns - */ -export const getUnnestTableSchema = async ({ - connection, - baseTableSchema, - resolutionConfig, - contextParams, -}: { - connection: AsyncDuckDBConnection; - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const updatedBaseTableSchema = withArrayFlattenModifier( - baseTableSchema, - resolutionConfig - ); - - const unnestedSql = await cubeQueryToSQL({ - connection, - query: { - measures: [], - dimensions: [ - ...updatedBaseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - ], - }, - tableSchemas: [updatedBaseTableSchema], - contextParams, - }); - - const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( - unnestedSql, - baseTableSchema - ); - - return unnestedBaseTableSchema; -}; - -/** - * Apply resolution (join with lookup tables) - * - * This function: - * 1. Uses the base table schema from Phase 1 (source of truth) - * 2. Generates resolution schemas for array fields - * 3. Sets up join paths between unnested data and resolution tables - * @returns Table schema with resolved values from lookup tables - */ -export const getResolvedTableSchema = async ({ - connection, - baseTableSchema, - resolutionConfig, - contextParams, - columnProjections, -}: { - connection: AsyncDuckDBConnection; - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; - columnProjections?: string[]; -}): Promise => { - const updatedBaseTableSchema: TableSchema = baseTableSchema; - - // Generate resolution schemas for fields that need resolution - const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ - updatedBaseTableSchema, - ]); - - const joinPaths = generateResolutionJoinPaths( - updatedBaseTableSchema.name, - resolutionConfig, - [updatedBaseTableSchema] - ); - - const tempQuery: Query = { - measures: [], - dimensions: baseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - }; - - const updatedColumnProjections = columnProjections?.map((cp) => - memberKeyToSafeKey(cp) - ); - // Generate resolved dimensions using columnProjections - const resolvedDimensions = generateResolvedDimensions( - updatedBaseTableSchema.name, - tempQuery, - resolutionConfig, - updatedColumnProjections - ); - - // Create query and generate SQL - const resolutionQuery: Query = { - measures: [], - dimensions: resolvedDimensions, - joinPaths, - }; - - const resolvedSql = await cubeQueryToSQL({ - connection, - query: resolutionQuery, - tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], - contextParams, - }); - - // Use the baseTableSchema which already has all the column info - const resolvedTableSchema: TableSchema = createWrapperTableSchema( - resolvedSql, - updatedBaseTableSchema - ); - - // Create a map of resolution schema dimensions by original column name - const resolutionDimensionsByColumnName = new Map(); - resolutionConfig.columnConfigs.forEach((config) => { - const resSchema = resolutionSchemas.find((rs) => - rs.dimensions.some((dim) => dim.name.startsWith(config.name)) - ); - if (resSchema) { - resolutionDimensionsByColumnName.set( - config.name, - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ); - } - }); - - // Maintain the same order as baseTableSchema.dimensions - // Replace dimensions that need resolution with their resolved counterparts - resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { - const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); - if (resolvedDims) { - // Replace with resolved dimensions - return resolvedDims; - } else { - // Keep the original dimension with correct SQL reference - return [dim]; - } - }); - - return resolvedTableSchema; -}; - -/** - * Re-aggregate to reverse the unnest - * - * This function: - * 1. Groups by row_id - * 2. Uses MAX for non-array columns (they're duplicated) - * 3. Uses ARRAY_AGG for resolved array columns - * - * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) - * @param resolutionConfig - Resolution configuration - * @param contextParams - Optional context parameters - * @returns Final SQL with arrays containing resolved values - */ -export const getAggregatedSql = async ({ - connection, - resolvedTableSchema, - resolutionConfig, - contextParams, -}: { - connection: AsyncDuckDBConnection; - resolvedTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; - - // Identify which columns need ARRAY_AGG vs MAX - const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); - const baseTableName = aggregationBaseTableSchema.name; - - const isResolvedArrayColumn = (dimName: string) => { - return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); - }); - }; - - // Create aggregation measures with proper aggregation functions - // Get row_id dimension for GROUP BY - const rowIdDimension = aggregationBaseTableSchema.dimensions.find( - (d) => d.name === ROW_ID_DIMENSION_NAME - ); - - if (!rowIdDimension) { - throw new Error('Row id dimension not found'); - } - // Create measures with MAX or ARRAY_AGG based on column type - const aggregationMeasures: Measure[] = []; - - aggregationBaseTableSchema.dimensions - .filter((dim) => dim.name !== rowIdDimension?.name) - .forEach((dim) => { - const isArrayColumn = isResolvedArrayColumn(dim.name); - - // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") - // We just need to wrap it in the aggregation function - const columnRef = - dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; - - // Use ARRAY_AGG for resolved array columns, MAX for others - // Filter out null values for ARRAY_AGG using FILTER clause - const aggregationFn = isArrayColumn - ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` - : `MAX(${columnRef})`; - - aggregationMeasures.push({ - name: dim.name, - sql: aggregationFn, - type: dim.type, - alias: dim.alias, - }); - }); - - // Update the schema with aggregation measures - const schemaWithAggregation: TableSchema = { - ...aggregationBaseTableSchema, - measures: aggregationMeasures, - dimensions: [rowIdDimension], - }; - - // Generate the final SQL - const aggregatedSql = await cubeQueryToSQL({ - connection, - query: { - measures: aggregationMeasures.map((m) => - getNamespacedKey(baseTableName, m.name) - ), - dimensions: rowIdDimension - ? [getNamespacedKey(baseTableName, rowIdDimension.name)] - : [], - }, - tableSchemas: [schemaWithAggregation], - contextParams, - }); - - // Order by row_id to maintain consistent ordering before excluding it - return wrapWithRowIdOrderingAndExclusion( - aggregatedSql, - ROW_ID_DIMENSION_NAME - ); -}; diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts new file mode 100644 index 00000000..92fecf9a --- /dev/null +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts @@ -0,0 +1,114 @@ +import { + ContextParams, + getArrayTypeResolutionColumnConfigs, + getNamespacedKey, + Measure, + MEERKAT_OUTPUT_DELIMITER, + ResolutionConfig, + ROW_ID_DIMENSION_NAME, + TableSchema, + wrapWithRowIdOrderingAndExclusion, +} from '@devrev/meerkat-core'; +import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; +import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; + +/** + * Re-aggregate to reverse the unnest + * + * This function: + * 1. Groups by row_id + * 2. Uses MAX for non-array columns (they're duplicated) + * 3. Uses ARRAY_AGG for resolved array columns + * + * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) + * @param resolutionConfig - Resolution configuration + * @param contextParams - Optional context parameters + * @returns Final SQL with arrays containing resolved values + */ +export const getAggregatedSql = async ({ + connection, + resolvedTableSchema, + resolutionConfig, + contextParams, +}: { + connection: AsyncDuckDBConnection; + resolvedTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; + + // Identify which columns need ARRAY_AGG vs MAX + const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); + const baseTableName = aggregationBaseTableSchema.name; + + const isResolvedArrayColumn = (dimName: string) => { + return arrayColumns.some((arrayCol) => { + return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); + }); + }; + + // Create aggregation measures with proper aggregation functions + // Get row_id dimension for GROUP BY + const rowIdDimension = aggregationBaseTableSchema.dimensions.find( + (d) => d.name === ROW_ID_DIMENSION_NAME + ); + + if (!rowIdDimension) { + throw new Error('Row id dimension not found'); + } + // Create measures with MAX or ARRAY_AGG based on column type + const aggregationMeasures: Measure[] = []; + + aggregationBaseTableSchema.dimensions + .filter((dim) => dim.name !== rowIdDimension?.name) + .forEach((dim) => { + const isArrayColumn = isResolvedArrayColumn(dim.name); + + // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") + // We just need to wrap it in the aggregation function + const columnRef = + dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; + + // Use ARRAY_AGG for resolved array columns, MAX for others + // Filter out null values for ARRAY_AGG using FILTER clause + const aggregationFn = isArrayColumn + ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` + : `MAX(${columnRef})`; + + aggregationMeasures.push({ + name: dim.name, + sql: aggregationFn, + type: dim.type, + alias: dim.alias, + }); + }); + + // Update the schema with aggregation measures + const schemaWithAggregation: TableSchema = { + ...aggregationBaseTableSchema, + measures: aggregationMeasures, + dimensions: [rowIdDimension], + }; + + // Generate the final SQL + const aggregatedSql = await cubeQueryToSQL({ + connection, + query: { + measures: aggregationMeasures.map((m) => + getNamespacedKey(baseTableName, m.name) + ), + dimensions: rowIdDimension + ? [getNamespacedKey(baseTableName, rowIdDimension.name)] + : [], + }, + tableSchemas: [schemaWithAggregation], + contextParams, + }); + + // Order by row_id to maintain consistent ordering before excluding it + return wrapWithRowIdOrderingAndExclusion( + aggregatedSql, + ROW_ID_DIMENSION_NAME + ); +}; diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts new file mode 100644 index 00000000..2383c2d7 --- /dev/null +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts @@ -0,0 +1,123 @@ +import { + ContextParams, + createWrapperTableSchema, + generateResolutionJoinPaths, + generateResolutionSchemas, + generateResolvedDimensions, + getNamespacedKey, + memberKeyToSafeKey, + Query, + ResolutionConfig, + TableSchema, +} from '@devrev/meerkat-core'; +import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; +import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; + +/** + * Apply resolution (join with lookup tables) + * + * This function: + * 1. Uses the base table schema from Phase 1 (source of truth) + * 2. Generates resolution schemas for array fields + * 3. Sets up join paths between unnested data and resolution tables + * @returns Table schema with resolved values from lookup tables + */ +export const getResolvedTableSchema = async ({ + connection, + baseTableSchema, + resolutionConfig, + contextParams, + columnProjections, +}: { + connection: AsyncDuckDBConnection; + baseTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; + columnProjections?: string[]; +}): Promise => { + const updatedBaseTableSchema: TableSchema = baseTableSchema; + + // Generate resolution schemas for fields that need resolution + const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ + updatedBaseTableSchema, + ]); + + const joinPaths = generateResolutionJoinPaths( + updatedBaseTableSchema.name, + resolutionConfig, + [updatedBaseTableSchema] + ); + + const tempQuery: Query = { + measures: [], + dimensions: baseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), + }; + + const updatedColumnProjections = columnProjections?.map((cp) => + memberKeyToSafeKey(cp) + ); + // Generate resolved dimensions using columnProjections + const resolvedDimensions = generateResolvedDimensions( + updatedBaseTableSchema.name, + tempQuery, + resolutionConfig, + updatedColumnProjections + ); + + // Create query and generate SQL + const resolutionQuery: Query = { + measures: [], + dimensions: resolvedDimensions, + joinPaths, + }; + + const resolvedSql = await cubeQueryToSQL({ + connection, + query: resolutionQuery, + tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], + contextParams, + }); + + // Use the baseTableSchema which already has all the column info + const resolvedTableSchema: TableSchema = createWrapperTableSchema( + resolvedSql, + updatedBaseTableSchema + ); + + // Create a map of resolution schema dimensions by original column name + const resolutionDimensionsByColumnName = new Map(); + resolutionConfig.columnConfigs.forEach((config) => { + const resSchema = resolutionSchemas.find((rs) => + rs.dimensions.some((dim) => dim.name.startsWith(config.name)) + ); + if (resSchema) { + resolutionDimensionsByColumnName.set( + config.name, + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ); + } + }); + + // Maintain the same order as baseTableSchema.dimensions + // Replace dimensions that need resolution with their resolved counterparts + resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { + const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); + if (resolvedDims) { + // Replace with resolved dimensions + return resolvedDims; + } else { + // Keep the original dimension with correct SQL reference + return [dim]; + } + }); + + return resolvedTableSchema; +}; + diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts new file mode 100644 index 00000000..8c5cf21e --- /dev/null +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts @@ -0,0 +1,58 @@ +import { + ContextParams, + createWrapperTableSchema, + getNamespacedKey, + Query, + ResolutionConfig, + TableSchema, + withArrayFlattenModifier, +} from '@devrev/meerkat-core'; +import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; +import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; + +/** + * Apply unnesting + * + * This function performs 1 step: + * 1. Create schema with unnest modifiers for array columns + * 2. Generate final unnested SQL + * @returns Table schema with unnest modifiers for array columns + */ +export const getUnnestTableSchema = async ({ + connection, + baseTableSchema, + resolutionConfig, + contextParams, +}: { + connection: AsyncDuckDBConnection; + baseTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + const updatedBaseTableSchema = withArrayFlattenModifier( + baseTableSchema, + resolutionConfig + ); + + const unnestedSql = await cubeQueryToSQL({ + connection, + query: { + measures: [], + dimensions: [ + ...updatedBaseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), + ], + }, + tableSchemas: [updatedBaseTableSchema], + contextParams, + }); + + const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( + unnestedSql, + baseTableSchema + ); + + return unnestedBaseTableSchema; +}; + diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts index 3d66e914..ab2eb214 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/types.ts @@ -12,4 +12,3 @@ export type Modifier = { matcher: (modifier: DimensionModifier) => boolean; modifier: (modifier: DimensionModifier) => string; }; - diff --git a/meerkat-node/package.json b/meerkat-node/package.json index f56f91ee..911eb07d 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.105", + "version": "0.0.106", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 6fc152fc..38ce29a6 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -2,28 +2,26 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, createBaseTableSchema, - createWrapperTableSchema, Dimension, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, generateRowNumberSql, - getArrayTypeResolutionColumnConfigs, getNamespacedKey, - Measure, - MEERKAT_OUTPUT_DELIMITER, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - withArrayFlattenModifier, wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { cubeQueryToSQL, CubeQueryToSQLParams, } from '../cube-to-sql/cube-to-sql'; +import { getAggregatedSql } from './steps/aggregation-step'; +import { getResolvedTableSchema } from './steps/resolution-step'; +import { getUnnestTableSchema } from './steps/unnest-step'; export interface CubeQueryToSQLWithResolutionParams { query: Query; @@ -46,7 +44,7 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0 && !query.order) { + if (resolutionConfig.columnConfigs.length === 0) { return baseSql; } @@ -187,249 +185,3 @@ export const cubeQueryToSQLWithResolutionWithArray = async ({ return aggregatedSql; }; -/** - * Apply unnesting - * - * This function performs 1 step: - * 1. Create schema with unnest modifiers for array columns - * 2. Generate final unnested SQL - * @returns Table schema with unnest modifiers for array columns - */ -export const getUnnestTableSchema = async ({ - baseTableSchema, - resolutionConfig, - contextParams, -}: { - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const updatedBaseTableSchema = withArrayFlattenModifier( - baseTableSchema, - resolutionConfig - ); - - const unnestedSql = await cubeQueryToSQL({ - query: { - measures: [], - dimensions: [ - ...updatedBaseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - ], - }, - tableSchemas: [updatedBaseTableSchema], - contextParams, - }); - - const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( - unnestedSql, - baseTableSchema - ); - - return unnestedBaseTableSchema; -}; - -/** - * Apply resolution (join with lookup tables) - * - * This function: - * 1. Uses the base table schema from Phase 1 (source of truth) - * 2. Generates resolution schemas for array fields - * 3. Sets up join paths between unnested data and resolution tables - * @returns Table schema with resolved values from lookup tables - */ -export const getResolvedTableSchema = async ({ - baseTableSchema, - resolutionConfig, - contextParams, - columnProjections, -}: { - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; - columnProjections?: string[]; -}): Promise => { - const updatedBaseTableSchema: TableSchema = baseTableSchema; - - // Generate resolution schemas for fields that need resolution - const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ - updatedBaseTableSchema, - ]); - - const joinPaths = generateResolutionJoinPaths( - updatedBaseTableSchema.name, - resolutionConfig, - [updatedBaseTableSchema] - ); - - const tempQuery: Query = { - measures: [], - dimensions: baseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - }; - - const updatedColumnProjections = columnProjections?.map((cp) => - memberKeyToSafeKey(cp) - ); - // Generate resolved dimensions using columnProjections - const resolvedDimensions = generateResolvedDimensions( - updatedBaseTableSchema.name, - tempQuery, - resolutionConfig, - updatedColumnProjections - ); - - // Create query and generate SQL - const resolutionQuery: Query = { - measures: [], - dimensions: resolvedDimensions, - joinPaths, - }; - - const resolvedSql = await cubeQueryToSQL({ - query: resolutionQuery, - tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], - contextParams, - }); - - // Use the baseTableSchema which already has all the column info - const resolvedTableSchema: TableSchema = createWrapperTableSchema( - resolvedSql, - updatedBaseTableSchema - ); - - // Create a map of resolution schema dimensions by original column name - const resolutionDimensionsByColumnName = new Map(); - resolutionConfig.columnConfigs.forEach((config) => { - const resSchema = resolutionSchemas.find((rs) => - rs.dimensions.some((dim) => dim.name.startsWith(config.name)) - ); - if (resSchema) { - resolutionDimensionsByColumnName.set( - config.name, - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ); - } - }); - - // Maintain the same order as baseTableSchema.dimensions - // Replace dimensions that need resolution with their resolved counterparts - resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { - const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); - if (resolvedDims) { - // Replace with resolved dimensions - return resolvedDims; - } else { - // Keep the original dimension with correct SQL reference - return [dim]; - } - }); - - return resolvedTableSchema; -}; - -/** - * Re-aggregate to reverse the unnest - * - * This function: - * 1. Groups by row_id - * 2. Uses MAX for non-array columns (they're duplicated) - * 3. Uses ARRAY_AGG for resolved array columns - * - * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) - * @param resolutionConfig - Resolution configuration - * @param contextParams - Optional context parameters - * @returns Final SQL with arrays containing resolved values - */ -export const getAggregatedSql = async ({ - resolvedTableSchema, - resolutionConfig, - contextParams, -}: { - resolvedTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; - - // Identify which columns need ARRAY_AGG vs MAX - const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); - const baseTableName = aggregationBaseTableSchema.name; - - const isResolvedArrayColumn = (dimName: string) => { - return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); - }); - }; - - // Create aggregation measures with proper aggregation functions - // Get row_id dimension for GROUP BY - const rowIdDimension = aggregationBaseTableSchema.dimensions.find( - (d) => d.name === ROW_ID_DIMENSION_NAME - ); - - if (!rowIdDimension) { - throw new Error('Row id dimension not found'); - } - - // Create measures with MAX or ARRAY_AGG based on column type - const aggregationMeasures: Measure[] = []; - - aggregationBaseTableSchema.dimensions - .filter((dim) => dim.name !== rowIdDimension?.name) - .forEach((dim) => { - const isArrayColumn = isResolvedArrayColumn(dim.name); - - // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") - // We just need to wrap it in the aggregation function - const columnRef = - dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; - - // Use ARRAY_AGG for resolved array columns, MAX for others - // Filter out null values for ARRAY_AGG using FILTER clause - const aggregationFn = isArrayColumn - ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` - : `MAX(${columnRef})`; - - aggregationMeasures.push({ - name: dim.name, - sql: aggregationFn, - type: dim.type, - alias: dim.alias, - }); - }); - - // Update the schema with aggregation measures - const schemaWithAggregation: TableSchema = { - ...aggregationBaseTableSchema, - measures: aggregationMeasures, - dimensions: [rowIdDimension], - }; - - // Generate the final SQL - const aggregatedSql = await cubeQueryToSQL({ - query: { - measures: aggregationMeasures.map((m) => - getNamespacedKey(baseTableName, m.name) - ), - dimensions: rowIdDimension - ? [getNamespacedKey(baseTableName, rowIdDimension.name)] - : [], - }, - tableSchemas: [schemaWithAggregation], - contextParams, - }); - - // Order by row_id to maintain consistent ordering before excluding it - return wrapWithRowIdOrderingAndExclusion( - aggregatedSql, - ROW_ID_DIMENSION_NAME - ); -}; diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts new file mode 100644 index 00000000..83f87c55 --- /dev/null +++ b/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts @@ -0,0 +1,111 @@ +import { + ContextParams, + getArrayTypeResolutionColumnConfigs, + getNamespacedKey, + Measure, + MEERKAT_OUTPUT_DELIMITER, + ResolutionConfig, + ROW_ID_DIMENSION_NAME, + TableSchema, + wrapWithRowIdOrderingAndExclusion, +} from '@devrev/meerkat-core'; +import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; + +/** + * Re-aggregate to reverse the unnest + * + * This function: + * 1. Groups by row_id + * 2. Uses MAX for non-array columns (they're duplicated) + * 3. Uses ARRAY_AGG for resolved array columns + * + * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) + * @param resolutionConfig - Resolution configuration + * @param contextParams - Optional context parameters + * @returns Final SQL with arrays containing resolved values + */ +export const getAggregatedSql = async ({ + resolvedTableSchema, + resolutionConfig, + contextParams, +}: { + resolvedTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; + + // Identify which columns need ARRAY_AGG vs MAX + const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); + const baseTableName = aggregationBaseTableSchema.name; + + const isResolvedArrayColumn = (dimName: string) => { + return arrayColumns.some((arrayCol) => { + return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); + }); + }; + + // Create aggregation measures with proper aggregation functions + // Get row_id dimension for GROUP BY + const rowIdDimension = aggregationBaseTableSchema.dimensions.find( + (d) => d.name === ROW_ID_DIMENSION_NAME + ); + + if (!rowIdDimension) { + throw new Error('Row id dimension not found'); + } + // Create measures with MAX or ARRAY_AGG based on column type + const aggregationMeasures: Measure[] = []; + + aggregationBaseTableSchema.dimensions + .filter((dim) => dim.name !== rowIdDimension?.name) + .forEach((dim) => { + const isArrayColumn = isResolvedArrayColumn(dim.name); + + // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") + // We just need to wrap it in the aggregation function + const columnRef = + dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; + + // Use ARRAY_AGG for resolved array columns, MAX for others + // Filter out null values for ARRAY_AGG using FILTER clause + const aggregationFn = isArrayColumn + ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` + : `MAX(${columnRef})`; + + aggregationMeasures.push({ + name: dim.name, + sql: aggregationFn, + type: dim.type, + alias: dim.alias, + }); + }); + + // Update the schema with aggregation measures + const schemaWithAggregation: TableSchema = { + ...aggregationBaseTableSchema, + measures: aggregationMeasures, + dimensions: [rowIdDimension], + }; + + // Generate the final SQL + const aggregatedSql = await cubeQueryToSQL({ + query: { + measures: aggregationMeasures.map((m) => + getNamespacedKey(baseTableName, m.name) + ), + dimensions: rowIdDimension + ? [getNamespacedKey(baseTableName, rowIdDimension.name)] + : [], + }, + tableSchemas: [schemaWithAggregation], + contextParams, + }); + + // Order by row_id to maintain consistent ordering before excluding it + return wrapWithRowIdOrderingAndExclusion( + aggregatedSql, + ROW_ID_DIMENSION_NAME + ); +}; + diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts new file mode 100644 index 00000000..03e98b71 --- /dev/null +++ b/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts @@ -0,0 +1,118 @@ +import { + ContextParams, + createWrapperTableSchema, + generateResolutionJoinPaths, + generateResolutionSchemas, + generateResolvedDimensions, + getNamespacedKey, + memberKeyToSafeKey, + Query, + ResolutionConfig, + TableSchema, +} from '@devrev/meerkat-core'; +import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; + +/** + * Apply resolution (join with lookup tables) + * + * This function: + * 1. Uses the base table schema from Phase 1 (source of truth) + * 2. Generates resolution schemas for array fields + * 3. Sets up join paths between unnested data and resolution tables + * @returns Table schema with resolved values from lookup tables + */ +export const getResolvedTableSchema = async ({ + baseTableSchema, + resolutionConfig, + contextParams, + columnProjections, +}: { + baseTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; + columnProjections?: string[]; +}): Promise => { + const updatedBaseTableSchema: TableSchema = baseTableSchema; + + // Generate resolution schemas for fields that need resolution + const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ + updatedBaseTableSchema, + ]); + + const joinPaths = generateResolutionJoinPaths( + updatedBaseTableSchema.name, + resolutionConfig, + [updatedBaseTableSchema] + ); + + const tempQuery: Query = { + measures: [], + dimensions: baseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), + }; + + const updatedColumnProjections = columnProjections?.map((cp) => + memberKeyToSafeKey(cp) + ); + // Generate resolved dimensions using columnProjections + const resolvedDimensions = generateResolvedDimensions( + updatedBaseTableSchema.name, + tempQuery, + resolutionConfig, + updatedColumnProjections + ); + + // Create query and generate SQL + const resolutionQuery: Query = { + measures: [], + dimensions: resolvedDimensions, + joinPaths, + }; + + const resolvedSql = await cubeQueryToSQL({ + query: resolutionQuery, + tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], + contextParams, + }); + + // Use the baseTableSchema which already has all the column info + const resolvedTableSchema: TableSchema = createWrapperTableSchema( + resolvedSql, + updatedBaseTableSchema + ); + + // Create a map of resolution schema dimensions by original column name + const resolutionDimensionsByColumnName = new Map(); + resolutionConfig.columnConfigs.forEach((config) => { + const resSchema = resolutionSchemas.find((rs) => + rs.dimensions.some((dim) => dim.name.startsWith(config.name)) + ); + if (resSchema) { + resolutionDimensionsByColumnName.set( + config.name, + resSchema.dimensions.map((dim) => ({ + name: dim.name, + sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + type: dim.type, + alias: dim.alias, + })) + ); + } + }); + + // Maintain the same order as baseTableSchema.dimensions + // Replace dimensions that need resolution with their resolved counterparts + resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { + const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); + if (resolvedDims) { + // Replace with resolved dimensions + return resolvedDims; + } else { + // Keep the original dimension with correct SQL reference + return [dim]; + } + }); + + return resolvedTableSchema; +}; diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts new file mode 100644 index 00000000..943bb07a --- /dev/null +++ b/meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts @@ -0,0 +1,52 @@ +import { + ContextParams, + createWrapperTableSchema, + getNamespacedKey, + ResolutionConfig, + TableSchema, + withArrayFlattenModifier, +} from '@devrev/meerkat-core'; +import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; + +/** + * Apply unnesting + * + * This function performs 1 step: + * 1. Create schema with unnest modifiers for array columns + * 2. Generate final unnested SQL + * @returns Table schema with unnest modifiers for array columns + */ +export const getUnnestTableSchema = async ({ + baseTableSchema, + resolutionConfig, + contextParams, +}: { + baseTableSchema: TableSchema; + resolutionConfig: ResolutionConfig; + contextParams?: ContextParams; +}): Promise => { + const updatedBaseTableSchema = withArrayFlattenModifier( + baseTableSchema, + resolutionConfig + ); + + const unnestedSql = await cubeQueryToSQL({ + query: { + measures: [], + dimensions: [ + ...updatedBaseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), + ], + }, + tableSchemas: [updatedBaseTableSchema], + contextParams, + }); + + const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( + unnestedSql, + baseTableSchema + ); + + return unnestedBaseTableSchema; +}; From b72c6580a590ca6416b838fee2b80a13010454b3 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 21:42:43 +0530 Subject: [PATCH 28/38] moving to use a merged flow --- .../browser-cube-to-sql-with-resolution.ts | 113 +++++---------- .../cube-to-sql-with-resolution.spec.ts | 4 +- meerkat-node/src/__tests__/resolution.spec.ts | 133 +++++++++--------- .../cube-to-sql-with-resolution.ts | 110 ++++----------- 4 files changed, 131 insertions(+), 229 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 1ea82670..606b2a81 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -3,23 +3,15 @@ import { ContextParams, createBaseTableSchema, Dimension, - generateResolutionJoinPaths, - generateResolutionSchemas, - generateResolvedDimensions, generateRowNumberSql, - getNamespacedKey, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; -import { - cubeQueryToSQL, - CubeQueryToSQLParams, -} from '../browser-cube-to-sql/browser-cube-to-sql'; +import { cubeQueryToSQL } from '../browser-cube-to-sql/browser-cube-to-sql'; import { getAggregatedSql } from './steps/aggregation-step'; import { getResolvedTableSchema } from './steps/resolution-step'; import { getUnnestTableSchema } from './steps/unnest-step'; @@ -48,86 +40,45 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0) { + // We have columnProjections check here to ensure that, we are using the same + // order in the final query + if ( + resolutionConfig.columnConfigs.length === 0 && + columnProjections?.length === 0 + ) { return baseSql; } - // Check if any columns to be resolved are array types - if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - // This is to ensure that, only the column projection columns - // are being resolved and other definitions are ignored. - resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( - (config) => { - return columnProjections?.includes(config.name); - } - ); - return cubeQueryToSQLWithResolutionWithArray({ - connection, - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, + // If column projections are provided, filter the query to only include the columns that are being projected. + if (columnProjections) { + query.dimensions = query.dimensions?.filter((dimension) => { + return columnProjections?.includes(dimension); + }); + query.measures = query.measures?.filter((measure) => { + return columnProjections?.includes(measure); }); } else { - // Create a table schema for the base query. - const baseTable: TableSchema = createBaseTableSchema( - baseSql, - tableSchemas, - resolutionConfig, - query.measures, - query.dimensions - ); - - // Add row_id dimension to preserve ordering from base SQL - const rowIdDimension: Dimension = { - name: ROW_ID_DIMENSION_NAME, - sql: generateRowNumberSql( - query, - baseTable.dimensions, - BASE_DATA_SOURCE_NAME - ), - type: 'number', - alias: ROW_ID_DIMENSION_NAME, - }; - baseTable.dimensions.push(rowIdDimension); - - const resolutionSchemas: TableSchema[] = generateResolutionSchemas( - resolutionConfig, - tableSchemas - ); - - const resolveParams: CubeQueryToSQLParams = { - connection, - query: { - measures: [], - dimensions: [ - ...generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), - // Include row_id in dimensions to preserve it through the query - getNamespacedKey(BASE_DATA_SOURCE_NAME, ROW_ID_DIMENSION_NAME), - ], - joinPaths: generateResolutionJoinPaths( - BASE_DATA_SOURCE_NAME, - resolutionConfig, - tableSchemas - ), - }, - tableSchemas: [baseTable, ...resolutionSchemas], - }; - const sql = await cubeQueryToSQL(resolveParams); - - // Order by row_id to maintain base SQL ordering, then exclude it - return wrapWithRowIdOrderingAndExclusion(sql, ROW_ID_DIMENSION_NAME); + columnProjections = [...(query.dimensions || []), ...query.measures]; } + // This is to ensure that, only the column projection columns + // are being resolved and other definitions are ignored. + resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( + (config) => { + return columnProjections?.includes(config.name); + } + ); + return getCubeQueryToSQLWithResolution({ + connection, + baseSql, + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, + }); }; -export const cubeQueryToSQLWithResolutionWithArray = async ({ +const getCubeQueryToSQLWithResolution = async ({ connection, baseSql, query, diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts index c9f7a66b..7bb19cab 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -429,7 +429,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { expect(ticket3['Tags']).toEqual(['tag1', 'tag4', 'tag3']); }); - it('Should return regular SQL when no resolution is configured', async () => { + it('Should return aggregated SQL even when no resolution is configured', async () => { const query: Query = { measures: ['tickets.count'], dimensions: ['tickets.id', 'tickets.created_by'], @@ -449,7 +449,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { console.log('SQL without resolution:', sql); // Should not have resolution-specific features when no resolution is configured - expect(sql).not.toContain('__row_id'); + expect(sql).toContain('__row_id'); expect(sql).not.toContain('unnest'); expect(sql).not.toContain('ARRAY_AGG'); diff --git a/meerkat-node/src/__tests__/resolution.spec.ts b/meerkat-node/src/__tests__/resolution.spec.ts index f336a591..d73261bb 100644 --- a/meerkat-node/src/__tests__/resolution.spec.ts +++ b/meerkat-node/src/__tests__/resolution.spec.ts @@ -105,6 +105,12 @@ export const BASE_TABLE_SCHEMA_WITH_ALIASES = { type: 'string', alias: 'Part ID 2', }, + { + name: 'work_id', + sql: 'base_table.work_id', + type: 'string', + alias: 'Work ID', + }, ], }; @@ -128,6 +134,33 @@ export const DIM_PART_SCHEMA_WITH_ALIASES = { ], }; +export const DIM_WORK_SCHEMA_WITH_ALIASES = { + name: 'dim_work', + sql: 'select id, display_id, title from system.dim_issue', + measures: [], + dimensions: [ + { + name: 'id', + sql: 'dim_work.id', + type: 'string', + alias: 'ID', + }, + { + name: 'display_id', + sql: 'dim_work.display_id', + type: 'string', + alias: 'Display ID', + }, + + { + name: 'title', + sql: 'dim_work.title', + type: 'string', + alias: 'Title', + }, + ], +}; + describe('Resolution Tests', () => { it('No Resolution Config', async () => { const query = { @@ -150,22 +183,16 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - SELECT - base_table__part_id_1, - base_table__random_column, - base_table__work_id, - base_table__part_id_2 - FROM - (SELECT - base_table.part_id_1 AS base_table__part_id_1, - base_table.random_column AS base_table__random_column, - base_table.work_id AS base_table__work_id, - base_table.part_id_2 AS base_table__part_id_2, - * + select * exclude(__row_id) from + (SELECT + MAX(__base_query."base_table__part_id_1") AS "base_table__part_id_1" , + MAX(__base_query."base_table__random_column") AS "base_table__random_column" , + MAX(__base_query."base_table__work_id") AS "base_table__work_id" , + MAX(__base_query."base_table__part_id_2") AS "base_table__part_id_2" , + "__row_id" FROM (SELECT __base_query."__row_id" AS "__row_id", * FROM - (select * from base_table) - AS base_table) - AS base_table + (SELECT "base_table__part_id_1", "base_table__random_column", "base_table__work_id", "base_table__part_id_2", "__row_id" FROM (SELECT __base_query."base_table__part_id_1" AS "base_table__part_id_1", __base_query."base_table__random_column" AS "base_table__random_column", __base_query."base_table__work_id" AS "base_table__work_id", __base_query."base_table__part_id_2" AS "base_table__part_id_2", __base_query."__row_id" AS "__row_id", * FROM (SELECT "base_table__part_id_1", "base_table__random_column", "base_table__work_id", "base_table__part_id_2", "__row_id" FROM (SELECT __base_query.base_table__part_id_1 AS "base_table__part_id_1", __base_query.base_table__random_column AS "base_table__random_column", __base_query.base_table__work_id AS "base_table__work_id", __base_query.base_table__part_id_2 AS "base_table__part_id_2", row_number() OVER () AS "__row_id", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query) AS __base_query) AS __base_query) AS __base_query GROUP BY __row_id) + order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -215,7 +242,7 @@ describe('Resolution Tests', () => { const sql = await cubeQueryToSQLWithResolution({ query, - tableSchemas: [BASE_TABLE_SCHEMA], + tableSchemas: [BASE_TABLE_SCHEMA_WITH_ALIASES], resolutionConfig: { columnConfigs: [ { @@ -237,27 +264,26 @@ describe('Resolution Tests', () => { resolutionColumns: ['display_id'], }, ], - tableSchemas: [DIM_PART_SCHEMA, DIM_WORK_SCHEMA], + tableSchemas: [ + DIM_PART_SCHEMA_WITH_ALIASES, + DIM_WORK_SCHEMA_WITH_ALIASES, + ], }, }); console.info(`SQL: `, sql); const expectedSQL = ` - select * exclude(__row_id) from (SELECT - "base_table__part_id_1 - display_id", - "base_table__random_column", - "base_table__work_id - display_id", - "base_table__work_id - title", - "base_table__part_id_2 - display_id", - "__row_id" - FROM - (SELECT __base_query.base_table__random_column AS "base_table__random_column", row_number() OVER () AS "__row_id", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query - LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 - ON __base_query.base_table__part_id_1 = base_table__part_id_1.id - LEFT JOIN (SELECT base_table__work_id.display_id AS "base_table__work_id - display_id", base_table__work_id.title AS "base_table__work_id - title", * FROM (select id, display_id, title from system.dim_issue) AS base_table__work_id) AS base_table__work_id - ON __base_query.base_table__work_id = base_table__work_id.id - LEFT JOIN (SELECT base_table__part_id_2.display_id AS "base_table__part_id_2 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 - ON __base_query.base_table__part_id_2 = base_table__part_id_2.id) - AS MEERKAT_GENERATED_TABLE) order by __row_id + select * exclude(__row_id) from + (SELECT + MAX(__base_query."Part ID 1 - Display ID") AS "Part ID 1 - Display ID" , + MAX(__base_query."Random Column") AS "Random Column" , + MAX(__base_query."Work ID - Display ID") AS "Work ID - Display ID" , + MAX(__base_query."Work ID - Title") AS "Work ID - Title" , + MAX(__base_query."Part ID 2 - Display ID") AS "Part ID 2 - Display ID" , + "__row_id" + FROM (SELECT __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1 - Display ID", "Random Column", "Work ID - Display ID", "Work ID - Title", "Part ID 2 - Display ID", "__row_id" FROM (SELECT __base_query."Random Column" AS "Random Column", __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Work ID", "Part ID 2", "__row_id" FROM (SELECT __base_query."Part ID 1" AS "Part ID 1", __base_query."Random Column" AS "Random Column", __base_query."Work ID" AS "Work ID", __base_query."Part ID 2" AS "Part ID 2", row_number() OVER () AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Work ID", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.work_id AS "Work ID", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query + LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query."Part ID 1" = base_table__part_id_1.id LEFT JOIN (SELECT base_table__work_id.display_id AS "Work ID - Display ID", base_table__work_id.title AS "Work ID - Title", * FROM (select id, display_id, title from system.dim_issue) AS base_table__work_id) AS base_table__work_id ON __base_query."Work ID" = base_table__work_id.id + LEFT JOIN (SELECT base_table__part_id_2.display_id AS "Part ID 2 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 ON __base_query."Part ID 2" = base_table__part_id_2.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) + order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -271,7 +297,7 @@ describe('Resolution Tests', () => { }; const sql = await cubeQueryToSQLWithResolution({ query, - tableSchemas: [BASE_TABLE_SCHEMA], + tableSchemas: [BASE_TABLE_SCHEMA_WITH_ALIASES], resolutionConfig: { columnConfigs: [ { @@ -281,20 +307,12 @@ describe('Resolution Tests', () => { resolutionColumns: ['display_id'], }, ], - tableSchemas: [DIM_PART_SCHEMA], + tableSchemas: [DIM_PART_SCHEMA_WITH_ALIASES], }, }); console.info(`SQL: `, sql); const expectedSQL = ` - select * exclude(__row_id) from (SELECT - "base_table__count", - "base_table__part_id_1 - display_id", - "__row_id" - FROM - (SELECT __base_query.base_table__count AS "base_table__count", row_number() OVER () AS "__row_id", * FROM (SELECT count(*) AS base_table__count , base_table__part_id_1 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, * FROM (select * from base_table) AS base_table) AS base_table GROUP BY base_table__part_id_1) AS __base_query - LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 - ON __base_query.base_table__part_id_1 = base_table__part_id_1.id) - AS MEERKAT_GENERATED_TABLE) order by __row_id + select * exclude(__row_id) from (SELECT MAX(__base_query."Part ID 1 - Display ID") AS "Part ID 1 - Display ID" , MAX(__base_query."Count") AS "Count" , "__row_id" FROM (SELECT __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1 - Display ID", "Count", "__row_id" FROM (SELECT __base_query."Count" AS "Count", __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1", "Count", "__row_id" FROM (SELECT __base_query."Part ID 1" AS "Part ID 1", __base_query."Count" AS "Count", row_number() OVER () AS "__row_id", * FROM (SELECT count(*) AS "Count" , "Part ID 1" FROM (SELECT base_table.part_id_1 AS "Part ID 1", * FROM (select * from base_table) AS base_table) AS base_table GROUP BY "Part ID 1") AS __base_query) AS __base_query) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query."Part ID 1" = base_table__part_id_1.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -334,18 +352,7 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - select * exclude(__row_id) from (SELECT - "Part ID 1 - Display ID", - "Random Column", - "Part ID 2 - Display ID", - "__row_id" - FROM - (SELECT __base_query."Random Column" AS "Random Column", row_number() OVER () AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query - LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 - ON __base_query."Part ID 1" = base_table__part_id_1.id - LEFT JOIN (SELECT base_table__part_id_2.display_id AS "Part ID 2 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 - ON __base_query."Part ID 2" = base_table__part_id_2.id) - AS MEERKAT_GENERATED_TABLE) order by __row_id + select * exclude(__row_id) from (SELECT MAX(__base_query."Part ID 1 - Display ID") AS "Part ID 1 - Display ID" , MAX(__base_query."Random Column") AS "Random Column" , MAX(__base_query."Part ID 2 - Display ID") AS "Part ID 2 - Display ID" , "__row_id" FROM (SELECT __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1 - Display ID", "Random Column", "Part ID 2 - Display ID", "__row_id" FROM (SELECT __base_query."Random Column" AS "Random Column", __base_query."__row_id" AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Part ID 2", "__row_id" FROM (SELECT __base_query."Part ID 1" AS "Part ID 1", __base_query."Random Column" AS "Random Column", __base_query."Part ID 2" AS "Part ID 2", row_number() OVER () AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query."Part ID 1" = base_table__part_id_1.id LEFT JOIN (SELECT base_table__part_id_2.display_id AS "Part ID 2 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_2) AS base_table__part_id_2 ON __base_query."Part ID 2" = base_table__part_id_2.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() @@ -365,7 +372,7 @@ describe('Resolution Tests', () => { const sql = await cubeQueryToSQLWithResolution({ query, - tableSchemas: [BASE_TABLE_SCHEMA], + tableSchemas: [BASE_TABLE_SCHEMA_WITH_ALIASES], resolutionConfig: { columnConfigs: [ { @@ -375,21 +382,13 @@ describe('Resolution Tests', () => { resolutionColumns: ['display_id'], }, ], - tableSchemas: [DIM_PART_SCHEMA, DIM_WORK_SCHEMA], + tableSchemas: [DIM_PART_SCHEMA_WITH_ALIASES], }, columnProjections: ['base_table.random_column', 'base_table.part_id_1'], }); console.info(`SQL: `, sql); const expectedSQL = ` - select * exclude(__row_id) from (SELECT - "base_table__random_column", - "base_table__part_id_1 - display_id", - "__row_id" - FROM - (SELECT __base_query.base_table__random_column AS "base_table__random_column", row_number() OVER () AS "__row_id", * FROM (SELECT base_table__part_id_1, base_table__random_column, base_table__work_id, base_table__part_id_2 FROM (SELECT base_table.part_id_1 AS base_table__part_id_1, base_table.random_column AS base_table__random_column, base_table.work_id AS base_table__work_id, base_table.part_id_2 AS base_table__part_id_2, * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query - LEFT JOIN (SELECT base_table__part_id_1.display_id AS "base_table__part_id_1 - display_id", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 - ON __base_query.base_table__part_id_1 = base_table__part_id_1.id) - AS MEERKAT_GENERATED_TABLE) order by __row_id + select * exclude(__row_id) from (SELECT MAX(__base_query.\"Random Column\") AS \"Random Column\" , MAX(__base_query.\"Part ID 1 - Display ID\") AS \"Part ID 1 - Display ID\" , \"__row_id\" FROM (SELECT __base_query.\"__row_id\" AS \"__row_id\", * FROM (SELECT \"Random Column\", \"Part ID 1 - Display ID\", \"__row_id\" FROM (SELECT __base_query.\"Random Column\" AS \"Random Column\", __base_query.\"__row_id\" AS \"__row_id\", * FROM (SELECT \"Random Column\", \"Part ID 1\", \"__row_id\" FROM (SELECT __base_query.\"Random Column\" AS \"Random Column\", __base_query.\"Part ID 1\" AS \"Part ID 1\", row_number() OVER () AS \"__row_id\", * FROM (SELECT \"Part ID 1\", \"Random Column\", \"Work ID\", \"Part ID 2\" FROM (SELECT base_table.part_id_1 AS \"Part ID 1\", base_table.random_column AS \"Random Column\", base_table.work_id AS \"Work ID\", base_table.part_id_2 AS \"Part ID 2\", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS \"Part ID 1 - Display ID\", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query.\"Part ID 1\" = base_table__part_id_1.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 38ce29a6..839d5fc5 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -3,22 +3,14 @@ import { ContextParams, createBaseTableSchema, Dimension, - generateResolutionJoinPaths, - generateResolutionSchemas, - generateResolvedDimensions, generateRowNumberSql, - getNamespacedKey, memberKeyToSafeKey, Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, - wrapWithRowIdOrderingAndExclusion, } from '@devrev/meerkat-core'; -import { - cubeQueryToSQL, - CubeQueryToSQLParams, -} from '../cube-to-sql/cube-to-sql'; +import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; import { getAggregatedSql } from './steps/aggregation-step'; import { getResolvedTableSchema } from './steps/resolution-step'; import { getUnnestTableSchema } from './steps/unnest-step'; @@ -44,84 +36,44 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - if (resolutionConfig.columnConfigs.length === 0) { + // We have columnProjections check here to ensure that, we are using the same + // order in the final query + if ( + resolutionConfig.columnConfigs.length === 0 && + columnProjections?.length === 0 + ) { return baseSql; } - // Check if any columns to be resolved are array types - if (resolutionConfig.columnConfigs.some((config) => config.isArrayType)) { - // This is to ensure that, only the column projection columns - // are being resolved and other definitions are ignored. - resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( - (config) => { - return columnProjections?.includes(config.name); - } - ); - return cubeQueryToSQLWithResolutionWithArray({ - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, + // If column projections are provided, filter the query to only include the columns that are being projected. + if (columnProjections) { + query.dimensions = query.dimensions?.filter((dimension) => { + return columnProjections?.includes(dimension); + }); + query.measures = query.measures?.filter((measure) => { + return columnProjections?.includes(measure); }); } else { - // Create a table schema for the base query. - const baseTable: TableSchema = createBaseTableSchema( - baseSql, - tableSchemas, - resolutionConfig, - query.measures, - query.dimensions - ); - - // Add row_id dimension to preserve ordering from base SQL - const rowIdDimension: Dimension = { - name: ROW_ID_DIMENSION_NAME, - sql: generateRowNumberSql( - query, - baseTable.dimensions, - BASE_DATA_SOURCE_NAME - ), - type: 'number', - alias: ROW_ID_DIMENSION_NAME, - }; - baseTable.dimensions.push(rowIdDimension); - - const resolutionSchemas: TableSchema[] = generateResolutionSchemas( - resolutionConfig, - tableSchemas - ); - - const resolveParams: CubeQueryToSQLParams = { - query: { - measures: [], - dimensions: [ - ...generateResolvedDimensions( - BASE_DATA_SOURCE_NAME, - query, - resolutionConfig, - columnProjections - ), - // Include row_id in dimensions to preserve it through the query - getNamespacedKey(BASE_DATA_SOURCE_NAME, ROW_ID_DIMENSION_NAME), - ], - joinPaths: generateResolutionJoinPaths( - BASE_DATA_SOURCE_NAME, - resolutionConfig, - tableSchemas - ), - }, - tableSchemas: [baseTable, ...resolutionSchemas], - }; - const sql = await cubeQueryToSQL(resolveParams); - - // Order by row_id to maintain base SQL ordering, then exclude it - return wrapWithRowIdOrderingAndExclusion(sql, ROW_ID_DIMENSION_NAME); + columnProjections = [...(query.dimensions || []), ...query.measures]; } + // This is to ensure that, only the column projection columns + // are being resolved and other definitions are ignored. + resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter( + (config) => { + return columnProjections?.includes(config.name); + } + ); + return getCubeQueryToSQLWithResolution({ + baseSql, + query, + tableSchemas, + resolutionConfig, + columnProjections, + contextParams, + }); }; -export const cubeQueryToSQLWithResolutionWithArray = async ({ +const getCubeQueryToSQLWithResolution = async ({ baseSql, query, tableSchemas, From 30f63dca736c3c068b67e153870b2e8988757ac6 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 22:47:10 +0530 Subject: [PATCH 29/38] changes after testing --- .../browser-cube-to-sql-with-resolution.ts | 18 +++------ .../steps/resolution-step.ts | 36 ++++++++++++------ .../cube-to-sql-with-resolution.ts | 14 ++----- .../steps/resolution-step.ts | 37 +++++++++++++------ 4 files changed, 57 insertions(+), 48 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 606b2a81..5055747f 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -49,15 +49,7 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } - // If column projections are provided, filter the query to only include the columns that are being projected. - if (columnProjections) { - query.dimensions = query.dimensions?.filter((dimension) => { - return columnProjections?.includes(dimension); - }); - query.measures = query.measures?.filter((measure) => { - return columnProjections?.includes(measure); - }); - } else { + if (!columnProjections) { columnProjections = [...(query.dimensions || []), ...query.measures]; } // This is to ensure that, only the column projection columns @@ -92,15 +84,15 @@ const getCubeQueryToSQLWithResolution = async ({ query: Query; tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; - columnProjections?: string[]; + columnProjections: string[]; contextParams?: ContextParams; }): Promise => { const baseSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, resolutionConfig, - [], - columnProjections + query.measures, + query.dimensions ); baseSchema.dimensions.push({ @@ -113,7 +105,7 @@ const getCubeQueryToSQLWithResolution = async ({ type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); - columnProjections?.push(ROW_ID_DIMENSION_NAME); + columnProjections.push(ROW_ID_DIMENSION_NAME); // Doing this because we need to use the original name of the column in the base table schema. resolutionConfig.columnConfigs.forEach((config) => { diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts index 2383c2d7..a81de5c5 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts @@ -33,7 +33,7 @@ export const getResolvedTableSchema = async ({ baseTableSchema: TableSchema; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; - columnProjections?: string[]; + columnProjections: string[]; }): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; @@ -105,19 +105,31 @@ export const getResolvedTableSchema = async ({ } }); - // Maintain the same order as baseTableSchema.dimensions + // Maintain the same order as columnProjections // Replace dimensions that need resolution with their resolved counterparts - resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { - const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); - if (resolvedDims) { - // Replace with resolved dimensions - return resolvedDims; - } else { - // Keep the original dimension with correct SQL reference - return [dim]; + resolvedTableSchema.dimensions = (updatedColumnProjections || []).flatMap( + (projectionName) => { + // Check if this column has resolved dimensions + const resolvedDims = resolutionDimensionsByColumnName.get(projectionName); + if (resolvedDims) { + // Use resolved dimensions + return resolvedDims; + } + + // Otherwise, find the original dimension from baseTableSchema + const originalDim = baseTableSchema.dimensions.find( + (d) => d.name === projectionName + ); + if (originalDim) { + return [originalDim]; + } + + // If not found, throw an error + throw new Error( + `Column projection '${projectionName}' not found in base table schema dimensions` + ); } - }); + ); return resolvedTableSchema; }; - diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 839d5fc5..65c2f9e1 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -45,15 +45,7 @@ export const cubeQueryToSQLWithResolution = async ({ return baseSql; } - // If column projections are provided, filter the query to only include the columns that are being projected. - if (columnProjections) { - query.dimensions = query.dimensions?.filter((dimension) => { - return columnProjections?.includes(dimension); - }); - query.measures = query.measures?.filter((measure) => { - return columnProjections?.includes(measure); - }); - } else { + if (!columnProjections) { columnProjections = [...(query.dimensions || []), ...query.measures]; } // This is to ensure that, only the column projection columns @@ -85,7 +77,7 @@ const getCubeQueryToSQLWithResolution = async ({ query: Query; tableSchemas: TableSchema[]; resolutionConfig: ResolutionConfig; - columnProjections?: string[]; + columnProjections: string[]; contextParams?: ContextParams; }): Promise => { const baseSchema: TableSchema = createBaseTableSchema( @@ -106,7 +98,7 @@ const getCubeQueryToSQLWithResolution = async ({ type: 'number', alias: ROW_ID_DIMENSION_NAME, } as Dimension); - columnProjections?.push(ROW_ID_DIMENSION_NAME); + columnProjections.push(ROW_ID_DIMENSION_NAME); // Doing this because we need to use the original name of the column in the base table schema. resolutionConfig.columnConfigs.forEach((config) => { diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts index 03e98b71..be06c372 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts @@ -24,13 +24,13 @@ import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; export const getResolvedTableSchema = async ({ baseTableSchema, resolutionConfig, - contextParams, columnProjections, + contextParams, }: { baseTableSchema: TableSchema; resolutionConfig: ResolutionConfig; + columnProjections: string[]; contextParams?: ContextParams; - columnProjections?: string[]; }): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; @@ -101,18 +101,31 @@ export const getResolvedTableSchema = async ({ } }); - // Maintain the same order as baseTableSchema.dimensions + // Maintain the same order as columnProjections // Replace dimensions that need resolution with their resolved counterparts - resolvedTableSchema.dimensions = baseTableSchema.dimensions.flatMap((dim) => { - const resolvedDims = resolutionDimensionsByColumnName.get(dim.name); - if (resolvedDims) { - // Replace with resolved dimensions - return resolvedDims; - } else { - // Keep the original dimension with correct SQL reference - return [dim]; + resolvedTableSchema.dimensions = (updatedColumnProjections || []).flatMap( + (projectionName) => { + // Check if this column has resolved dimensions + const resolvedDims = resolutionDimensionsByColumnName.get(projectionName); + if (resolvedDims) { + // Use resolved dimensions + return resolvedDims; + } + + // Otherwise, find the original dimension from baseTableSchema + const originalDim = baseTableSchema.dimensions.find( + (d) => d.name === projectionName + ); + if (originalDim) { + return [originalDim]; + } + + // If not found, throw an error + throw new Error( + `Column projection '${projectionName}' not found in base table schema dimensions` + ); } - }); + ); return resolvedTableSchema; }; From 1cb3e1bac73af8fcea2ea6355cf17e88c0041b6f Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Tue, 11 Nov 2025 23:00:59 +0530 Subject: [PATCH 30/38] fixing lint error --- meerkat-node/src/__tests__/resolution.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meerkat-node/src/__tests__/resolution.spec.ts b/meerkat-node/src/__tests__/resolution.spec.ts index d73261bb..3f31268a 100644 --- a/meerkat-node/src/__tests__/resolution.spec.ts +++ b/meerkat-node/src/__tests__/resolution.spec.ts @@ -388,7 +388,7 @@ describe('Resolution Tests', () => { }); console.info(`SQL: `, sql); const expectedSQL = ` - select * exclude(__row_id) from (SELECT MAX(__base_query.\"Random Column\") AS \"Random Column\" , MAX(__base_query.\"Part ID 1 - Display ID\") AS \"Part ID 1 - Display ID\" , \"__row_id\" FROM (SELECT __base_query.\"__row_id\" AS \"__row_id\", * FROM (SELECT \"Random Column\", \"Part ID 1 - Display ID\", \"__row_id\" FROM (SELECT __base_query.\"Random Column\" AS \"Random Column\", __base_query.\"__row_id\" AS \"__row_id\", * FROM (SELECT \"Random Column\", \"Part ID 1\", \"__row_id\" FROM (SELECT __base_query.\"Random Column\" AS \"Random Column\", __base_query.\"Part ID 1\" AS \"Part ID 1\", row_number() OVER () AS \"__row_id\", * FROM (SELECT \"Part ID 1\", \"Random Column\", \"Work ID\", \"Part ID 2\" FROM (SELECT base_table.part_id_1 AS \"Part ID 1\", base_table.random_column AS \"Random Column\", base_table.work_id AS \"Work ID\", base_table.part_id_2 AS \"Part ID 2\", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS \"Part ID 1 - Display ID\", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query.\"Part ID 1\" = base_table__part_id_1.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) order by __row_id + select * exclude(__row_id) from (SELECT MAX(__base_query."Random Column") AS "Random Column" , MAX(__base_query."Part ID 1 - Display ID") AS "Part ID 1 - Display ID" , "__row_id" FROM (SELECT __base_query."__row_id" AS "__row_id", * FROM (SELECT "Random Column", "Part ID 1 - Display ID", "__row_id" FROM (SELECT __base_query."Random Column" AS "Random Column", __base_query."__row_id" AS "__row_id", * FROM (SELECT "Random Column", "Part ID 1", "__row_id" FROM (SELECT __base_query."Random Column" AS "Random Column", __base_query."Part ID 1" AS "Part ID 1", row_number() OVER () AS "__row_id", * FROM (SELECT "Part ID 1", "Random Column", "Work ID", "Part ID 2" FROM (SELECT base_table.part_id_1 AS "Part ID 1", base_table.random_column AS "Random Column", base_table.work_id AS "Work ID", base_table.part_id_2 AS "Part ID 2", * FROM (select * from base_table) AS base_table) AS base_table) AS __base_query) AS __base_query) AS __base_query LEFT JOIN (SELECT base_table__part_id_1.display_id AS "Part ID 1 - Display ID", * FROM (select id, display_id from system.dim_feature UNION ALL select id, display_id from system.dim_product) AS base_table__part_id_1) AS base_table__part_id_1 ON __base_query."Part ID 1" = base_table__part_id_1.id) AS MEERKAT_GENERATED_TABLE) AS __base_query) AS __base_query GROUP BY __row_id) order by __row_id `; expect(sql.replace(/\s+/g, ' ').trim()).toBe( expectedSQL.replace(/\s+/g, ' ').trim() From c8c7f61bc1b03d1a1f09a8c2c98e871bfa006590 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 11:39:08 +0530 Subject: [PATCH 31/38] cr comments --- .../browser-cube-to-sql-with-resolution.ts | 59 ++----- .../steps/unnest-step.ts | 58 ------- .../src/constants/{ => exports}/index.ts | 0 .../__tests__/sql-expression-modifier.spec.ts | 152 ++++++++++-------- .../index.ts | 6 + .../sql-expression-modifiers.ts | 23 +-- meerkat-core/src/index.ts | 7 +- meerkat-core/src/resolution/resolution.ts | 36 ++++- .../src/resolution}/steps/aggregation-step.ts | 34 ++-- .../src/resolution}/steps/resolution-step.ts | 41 +++-- .../src/resolution}/steps/unnest-step.ts | 18 ++- meerkat-node/src/__tests__/working_query.txt | 117 -------------- .../cube-to-sql-with-resolution.ts | 54 ++----- .../steps/aggregation-step.ts | 111 ------------- .../steps/resolution-step.ts | 131 --------------- 15 files changed, 221 insertions(+), 626 deletions(-) delete mode 100644 meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts rename meerkat-core/src/constants/{ => exports}/index.ts (100%) create mode 100644 meerkat-core/src/get-wrapped-base-query-with-projections/index.ts rename {meerkat-browser/src/browser-cube-to-sql-with-resolution => meerkat-core/src/resolution}/steps/aggregation-step.ts (80%) rename {meerkat-browser/src/browser-cube-to-sql-with-resolution => meerkat-core/src/resolution}/steps/resolution-step.ts (80%) rename {meerkat-node/src/cube-to-sql-with-resolution => meerkat-core/src/resolution}/steps/unnest-step.ts (75%) delete mode 100644 meerkat-node/src/__tests__/working_query.txt delete mode 100644 meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts delete mode 100644 meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 5055747f..7abd7c50 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -1,6 +1,9 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, + getAggregatedSql as coreGetAggregatedSql, + getResolvedTableSchema as coreGetResolvedTableSchema, + getUnnestTableSchema as coreGetUnnestTableSchema, createBaseTableSchema, Dimension, generateRowNumberSql, @@ -8,13 +11,11 @@ import { Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, + shouldSkipResolution, TableSchema, } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; import { cubeQueryToSQL } from '../browser-cube-to-sql/browser-cube-to-sql'; -import { getAggregatedSql } from './steps/aggregation-step'; -import { getResolvedTableSchema } from './steps/resolution-step'; -import { getUnnestTableSchema } from './steps/unnest-step'; export interface CubeQueryToSQLWithResolutionParams { connection: AsyncDuckDBConnection; @@ -40,12 +41,8 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - // We have columnProjections check here to ensure that, we are using the same - // order in the final query - if ( - resolutionConfig.columnConfigs.length === 0 && - columnProjections?.length === 0 - ) { + // Check if resolution should be skipped + if (shouldSkipResolution(resolutionConfig, columnProjections)) { return baseSql; } @@ -59,34 +56,7 @@ export const cubeQueryToSQLWithResolution = async ({ return columnProjections?.includes(config.name); } ); - return getCubeQueryToSQLWithResolution({ - connection, - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, - }); -}; -const getCubeQueryToSQLWithResolution = async ({ - connection, - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, -}: { - connection: AsyncDuckDBConnection; - baseSql: string; - query: Query; - tableSchemas: TableSchema[]; - resolutionConfig: ResolutionConfig; - columnProjections: string[]; - contextParams?: ContextParams; -}): Promise => { const baseSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, @@ -95,7 +65,7 @@ const getCubeQueryToSQLWithResolution = async ({ query.dimensions ); - baseSchema.dimensions.push({ + const rowIdDimension: Dimension = { name: ROW_ID_DIMENSION_NAME, sql: generateRowNumberSql( query, @@ -104,7 +74,8 @@ const getCubeQueryToSQLWithResolution = async ({ ), type: 'number', alias: ROW_ID_DIMENSION_NAME, - } as Dimension); + }; + baseSchema.dimensions.push(rowIdDimension); columnProjections.push(ROW_ID_DIMENSION_NAME); // Doing this because we need to use the original name of the column in the base table schema. @@ -113,28 +84,28 @@ const getCubeQueryToSQLWithResolution = async ({ }); // Generate SQL with row_id and unnested arrays - const unnestTableSchema = await getUnnestTableSchema({ - connection, + const unnestTableSchema = await coreGetUnnestTableSchema({ baseTableSchema: baseSchema, resolutionConfig, contextParams, + cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }), }); // Apply resolution (join with lookup tables) - const resolvedTableSchema = await getResolvedTableSchema({ - connection, + const resolvedTableSchema = await coreGetResolvedTableSchema({ baseTableSchema: unnestTableSchema, resolutionConfig, contextParams, columnProjections, + cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }), }); // Re-aggregate to reverse the unnest - const aggregatedSql = await getAggregatedSql({ - connection, + const aggregatedSql = await coreGetAggregatedSql({ resolvedTableSchema, resolutionConfig, contextParams, + cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }), }); return aggregatedSql; diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts deleted file mode 100644 index 8c5cf21e..00000000 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/unnest-step.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - ContextParams, - createWrapperTableSchema, - getNamespacedKey, - Query, - ResolutionConfig, - TableSchema, - withArrayFlattenModifier, -} from '@devrev/meerkat-core'; -import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; -import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; - -/** - * Apply unnesting - * - * This function performs 1 step: - * 1. Create schema with unnest modifiers for array columns - * 2. Generate final unnested SQL - * @returns Table schema with unnest modifiers for array columns - */ -export const getUnnestTableSchema = async ({ - connection, - baseTableSchema, - resolutionConfig, - contextParams, -}: { - connection: AsyncDuckDBConnection; - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const updatedBaseTableSchema = withArrayFlattenModifier( - baseTableSchema, - resolutionConfig - ); - - const unnestedSql = await cubeQueryToSQL({ - connection, - query: { - measures: [], - dimensions: [ - ...updatedBaseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - ], - }, - tableSchemas: [updatedBaseTableSchema], - contextParams, - }); - - const unnestedBaseTableSchema: TableSchema = createWrapperTableSchema( - unnestedSql, - baseTableSchema - ); - - return unnestedBaseTableSchema; -}; - diff --git a/meerkat-core/src/constants/index.ts b/meerkat-core/src/constants/exports/index.ts similarity index 100% rename from meerkat-core/src/constants/index.ts rename to meerkat-core/src/constants/exports/index.ts diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/__tests__/sql-expression-modifier.spec.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/__tests__/sql-expression-modifier.spec.ts index 945dd43e..9bc4b687 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/__tests__/sql-expression-modifier.spec.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/__tests__/sql-expression-modifier.spec.ts @@ -1,56 +1,66 @@ import { Dimension } from '../../types/cube-types/table'; import { isArrayTypeMember } from '../../utils/is-array-member-type'; -import { arrayFieldUnNestModifier, DimensionModifier, getModifiedSqlExpression, MODIFIERS, shouldUnnest } from "../sql-expression-modifiers"; +import { + arrayFieldUnNestModifier, + shouldUnnest, +} from '../modifiers/array-unnest-modifier'; +import { + getModifiedSqlExpression, + MODIFIERS, +} from '../sql-expression-modifiers'; +import { DimensionModifier } from '../types'; -jest.mock("../../utils/is-array-member-type", () => { +jest.mock('../../utils/is-array-member-type', () => { return { - isArrayTypeMember: jest.fn() - } + isArrayTypeMember: jest.fn(), + }; }); const QUERY = { - measures: ["test_measure"], - dimensions: ["test_dimension"] -} + measures: ['test_measure'], + dimensions: ['test_dimension'], +}; -describe("Dimension Modifier", () => { - describe("arrayFieldUnNestModifier", () => { - it("should return the correct unnested SQL expression", () => { +describe('Dimension Modifier', () => { + describe('arrayFieldUnNestModifier', () => { + it('should return the correct unnested SQL expression', () => { const modifier: DimensionModifier = { - sqlExpression: "some_array_field", + sqlExpression: 'some_array_field', dimension: {} as Dimension, - key: "test_key", - query: QUERY + key: 'test_key', + query: QUERY, }; - expect(arrayFieldUnNestModifier(modifier)).toBe("array[unnest(some_array_field)]"); + expect(arrayFieldUnNestModifier(modifier)).toBe( + 'array[unnest(some_array_field)]' + ); }); }); - describe("shouldUnnest", () => { - it("should return true when dimension is array type and has shouldUnnestGroupBy modifier", () => { + describe('shouldUnnest', () => { + it('should return true when dimension is array type and has shouldUnnestGroupBy modifier', () => { (isArrayTypeMember as jest.Mock).mockReturnValue(true); const modifier: DimensionModifier = { - sqlExpression: "some_expression", - dimension: { - type: "array", - modifier: { shouldUnnestGroupBy: true } + sqlExpression: 'some_expression', + dimension: { + type: 'array', + modifier: { shouldUnnestGroupBy: true }, } as Dimension, - key: "test_key", - query: QUERY + key: 'test_key', + query: QUERY, }; expect(shouldUnnest(modifier)).toBe(true); }); - it("should return false when dimension is not array type", () => { + it('should return false when dimension is not array type', () => { (isArrayTypeMember as jest.Mock).mockReturnValue(false); const modifier: DimensionModifier = { - sqlExpression: "some_expression", - dimension: { - type: "string", - modifier: { shouldUnnestGroupBy: true } + sqlExpression: 'some_expression', + dimension: { + type: 'string', + modifier: { shouldUnnestGroupBy: true }, } as Dimension, - key: "test_key", - query: QUERY + key: 'test_key', + query: QUERY, }; expect(shouldUnnest(modifier)).toBe(false); }); @@ -58,73 +68,75 @@ describe("Dimension Modifier", () => { it("should return false when dimension doesn't have shouldUnnestGroupBy modifier", () => { (isArrayTypeMember as jest.Mock).mockReturnValue(true); const modifier: DimensionModifier = { - sqlExpression: "some_expression", - dimension: { - type: "array", - modifier: {} + sqlExpression: 'some_expression', + dimension: { + type: 'array', + modifier: {}, + } as Dimension, + key: 'test_key', + query: QUERY, + }; + expect(shouldUnnest(modifier)).toBe(false); + }); + it('should return false when dimension when modifier undefined', () => { + (isArrayTypeMember as jest.Mock).mockReturnValue(true); + const modifier: DimensionModifier = { + sqlExpression: 'some_expression', + dimension: { + type: 'array', } as Dimension, - key: "test_key", - query: QUERY + key: 'test_key', + query: QUERY, }; expect(shouldUnnest(modifier)).toBe(false); }); - it("should return false when dimension when modifier undefined", () => { - (isArrayTypeMember as jest.Mock).mockReturnValue(true); - const modifier: DimensionModifier = { - sqlExpression: "some_expression", - dimension: { - type: "array", - } as Dimension, - key: "test_key", - query: QUERY - }; - expect(shouldUnnest(modifier)).toBe(false); - }); }); - describe("getModifiedSqlExpression", () => { - it("should not modify if no modifiers passed", () => { + describe('getModifiedSqlExpression', () => { + it('should not modify if no modifiers passed', () => { (isArrayTypeMember as jest.Mock).mockReturnValue(true); const input = { - sqlExpression: "array_field", + sqlExpression: 'array_field', dimension: { - type: "array", - modifier: { shouldUnnestGroupBy: true } + type: 'array', + modifier: { shouldUnnestGroupBy: true }, } as Dimension, query: QUERY, - key: "test_key", - modifiers: [] + key: 'test_key', + modifiers: [], }; - expect(getModifiedSqlExpression(input)).toBe("array_field"); + expect(getModifiedSqlExpression(input)).toBe('array_field'); }); - it("should apply the modifier when conditions are met", () => { + it('should apply the modifier when conditions are met', () => { (isArrayTypeMember as jest.Mock).mockReturnValue(true); const input = { - sqlExpression: "array_field", + sqlExpression: 'array_field', dimension: { - type: "array", - modifier: { shouldUnnestGroupBy: true } + type: 'array', + modifier: { shouldUnnestGroupBy: true }, } as Dimension, query: QUERY, - key: "test_key", - modifiers: MODIFIERS + key: 'test_key', + modifiers: MODIFIERS, }; - expect(getModifiedSqlExpression(input)).toBe("array[unnest(array_field)]"); + expect(getModifiedSqlExpression(input)).toBe( + 'array[unnest(array_field)]' + ); }); - it("should not apply the modifier when conditions are not met", () => { + it('should not apply the modifier when conditions are not met', () => { (isArrayTypeMember as jest.Mock).mockReturnValue(false); const input = { - sqlExpression: "non_array_field", + sqlExpression: 'non_array_field', dimension: { - type: "string", - modifier: {} + type: 'string', + modifier: {}, } as Dimension, query: QUERY, - key: "test_key", - modifiers: MODIFIERS + key: 'test_key', + modifiers: MODIFIERS, }; - expect(getModifiedSqlExpression(input)).toBe("non_array_field"); + expect(getModifiedSqlExpression(input)).toBe('non_array_field'); }); }); -}); \ No newline at end of file +}); diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/index.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/index.ts new file mode 100644 index 00000000..c4aae4e7 --- /dev/null +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/index.ts @@ -0,0 +1,6 @@ +export { getWrappedBaseQueryWithProjections } from './get-wrapped-base-query-with-projections'; +export { + getModifiedSqlExpression, + MODIFIERS, +} from './sql-expression-modifiers'; +export type { DimensionModifier, Modifier } from './types'; diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts index 9d89a18b..1dcab304 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/sql-expression-modifiers.ts @@ -1,28 +1,9 @@ -import { - arrayFlattenModifier, - arrayFlattenModifierConfig, - shouldFlattenArray, -} from './modifiers/array-flatten-modifier'; -import { - arrayFieldUnNestModifier, - arrayUnnestModifier, - shouldUnnest, -} from './modifiers/array-unnest-modifier'; +import { arrayFlattenModifierConfig } from './modifiers/array-flatten-modifier'; +import { arrayUnnestModifier } from './modifiers/array-unnest-modifier'; import { DimensionModifier, Modifier } from './types'; -// Re-export types for backward compatibility -export type { DimensionModifier, Modifier }; - export const MODIFIERS = [arrayUnnestModifier, arrayFlattenModifierConfig]; -// Export individual modifier functions for backward compatibility -export { - arrayFieldUnNestModifier, - arrayFlattenModifier, - shouldFlattenArray, - shouldUnnest, -}; - export const getModifiedSqlExpression = ({ sqlExpression, dimension, diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index 9b0541cd..80881e75 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -2,7 +2,7 @@ export * from './ast-builder/ast-builder'; export * from './ast-deserializer/ast-deserializer'; export * from './ast-serializer/ast-serializer'; export * from './ast-validator'; -export * from './constants'; +export * from './constants/exports'; export { detectApplyContextParamsToBaseSQL } from './context-params/context-params-ast'; export * from './cube-measure-transformer/cube-measure-transformer'; export * from './cube-to-duckdb/cube-filter-to-duckdb'; @@ -13,10 +13,13 @@ export { } from './filter-params/filter-params-ast'; export { getFilterParamsSQL } from './get-filter-params-sql/get-filter-params-sql'; export { getFinalBaseSQL } from './get-final-base-sql/get-final-base-sql'; -export { getWrappedBaseQueryWithProjections } from './get-wrapped-base-query-with-projections/get-wrapped-base-query-with-projections'; +export * from './get-wrapped-base-query-with-projections'; export * from './joins/joins'; export * from './member-formatters'; export * from './resolution/resolution'; +export * from './resolution/steps/aggregation-step'; +export * from './resolution/steps/resolution-step'; +export * from './resolution/steps/unnest-step'; export * from './resolution/types'; export { FilterType } from './types/cube-types'; export * from './types/cube-types/index'; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 3e3c8a64..e9178334 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -15,6 +15,38 @@ import { ResolutionConfig, } from './types'; +/** + * Constructs a SQL column reference from a table name and a dimension/measure. + * + * @param tableName - The name of the table + * @param member - The dimension or measure object with name and optional alias + * @returns Formatted SQL column reference like: tableName."columnName" + */ +export const getColumnReference = ( + tableName: string, + member: { name: string; alias?: string } +): string => { + return `${tableName}."${member.alias || member.name}"`; +}; + +/** + * Checks if resolution should be skipped based on the resolution configuration and column projections. + * Resolution is skipped when there are no columns to resolve and no column projections. + * + * @param resolutionConfig - The resolution configuration + * @param columnProjections - Optional array of column projections + * @returns true if resolution should be skipped, false otherwise + */ +export const shouldSkipResolution = ( + resolutionConfig: ResolutionConfig, + columnProjections?: string[] +): boolean => { + return ( + resolutionConfig.columnConfigs.length === 0 && + columnProjections?.length === 0 + ); +}; + const constructBaseDimension = (name: string, schema: Measure | Dimension) => { return { name: memberKeyToSafeKey(name), @@ -82,13 +114,13 @@ export const createWrapperTableSchema = ( sql: sql, dimensions: baseTableSchema.dimensions.map((d) => ({ name: d.name, - sql: `${BASE_DATA_SOURCE_NAME}."${d.alias || d.name}"`, + sql: getColumnReference(BASE_DATA_SOURCE_NAME, d), type: d.type, alias: d.alias, })), measures: baseTableSchema.measures.map((m) => ({ name: m.name, - sql: `${BASE_DATA_SOURCE_NAME}."${m.alias || m.name}"`, + sql: getColumnReference(BASE_DATA_SOURCE_NAME, m), type: m.type, alias: m.alias, })), diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts b/meerkat-core/src/resolution/steps/aggregation-step.ts similarity index 80% rename from meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts rename to meerkat-core/src/resolution/steps/aggregation-step.ts index 92fecf9a..1161fbe6 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/aggregation-step.ts +++ b/meerkat-core/src/resolution/steps/aggregation-step.ts @@ -4,13 +4,23 @@ import { getNamespacedKey, Measure, MEERKAT_OUTPUT_DELIMITER, + Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, TableSchema, wrapWithRowIdOrderingAndExclusion, -} from '@devrev/meerkat-core'; -import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; -import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; +} from '../../index'; + +/** + * Constructs the resolved column name prefix for array resolution. + * This is used to identify which columns in the resolved schema correspond to array fields. + * + * @param columnName - The original column name + * @returns The prefixed column name used in resolution + */ +const getResolvedArrayColumnPrefix = (columnName: string): string => { + return `${columnName}${MEERKAT_OUTPUT_DELIMITER}`; +}; /** * Re-aggregate to reverse the unnest @@ -26,15 +36,19 @@ import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; * @returns Final SQL with arrays containing resolved values */ export const getAggregatedSql = async ({ - connection, resolvedTableSchema, resolutionConfig, contextParams, + cubeQueryToSQL, }: { - connection: AsyncDuckDBConnection; resolvedTableSchema: TableSchema; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; + cubeQueryToSQL: (params: { + query: Query; + tableSchemas: TableSchema[]; + contextParams?: ContextParams; + }) => Promise; }): Promise => { const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; @@ -44,7 +58,7 @@ export const getAggregatedSql = async ({ const isResolvedArrayColumn = (dimName: string) => { return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); + return dimName.includes(getResolvedArrayColumnPrefix(arrayCol.name)); }); }; @@ -67,8 +81,7 @@ export const getAggregatedSql = async ({ // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") // We just need to wrap it in the aggregation function - const columnRef = - dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; + const columnRef = dim.sql; // Use ARRAY_AGG for resolved array columns, MAX for others // Filter out null values for ARRAY_AGG using FILTER clause @@ -93,14 +106,11 @@ export const getAggregatedSql = async ({ // Generate the final SQL const aggregatedSql = await cubeQueryToSQL({ - connection, query: { measures: aggregationMeasures.map((m) => getNamespacedKey(baseTableName, m.name) ), - dimensions: rowIdDimension - ? [getNamespacedKey(baseTableName, rowIdDimension.name)] - : [], + dimensions: [getNamespacedKey(baseTableName, rowIdDimension.name)], }, tableSchemas: [schemaWithAggregation], contextParams, diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-core/src/resolution/steps/resolution-step.ts similarity index 80% rename from meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts rename to meerkat-core/src/resolution/steps/resolution-step.ts index a81de5c5..186ad134 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/steps/resolution-step.ts +++ b/meerkat-core/src/resolution/steps/resolution-step.ts @@ -4,14 +4,13 @@ import { generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, + getColumnReference, getNamespacedKey, memberKeyToSafeKey, Query, ResolutionConfig, TableSchema, -} from '@devrev/meerkat-core'; -import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; -import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; +} from '../../index'; /** * Apply resolution (join with lookup tables) @@ -23,17 +22,21 @@ import { cubeQueryToSQL } from '../../browser-cube-to-sql/browser-cube-to-sql'; * @returns Table schema with resolved values from lookup tables */ export const getResolvedTableSchema = async ({ - connection, baseTableSchema, resolutionConfig, - contextParams, columnProjections, + contextParams, + cubeQueryToSQL, }: { - connection: AsyncDuckDBConnection; baseTableSchema: TableSchema; resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; columnProjections: string[]; + contextParams?: ContextParams; + cubeQueryToSQL: (params: { + query: Query; + tableSchemas: TableSchema[]; + contextParams?: ContextParams; + }) => Promise; }): Promise => { const updatedBaseTableSchema: TableSchema = baseTableSchema; @@ -74,7 +77,6 @@ export const getResolvedTableSchema = async ({ }; const resolvedSql = await cubeQueryToSQL({ - connection, query: resolutionQuery, tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], contextParams, @@ -88,16 +90,31 @@ export const getResolvedTableSchema = async ({ // Create a map of resolution schema dimensions by original column name const resolutionDimensionsByColumnName = new Map(); + + // Create a map of resolution schemas by config name for efficient lookup + const resolutionSchemaByConfigName = new Map< + string, + (typeof resolutionSchemas)[0] + >(); + resolutionSchemas.forEach((resSchema) => { + resolutionConfig.columnConfigs.forEach((config) => { + if ( + resSchema.dimensions.some((dim) => dim.name.startsWith(config.name)) + ) { + resolutionSchemaByConfigName.set(config.name, resSchema); + } + }); + }); + + // Build the dimension map using the pre-indexed schemas resolutionConfig.columnConfigs.forEach((config) => { - const resSchema = resolutionSchemas.find((rs) => - rs.dimensions.some((dim) => dim.name.startsWith(config.name)) - ); + const resSchema = resolutionSchemaByConfigName.get(config.name); if (resSchema) { resolutionDimensionsByColumnName.set( config.name, resSchema.dimensions.map((dim) => ({ name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, + sql: getColumnReference(resolvedTableSchema.name, dim), type: dim.type, alias: dim.alias, })) diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts b/meerkat-core/src/resolution/steps/unnest-step.ts similarity index 75% rename from meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts rename to meerkat-core/src/resolution/steps/unnest-step.ts index 943bb07a..a04ee1db 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/steps/unnest-step.ts +++ b/meerkat-core/src/resolution/steps/unnest-step.ts @@ -2,11 +2,11 @@ import { ContextParams, createWrapperTableSchema, getNamespacedKey, + Query, ResolutionConfig, TableSchema, withArrayFlattenModifier, -} from '@devrev/meerkat-core'; -import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; +} from '../../index'; /** * Apply unnesting @@ -20,10 +20,16 @@ export const getUnnestTableSchema = async ({ baseTableSchema, resolutionConfig, contextParams, + cubeQueryToSQL, }: { baseTableSchema: TableSchema; resolutionConfig: ResolutionConfig; contextParams?: ContextParams; + cubeQueryToSQL: (params: { + query: Query; + tableSchemas: TableSchema[]; + contextParams?: ContextParams; + }) => Promise; }): Promise => { const updatedBaseTableSchema = withArrayFlattenModifier( baseTableSchema, @@ -33,11 +39,9 @@ export const getUnnestTableSchema = async ({ const unnestedSql = await cubeQueryToSQL({ query: { measures: [], - dimensions: [ - ...updatedBaseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - ], + dimensions: updatedBaseTableSchema.dimensions.map((d) => + getNamespacedKey(updatedBaseTableSchema.name, d.name) + ), }, tableSchemas: [updatedBaseTableSchema], contextParams, diff --git a/meerkat-node/src/__tests__/working_query.txt b/meerkat-node/src/__tests__/working_query.txt deleted file mode 100644 index 02298503..00000000 --- a/meerkat-node/src/__tests__/working_query.txt +++ /dev/null @@ -1,117 +0,0 @@ -SELECT - * exclude (__row_id) -FROM - ( - SELECT - MAX(__resolved_query."Ticket Id") AS "Ticket Id", - ARRAY_AGG(DISTINCT __resolved_query."Tags - Tag name") AS "Tags - Tag name", - "__row_id" - FROM - ( - SELECT - __resolved_query."__row_id" AS "__row_id", - * - FROM - ( - SELECT - "Tags - Tag name", - "Ticket Id", - "__row_id" - FROM - ( - SELECT - __unnested_base_query."Ticket Id" AS "Ticket Id", - __unnested_base_query."__row_id" AS "__row_id", - * - FROM - ( - SELECT - "Ticket Id", - "Tags", - "__row_id" - FROM - ( - SELECT - __base_query."Ticket Id" AS "Ticket Id", - unnest(__base_query."Tags") AS "Tags", - row_number() OVER () AS "__row_id", - * - FROM - ( - SELECT - count("ID") AS "Ticket Id", - "Tags" - FROM - ( - SELECT - dim_ticket.created_date AS "Created date", - 'ticket' AS "Work Type", - json_extract_string(dim_ticket.stage_json, '$.stage_id') AS "Stage", - CAST( - json_extract_string(dim_ticket.tags_json, '$[*].tag_id') AS VARCHAR[] - ) AS "Tags", - dim_ticket.id AS "ID", - * - FROM - ( - SELECT - * - FROM - devrev.dim_ticket - ) AS dim_ticket - ) AS dim_ticket - WHERE - ( - ( - ( - ("Created date" >= '2025-08-02T09:30:00.000Z') - AND ("Created date" <= '2025-10-31T10:29:59.999Z') - ) - AND ("Work Type" IN ('ticket')) - AND ( - Stage IN ( - 'don:core:dvrv-us-1:devo/787:custom_stage/514', - 'don:core:dvrv-us-1:devo/787:custom_stage/512', - 'don:core:dvrv-us-1:devo/787:custom_stage/501', - 'don:core:dvrv-us-1:devo/787:custom_stage/485', - 'don:core:dvrv-us-1:devo/787:custom_stage/440', - 'don:core:dvrv-us-1:devo/787:custom_stage/25', - 'don:core:dvrv-us-1:devo/787:custom_stage/24', - 'don:core:dvrv-us-1:devo/787:custom_stage/22', - 'don:core:dvrv-us-1:devo/787:custom_stage/21', - 'don:core:dvrv-us-1:devo/787:custom_stage/19', - 'don:core:dvrv-us-1:devo/787:custom_stage/13', - 'don:core:dvrv-us-1:devo/787:custom_stage/10', - 'don:core:dvrv-us-1:devo/787:custom_stage/8', - 'don:core:dvrv-us-1:devo/787:custom_stage/7', - 'don:core:dvrv-us-1:devo/787:custom_stage/6', - 'don:core:dvrv-us-1:devo/787:custom_stage/4' - ) - ) - ) - ) - GROUP BY - Tags - LIMIT - 50000 - ) AS __base_query - ) AS __base_query - ) AS __unnested_base_query - LEFT JOIN ( - SELECT - __unnested_base_query__dim_ticket__tags_json.name AS "Tags - Tag name", - * - FROM - ( - SELECT - * - FROM - devrev.dim_tag - ) AS __unnested_base_query__dim_ticket__tags_json - ) AS __unnested_base_query__dim_ticket__tags_json ON __unnested_base_query."Tags" = __unnested_base_query__dim_ticket__tags_json.id - ) AS MEERKAT_GENERATED_TABLE - ) AS __resolved_query - ) AS __resolved_query - GROUP BY - __row_id - ) \ No newline at end of file diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 65c2f9e1..3cb83195 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -1,6 +1,9 @@ import { BASE_DATA_SOURCE_NAME, ContextParams, + getAggregatedSql as coreGetAggregatedSql, + getResolvedTableSchema as coreGetResolvedTableSchema, + getUnnestTableSchema as coreGetUnnestTableSchema, createBaseTableSchema, Dimension, generateRowNumberSql, @@ -8,12 +11,10 @@ import { Query, ResolutionConfig, ROW_ID_DIMENSION_NAME, + shouldSkipResolution, TableSchema, } from '@devrev/meerkat-core'; import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; -import { getAggregatedSql } from './steps/aggregation-step'; -import { getResolvedTableSchema } from './steps/resolution-step'; -import { getUnnestTableSchema } from './steps/unnest-step'; export interface CubeQueryToSQLWithResolutionParams { query: Query; @@ -36,12 +37,8 @@ export const cubeQueryToSQLWithResolution = async ({ contextParams, }); - // We have columnProjections check here to ensure that, we are using the same - // order in the final query - if ( - resolutionConfig.columnConfigs.length === 0 && - columnProjections?.length === 0 - ) { + // Check if resolution should be skipped + if (shouldSkipResolution(resolutionConfig, columnProjections)) { return baseSql; } @@ -55,31 +52,7 @@ export const cubeQueryToSQLWithResolution = async ({ return columnProjections?.includes(config.name); } ); - return getCubeQueryToSQLWithResolution({ - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, - }); -}; -const getCubeQueryToSQLWithResolution = async ({ - baseSql, - query, - tableSchemas, - resolutionConfig, - columnProjections, - contextParams, -}: { - baseSql: string; - query: Query; - tableSchemas: TableSchema[]; - resolutionConfig: ResolutionConfig; - columnProjections: string[]; - contextParams?: ContextParams; -}): Promise => { const baseSchema: TableSchema = createBaseTableSchema( baseSql, tableSchemas, @@ -87,8 +60,7 @@ const getCubeQueryToSQLWithResolution = async ({ [], columnProjections ); - - baseSchema.dimensions.push({ + const rowIdDimension: Dimension = { name: ROW_ID_DIMENSION_NAME, sql: generateRowNumberSql( query, @@ -97,7 +69,8 @@ const getCubeQueryToSQLWithResolution = async ({ ), type: 'number', alias: ROW_ID_DIMENSION_NAME, - } as Dimension); + }; + baseSchema.dimensions.push(rowIdDimension); columnProjections.push(ROW_ID_DIMENSION_NAME); // Doing this because we need to use the original name of the column in the base table schema. @@ -106,25 +79,28 @@ const getCubeQueryToSQLWithResolution = async ({ }); // Generate SQL with row_id and unnested arrays - const unnestTableSchema = await getUnnestTableSchema({ + const unnestTableSchema = await coreGetUnnestTableSchema({ baseTableSchema: baseSchema, resolutionConfig, contextParams, + cubeQueryToSQL: async (params) => cubeQueryToSQL(params), }); // Apply resolution (join with lookup tables) - const resolvedTableSchema = await getResolvedTableSchema({ + const resolvedTableSchema = await coreGetResolvedTableSchema({ baseTableSchema: unnestTableSchema, resolutionConfig, contextParams, columnProjections, + cubeQueryToSQL: async (params) => cubeQueryToSQL(params), }); // Re-aggregate to reverse the unnest - const aggregatedSql = await getAggregatedSql({ + const aggregatedSql = await coreGetAggregatedSql({ resolvedTableSchema, resolutionConfig, contextParams, + cubeQueryToSQL: async (params) => cubeQueryToSQL(params), }); return aggregatedSql; diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts deleted file mode 100644 index 83f87c55..00000000 --- a/meerkat-node/src/cube-to-sql-with-resolution/steps/aggregation-step.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { - ContextParams, - getArrayTypeResolutionColumnConfigs, - getNamespacedKey, - Measure, - MEERKAT_OUTPUT_DELIMITER, - ResolutionConfig, - ROW_ID_DIMENSION_NAME, - TableSchema, - wrapWithRowIdOrderingAndExclusion, -} from '@devrev/meerkat-core'; -import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; - -/** - * Re-aggregate to reverse the unnest - * - * This function: - * 1. Groups by row_id - * 2. Uses MAX for non-array columns (they're duplicated) - * 3. Uses ARRAY_AGG for resolved array columns - * - * @param resolvedTableSchema - Schema from Phase 2 (contains all column info) - * @param resolutionConfig - Resolution configuration - * @param contextParams - Optional context parameters - * @returns Final SQL with arrays containing resolved values - */ -export const getAggregatedSql = async ({ - resolvedTableSchema, - resolutionConfig, - contextParams, -}: { - resolvedTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - contextParams?: ContextParams; -}): Promise => { - const aggregationBaseTableSchema: TableSchema = resolvedTableSchema; - - // Identify which columns need ARRAY_AGG vs MAX - const arrayColumns = getArrayTypeResolutionColumnConfigs(resolutionConfig); - const baseTableName = aggregationBaseTableSchema.name; - - const isResolvedArrayColumn = (dimName: string) => { - return arrayColumns.some((arrayCol) => { - return dimName.includes(`${arrayCol.name}${MEERKAT_OUTPUT_DELIMITER}`); - }); - }; - - // Create aggregation measures with proper aggregation functions - // Get row_id dimension for GROUP BY - const rowIdDimension = aggregationBaseTableSchema.dimensions.find( - (d) => d.name === ROW_ID_DIMENSION_NAME - ); - - if (!rowIdDimension) { - throw new Error('Row id dimension not found'); - } - // Create measures with MAX or ARRAY_AGG based on column type - const aggregationMeasures: Measure[] = []; - - aggregationBaseTableSchema.dimensions - .filter((dim) => dim.name !== rowIdDimension?.name) - .forEach((dim) => { - const isArrayColumn = isResolvedArrayColumn(dim.name); - - // The dimension's sql field already has the correct reference (e.g., __resolved_query."__row_id") - // We just need to wrap it in the aggregation function - const columnRef = - dim.sql || `${baseTableName}."${dim.alias || dim.name}"`; - - // Use ARRAY_AGG for resolved array columns, MAX for others - // Filter out null values for ARRAY_AGG using FILTER clause - const aggregationFn = isArrayColumn - ? `COALESCE(ARRAY_AGG(DISTINCT ${columnRef}) FILTER (WHERE ${columnRef} IS NOT NULL), [])` - : `MAX(${columnRef})`; - - aggregationMeasures.push({ - name: dim.name, - sql: aggregationFn, - type: dim.type, - alias: dim.alias, - }); - }); - - // Update the schema with aggregation measures - const schemaWithAggregation: TableSchema = { - ...aggregationBaseTableSchema, - measures: aggregationMeasures, - dimensions: [rowIdDimension], - }; - - // Generate the final SQL - const aggregatedSql = await cubeQueryToSQL({ - query: { - measures: aggregationMeasures.map((m) => - getNamespacedKey(baseTableName, m.name) - ), - dimensions: rowIdDimension - ? [getNamespacedKey(baseTableName, rowIdDimension.name)] - : [], - }, - tableSchemas: [schemaWithAggregation], - contextParams, - }); - - // Order by row_id to maintain consistent ordering before excluding it - return wrapWithRowIdOrderingAndExclusion( - aggregatedSql, - ROW_ID_DIMENSION_NAME - ); -}; - diff --git a/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts b/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts deleted file mode 100644 index be06c372..00000000 --- a/meerkat-node/src/cube-to-sql-with-resolution/steps/resolution-step.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { - ContextParams, - createWrapperTableSchema, - generateResolutionJoinPaths, - generateResolutionSchemas, - generateResolvedDimensions, - getNamespacedKey, - memberKeyToSafeKey, - Query, - ResolutionConfig, - TableSchema, -} from '@devrev/meerkat-core'; -import { cubeQueryToSQL } from '../../cube-to-sql/cube-to-sql'; - -/** - * Apply resolution (join with lookup tables) - * - * This function: - * 1. Uses the base table schema from Phase 1 (source of truth) - * 2. Generates resolution schemas for array fields - * 3. Sets up join paths between unnested data and resolution tables - * @returns Table schema with resolved values from lookup tables - */ -export const getResolvedTableSchema = async ({ - baseTableSchema, - resolutionConfig, - columnProjections, - contextParams, -}: { - baseTableSchema: TableSchema; - resolutionConfig: ResolutionConfig; - columnProjections: string[]; - contextParams?: ContextParams; -}): Promise => { - const updatedBaseTableSchema: TableSchema = baseTableSchema; - - // Generate resolution schemas for fields that need resolution - const resolutionSchemas = generateResolutionSchemas(resolutionConfig, [ - updatedBaseTableSchema, - ]); - - const joinPaths = generateResolutionJoinPaths( - updatedBaseTableSchema.name, - resolutionConfig, - [updatedBaseTableSchema] - ); - - const tempQuery: Query = { - measures: [], - dimensions: baseTableSchema.dimensions.map((d) => - getNamespacedKey(updatedBaseTableSchema.name, d.name) - ), - }; - - const updatedColumnProjections = columnProjections?.map((cp) => - memberKeyToSafeKey(cp) - ); - // Generate resolved dimensions using columnProjections - const resolvedDimensions = generateResolvedDimensions( - updatedBaseTableSchema.name, - tempQuery, - resolutionConfig, - updatedColumnProjections - ); - - // Create query and generate SQL - const resolutionQuery: Query = { - measures: [], - dimensions: resolvedDimensions, - joinPaths, - }; - - const resolvedSql = await cubeQueryToSQL({ - query: resolutionQuery, - tableSchemas: [updatedBaseTableSchema, ...resolutionSchemas], - contextParams, - }); - - // Use the baseTableSchema which already has all the column info - const resolvedTableSchema: TableSchema = createWrapperTableSchema( - resolvedSql, - updatedBaseTableSchema - ); - - // Create a map of resolution schema dimensions by original column name - const resolutionDimensionsByColumnName = new Map(); - resolutionConfig.columnConfigs.forEach((config) => { - const resSchema = resolutionSchemas.find((rs) => - rs.dimensions.some((dim) => dim.name.startsWith(config.name)) - ); - if (resSchema) { - resolutionDimensionsByColumnName.set( - config.name, - resSchema.dimensions.map((dim) => ({ - name: dim.name, - sql: `${resolvedTableSchema.name}."${dim.alias || dim.name}"`, - type: dim.type, - alias: dim.alias, - })) - ); - } - }); - - // Maintain the same order as columnProjections - // Replace dimensions that need resolution with their resolved counterparts - resolvedTableSchema.dimensions = (updatedColumnProjections || []).flatMap( - (projectionName) => { - // Check if this column has resolved dimensions - const resolvedDims = resolutionDimensionsByColumnName.get(projectionName); - if (resolvedDims) { - // Use resolved dimensions - return resolvedDims; - } - - // Otherwise, find the original dimension from baseTableSchema - const originalDim = baseTableSchema.dimensions.find( - (d) => d.name === projectionName - ); - if (originalDim) { - return [originalDim]; - } - - // If not found, throw an error - throw new Error( - `Column projection '${projectionName}' not found in base table schema dimensions` - ); - } - ); - - return resolvedTableSchema; -}; From b236c7891b0bcf722790e32963826eed04d2fed3 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 12:07:36 +0530 Subject: [PATCH 32/38] changing type of resolutionConfig isArrayType --- .../src/resolution/resolution.spec.ts | 45 +++++++++++++------ meerkat-core/src/resolution/resolution.ts | 5 ++- meerkat-core/src/resolution/types.ts | 12 +++-- .../cube-to-sql-with-resolution.spec.ts | 11 ++--- meerkat-node/src/__tests__/resolution.spec.ts | 8 ++++ 5 files changed, 56 insertions(+), 25 deletions(-) diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index 0ed7deef..ccda17c2 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -1,3 +1,4 @@ +import { isArrayTypeMember } from '../utils/is-array-member-type'; import { createBaseTableSchema, createWrapperTableSchema, @@ -108,13 +109,14 @@ describe('Create base table schema', () => { { name: 'base_table.column1', source: 'resolution_table', - isArrayType: false, + type: 'string' as const, joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', source: 'resolution_table', + type: 'string' as const, joinColumn: 'id', resolutionColumns: ['display_name'], }, @@ -220,14 +222,14 @@ describe('Create base table schema', () => { columnConfigs: [ { name: 'base_table.column1', - isArrayType: false, + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', - isArrayType: false, + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_name'], @@ -301,12 +303,14 @@ describe('Generate resolution schemas', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['id', 'display_name'], @@ -384,12 +388,14 @@ describe('Generate resolution schemas', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + type: 'string' as const, source: 'resolution_table1', // does not exist joinColumn: 'id', resolutionColumns: ['id', 'display_name'], @@ -431,6 +437,7 @@ describe('Generate resolution schemas', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -468,6 +475,7 @@ describe('Generate resolution schemas', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -533,6 +541,7 @@ describe('Generate resolution schemas', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -593,12 +602,14 @@ describe('Generate resolved dimensions', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_name'], @@ -628,6 +639,7 @@ describe('Generate resolved dimensions', () => { columnConfigs: [ { name: 'base_table.column3', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -657,12 +669,14 @@ describe('Generate resolved dimensions', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['id', 'display_name'], @@ -698,12 +712,14 @@ describe('Generate resolution join paths', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.column2', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_name'], @@ -763,6 +779,7 @@ describe('Generate resolution join paths', () => { columnConfigs: [ { name: 'base_table.column1', + type: 'string' as const, source: 'resolution_table', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -907,21 +924,21 @@ describe('getArrayTypeResolutionColumnConfigs', () => { columnConfigs: [ { name: 'table.array_column', - isArrayType: true, + type: 'string_array' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], }, { name: 'table.scalar_column', - isArrayType: false, + type: 'string' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], }, { name: 'table.another_array', - isArrayType: true, + type: 'number_array' as const, source: 'lookup_table2', joinColumn: 'id', resolutionColumns: ['value'], @@ -935,7 +952,7 @@ describe('getArrayTypeResolutionColumnConfigs', () => { expect(result).toHaveLength(2); expect(result[0].name).toBe('table.array_column'); expect(result[1].name).toBe('table.another_array'); - expect(result.every((config) => config.isArrayType === true)).toBe(true); + expect(result.every((config) => isArrayTypeMember(config.type))).toBe(true); }); it('should return empty array when no array type configs exist', () => { @@ -943,14 +960,14 @@ describe('getArrayTypeResolutionColumnConfigs', () => { columnConfigs: [ { name: 'table.scalar_column1', - isArrayType: false, + type: 'string' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], }, { name: 'table.scalar_column2', - isArrayType: false, + type: 'number' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], @@ -1000,7 +1017,7 @@ describe('withArrayFlattenModifier', () => { columnConfigs: [ { name: 'array_column', - isArrayType: true, + type: 'string_array' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], @@ -1047,14 +1064,14 @@ describe('withArrayFlattenModifier', () => { columnConfigs: [ { name: 'array_column1', - isArrayType: true, + type: 'string_array' as const, source: 'lookup_table1', joinColumn: 'id', resolutionColumns: ['name'], }, { name: 'array_column2', - isArrayType: true, + type: 'number_array' as const, source: 'lookup_table2', joinColumn: 'id', resolutionColumns: ['value'], @@ -1099,7 +1116,7 @@ describe('withArrayFlattenModifier', () => { columnConfigs: [ { name: 'column1', - isArrayType: false, + type: 'string' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], @@ -1126,7 +1143,7 @@ describe('withArrayFlattenModifier', () => { columnConfigs: [ { name: 'array_column', - isArrayType: true, + type: 'string_array' as const, source: 'lookup_table', joinColumn: 'id', resolutionColumns: ['name'], diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index e9178334..d3517466 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -9,6 +9,7 @@ import { findInDimensionSchemas, findInSchemas, } from '../utils/find-in-table-schema'; +import { isArrayTypeMember } from '../utils/is-array-member-type'; import { BASE_DATA_SOURCE_NAME, ResolutionColumnConfig, @@ -156,8 +157,8 @@ export const withArrayFlattenModifier = ( export const getArrayTypeResolutionColumnConfigs = ( resolutionConfig: ResolutionConfig ) => { - return resolutionConfig.columnConfigs.filter( - (config) => config.isArrayType === true + return resolutionConfig.columnConfigs.filter((config) => + isArrayTypeMember(config.type) ); }; diff --git a/meerkat-core/src/resolution/types.ts b/meerkat-core/src/resolution/types.ts index 31cfb58d..3f7824b7 100644 --- a/meerkat-core/src/resolution/types.ts +++ b/meerkat-core/src/resolution/types.ts @@ -1,12 +1,16 @@ -import { TableSchema } from '../types/cube-types/table'; +import { + DimensionType, + MeasureType, + TableSchema, +} from '../types/cube-types/table'; export interface ResolutionColumnConfig { // Name of the column that needs resolution. // Should match a measure or dimension in the query. name: string; - // Indicates if this column is an array type that needs special handling (UNNEST/ARRAY_AGG). - // This is essential metadata for the multi-phase resolution process. - isArrayType?: boolean; + // Type of the dimension/measure (e.g., 'string', 'number', 'string_array') + // Used to determine if special array handling (UNNEST/ARRAY_AGG) is needed. + type: DimensionType | MeasureType; // Name of the data source to use for resolution. source: string; // Name of the column in the data source to join on. diff --git a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts index 7bb19cab..1455f6a0 100644 --- a/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts +++ b/meerkat-node/src/__tests__/cube-to-sql-with-resolution.spec.ts @@ -172,7 +172,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { columnConfigs: [ { name: 'tickets.owners', - isArrayType: true, + type: 'string_array' as const, source: 'owners_lookup', joinColumn: 'id', resolutionColumns: ['display_name', 'email'], @@ -245,21 +245,21 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { columnConfigs: [ { name: 'tickets.owners', - isArrayType: true, + type: 'string_array' as const, source: 'owners_lookup', joinColumn: 'id', resolutionColumns: ['display_name'], }, { name: 'tickets.tags', - isArrayType: true, + type: 'string_array' as const, source: 'tags_lookup', joinColumn: 'id', resolutionColumns: ['tag_name'], }, { name: 'tickets.created_by', - isArrayType: false, + type: 'string' as const, source: 'created_by_lookup', joinColumn: 'id', resolutionColumns: ['name'], @@ -362,7 +362,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { columnConfigs: [ { name: 'tickets.created_by', - isArrayType: false, + type: 'string' as const, source: 'created_by_lookup', joinColumn: 'id', resolutionColumns: ['name'], @@ -492,6 +492,7 @@ describe('cubeQueryToSQLWithResolution - Array field resolution', () => { columnConfigs: [ { name: 'tickets.created_by', + type: 'string' as const, source: 'created_by_lookup', joinColumn: 'id', resolutionColumns: ['name'], diff --git a/meerkat-node/src/__tests__/resolution.spec.ts b/meerkat-node/src/__tests__/resolution.spec.ts index 3f31268a..c909afc3 100644 --- a/meerkat-node/src/__tests__/resolution.spec.ts +++ b/meerkat-node/src/__tests__/resolution.spec.ts @@ -218,6 +218,7 @@ describe('Resolution Tests', () => { columnConfigs: [ { name: 'base_table.part_id_1', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -247,18 +248,21 @@ describe('Resolution Tests', () => { columnConfigs: [ { name: 'base_table.part_id_1', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.work_id', + type: 'string' as const, source: 'dim_work', joinColumn: 'id', resolutionColumns: ['display_id', 'title'], }, { name: 'base_table.part_id_2', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -302,6 +306,7 @@ describe('Resolution Tests', () => { columnConfigs: [ { name: 'base_table.part_id_1', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -336,12 +341,14 @@ describe('Resolution Tests', () => { columnConfigs: [ { name: 'base_table.part_id_1', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], }, { name: 'base_table.part_id_2', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], @@ -377,6 +384,7 @@ describe('Resolution Tests', () => { columnConfigs: [ { name: 'base_table.part_id_1', + type: 'string' as const, source: 'dim_part', joinColumn: 'id', resolutionColumns: ['display_id'], From 2f350b77ee25775d5fae8a91d9f1256fdfd41bdf Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 12:44:30 +0530 Subject: [PATCH 33/38] minor updates --- .../get-aliased-columns-from-filters.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meerkat-core/src/get-wrapped-base-query-with-projections/get-aliased-columns-from-filters.ts b/meerkat-core/src/get-wrapped-base-query-with-projections/get-aliased-columns-from-filters.ts index 2ada853e..8c21347c 100644 --- a/meerkat-core/src/get-wrapped-base-query-with-projections/get-aliased-columns-from-filters.ts +++ b/meerkat-core/src/get-wrapped-base-query-with-projections/get-aliased-columns-from-filters.ts @@ -5,7 +5,8 @@ import { findInDimensionSchema, findInMeasureSchema, } from '../utils/find-in-table-schema'; -import { getModifiedSqlExpression, Modifier } from './sql-expression-modifiers'; +import { getModifiedSqlExpression } from './sql-expression-modifiers'; +import { Modifier } from './types'; export const getDimensionProjection = ({ key, From 46d4884748cbc64f16e7b8b5b3de72966c45cb50 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 14:17:14 +0530 Subject: [PATCH 34/38] splitting resolution file into multiple generators --- meerkat-core/src/index.ts | 1 + .../generate-resolution-join-paths.ts | 23 +++ .../generators/generate-resolution-schemas.ts | 67 ++++++++ .../generate-resolved-dimensions.ts | 38 +++++ .../generators/generate-row-number-sql.ts | 36 +++++ .../src/resolution/generators/index.ts | 4 + .../src/resolution/resolution.spec.ts | 6 +- meerkat-core/src/resolution/resolution.ts | 150 +----------------- 8 files changed, 174 insertions(+), 151 deletions(-) create mode 100644 meerkat-core/src/resolution/generators/generate-resolution-join-paths.ts create mode 100644 meerkat-core/src/resolution/generators/generate-resolution-schemas.ts create mode 100644 meerkat-core/src/resolution/generators/generate-resolved-dimensions.ts create mode 100644 meerkat-core/src/resolution/generators/generate-row-number-sql.ts create mode 100644 meerkat-core/src/resolution/generators/index.ts diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index 80881e75..0699e47a 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -16,6 +16,7 @@ export { getFinalBaseSQL } from './get-final-base-sql/get-final-base-sql'; export * from './get-wrapped-base-query-with-projections'; export * from './joins/joins'; export * from './member-formatters'; +export * from './resolution/generators'; export * from './resolution/resolution'; export * from './resolution/steps/aggregation-step'; export * from './resolution/steps/resolution-step'; diff --git a/meerkat-core/src/resolution/generators/generate-resolution-join-paths.ts b/meerkat-core/src/resolution/generators/generate-resolution-join-paths.ts new file mode 100644 index 00000000..d6c5ab1a --- /dev/null +++ b/meerkat-core/src/resolution/generators/generate-resolution-join-paths.ts @@ -0,0 +1,23 @@ +import { constructAlias, memberKeyToSafeKey } from '../../member-formatters'; +import { JoinPath } from '../../types/cube-types/query'; +import { TableSchema } from '../../types/cube-types/table'; +import { findInSchemas } from '../../utils/find-in-table-schema'; +import { ResolutionConfig } from '../types'; + +export const generateResolutionJoinPaths = ( + baseDataSourceName: string, + resolutionConfig: ResolutionConfig, + baseTableSchemas: TableSchema[] +): JoinPath[] => { + return resolutionConfig.columnConfigs.map((config) => [ + { + left: baseDataSourceName, + right: memberKeyToSafeKey(config.name), + on: constructAlias({ + name: config.name, + alias: findInSchemas(config.name, baseTableSchemas)?.alias, + aliasContext: { isAstIdentifier: false }, + }), + }, + ]); +}; diff --git a/meerkat-core/src/resolution/generators/generate-resolution-schemas.ts b/meerkat-core/src/resolution/generators/generate-resolution-schemas.ts new file mode 100644 index 00000000..7adc799e --- /dev/null +++ b/meerkat-core/src/resolution/generators/generate-resolution-schemas.ts @@ -0,0 +1,67 @@ +import { + constructAlias, + getNamespacedKey, + memberKeyToSafeKey, +} from '../../member-formatters'; +import { TableSchema } from '../../types/cube-types/table'; +import { + findInDimensionSchemas, + findInSchemas, +} from '../../utils/find-in-table-schema'; +import { ResolutionConfig } from '../types'; + +export const generateResolutionSchemas = ( + config: ResolutionConfig, + baseTableSchemas: TableSchema[] +) => { + const resolutionSchemas: TableSchema[] = []; + config.columnConfigs.forEach((colConfig) => { + const tableSchema = config.tableSchemas.find( + (ts) => ts.name === colConfig.source + ); + if (!tableSchema) { + throw new Error(`Table schema not found for ${colConfig.source}`); + } + + const baseName = memberKeyToSafeKey(colConfig.name); + const baseAlias = constructAlias({ + name: colConfig.name, + alias: findInSchemas(colConfig.name, baseTableSchemas)?.alias, + aliasContext: { isTableSchemaAlias: true }, + }); + + // For each column that needs to be resolved, create a copy of the relevant table schema. + // We use the name of the column in the base query as the table schema name + // to avoid conflicts. + const resolutionSchema: TableSchema = { + name: baseName, + sql: tableSchema.sql, + measures: [], + dimensions: colConfig.resolutionColumns.map((col) => { + const dimension = findInDimensionSchemas( + getNamespacedKey(colConfig.source, col), + config.tableSchemas + ); + if (!dimension) { + throw new Error(`Dimension not found: ${col}`); + } + return { + // Need to create a new name due to limitations with how + // CubeToSql handles duplicate dimension names between different sources. + name: memberKeyToSafeKey(getNamespacedKey(colConfig.name, col)), + sql: `${baseName}.${col}`, + type: dimension.type, + alias: `${baseAlias} - ${constructAlias({ + name: col, + alias: dimension.alias, + aliasContext: { isTableSchemaAlias: true }, + })}`, + }; + }), + }; + + resolutionSchemas.push(resolutionSchema); + }); + + return resolutionSchemas; +}; diff --git a/meerkat-core/src/resolution/generators/generate-resolved-dimensions.ts b/meerkat-core/src/resolution/generators/generate-resolved-dimensions.ts new file mode 100644 index 00000000..788b9703 --- /dev/null +++ b/meerkat-core/src/resolution/generators/generate-resolved-dimensions.ts @@ -0,0 +1,38 @@ +import { getNamespacedKey, memberKeyToSafeKey } from '../../member-formatters'; +import { Member, Query } from '../../types/cube-types/query'; +import { ResolutionConfig } from '../types'; + +export const generateResolvedDimensions = ( + baseDataSourceName: string, + query: Query, + config: ResolutionConfig, + columnProjections?: string[] +): Member[] => { + // If column projections are provided, use those. + // Otherwise, use all measures and dimensions from the original query. + const aggregatedDimensions = columnProjections + ? columnProjections + : [...query.measures, ...(query.dimensions || [])]; + + const resolvedDimensions: Member[] = aggregatedDimensions.flatMap( + (dimension) => { + const columnConfig = config.columnConfigs.find( + (c) => c.name === dimension + ); + + if (!columnConfig) { + return [ + getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), + ]; + } else { + return columnConfig.resolutionColumns.map((col) => + getNamespacedKey( + memberKeyToSafeKey(dimension), + memberKeyToSafeKey(getNamespacedKey(columnConfig.name, col)) + ) + ); + } + } + ); + return resolvedDimensions; +}; diff --git a/meerkat-core/src/resolution/generators/generate-row-number-sql.ts b/meerkat-core/src/resolution/generators/generate-row-number-sql.ts new file mode 100644 index 00000000..653df02a --- /dev/null +++ b/meerkat-core/src/resolution/generators/generate-row-number-sql.ts @@ -0,0 +1,36 @@ +import { memberKeyToSafeKey } from '../../member-formatters'; + +/** + * Generates row_number() OVER (ORDER BY ...) SQL based on query order. + * This is used to preserve the original query ordering through resolution operations. + * + * @param query - The query object that may contain an order clause + * @param dimensions - The dimensions array from the base table schema + * @param baseTableName - The base table name to use in column references + * @returns SQL expression for row_number() OVER (ORDER BY ...) + */ +export const generateRowNumberSql = ( + query: { order?: Record }, + dimensions: { name: string; alias?: string }[], + baseTableName: string +): string => { + let rowNumberSql = 'row_number() OVER ('; + if (query.order && Object.keys(query.order).length > 0) { + const orderClauses = Object.entries(query.order).map( + ([member, direction]) => { + // Find the actual column name/alias in the base table dimensions + const safeMember = memberKeyToSafeKey(member); + const dimension = dimensions.find( + (d) => d.name === safeMember || d.alias === safeMember + ); + const columnName = dimension + ? dimension.alias || dimension.name + : safeMember; + return `${baseTableName}."${columnName}" ${direction.toUpperCase()}`; + } + ); + rowNumberSql += `ORDER BY ${orderClauses.join(', ')}`; + } + rowNumberSql += ')'; + return rowNumberSql; +}; diff --git a/meerkat-core/src/resolution/generators/index.ts b/meerkat-core/src/resolution/generators/index.ts new file mode 100644 index 00000000..dea7f48d --- /dev/null +++ b/meerkat-core/src/resolution/generators/index.ts @@ -0,0 +1,4 @@ +export * from './generate-resolution-join-paths'; +export * from './generate-resolution-schemas'; +export * from './generate-resolved-dimensions'; +export * from './generate-row-number-sql'; diff --git a/meerkat-core/src/resolution/resolution.spec.ts b/meerkat-core/src/resolution/resolution.spec.ts index ccda17c2..4f432bb8 100644 --- a/meerkat-core/src/resolution/resolution.spec.ts +++ b/meerkat-core/src/resolution/resolution.spec.ts @@ -1,11 +1,13 @@ import { isArrayTypeMember } from '../utils/is-array-member-type'; import { - createBaseTableSchema, - createWrapperTableSchema, generateResolutionJoinPaths, generateResolutionSchemas, generateResolvedDimensions, generateRowNumberSql, +} from './generators'; +import { + createBaseTableSchema, + createWrapperTableSchema, getArrayTypeResolutionColumnConfigs, withArrayFlattenModifier, } from './resolution'; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index d3517466..72cbb33c 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -3,12 +3,8 @@ import { getNamespacedKey, memberKeyToSafeKey, } from '../member-formatters'; -import { JoinPath, Member, Query } from '../types/cube-types/query'; +import { Member } from '../types/cube-types/query'; import { Dimension, Measure, TableSchema } from '../types/cube-types/table'; -import { - findInDimensionSchemas, - findInSchemas, -} from '../utils/find-in-table-schema'; import { isArrayTypeMember } from '../utils/is-array-member-type'; import { BASE_DATA_SOURCE_NAME, @@ -162,150 +158,6 @@ export const getArrayTypeResolutionColumnConfigs = ( ); }; -export const generateResolutionSchemas = ( - config: ResolutionConfig, - baseTableSchemas: TableSchema[] -) => { - const resolutionSchemas: TableSchema[] = []; - config.columnConfigs.forEach((colConfig) => { - const tableSchema = config.tableSchemas.find( - (ts) => ts.name === colConfig.source - ); - if (!tableSchema) { - throw new Error(`Table schema not found for ${colConfig.source}`); - } - - const baseName = memberKeyToSafeKey(colConfig.name); - const baseAlias = constructAlias({ - name: colConfig.name, - alias: findInSchemas(colConfig.name, baseTableSchemas)?.alias, - aliasContext: { isTableSchemaAlias: true }, - }); - - // For each column that needs to be resolved, create a copy of the relevant table schema. - // We use the name of the column in the base query as the table schema name - // to avoid conflicts. - const resolutionSchema: TableSchema = { - name: baseName, - sql: tableSchema.sql, - measures: [], - dimensions: colConfig.resolutionColumns.map((col) => { - const dimension = findInDimensionSchemas( - getNamespacedKey(colConfig.source, col), - config.tableSchemas - ); - if (!dimension) { - throw new Error(`Dimension not found: ${col}`); - } - return { - // Need to create a new name due to limitations with how - // CubeToSql handles duplicate dimension names between different sources. - name: memberKeyToSafeKey(getNamespacedKey(colConfig.name, col)), - sql: `${baseName}.${col}`, - type: dimension.type, - alias: `${baseAlias} - ${constructAlias({ - name: col, - alias: dimension.alias, - aliasContext: { isTableSchemaAlias: true }, - })}`, - }; - }), - }; - - resolutionSchemas.push(resolutionSchema); - }); - - return resolutionSchemas; -}; - -export const generateResolvedDimensions = ( - baseDataSourceName: string, - query: Query, - config: ResolutionConfig, - columnProjections?: string[] -): Member[] => { - // If column projections are provided, use those. - // Otherwise, use all measures and dimensions from the original query. - const aggregatedDimensions = columnProjections - ? columnProjections - : [...query.measures, ...(query.dimensions || [])]; - - const resolvedDimensions: Member[] = aggregatedDimensions.flatMap( - (dimension) => { - const columnConfig = config.columnConfigs.find( - (c) => c.name === dimension - ); - - if (!columnConfig) { - return [ - getNamespacedKey(baseDataSourceName, memberKeyToSafeKey(dimension)), - ]; - } else { - return columnConfig.resolutionColumns.map((col) => - getNamespacedKey( - memberKeyToSafeKey(dimension), - memberKeyToSafeKey(getNamespacedKey(columnConfig.name, col)) - ) - ); - } - } - ); - return resolvedDimensions; -}; - -export const generateResolutionJoinPaths = ( - baseDataSourceName: string, - resolutionConfig: ResolutionConfig, - baseTableSchemas: TableSchema[] -): JoinPath[] => { - return resolutionConfig.columnConfigs.map((config) => [ - { - left: baseDataSourceName, - right: memberKeyToSafeKey(config.name), - on: constructAlias({ - name: config.name, - alias: findInSchemas(config.name, baseTableSchemas)?.alias, - aliasContext: { isAstIdentifier: false }, - }), - }, - ]); -}; - -/** - * Generates row_number() OVER (ORDER BY ...) SQL based on query order. - * This is used to preserve the original query ordering through resolution operations. - * - * @param query - The query object that may contain an order clause - * @param dimensions - The dimensions array from the base table schema - * @param baseTableName - The base table name to use in column references - * @returns SQL expression for row_number() OVER (ORDER BY ...) - */ -export const generateRowNumberSql = ( - query: { order?: Record }, - dimensions: { name: string; alias?: string }[], - baseTableName: string -): string => { - let rowNumberSql = 'row_number() OVER ('; - if (query.order && Object.keys(query.order).length > 0) { - const orderClauses = Object.entries(query.order).map( - ([member, direction]) => { - // Find the actual column name/alias in the base table dimensions - const safeMember = memberKeyToSafeKey(member); - const dimension = dimensions.find( - (d) => d.name === safeMember || d.alias === safeMember - ); - const columnName = dimension - ? dimension.alias || dimension.name - : safeMember; - return `${baseTableName}."${columnName}" ${direction.toUpperCase()}`; - } - ); - rowNumberSql += `ORDER BY ${orderClauses.join(', ')}`; - } - rowNumberSql += ')'; - return rowNumberSql; -}; - /** * Wraps SQL to order by row_id and then exclude it from results. * This maintains the ordering from the base query while removing the internal row_id column. From 4a32ce4fcc9ce1cc45fdd6f9d2af34817320dd94 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 14:52:11 +0530 Subject: [PATCH 35/38] minor update --- .../browser-cube-to-sql-with-resolution.ts | 2 +- .../cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts index 7abd7c50..33e02d6e 100644 --- a/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts +++ b/meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts @@ -42,7 +42,7 @@ export const cubeQueryToSQLWithResolution = async ({ }); // Check if resolution should be skipped - if (shouldSkipResolution(resolutionConfig, columnProjections)) { + if (shouldSkipResolution(resolutionConfig, query, columnProjections)) { return baseSql; } diff --git a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts index 3cb83195..44145be6 100644 --- a/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts +++ b/meerkat-node/src/cube-to-sql-with-resolution/cube-to-sql-with-resolution.ts @@ -38,7 +38,7 @@ export const cubeQueryToSQLWithResolution = async ({ }); // Check if resolution should be skipped - if (shouldSkipResolution(resolutionConfig, columnProjections)) { + if (shouldSkipResolution(resolutionConfig, query, columnProjections)) { return baseSql; } From aca247b95097a7a11204fbb35eab6ff4cb391aba Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 15:02:31 +0530 Subject: [PATCH 36/38] cr comments --- .../resolution/generators/generate-row-number-sql.ts | 10 +++++++++- meerkat-core/src/resolution/resolution.ts | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/meerkat-core/src/resolution/generators/generate-row-number-sql.ts b/meerkat-core/src/resolution/generators/generate-row-number-sql.ts index 653df02a..43555674 100644 --- a/meerkat-core/src/resolution/generators/generate-row-number-sql.ts +++ b/meerkat-core/src/resolution/generators/generate-row-number-sql.ts @@ -26,7 +26,7 @@ export const generateRowNumberSql = ( const columnName = dimension ? dimension.alias || dimension.name : safeMember; - return `${baseTableName}."${columnName}" ${direction.toUpperCase()}`; + return generateOrderClause(baseTableName, columnName, direction); } ); rowNumberSql += `ORDER BY ${orderClauses.join(', ')}`; @@ -34,3 +34,11 @@ export const generateRowNumberSql = ( rowNumberSql += ')'; return rowNumberSql; }; + +const generateOrderClause = ( + baseTableName: string, + columnName: string, + direction: string +) => { + return `${baseTableName}."${columnName}" ${direction.toUpperCase()}`; +}; diff --git a/meerkat-core/src/resolution/resolution.ts b/meerkat-core/src/resolution/resolution.ts index 72cbb33c..dbf61a05 100644 --- a/meerkat-core/src/resolution/resolution.ts +++ b/meerkat-core/src/resolution/resolution.ts @@ -3,7 +3,7 @@ import { getNamespacedKey, memberKeyToSafeKey, } from '../member-formatters'; -import { Member } from '../types/cube-types/query'; +import { Member, Query } from '../types/cube-types/query'; import { Dimension, Measure, TableSchema } from '../types/cube-types/table'; import { isArrayTypeMember } from '../utils/is-array-member-type'; import { @@ -36,11 +36,15 @@ export const getColumnReference = ( */ export const shouldSkipResolution = ( resolutionConfig: ResolutionConfig, + query: Query, columnProjections?: string[] ): boolean => { + // If no resolution required and no column projections to ensure order in which export is happening + // and explicit order is not provided, then skip resolution. return ( resolutionConfig.columnConfigs.length === 0 && - columnProjections?.length === 0 + columnProjections?.length === 0 && + !query.order ); }; From 5b33f511c439e7553a5d82671d8ad5b382229134 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 15:08:59 +0530 Subject: [PATCH 37/38] updating package version for meerkat-core --- meerkat-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meerkat-core/package.json b/meerkat-core/package.json index 74ef3cda..2b047d9f 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.105", + "version": "0.0.106", "dependencies": { "tslib": "^2.3.0" }, From a6351cc763cc5bf72d2b3213796d2b5413d14d91 Mon Sep 17 00:00:00 2001 From: senthilb-devrev Date: Wed, 12 Nov 2025 15:47:31 +0530 Subject: [PATCH 38/38] added a todo --- meerkat-core/src/utils/find-in-table-schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/meerkat-core/src/utils/find-in-table-schema.ts b/meerkat-core/src/utils/find-in-table-schema.ts index 8eaa5e5f..66065111 100644 --- a/meerkat-core/src/utils/find-in-table-schema.ts +++ b/meerkat-core/src/utils/find-in-table-schema.ts @@ -52,6 +52,7 @@ export const findInSchemas = (name: string, tableSchemas: TableSchema[]) => { ** Finds the dimension or measure in the provided table schemas. ** Handles both namespaced (`tableName.columnName`) and non-namespaced names. */ + // TODO: Move to only using namespaced keys. if (!name.includes('.')) { if (tableSchemas.length > 1) { throw new Error(