Array resolution

meerkat-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.105",
+  "version": "0.0.106",
   "dependencies": {
     "tslib": "^2.3.0",
     "@devrev/meerkat-core": "*",

meerkat-browser/src/browser-cube-to-sql-with-resolution/browser-cube-to-sql-with-resolution.ts

@@ -1,18 +1,21 @@
 import {
+  BASE_DATA_SOURCE_NAME,
   ContextParams,
+  getAggregatedSql as coreGetAggregatedSql,
+  getResolvedTableSchema as coreGetResolvedTableSchema,
+  getUnnestTableSchema as coreGetUnnestTableSchema,
   createBaseTableSchema,
-  generateResolutionJoinPaths,
-  generateResolutionSchemas,
-  generateResolvedDimensions,
+  Dimension,
+  generateRowNumberSql,
+  memberKeyToSafeKey,
   Query,
   ResolutionConfig,
+  ROW_ID_DIMENSION_NAME,
+  shouldSkipResolution,
   TableSchema,
 } from '@devrev/meerkat-core';
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
-import {
-  cubeQueryToSQL,
-  CubeQueryToSQLParams,
-} from '../browser-cube-to-sql/browser-cube-to-sql';
+import { cubeQueryToSQL } from '../browser-cube-to-sql/browser-cube-to-sql';
 
 export interface CubeQueryToSQLWithResolutionParams {
   connection: AsyncDuckDBConnection;
@@ -38,39 +41,72 @@ export const cubeQueryToSQLWithResolution = async ({
     contextParams,
   });
 
-  if (resolutionConfig.columnConfigs.length === 0) {
-    // If no resolution is needed, return the base SQL.
+  // Check if resolution should be skipped
+  if (shouldSkipResolution(resolutionConfig, query, columnProjections)) {
     return baseSql;
   }
 
-  // Create a table schema for the base query.
-  const baseTable: TableSchema = createBaseTableSchema(
+  if (!columnProjections) {
+    columnProjections = [...(query.dimensions || []), ...query.measures];
+  }
+  // This is to ensure that, only the column projection columns
+  // are being resolved and other definitions are ignored.
+  resolutionConfig.columnConfigs = resolutionConfig.columnConfigs.filter(
+    (config) => {
+      return columnProjections?.includes(config.name);
+    }
+  );
+
+  const baseSchema: TableSchema = createBaseTableSchema(
     baseSql,
     tableSchemas,
     resolutionConfig,
     query.measures,
     query.dimensions
   );
 
-  const resolutionSchemas: TableSchema[] = generateResolutionSchemas(
+  const rowIdDimension: Dimension = {
+    name: ROW_ID_DIMENSION_NAME,
+    sql: generateRowNumberSql(
+      query,
+      baseSchema.dimensions,
+      BASE_DATA_SOURCE_NAME
+    ),
+    type: 'number',
+    alias: ROW_ID_DIMENSION_NAME,
+  };
+  baseSchema.dimensions.push(rowIdDimension);
+  columnProjections.push(ROW_ID_DIMENSION_NAME);
+
+  // Doing this because we need to use the original name of the column in the base table schema.
+  resolutionConfig.columnConfigs.forEach((config) => {
+    config.name = memberKeyToSafeKey(config.name);
+  });
+
+  // Generate SQL with row_id and unnested arrays
+  const unnestTableSchema = await coreGetUnnestTableSchema({
+    baseTableSchema: baseSchema,
     resolutionConfig,
-    tableSchemas
-  );
+    contextParams,
+    cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }),
+  });
 
-  const resolveParams: CubeQueryToSQLParams = {
-    connection: connection,
-    query: {
-      measures: [],
-      dimensions: generateResolvedDimensions(
-        query,
-        resolutionConfig,
-        columnProjections
-      ),
-      joinPaths: generateResolutionJoinPaths(resolutionConfig, tableSchemas),
-    },
-    tableSchemas: [baseTable, ...resolutionSchemas],
-  };
-  const sql = await cubeQueryToSQL(resolveParams);
+  //  Apply resolution (join with lookup tables)
+  const resolvedTableSchema = await coreGetResolvedTableSchema({
+    baseTableSchema: unnestTableSchema,
+    resolutionConfig,
+    contextParams,
+    columnProjections,
+    cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }),
+  });
+
+  // Re-aggregate to reverse the unnest
+  const aggregatedSql = await coreGetAggregatedSql({
+    resolvedTableSchema,
+    resolutionConfig,
+    contextParams,
+    cubeQueryToSQL: async (params) => cubeQueryToSQL({ connection, ...params }),
+  });
 
-  return sql;
+  return aggregatedSql;
 };

meerkat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.105",
+  "version": "0.0.106",
   "dependencies": {
     "tslib": "^2.3.0"
   },

meerkat-core/src/constants/exports/index.ts

@@ -0,0 +1 @@
+export const ROW_ID_DIMENSION_NAME = '__row_id';

meerkat-core/src/get-wrapped-base-query-with-projections/__tests__/sql-expression-modifier.spec.ts

@@ -1,130 +1,142 @@
 import { Dimension } from '../../types/cube-types/table';
 import { isArrayTypeMember } from '../../utils/is-array-member-type';
-import { arrayFieldUnNestModifier, DimensionModifier, getModifiedSqlExpression, MODIFIERS, shouldUnnest } from "../sql-expression-modifiers";
+import {
+  arrayFieldUnNestModifier,
+  shouldUnnest,
+} from '../modifiers/array-unnest-modifier';
+import {
+  getModifiedSqlExpression,
+  MODIFIERS,
+} from '../sql-expression-modifiers';
+import { DimensionModifier } from '../types';
 
-jest.mock("../../utils/is-array-member-type", () => {
+jest.mock('../../utils/is-array-member-type', () => {
   return {
-    isArrayTypeMember: jest.fn()
-  }
+    isArrayTypeMember: jest.fn(),
+  };
 });
 
 const QUERY = {
-  measures: ["test_measure"],
-  dimensions: ["test_dimension"]
-}
+  measures: ['test_measure'],
+  dimensions: ['test_dimension'],
+};
 
-describe("Dimension Modifier", () => {
-  describe("arrayFieldUnNestModifier", () => {
-    it("should return the correct unnested SQL expression", () => {
+describe('Dimension Modifier', () => {
+  describe('arrayFieldUnNestModifier', () => {
+    it('should return the correct unnested SQL expression', () => {
       const modifier: DimensionModifier = {
-        sqlExpression: "some_array_field",
+        sqlExpression: 'some_array_field',
         dimension: {} as Dimension,
-        key: "test_key",
-        query: QUERY
+        key: 'test_key',
+        query: QUERY,
       };
-      expect(arrayFieldUnNestModifier(modifier)).toBe("array[unnest(some_array_field)]");
+      expect(arrayFieldUnNestModifier(modifier)).toBe(
+        'array[unnest(some_array_field)]'
+      );
     });
   });
 
-  describe("shouldUnnest", () => {
-    it("should return true when dimension is array type and has shouldUnnestGroupBy modifier", () => {
+  describe('shouldUnnest', () => {
+    it('should return true when dimension is array type and has shouldUnnestGroupBy modifier', () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(true);
       const modifier: DimensionModifier = {
-        sqlExpression: "some_expression",
-        dimension: { 
-          type: "array",
-          modifier: { shouldUnnestGroupBy: true }
+        sqlExpression: 'some_expression',
+        dimension: {
+          type: 'array',
+          modifier: { shouldUnnestGroupBy: true },
         } as Dimension,
-        key: "test_key",
-        query: QUERY
+        key: 'test_key',
+        query: QUERY,
       };
       expect(shouldUnnest(modifier)).toBe(true);
     });
 
-    it("should return false when dimension is not array type", () => {
+    it('should return false when dimension is not array type', () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(false);
       const modifier: DimensionModifier = {
-        sqlExpression: "some_expression",
-        dimension: { 
-          type: "string",
-          modifier: { shouldUnnestGroupBy: true }
+        sqlExpression: 'some_expression',
+        dimension: {
+          type: 'string',
+          modifier: { shouldUnnestGroupBy: true },
         } as Dimension,
-        key: "test_key",
-        query: QUERY
+        key: 'test_key',
+        query: QUERY,
       };
       expect(shouldUnnest(modifier)).toBe(false);
     });
 
     it("should return false when dimension doesn't have shouldUnnestGroupBy modifier", () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(true);
       const modifier: DimensionModifier = {
-        sqlExpression: "some_expression",
-        dimension: { 
-          type: "array",
-          modifier: {}
+        sqlExpression: 'some_expression',
+        dimension: {
+          type: 'array',
+          modifier: {},
+        } as Dimension,
+        key: 'test_key',
+        query: QUERY,
+      };
+      expect(shouldUnnest(modifier)).toBe(false);
+    });
+    it('should return false when dimension when modifier undefined', () => {
+      (isArrayTypeMember as jest.Mock).mockReturnValue(true);
+      const modifier: DimensionModifier = {
+        sqlExpression: 'some_expression',
+        dimension: {
+          type: 'array',
         } as Dimension,
-        key: "test_key",
-        query: QUERY
+        key: 'test_key',
+        query: QUERY,
       };
       expect(shouldUnnest(modifier)).toBe(false);
     });
-    it("should return false when dimension when modifier undefined", () => {
-        (isArrayTypeMember as jest.Mock).mockReturnValue(true);
-        const modifier: DimensionModifier = {
-          sqlExpression: "some_expression",
-          dimension: { 
-            type: "array",
-          } as Dimension,
-          key: "test_key",
-          query: QUERY
-        };
-        expect(shouldUnnest(modifier)).toBe(false);
-      });
   });
 
-  describe("getModifiedSqlExpression", () => {
-    it("should not modify if no modifiers passed", () => {
+  describe('getModifiedSqlExpression', () => {
+    it('should not modify if no modifiers passed', () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(true);
       const input = {
-        sqlExpression: "array_field",
+        sqlExpression: 'array_field',
         dimension: {
-          type: "array",
-          modifier: { shouldUnnestGroupBy: true }
+          type: 'array',
+          modifier: { shouldUnnestGroupBy: true },
         } as Dimension,
         query: QUERY,
-        key: "test_key",
-        modifiers: []
+        key: 'test_key',
+        modifiers: [],
       };
-      expect(getModifiedSqlExpression(input)).toBe("array_field");
+      expect(getModifiedSqlExpression(input)).toBe('array_field');
     });
-    it("should apply the modifier when conditions are met", () => {
+    it('should apply the modifier when conditions are met', () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(true);
       const input = {
-        sqlExpression: "array_field",
+        sqlExpression: 'array_field',
         dimension: {
-          type: "array",
-          modifier: { shouldUnnestGroupBy: true }
+          type: 'array',
+          modifier: { shouldUnnestGroupBy: true },
         } as Dimension,
         query: QUERY,
-        key: "test_key",
-        modifiers: MODIFIERS
+        key: 'test_key',
+        modifiers: MODIFIERS,
       };
-      expect(getModifiedSqlExpression(input)).toBe("array[unnest(array_field)]");
+      expect(getModifiedSqlExpression(input)).toBe(
+        'array[unnest(array_field)]'
+      );
     });
 
-    it("should not apply the modifier when conditions are not met", () => {
+    it('should not apply the modifier when conditions are not met', () => {
       (isArrayTypeMember as jest.Mock).mockReturnValue(false);
       const input = {
-        sqlExpression: "non_array_field",
+        sqlExpression: 'non_array_field',
         dimension: {
-          type: "string",
-          modifier: {}
+          type: 'string',
+          modifier: {},
         } as Dimension,
         query: QUERY,
-        key: "test_key",
-        modifiers: MODIFIERS
+        key: 'test_key',
+        modifiers: MODIFIERS,
       };
-      expect(getModifiedSqlExpression(input)).toBe("non_array_field");
+      expect(getModifiedSqlExpression(input)).toBe('non_array_field');
     });
   });
-});
+});

meerkat-core/src/get-wrapped-base-query-with-projections/get-aliased-columns-from-filters.ts

@@ -5,7 +5,8 @@ import {
   findInDimensionSchema,
   findInMeasureSchema,
 } from '../utils/find-in-table-schema';
-import { getModifiedSqlExpression, Modifier } from './sql-expression-modifiers';
+import { getModifiedSqlExpression } from './sql-expression-modifiers';
+import { Modifier } from './types';
 
 export const getDimensionProjection = ({
   key,

meerkat-core/src/get-wrapped-base-query-with-projections/index.ts

@@ -0,0 +1,6 @@
+export { getWrappedBaseQueryWithProjections } from './get-wrapped-base-query-with-projections';
+export {
+  getModifiedSqlExpression,
+  MODIFIERS,
+} from './sql-expression-modifiers';
+export type { DimensionModifier, Modifier } from './types';

meerkat-core/src/get-wrapped-base-query-with-projections/modifiers/array-flatten-modifier.ts

@@ -0,0 +1,25 @@
+import { isArrayTypeMember } from '../../utils/is-array-member-type';
+import { DimensionModifier, Modifier } from '../types';
+
+export const arrayFlattenModifier = ({
+  sqlExpression,
+}: DimensionModifier): string => {
+  // Ensure NULL or empty arrays produce at least one row with NULL value
+  // This prevents rows from being dropped when arrays are NULL or empty
+  // COALESCE handles NULL, and len() = 0 check handles empty arrays []
+  return `unnest(CASE WHEN ${sqlExpression} IS NULL OR len(COALESCE(${sqlExpression}, [])) = 0 THEN [NULL] ELSE ${sqlExpression} END)`;
+};
+
+export const shouldFlattenArray = ({
+  dimension,
+}: DimensionModifier): boolean => {
+  const isArrayType = isArrayTypeMember(dimension.type);
+  const shouldFlattenArray = dimension.modifier?.shouldFlattenArray;
+  return !!(isArrayType && shouldFlattenArray);
+};
+
+export const arrayFlattenModifierConfig: Modifier = {
+  name: 'shouldFlattenArray',
+  matcher: shouldFlattenArray,
+  modifier: arrayFlattenModifier,
+};