feat(cubesql): Group By Rollup support

.github/workflows/drivers-tests.yml

@@ -268,4 +268,4 @@ jobs:
         run: |
           cd ./packages/cubejs-testing-drivers
           export DEBUG=testcontainers
-          yarn ${{ matrix.database }}-full
+          yarn ${{ matrix.database }}-full

packages/cubejs-api-gateway/src/gateway.ts

@@ -145,7 +145,7 @@
 
   protected readonly playgroundAuthSecret?: string;
 
   protected readonly event: (name: string, props?: object) => void;
 
   public constructor(
     protected readonly apiSecret: string,
@@ -178,7 +178,7 @@
     this.contextRejectionMiddleware = options.contextRejectionMiddleware || (async (req, res, next) => next());
     this.wsContextAcceptor = options.wsContextAcceptor || (() => ({ accepted: true }));
     // eslint-disable-next-line @typescript-eslint/no-empty-function
     this.event = options.event || function () {};
   }
 
   public initApp(app: ExpressApplication) {
@@ -1274,12 +1274,20 @@
         const obj = JSON.parse(memberExpression);
         const args = obj.cube_params;
         args.push(`return \`${obj.expr}\``);
+
+        const groupingSet = obj.grouping_set ? {
+          groupType: obj.grouping_set.group_type,
+          id: obj.grouping_set.id,
+          subId: obj.grouping_set.sub_id ? obj.grouping_set.sub_id : undefined
+        } : undefined;
+
         return {
           cubeName: obj.cube_name,
           name: obj.alias,
           expressionName: obj.alias,
           expression: Function.constructor.apply(null, args),
           definition: memberExpression,
+          groupingSet,
         };
       } else {
         return memberExpression;
@@ -1726,7 +1734,7 @@
     const {
       context,
       res,
       apiType = 'sql',
     } = request;
     const requestStarted = new Date();
 

packages/cubejs-api-gateway/src/query.js

@@ -47,6 +47,11 @@ const memberExpression = Joi.object().keys({
   name: Joi.string().required(),
   expressionName: Joi.string(),
   definition: Joi.string(),
+  groupingSet: Joi.object().keys({
+    groupType: Joi.valid('Rollup', 'Cube').required(),
+    id: Joi.number().required(),
+    subId: Joi.number()
+  })
 });
 
 const operators = [

packages/cubejs-api-gateway/src/types/query.ts

@@ -10,7 +10,7 @@
   TimeMember,
   FilterOperator,
   QueryTimeDimensionGranularity,
   QueryOrderType,
 } from './strings';
 import { ResultType } from './enums';
 
@@ -39,12 +39,19 @@
   or: (QueryFilter | LogicalAndFilter)[]
 };
 
+type GroupingSet = {
+    groupType: string,
+    id: number,
+    subId?: null | number
+};
+
 type MemberExpression = {
   expression: Function;
   cubeName: string;
   name: string;
   expressionName: string;
   definition: string;
+  groupingSet?: GroupingSet
 };
 
 /**

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -1899,7 +1899,11 @@ export class BaseQuery {
       return '';
     }
     const dimensionColumns = this.dimensionColumns();
-    return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.map((c, i) => `${i + 1}`).join(', ')}` : '';
+    if (!dimensionColumns.length) {
+      return '';
+    }
+    const dimensionNames = dimensionColumns.map((c, i) => `${i + 1}`);
+    return this.rollupGroupByClause(dimensionNames);
   }
 
   getFieldIndex(id) {
@@ -1993,6 +1997,53 @@ export class BaseQuery {
     return this.limitOffsetClause(limit, offset);
   }
 
+  /**
+   * @protected
+   * @param {Array<string>} dimensionNames
+   * @returns {string}
+   */
+  rollupGroupByClause(dimensionNames) {
+    if (this.ungrouped) {
+      return '';
+    }
+    const dimensionColumns = this.dimensionColumns();
+    if (!dimensionColumns.length) {
+      return '';
+    }
+
+    const groupingSets = R.flatten(this.dimensionsForSelect().map(d => d.dimension).filter(d => !!d)).map(d => d.groupingSet);
+
+    let result = ' GROUP BY ';
+
+    dimensionColumns.forEach((c, i) => {
+      const groupingSet = groupingSets[i];
+      const comma = i > 0 ? ', ' : '';
+      const prevId = i > 0 ? (groupingSets[i - 1] || { id: null }).id : null;
+      const currId = (groupingSet || { id: null }).id;
+
+      if (prevId !== null && currId !== prevId) {
+        result += ')';
+      }
+
+      if ((prevId === null || currId !== prevId) && groupingSet != null) {
+        if (groupingSet.groupType === 'Rollup') {
+          result += `${comma}ROLLUP(`;
+        } else if (groupingSet.groupType === 'Cube') {
+          result += `${comma}CUBE(`;
+        }
+      } else {
+        result += `${comma}`;
+      }
+
+      result += dimensionNames[i];
+    });
+    if (groupingSets[groupingSets.length - 1] != null) {
+      result += ')';
+    }
+
+    return result;
+  }
+
   /**
    * @protected
    * @param limit
@@ -2937,10 +2988,11 @@ export class BaseQuery {
           '{{ from | indent(2, true) }}\n' +
           ') AS {{ from_alias }}{% endif %}' +
           '{% if filter %}\nWHERE {{ filter }}{% endif %}' +
-          '{% if group_by %}\nGROUP BY {{ group_by | map(attribute=\'index\') | join(\', \') }}{% endif %}' +
+          '{% if group_by %}\nGROUP BY {{ group_by }}{% endif %}' +
           '{% if order_by %}\nORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}' +
           '{% if limit %}\nLIMIT {{ limit }}{% endif %}' +
           '{% if offset %}\nOFFSET {{ offset }}{% endif %}',
+        group_by_exprs: '{{ group_by | map(attribute=\'index\') | join(\', \') }}',
       },
       expressions: {
         column_aliased: '{{expr}} {{quoted_alias}}',
@@ -2954,6 +3006,8 @@ export class BaseQuery {
         in_list: '{{ expr }} {% if negated %}NOT {% endif %}IN ({{ in_exprs_concat }})',
         subquery: '({{ expr }})',
         in_subquery: '{{ expr }} {% if negated %}NOT {% endif %}IN {{ subquery_expr }}',
+        rollup: 'ROLLUP({{ exprs_concat }})',
+        cube: 'CUBE({{ exprs_concat }})',
         negative: '-({{ expr }})',
         not: 'NOT ({{ expr }})',
         true: 'TRUE',

packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts

@@ -114,7 +114,7 @@ export class PrestodbQuery extends BaseQuery {
     templates.functions.DATEPART = 'DATE_PART({{ args_concat }})';
     templates.statements.select = 'SELECT {{ select_concat | map(attribute=\'aliased\') | join(\', \') }} \n' +
       'FROM (\n  {{ from }}\n) AS {{ from_alias }} \n' +
-      '{% if group_by %} GROUP BY {{ group_by | map(attribute=\'index\') | join(\', \') }}{% endif %}' +
+      '{% if group_by %} GROUP BY {{ group_by }}{% endif %}' +
       '{% if order_by %} ORDER BY {{ order_by | map(attribute=\'expr\') | join(\', \') }}{% endif %}' +
       '{% if offset %}\nOFFSET {{ offset }}{% endif %}' +
       '{% if limit %}\nLIMIT {{ limit }}{% endif %}';

packages/cubejs-testing-drivers/fixtures/athena.json

@@ -143,8 +143,15 @@
     "querying BigECommerce: null sum",
     "querying BigECommerce: null boolean",
     "--------------------",
+
     "week granularity is not supported for intervals",
     "--------------------",
-    "querying BigECommerce: rolling window by 2 week"
+    "querying BigECommerce: rolling window by 2 week",
+
+    "SKIPPED SQL API (Need work)",
+    "---------------------------------------",
+    "SQL API: Simple Rollup",
+    "SQL API: Complex Rollup",
+    "SQL API: Nested Rollup"
   ]
 }

packages/cubejs-testing-drivers/fixtures/bigquery.json

@@ -165,6 +165,7 @@
 
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",
-    "SQL API: reuse params"
+    "SQL API: reuse params",
+    "SQL API: Complex Rollup"
   ]
 }

packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json

@@ -156,6 +156,10 @@
     "querying Products: dimensions -- doesn't work wo ordering",
     "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- rounding in athena",
     "querying ECommerce: total sales, total profit by month + order (date) + total -- doesn't work with the BigQuery",
-    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test"
+    "querying ECommerce: total quantity, avg discount, total sales, total profit by product + order + total -- noisy test",
+
+    "SKIPPED SQL API (Need work)",
+    "---------------------------------------",
+    "SQL API: Nested Rollup"
   ]
-}
+}

packages/cubejs-testing-drivers/fixtures/mssql.json

@@ -138,6 +138,9 @@
     "SQL API: reuse params",
     "SQL API: post-aggregate percentage of total",
     "SQL API: powerbi min max ungrouped flag",
-    "SQL API: powerbi min max push down"
+    "SQL API: powerbi min max push down",
+    "SQL API: Simple Rollup",
+    "SQL API: Complex Rollup",
+    "SQL API: Nested Rollup"
   ]
 }

packages/cubejs-testing-drivers/fixtures/mysql.json

@@ -130,6 +130,9 @@
 
     "SKIPPED SQL API (Need work)",
     "---------------------------------------",
-    "SQL API: reuse params"
+    "SQL API: reuse params",
+    "SQL API: Simple Rollup",
+    "SQL API: Complex Rollup",
+    "SQL API: Nested Rollup"
   ]
 }

packages/cubejs-testing-drivers/src/tests/testQueries.ts

@@ -1628,5 +1628,54 @@ from
   `);
       expect(res.rows).toMatchSnapshot('reuse_params');
     });
+
+    executePg('SQL API: Simple Rollup', async () => {
+      const connection = await createPostgresClient();
+      const res = await connection.query(`
+    select
+        rowId, orderId, orderDate, sum(count)
+    from
+      "ECommerce" as "ECommerce"
+    group by
+      ROLLUP(1, 2, 3)
+    order by 1, 2, 3
+
+  `);
+      expect(res.rows).toMatchSnapshot('simple_rollup');
+    });
+
+    executePg('SQL API: Complex Rollup', async () => {
+      const connection = await createPostgresClient();
+      const res = await connection.query(`
+    select
+        rowId, orderId, orderDate, city, sum(count)
+    from
+      "ECommerce" as "ECommerce"
+    group by
+      ROLLUP(1, 2), 3, ROLLUP(4) 
+    order by 1, 2, 3, 4
+
+  `);
+      expect(res.rows).toMatchSnapshot('complex_rollup');
+    });
+    executePg('SQL API: Nested Rollup', async () => {
+      const connection = await createPostgresClient();
+      const res = await connection.query(`
+    select rowId, orderId, orderDate, sum(cnt)
+    from (
+        select
+            rowId, orderId, orderDate, sum(count) as cnt
+        from
+        "ECommerce" as "ECommerce"
+        group by 1, 2, 3
+
+    ) a
+    group by
+      ROLLUP(1, 2, 3)
+    order by 1, 2, 3
+
+  `);
+      expect(res.rows).toMatchSnapshot('nested_rollup');
+    });
   });
 }