feat: Ungrouped query pre-aggregation support (#8058)

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -463,7 +463,7 @@ export class BaseQuery {
   buildParamAnnotatedSql() {
     let sql;
     let preAggForQuery;
-    if (!this.options.preAggregationQuery && !this.ungrouped) {
+    if (!this.options.preAggregationQuery) {
       preAggForQuery =
         this.preAggregations.findPreAggregationForQuery();
       if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) {
@@ -539,7 +539,7 @@ export class BaseQuery {
   }
 
   externalPreAggregationQuery() {
-    if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && !this.ungrouped && this.externalQueryClass) {
+    if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
       const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery();
       if (preAggregationForQuery && preAggregationForQuery.preAggregation.external) {
         return true;
@@ -558,7 +558,7 @@ export class BaseQuery {
    * @returns {Array<string>}
    */
   buildSqlAndParams(exportAnnotatedSql) {
-    if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && !this.ungrouped && this.externalQueryClass) {
+    if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
       if (this.externalPreAggregationQuery()) { // TODO performance
         return this.externalQuery().buildSqlAndParams(exportAnnotatedSql);
       }
@@ -1848,7 +1848,7 @@ export class BaseQuery {
   primaryKeyNames(cubeName) {
     const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName];
     if (!primaryKeys || !primaryKeys.length) {
-      throw new UserError(`One or more Primary key is required for '${cubeName}`);
+      throw new UserError(`One or more Primary key is required for '${cubeName}' cube`);
     }
     return primaryKeys.map((pk) => this.primaryKeyName(cubeName, pk));
   }

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -314,6 +314,17 @@ export class PreAggregations {
       });
     }).map(d => query.newTimeDimension(d));
 
+    let sortedAllCubeNames;
+    let sortedUsedCubePrimaryKeys;
+
+    if (query.ungrouped) {
+      const { allCubeNames } = query;
+      sortedAllCubeNames = allCubeNames.concat([]);
+      sortedAllCubeNames.sort();
+      sortedUsedCubePrimaryKeys = query.allCubeNames.flatMap(c => query.primaryKeyNames(c));
+      sortedUsedCubePrimaryKeys.sort();
+    }
+
     const measureToLeafMeasures = {};
 
     const leafMeasurePaths =
@@ -398,7 +409,10 @@ export class PreAggregations {
       ownedDimensions,
       ownedTimeDimensionsWithRollupGranularity,
       ownedTimeDimensionsAsIs,
-      allBackAliasMembers
+      allBackAliasMembers,
+      ungrouped: query.ungrouped,
+      sortedUsedCubePrimaryKeys,
+      sortedAllCubeNames
     };
   }
 
@@ -665,6 +679,20 @@ export class PreAggregations {
           (references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions))
       );
 
+      if (transformedQuery.ungrouped) {
+        const allReferenceCubes = R.pipe(R.map(m => (m.dimension || m).split('.')[0]), R.uniq, R.sortBy(R.identity))(
+          references.measures.concat(references.dimensions).concat(references.timeDimensions)
+        );
+        if (
+          !R.equals(transformedQuery.sortedAllCubeNames, allReferenceCubes) ||
+          !(
+            dimensionsMatch(transformedQuery.sortedUsedCubePrimaryKeys, true) || dimensionsMatch(transformedQuery.sortedUsedCubePrimaryKeys, false)
+          )
+        ) {
+          return false;
+        }
+      }
+
       const backAliasMeasures = backAlias(references.measures);
       return ((
         windowGranularityMatches(references)
@@ -687,9 +715,9 @@ export class PreAggregations {
      * @returns {boolean}
      */
     const canUseFn =
-      transformedQuery.leafMeasureAdditive &&
-      !transformedQuery.hasMultipliedMeasures
-        ? (r) => canUsePreAggregationLeafMeasureAdditive(r) ||
+      (
+        transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures || transformedQuery.ungrouped
+      ) ? (r) => canUsePreAggregationLeafMeasureAdditive(r) ||
           canUsePreAggregationNotAdditive(r)
         : canUsePreAggregationNotAdditive;
 
@@ -1293,12 +1321,12 @@ export class PreAggregations {
         const measure = this.query.newMeasure(path);
         return [
           path,
-          this.query.aggregateOnGroupedColumn(
+          this.query.ungrouped ? measure.aliasName() : (this.query.aggregateOnGroupedColumn(
             measure.measureDefinition(),
             measure.aliasName(),
             !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate,
             path,
-          ) || `sum(${measure.aliasName()})`,
+          ) || `sum(${measure.aliasName()})`),
         ];
       }),
       R.fromPairs,

packages/cubejs-testing-drivers/fixtures/_schemas.json

@@ -6,17 +6,23 @@
         {
           "name": "category",
           "type": "string",
-          "sql": "category"
+          "sql": "category",
+          "primary_key": true,
+          "shown": true
         },
         {
           "name": "subCategory",
           "type": "string",
-          "sql": "sub_category"
+          "sql": "sub_category",
+          "primary_key": true,
+          "shown": true
         },
         {
           "name": "productName",
           "type": "string",
-          "sql": "product_name"
+          "sql": "product_name",
+          "primary_key": true,
+          "shown": true
         }
       ]
     },
@@ -151,7 +157,6 @@
     },
     {
       "name": "BigECommerce",
-      "extends": "Products",
       "joins": [
         {
           "name": "Customers",
@@ -167,6 +172,21 @@
           "primary_key": true,
           "shown": true
         },
+        {
+          "name": "category",
+          "type": "string",
+          "sql": "category"
+        },
+        {
+          "name": "subCategory",
+          "type": "string",
+          "sql": "sub_category"
+        },
+        {
+          "name": "productName",
+          "type": "string",
+          "sql": "product_name"
+        },
         {
           "name": "orderId",
           "sql": "order_id",

packages/cubejs-testing-drivers/src/tests/testQueries.ts

@@ -1522,5 +1522,17 @@ from
   `);
       expect(res.rows).toMatchSnapshot('powerbi_min_max_ungrouped_flag');
     });
+
+    executePg('SQL API: ungrouped pre-agg', async () => {
+      const res = await connection.query(`
+    select
+      "productName",
+      "totalSales" 
+    from 
+      "public"."BigECommerce" "$Table" 
+    order by 2 desc, 1 asc
+  `);
+      expect(res.rows).toMatchSnapshot('ungrouped_pre_agg');
+    });
   });
 }

packages/cubejs-testing-drivers/test/__snapshots__/postgres-full.test.ts.snap

@@ -19,6 +19,187 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/postgres-driver SQL API: ungrouped pre-agg: ungrouped_pre_agg 1`] = `
+Array [
+  Object {
+    "productName": "Canon PC1080F Personal Copier",
+    "totalSales": 2399.96,
+  },
+  Object {
+    "productName": "Logitech diNovo Edge Keyboard",
+    "totalSales": 2249.91,
+  },
+  Object {
+    "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish",
+    "totalSales": 2154.9,
+  },
+  Object {
+    "productName": "Google Nexus 5",
+    "totalSales": 1979.89,
+  },
+  Object {
+    "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish",
+    "totalSales": 1292.94,
+  },
+  Object {
+    "productName": "Canon PC1080F Personal Copier",
+    "totalSales": 1199.98,
+  },
+  Object {
+    "productName": "Hewlett Packard 610 Color Digital Copier / Printer",
+    "totalSales": 899.982,
+  },
+  Object {
+    "productName": "Okidata C610n Printer",
+    "totalSales": 649,
+  },
+  Object {
+    "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer",
+    "totalSales": 600,
+  },
+  Object {
+    "productName": "Google Nexus 6",
+    "totalSales": 539.97,
+  },
+  Object {
+    "productName": "Google Nexus 7",
+    "totalSales": 539.97,
+  },
+  Object {
+    "productName": "Harbour Creations 67200 Series Stacking Chairs",
+    "totalSales": 498.26,
+  },
+  Object {
+    "productName": "DMI Eclipse Executive Suite Bookcases",
+    "totalSales": 400.784,
+  },
+  Object {
+    "productName": "HTC One",
+    "totalSales": 239.976,
+  },
+  Object {
+    "productName": "Iceberg Nesting Folding Chair, 19w x 6d x 43h",
+    "totalSales": 232.88,
+  },
+  Object {
+    "productName": "Balt Solid Wood Rectangular Table",
+    "totalSales": 210.98,
+  },
+  Object {
+    "productName": "Tyvek Side-Opening Peel & Seel Expanding Envelopes",
+    "totalSales": 180.96,
+  },
+  Object {
+    "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener",
+    "totalSales": 179.9,
+  },
+  Object {
+    "productName": "Harbour Creations 67200 Series Stacking Chairs",
+    "totalSales": 128.124,
+  },
+  Object {
+    "productName": "Harbour Creations 67200 Series Stacking Chairs",
+    "totalSales": 113.888,
+  },
+  Object {
+    "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener",
+    "totalSales": 86.352,
+  },
+  Object {
+    "productName": "Kingston Digital DataTraveler 16GB USB 2.2",
+    "totalSales": 71.6,
+  },
+  Object {
+    "productName": "Linden 10 Round Wall Clock, Black",
+    "totalSales": 48.896,
+  },
+  Object {
+    "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box",
+    "totalSales": 45.92,
+  },
+  Object {
+    "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box",
+    "totalSales": 45.92,
+  },
+  Object {
+    "productName": "Kingston Digital DataTraveler 16GB USB 2.0",
+    "totalSales": 44.75,
+  },
+  Object {
+    "productName": "Kingston Digital DataTraveler 16GB USB 2.1",
+    "totalSales": 44.75,
+  },
+  Object {
+    "productName": "Recycled Eldon Regeneration Jumbo File",
+    "totalSales": 39.296,
+  },
+  Object {
+    "productName": "Linden 10 Round Wall Clock, Black",
+    "totalSales": 36.672,
+  },
+  Object {
+    "productName": "Linden 10 Round Wall Clock, Black",
+    "totalSales": 30.56,
+  },
+  Object {
+    "productName": "Linden 10 Round Wall Clock, Black",
+    "totalSales": 30.56,
+  },
+  Object {
+    "productName": "Anderson Hickey Conga Table Tops & Accessories",
+    "totalSales": 24.368,
+  },
+  Object {
+    "productName": "Plymouth Boxed Rubber Bands by Plymouth",
+    "totalSales": 23.55,
+  },
+  Object {
+    "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box",
+    "totalSales": 18.368,
+  },
+  Object {
+    "productName": "Wausau Papers Astrobrights Colored Envelopes",
+    "totalSales": 14.352,
+  },
+  Object {
+    "productName": "Plymouth Boxed Rubber Bands by Plymouth",
+    "totalSales": 14.13,
+  },
+  Object {
+    "productName": "Project Tote Personal File",
+    "totalSales": 14.03,
+  },
+  Object {
+    "productName": "Plymouth Boxed Rubber Bands by Plymouth",
+    "totalSales": 11.304,
+  },
+  Object {
+    "productName": "Magna Visual Magnetic Picture Hangers",
+    "totalSales": 9.64,
+  },
+  Object {
+    "productName": "OIC #2 Pencils, Medium Soft",
+    "totalSales": 9.4,
+  },
+  Object {
+    "productName": "Magna Visual Magnetic Picture Hangers",
+    "totalSales": 7.712,
+  },
+  Object {
+    "productName": "OIC #2 Pencils, Medium Soft",
+    "totalSales": 3.76,
+  },
+  Object {
+    "productName": "OIC #2 Pencils, Medium Soft",
+    "totalSales": 3.76,
+  },
+  Object {
+    "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer",
+    "totalSales": null,
+  },
+]
+`;
+
 exports[`Queries with the @cubejs-backend/postgres-driver filtering Customers: contains + dimensions, first 1`] = `
 Array [
   Object {