fix: Apply time shift after timezone conversions to avoid double casts (

#8229) * fix: Apply time shift after timezone conversions to avoid double casts * Tests * Fix mysql * Fix mysql and bigquery
cube-js · May 3, 2024 · 651b9e0 · 651b9e0
1 parent 4065bde
commit 651b9e0
Show file tree

Hide file tree

Showing 20 changed files with 385 additions and 15 deletions.
diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts
@@ -45,6 +45,7 @@ export class BaseDimension {
     return this.convertTzForRawTimeDimensionIfNeeded(() => this.query.dimensionSql(this));
   }
 
+  // We need this for dimensions however we don't for filters for performance reasons
   public convertTzForRawTimeDimensionIfNeeded(sql) {
     if (this.query.options.convertTzForRawTimeDimension) {
       return this.query.evaluateSymbolSqlWithContext(sql, {

diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts
@@ -107,6 +107,9 @@ export class BaseMeasure {
       return false;
     }
     const definition = this.measureDefinition();
+    if (definition.postAggregate) {
+      return false;
+    }
     return definition.type === 'sum' || definition.type === 'count' || definition.type === 'countDistinctApprox' ||
       definition.type === 'min' || definition.type === 'max';
   }

diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
@@ -1204,7 +1204,8 @@ export class BaseQuery {
       postAggregateTimeDimensions: withQuery.postAggregateTimeDimensions,
       filters: withQuery.filters,
       // TODO do we need it?
-      postAggregateQuery: true // !!fromDimensions.find(d => this.newDimension(d).isPostAggregate())
+      postAggregateQuery: true, // !!fromDimensions.find(d => this.newDimension(d).isPostAggregate())
+      disableExternalPreAggregations: true,
     });
 
     const measures = fromSubQuery && fromMeasures.map(m => fromSubQuery.newMeasure(m));
@@ -1232,6 +1233,7 @@ export class BaseQuery {
         const { type } = this.newMeasure(d).definition();
         return type === 'rank' || BaseQuery.isCalculatedMeasureType(type);
       }),
+      disableExternalPreAggregations: true,
     };
     const subQuery = this.newSubQuery(subQueryOptions);
 
@@ -2184,7 +2186,7 @@ export class BaseQuery {
         } else {
           let res = this.autoPrefixAndEvaluateSql(cubeName, symbol.sql);
           if (symbol.shiftInterval) {
-            res = `(${this.addTimestampInterval(this.timeStampCast(res), symbol.shiftInterval)})`;
+            res = `(${this.addTimestampInterval(res, symbol.shiftInterval)})`;
           }
           if (this.safeEvaluateSymbolContext().convertTzForRawTimeDimension &&
             !memberExpressionType &&

diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js
@@ -371,6 +371,7 @@ export class PreAggregations {
       );
 
     const isAdditive = R.all(m => m.isAdditive(), query.measures);
+    const hasPostAggregate = R.any(m => m.isPostAggregate(), query.measures);
     const leafMeasures = leafMeasurePaths.map(path => query.newMeasure(path));
     const leafMeasureAdditive = R.all(m => m.isAdditive(), leafMeasures);
     const cumulativeMeasures = leafMeasures
@@ -412,7 +413,8 @@ export class PreAggregations {
       allBackAliasMembers,
       ungrouped: query.ungrouped,
       sortedUsedCubePrimaryKeys,
-      sortedAllCubeNames
+      sortedAllCubeNames,
+      hasPostAggregate
     };
   }
 
@@ -716,7 +718,7 @@ export class PreAggregations {
      */
     const canUseFn =
       (
-        transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures || transformedQuery.ungrouped
+        transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures && !transformedQuery.hasPostAggregate || transformedQuery.ungrouped
       ) ? (r) => canUsePreAggregationLeafMeasureAdditive(r) ||
           canUsePreAggregationNotAdditive(r)
         : canUsePreAggregationNotAdditive;

diff --git a/packages/cubejs-testing-drivers/fixtures/_schemas.json b/packages/cubejs-testing-drivers/fixtures/_schemas.json
@@ -264,6 +264,17 @@
           "sql": "profit",
           "type": "sum"
         },
+        {
+          "name": "totalProfitYearAgo",
+          "type": "number",
+          "sql": "{totalProfit}",
+          "post_aggregate": true,
+          "time_shift": [{
+            "time_dimension": "orderDate",
+            "interval": "366 day",
+            "type": "prior"
+          }]
+        },
         {
           "name": "hiddenSum",
           "sql": "profit",

diff --git a/packages/cubejs-testing-drivers/fixtures/bigquery.json b/packages/cubejs-testing-drivers/fixtures/bigquery.json
@@ -19,7 +19,7 @@
   "cast": {
     "SELECT_PREFIX": "",
     "SELECT_SUFFIX": "",
-    "DATE_PREFIX": "PARSE_DATE('%Y-%m-%d', ",
+    "DATE_PREFIX": "PARSE_TIMESTAMP('%Y-%m-%d', ",
     "DATE_SUFFIX": ")",
     "CREATE_TBL_PREFIX": "CREATE TABLE ",
     "CREATE_TBL_SUFFIX": " AS ",

diff --git a/packages/cubejs-testing-drivers/fixtures/mysql.json b/packages/cubejs-testing-drivers/fixtures/mysql.json
@@ -15,7 +15,7 @@
     "ports" : ["4000"]
   },
   "data": {
-    "image": "mysql:8",
+    "image": "mysql:8.0.22",
     "environment": {
       "MYSQL_DATABASE": "test",
       "MYSQL_ROOT_PASSWORD": "123-Strong-Password-321"

diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts
@@ -1481,6 +1481,20 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
       expect(response.rawData()).toMatchSnapshot();
     });
 
+    execute('querying BigECommerce: totalProfitYearAgo', async () => {
+      const response = await client.load({
+        measures: [
+          'BigECommerce.totalProfitYearAgo',
+        ],
+        timeDimensions: [{
+          dimension: 'BigECommerce.orderDate',
+          granularity: 'month',
+          dateRange: ['2020-01-01', '2020-12-31'],
+        }],
+      });
+      expect(response.rawData()).toMatchSnapshot();
+    });
+
     if (includeIncrementalSchemaSuite) {
       incrementalSchemaLoadingSuite(execute, () => driver, tables);
     }

diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/athena-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/athena-full.test.ts.snap
@@ -2275,6 +2275,138 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/athena-driver querying BigECommerce: rolling window by 2 day 1`] = `
+Array [
+  Object {
+    "BigECommerce.orderDate": "2020-01-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-01-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-02-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-02-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-03-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-03-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-04-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-04-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-05-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-05-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-06-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-06-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-07-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-07-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-08-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-08-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-09-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-09-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-10-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-10-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": "1",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-11-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-11-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-12-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-12-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Day": null,
+  },
+]
+`;
+
+exports[`Queries with the @cubejs-backend/athena-driver querying BigECommerce: rolling window by 2 month 1`] = `
+Array [
+  Object {
+    "BigECommerce.orderDate": "2020-01-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-01-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "2",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-02-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-02-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "3",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-03-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-03-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "3",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-04-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-04-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "3",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-05-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-05-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "6",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-06-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-06-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "12",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-07-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-07-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "7",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-08-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-08-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": null,
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-09-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-09-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "6",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-10-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-10-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "10",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-11-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-11-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "13",
+  },
+  Object {
+    "BigECommerce.orderDate": "2020-12-01T00:00:00.000",
+    "BigECommerce.orderDate.month": "2020-12-01T00:00:00.000",
+    "BigECommerce.rollingCountBy2Month": "16",
+  },
+]
+`;
+
+exports[`Queries with the @cubejs-backend/athena-driver querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/athena-driver querying Customers: dimensions 1`] = `
 Array [
   Object {

diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/bigquery-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/bigquery-full.test.ts.snap
@@ -606,6 +606,8 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/bigquery-driver querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/bigquery-driver filtering Customers: endsWith filter + dimensions, third 1`] = `Array []`;
 
 exports[`Queries with the @cubejs-backend/bigquery-driver filtering Customers: notEndsWith filter + dimensions, first 1`] = `

diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/clickhouse-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/clickhouse-full.test.ts.snap
@@ -1375,6 +1375,8 @@ Array [
 
 exports[`Queries with the @cubejs-backend/clickhouse-driver filtering Customers: startsWith + dimensions, third 1`] = `Array []`;
 
+exports[`Queries with the @cubejs-backend/clickhouse-driver querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/clickhouse-driver filtering ECommerce: contains dimensions, first 1`] = `
 Array [
   Object {

diff --git a/...cubejs-testing-drivers/test/__snapshots__/databricks-jdbc-export-bucket-full.test.ts.snap b/...cubejs-testing-drivers/test/__snapshots__/databricks-jdbc-export-bucket-full.test.ts.snap
@@ -2743,6 +2743,8 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/databricks-jdbc-driver export-bucket querying Customers: dimensions 1`] = `
 Array [
   Object {

diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/databricks-jdbc-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/databricks-jdbc-full.test.ts.snap
@@ -2743,6 +2743,8 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/databricks-jdbc-driver querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/databricks-jdbc-driver querying Customers: dimensions 1`] = `
 Array [
   Object {

diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/mssql-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/mssql-full.test.ts.snap
@@ -2727,6 +2727,8 @@ Array [
 ]
 `;
 
+exports[`Queries with the @cubejs-backend/mssql-driver querying BigECommerce: totalProfitYearAgo 1`] = `Array []`;
+
 exports[`Queries with the @cubejs-backend/mssql-driver querying Customers: dimensions 1`] = `
 Array [
   Object {