feat: Support every for refreshKey with SQL

cube-js · Jul 19, 2021 · 63cd8f4 · 63cd8f4
1 parent f22f71a
commit 63cd8f4
Show file tree

Hide file tree

Showing 3 changed files with 127 additions and 7 deletions.
diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
@@ -1740,7 +1740,9 @@ export class BaseQuery {
             this.evaluateSql(cube, cubeFromPath.refreshKey.sql),
             {
               external: false,
-              renewalThreshold: this.defaultRefreshKeyRenewalThreshold()
+              renewalThreshold: cubeFromPath.refreshKey.every
+                ? this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey, false)
+                : this.defaultRefreshKeyRenewalThreshold()
             },
             this
           ];
@@ -2126,7 +2128,9 @@ export class BaseQuery {
                 preAggregationQueryForSql.evaluateSql(cube, preAggregation.refreshKey.sql)
               ).concat({
                 external: false,
-                renewalThreshold: this.defaultRefreshKeyRenewalThreshold(),
+                renewalThreshold: preAggregation.refreshKey.every
+                  ? this.refreshKeyRenewalThresholdForInterval(preAggregation.refreshKey, false)
+                  : this.defaultRefreshKeyRenewalThreshold(),
               })
             ];
           }
@@ -2234,15 +2238,27 @@ export class BaseQuery {
     );
   }
 
-  refreshKeyRenewalThresholdForInterval(refreshKey) {
+  refreshKeyRenewalThresholdForInterval(refreshKey, limitedWithMax = true) {
     const { every } = refreshKey;
 
     if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) {
-      return Math.max(Math.min(Math.round(this.parseSecondDuration(every) / 10), 300), 1);
+      const threshold = Math.max(Math.round(this.parseSecondDuration(every) / 10), 1);
+
+      if (limitedWithMax) {
+        return Math.min(threshold, 300);
+      }
+
+      return threshold;
     }
 
     const { interval } = this.calcIntervalForCronString(refreshKey);
-    return Math.max(Math.min(Math.round(interval / 10), 300), 1);
+    const threshold = Math.max(Math.round(interval / 10), 1);
+
+    if (limitedWithMax) {
+      return Math.min(threshold, 300);
+    }
+
+    return threshold;
   }
 
   preAggregationStartEndQueries(cube, preAggregation) {

diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js
@@ -69,7 +69,10 @@ const BaseMeasure = {
 const BasePreAggregationWithoutPartitionGranularity = {
   refreshKey: Joi.alternatives().try(
     Joi.object().keys({
-      sql: Joi.func().required()
+      sql: Joi.func().required(),
+      // We dont support timezone for this, because it's useless
+      // We cannot support cron interval
+      every: Joi.alternatives().try(everyInterval),
     }),
     Joi.object().keys({
       every: Joi.alternatives().try(everyInterval, everyCronInterval),
@@ -116,7 +119,10 @@ const cubeSchema = Joi.object().keys({
   sql: Joi.func().required(),
   refreshKey: Joi.alternatives().try(
     Joi.object().keys({
-      sql: Joi.func().required()
+      sql: Joi.func().required(),
+      // We dont support timezone for this, because it's useless
+      // We cannot support cron interval
+      every: Joi.alternatives().try(everyInterval),
     }),
     Joi.object().keys({
       immutable: Joi.boolean().required()

diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts
@@ -502,4 +502,102 @@ describe('SQL Generation', () => {
       ]);
     });
   });
+
+  it('refreshKey (sql + every) in cube', async () => {
+    const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(
+      createCubeSchema({
+        name: 'cards',
+        refreshKey: `
+          refreshKey: {
+            sql: 'SELECT MAX(created) FROM cards',
+            every: '2 hours'
+          },
+        `,
+        preAggregations: `
+          countCreatedAt: {
+              type: 'rollup',
+              external: true,
+              measureReferences: [count],
+              timeDimensionReference: createdAt,
+              granularity: \`day\`,
+              partitionGranularity: \`month\`,
+              scheduledRefresh: true,
+          },
+        `
+      })
+    );
+    await compiler.compile();
+
+    const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: [
+        'cards.count'
+      ],
+      timeDimensions: [],
+      filters: [],
+      timezone: 'America/Los_Angeles',
+      externalQueryClass: MssqlQuery
+    });
+
+    const preAggregations: any = query.newPreAggregations().preAggregationsDescription();
+    expect(preAggregations.length).toEqual(1);
+    expect(preAggregations[0].invalidateKeyQueries).toEqual([
+      [
+        'SELECT MAX(created) FROM cards',
+        [],
+        {
+          external: false,
+          renewalThreshold: 720,
+        }
+      ]
+    ]);
+  });
+
+  it('refreshKey (sql + every) in preAggregation', async () => {
+    const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(
+      createCubeSchema({
+        name: 'cards',
+        refreshKey: ``,
+        preAggregations: `
+          countCreatedAt: {
+              type: 'rollup',
+              external: true,
+              measureReferences: [count],
+              timeDimensionReference: createdAt,
+              granularity: \`day\`,
+              partitionGranularity: \`month\`,
+              scheduledRefresh: true,
+              refreshKey: {
+                sql: 'SELECT MAX(created) FROM cards',
+                every: '2 hour'
+              },
+          },
+        `
+      })
+    );
+    await compiler.compile();
+
+    const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+      measures: [
+        'cards.count'
+      ],
+      timeDimensions: [],
+      filters: [],
+      timezone: 'America/Los_Angeles',
+      externalQueryClass: MssqlQuery
+    });
+
+    const preAggregations: any = query.newPreAggregations().preAggregationsDescription();
+    expect(preAggregations.length).toEqual(1);
+    expect(preAggregations[0].invalidateKeyQueries).toEqual([
+      [
+        'SELECT MAX(created) FROM cards',
+        [],
+        {
+          external: false,
+          // 60 * 60 *2
+          renewalThreshold: 720,
+        }
+      ]
+    ]);
+  });
 });