feat: refreshKey every support for CRON format interval (#1048)

* cron format interval

* cron interval

* cron string & and tests

* fixed bug with start time offset calculation

* docs for every and timezone

* Allow only timezone offset instead of timezone

* new docs and tests

* fix tests

* fix tests and docs

* Update cube.md

* Update cube.md

* code style

* code style

Author: VictorTrapenok

docs/Schema/cube.md

@@ -255,7 +255,24 @@ cube(`OrderFacts`, {
 });
 ```
 
-Available interval granularities are: `second`, `minute`, `hour`, `day` and `week`.
+
+`every` - can be set as an interval with granularities `second`, `minute`, `hour`, `day`, and `week` or accept CRON string with some limitations.
+If you set `every` as CRON string, you can use the `timezone` property.
+
+For example:
+
+```javascript
+cube(`OrderFacts`, {
+  sql: `SELECT * FROM orders`,
+  refreshKey: {
+    every: '30 5 * * 5',
+    timezone: 'America/Los_Angeles'
+  }
+});
+```
+
+`every` can accept only equal time intervals - so  "Day of month" and "month" intervals in CRON expressions are not supported.
+
 Such `refreshKey` is just a syntactic sugar over `refreshKey` SQL. 
 It's guaranteed that `refreshKey` change it's value at least once during `every` interval.
 It will be converted to appropriate SQL select which value will change over time based on interval value.

packages/cubejs-api-gateway/package.json

@@ -21,6 +21,7 @@
     "@hapi/joi": "^15.1.1",
     "body-parser": "^1.19.0",
     "chrono-node": "1.4.4",
+    "cron-parser": "^2.16.3",
     "jsonwebtoken": "^8.3.0",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.27",

packages/cubejs-api-gateway/yarn.lock

@@ -1018,6 +1018,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
+cron-parser@^2.16.3:
+  version "2.16.3"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a"
+  integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==
+  dependencies:
+    is-nan "^1.3.0"
+    moment-timezone "^0.5.31"
+
 cross-spawn@^6.0.0, cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -2253,6 +2261,13 @@ is-map@^2.0.1:
   resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1"
   integrity sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==
 
+is-nan@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.3.0.tgz#85d1f5482f7051c2019f5673ccebdb06f3b0db03"
+  integrity sha512-z7bbREymOqt2CCaZVly8aC4ML3Xhfi0ekuOnjO2L8vKdl+CttdVoGZQhd4adMFAsxQ5VeRVwORs4tU8RH+HFtQ==
+  dependencies:
+    define-properties "^1.1.3"
+
 is-number@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195"
@@ -3206,6 +3221,13 @@ moment-timezone@^0.5.27:
   dependencies:
     moment ">= 2.9.0"
 
+moment-timezone@^0.5.31:
+  version "0.5.31"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.31.tgz#9c40d8c5026f0c7ab46eda3d63e49c155148de05"
+  integrity sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==
+  dependencies:
+    moment ">= 2.9.0"
+
 "moment@>= 2.9.0", moment@^2.24.0:
   version "2.24.0"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"

packages/cubejs-schema-compiler/adapter/BaseQuery.js

@@ -1,5 +1,7 @@
 /* eslint-disable no-unused-vars,prefer-template */
 const R = require('ramda');
+const cronParser = require('cron-parser');
+ 
 const moment = require('moment-timezone');
 const inflection = require('inflection');
 
@@ -1653,7 +1655,7 @@ class BaseQuery {
           return this.evaluateSql(cube, cubeFromPath.refreshKey.sql);
         }
         if (cubeFromPath.refreshKey.every) {
-          return `SELECT ${this.everyRefreshKeySql(cubeFromPath.refreshKey.every)}`;
+          return `SELECT ${this.everyRefreshKeySql(cubeFromPath.refreshKey)}`;
         }
       }
       refreshKeyAllSetManually = false;
@@ -1687,7 +1689,7 @@ class BaseQuery {
       refreshKeyRenewalThresholds: cubes.map(c => {
         const cubeFromPath = this.cubeEvaluator.cubeFromPath(c);
         if (cubeFromPath.refreshKey && cubeFromPath.refreshKey.every) {
-          return this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey.every);
+          return this.refreshKeyRenewalThresholdForInterval(cubeFromPath.refreshKey);
         }
         return this.defaultRefreshKeyRenewalThreshold();
       })
@@ -1817,9 +1819,59 @@ class BaseQuery {
     }
   }
 
-  everyRefreshKeySql(interval) {
-    // cron-parser
-    return this.floorSql(`${this.unixTimestampSql()} / ${this.parseSecondDuration(interval)}`);
+  calcIntervalForCronString(refreshKey) {
+    const every = refreshKey.every || '1 hour';
+    // One of the years that start from monday (first day of week)
+    // Mon, 01 Jan 2018 00:00:00 GMT
+    const startDate = 1514764800000;
+    const opt = {
+      currentDate: new Date(startDate)
+    };
+    let utcOffset = 0;
+    if (refreshKey.timezone) {
+      utcOffset = moment.tz(refreshKey.timezone).utcOffset() * 60;
+    }
+    
+    let start;
+    let end;
+    let dayOffset;
+    let dayOffsetPrev;
+    try {
+      const interval = cronParser.parseExpression(every, opt);
+      dayOffset = interval.next().getTime();
+      dayOffsetPrev = interval.prev().getTime();
+      if (dayOffsetPrev === startDate) {
+        dayOffset = startDate;
+      }
+
+      start = interval.next().getTime();
+      end = interval.next().getTime();
+    } catch (err) {
+      throw new UserError(`Invalid cron string '${every}' in refreshKey (${err})`);
+    }
+    const delta = (end - start) / 1000;
+     
+    if (
+      !/^(\*|\d+)? ?(\*|\d+) (\*|\d+) \* \* (\*|\d+)$/g.test(every.replace(/ +/g, ' ').replace(/^ | $/g, ''))
+    ) {
+      throw new UserError(`Your cron string ('${every}') is correct, but we support only equal time intervals.`);
+    }
+    return {
+      utcOffset,
+      interval: delta,
+      dayOffset: (dayOffset - startDate) / 1000
+    };
+  }
+
+  everyRefreshKeySql(refreshKey) {
+    const every = refreshKey.every || '1 hour';
+
+    if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) {
+      return this.floorSql(`(${this.unixTimestampSql()}) / ${this.parseSecondDuration(every)}`);
+    }
+
+    const { dayOffset, utcOffset, interval } = this.calcIntervalForCronString(refreshKey);
+    return this.floorSql(`(${utcOffset} + ${dayOffset} + ${this.unixTimestampSql()}) / ${interval}`);
   }
 
   granularityFor(momentDate) {
@@ -1877,7 +1929,6 @@ class BaseQuery {
   }
 
   parseSecondDuration(interval) {
-    // cron-parser
     const intervalMatch = interval.match(/^(\d+) (second|minute|hour|day|week)s?$/);
     if (!intervalMatch) {
       throw new UserError(`Invalid interval: ${interval}`);
@@ -1922,8 +1973,8 @@ class BaseQuery {
               refreshKeyRenewalThresholds: [this.defaultRefreshKeyRenewalThreshold()]
             };
           }
-          const interval = preAggregation.refreshKey.every || '1 hour';
-          let refreshKey = this.everyRefreshKeySql(interval);
+
+          let refreshKey = this.everyRefreshKeySql(preAggregation.refreshKey);
           if (preAggregation.refreshKey.incremental) {
             if (!preAggregation.partitionGranularity) {
               throw new UserError('Incremental refresh key can only be used for partitioned pre-aggregations');
@@ -1944,7 +1995,7 @@ class BaseQuery {
           if (preAggregation.refreshKey.every || preAggregation.refreshKey.incremental) {
             return {
               queries: [this.paramAllocator.buildSqlAndParams(`SELECT ${refreshKey}`)],
-              refreshKeyRenewalThresholds: [this.refreshKeyRenewalThresholdForInterval(interval)]
+              refreshKeyRenewalThresholds: [this.refreshKeyRenewalThresholdForInterval(preAggregation.refreshKey)]
             };
           }
         }
@@ -1988,9 +2039,15 @@ class BaseQuery {
     );
   }
 
-  refreshKeyRenewalThresholdForInterval(interval) {
-    // cron-parser
-    return Math.max(Math.min(Math.round(this.parseSecondDuration(interval) / 10), 300), 1);
+  refreshKeyRenewalThresholdForInterval(refreshKey) {
+    const { every } = refreshKey;
+
+    if (/^(\d+) (second|minute|hour|day|week)s?$/.test(every)) {
+      return Math.max(Math.min(Math.round(this.parseSecondDuration(every) / 10), 300), 1);
+    }
+
+    const { interval } = this.calcIntervalForCronString(refreshKey);
+    return Math.max(Math.min(Math.round(interval / 10), 300), 1);
   }
 
   preAggregationStartEndQueries(cube, preAggregation) {

packages/cubejs-schema-compiler/compiler/CubeValidator.js

@@ -8,6 +8,8 @@ const timeInterval =
     Joi.any().valid('unbounded')
   ]);
 const everyInterval = Joi.string().regex(/^(\d+) (second|minute|hour|day|week)s?$/, 'refresh time interval');
+const everyCronInterval = Joi.string();
+const everyCronTimeZone = Joi.string();
 
 const BaseDimensionWithoutSubQuery = {
   aliases: Joi.array().items(Joi.string()),
@@ -69,7 +71,8 @@ const BasePreAggregationWithoutPartitionGranularity = {
       sql: Joi.func().required()
     }),
     Joi.object().keys({
-      every: everyInterval,
+      every: Joi.alternatives().try(everyInterval, everyCronInterval),
+      timezone: everyCronTimeZone,
       incremental: Joi.boolean(),
       updateWindow: timeInterval
     })
@@ -109,7 +112,8 @@ const cubeSchema = Joi.object().keys({
       immutable: Joi.boolean().required()
     }),
     Joi.object().keys({
-      every: everyInterval
+      every: Joi.alternatives().try(everyInterval, everyCronInterval),
+      timezone: everyCronTimeZone,
     })
   ),
   fileName: Joi.string().required(),

packages/cubejs-schema-compiler/package.json

@@ -25,6 +25,7 @@
     "@babel/types": "^7.4.0",
     "@hapi/joi": "^15.1.1",
     "antlr4": "^4.8.0",
+    "cron-parser": "^2.16.3",
     "humps": "^2.0.1",
     "inflection": "^1.12.0",
     "lru-cache": "^5.1.1",

packages/cubejs-schema-compiler/test/BaseQueryTest.js

@@ -0,0 +1,106 @@
+/* globals it, describe, after */
+/* eslint-disable quote-props */
+const moment = require('moment-timezone');
+const UserError = require('../compiler/UserError');
+const PostgresQuery = require('../adapter/PostgresQuery');
+const PrepareCompiler = require('./PrepareCompiler');
+require('should');
+
+const { prepareCompiler } = PrepareCompiler;
+const dbRunner = require('./DbRunner');
+
+describe('SQL Generation', function test() {
+  this.timeout(90000);
+
+  after(async () => {
+    await dbRunner.tearDown();
+  });
+
+  const { compiler, joinGraph, cubeEvaluator, transformer } = prepareCompiler(` 
+    cube('cards', {
+      sql: \`
+      select * from cards
+      \`,
+ 
+      measures: {
+        count: {
+          type: 'count'
+        }
+      },
+
+      dimensions: {
+        id: {
+          type: 'number',
+          sql: 'id',
+          primaryKey: true
+        }
+      }
+    }) 
+    `);
+
+    
+  it('Test for everyRefreshKeySql', () => {
+    const result = compiler.compile().then(() => {
+      const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
+        measures: [
+          'cards.count'
+        ],
+        timeDimensions: [],
+        filters: [],
+        timezone: 'America/Los_Angeles'
+      });
+
+      const utcOffset = moment.tz('America/Los_Angeles').utcOffset() * 60;
+      let r;
+      r = query.everyRefreshKeySql({
+        every: '1 hour'
+      });
+      r.should.be.equal('FLOOR((EXTRACT(EPOCH FROM NOW())) / 3600)');
+
+      r = query.everyRefreshKeySql({
+        every: '0 * * * * *',
+        timezone: 'America/Los_Angeles'
+      });
+      r.should.be.equal(`FLOOR((${utcOffset} + 0 + EXTRACT(EPOCH FROM NOW())) / 60)`);
+
+      r = query.everyRefreshKeySql({
+        every: '0 * * * *',
+        timezone: 'America/Los_Angeles'
+      });
+      r.should.be.equal(`FLOOR((${utcOffset} + 0 + EXTRACT(EPOCH FROM NOW())) / 3600)`);
+
+      r = query.everyRefreshKeySql({
+        every: '30 * * * *',
+        timezone: 'America/Los_Angeles'
+      });
+      r.should.be.equal(`FLOOR((${utcOffset} + 1800 + EXTRACT(EPOCH FROM NOW())) / 3600)`);
+
+      r = query.everyRefreshKeySql({
+        every: '30 5 * * 5',
+        timezone: 'America/Los_Angeles'
+      });
+      r.should.be.equal(`FLOOR((${utcOffset} + 365400 + EXTRACT(EPOCH FROM NOW())) / 604800)`);
+
+      for (let i = 1; i < 59; i++) {
+        r = query.everyRefreshKeySql({
+          every: `${i} * * * *`,
+          timezone: 'America/Los_Angeles'
+        });
+        r.should.be.equal(`FLOOR((${utcOffset} + ${i * 60} + EXTRACT(EPOCH FROM NOW())) / ${1 * 60 * 60})`);
+      }
+
+      try {
+        r = query.everyRefreshKeySql({
+          every: '*/9 */7 * * *',
+          timezone: 'America/Los_Angeles'
+        });
+        
+        throw new Error();
+      } catch (error) {
+        error.should.be.instanceof(UserError);
+      }
+    });
+
+    return result;
+  });
+});

packages/cubejs-schema-compiler/yarn.lock

@@ -1582,6 +1582,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
+cron-parser@^2.16.3:
+  version "2.16.3"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a"
+  integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==
+  dependencies:
+    is-nan "^1.3.0"
+    moment-timezone "^0.5.31"
+
 cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -2635,6 +2643,13 @@ is-map@^2.0.1:
   resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1"
   integrity sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==
 
+is-nan@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.3.0.tgz#85d1f5482f7051c2019f5673ccebdb06f3b0db03"
+  integrity sha512-z7bbREymOqt2CCaZVly8aC4ML3Xhfi0ekuOnjO2L8vKdl+CttdVoGZQhd4adMFAsxQ5VeRVwORs4tU8RH+HFtQ==
+  dependencies:
+    define-properties "^1.1.3"
+
 is-number@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
@@ -3103,6 +3118,13 @@ moment-timezone@^0.5.28:
   dependencies:
     moment ">= 2.9.0"
 
+moment-timezone@^0.5.31:
+  version "0.5.31"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.31.tgz#9c40d8c5026f0c7ab46eda3d63e49c155148de05"
+  integrity sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==
+  dependencies:
+    moment ">= 2.9.0"
+
 "moment@>= 2.9.0":
   version "2.23.0"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.23.0.tgz#759ea491ac97d54bac5ad776996e2a58cc1bc225"