feat: Minute and second granularities support

packages/cubejs-api-gateway/index.js

@@ -99,7 +99,7 @@ const querySchema = Joi.object().keys({
   }).xor('dimension', 'member')),
   timeDimensions: Joi.array().items(Joi.object().keys({
     dimension: id.required(),
-    granularity: Joi.valid('day', 'month', 'year', 'week', 'hour', null),
+    granularity: Joi.valid('day', 'month', 'year', 'week', 'hour', 'minute', 'second', null),
     dateRange: [
       Joi.array().items(Joi.string()).min(1).max(2),
       Joi.string()

packages/cubejs-schema-compiler/adapter/BaseMeasure.js

@@ -118,7 +118,7 @@ class BaseMeasure {
       return undefined;
     }
     if (interval.match(/day/)) {
-      return 'date';
+      return 'day';
     } else if (interval.match(/month/)) {
       return 'month';
     } else if (interval.match(/year/)) {

packages/cubejs-schema-compiler/adapter/BaseQuery.js

@@ -14,6 +14,15 @@ const PreAggregations = require('./PreAggregations');
 
 const DEFAULT_PREAGGREGATIONS_SCHEMA = `stb_pre_aggregations`;
 
+const standardGranularitiesParents = {
+  year: 'month',
+  week: 'day',
+  month: 'day',
+  day: 'hour',
+  hour: 'minute',
+  minute: 'second'
+};
+
 class BaseQuery {
   constructor(compilers, options) {
     this.compilers = compilers;
@@ -25,6 +34,8 @@ class BaseQuery {
     this.defaultOrder = this.defaultOrder.bind(this);
 
     this.initFromOptions();
+
+    this.granularityParentHierarchyCache = {};
   }
 
   initFromOptions() {
@@ -406,25 +417,43 @@ class BaseQuery {
     return this.convertTz(dateParam);
   }
 
+  granularityHierarchies() {
+    return R.fromPairs(Object.keys(standardGranularitiesParents).map(k => [k, this.granularityParentHierarchy(k)]));
+  }
+
+  granularityParent(granularity) {
+    return standardGranularitiesParents[granularity];
+  }
+
+  granularityParentHierarchy(granularity) {
+    if (!this.granularityParentHierarchyCache[granularity]) {
+      this.granularityParentHierarchyCache[granularity] = [granularity].concat(
+        this.granularityParent(granularity) ? this.granularityParentHierarchy(this.granularityParent(granularity)) : []
+      );
+    }
+    return this.granularityParentHierarchyCache[granularity];
+  }
+
   minGranularity(granularityA, granularityB) {
     if (!granularityA) {
       return granularityB;
     }
     if (!granularityB) {
       return granularityA;
     }
-    if (granularityA === 'hour' || granularityB === 'hour') {
-      return 'hour';
-    } else if (granularityA === 'date' || granularityB === 'date') {
-      return 'date';
-    } else if (granularityA === 'month' && granularityB === 'month') {
-      return 'month';
-    } else if (granularityA === 'year' && granularityB === 'year') {
-      return 'year';
-    } else if (granularityA === 'week' && granularityB === 'week') {
-      return 'week';
+    const aHierarchy = R.reverse(this.granularityParentHierarchy(granularityA));
+    const bHierarchy = R.reverse(this.granularityParentHierarchy(granularityB));
+    let lastIndex = Math.max(
+      aHierarchy.findIndex((g, i) => g !== bHierarchy[i]),
+      bHierarchy.findIndex((g, i) => g !== aHierarchy[i])
+    );
+    if (lastIndex === -1 && aHierarchy.length === bHierarchy.length) {
+      lastIndex = aHierarchy.length - 1;
+    }
+    if (lastIndex <= 0) {
+      throw new Error(`Can't find common parent for '${granularityA}' and '${granularityB}'`);
     }
-    return 'date';
+    return aHierarchy[lastIndex - 1];
   }
 
   overTimeSeriesQuery(baseQueryFn, cumulativeMeasure) {

packages/cubejs-schema-compiler/adapter/BaseTimeDimension.js

@@ -5,7 +5,7 @@ const BaseFilter = require('./BaseFilter');
 const UserError = require('../compiler/UserError');
 
 const TIME_SERIES = {
-  date: (range) =>
+  day: (range) =>
     Array.from(range.by('day'))
       .map(d => [d.format('YYYY-MM-DDT00:00:00.000'), d.format('YYYY-MM-DDT23:59:59.999')]),
   month: (range) =>
@@ -17,6 +17,12 @@ const TIME_SERIES = {
   hour: (range) =>
     Array.from(range.by('hour'))
       .map(d => [d.format('YYYY-MM-DDTHH:00:00.000'), d.format('YYYY-MM-DDTHH:59:59.999')]),
+  minute: (range) =>
+    Array.from(range.by('minute'))
+      .map(d => [d.format('YYYY-MM-DDTHH:MM:00.000'), d.format('YYYY-MM-DDTHH:MM:59.999')]),
+  second: (range) =>
+    Array.from(range.by('second'))
+      .map(d => [d.format('YYYY-MM-DDTHH:MM:SS.000'), d.format('YYYY-MM-DDTHH:MM:SS.999')]),
   week: (range) =>
     Array.from(range.snapTo('isoweek').by('week'))
       .map(d => [d.startOf('isoweek').format('YYYY-MM-DDT00:00:00.000'), d.endOf('isoweek').format('YYYY-MM-DDT23:59:59.999')])
@@ -49,7 +55,7 @@ class BaseTimeDimension extends BaseFilter {
   }
 
   unescapedAliasName(granularity) {
-    const actualGranularity = granularity || this.granularity || 'date';
+    const actualGranularity = granularity || this.granularity || 'day';
 
     return `${this.query.aliasName(this.dimension)}_${actualGranularity}`; // TODO date here for rollups
   }

packages/cubejs-schema-compiler/adapter/BigqueryQuery.js

@@ -2,9 +2,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'DAY',
+  day: 'DAY',
   week: 'WEEK(MONDAY)',
   hour: 'HOUR',
+  minute: 'MINUTE',
+  second: 'SECOND',
   month: 'MONTH',
   year: 'YEAR'
 };

packages/cubejs-schema-compiler/adapter/ClickHouseQuery.js

@@ -5,8 +5,10 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'Day',
+  day: 'Day',
   hour: 'Hour',
+  minute: 'Minute',
+  second: 'Second',
   month: 'Month',
   quarter: 'Quarter',
   year: 'Year',

packages/cubejs-schema-compiler/adapter/HiveQuery.js

@@ -4,9 +4,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd 00:00:00.000')`,
+  day: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd 00:00:00.000')`,
   week: (date) => `DATE_FORMAT(from_unixtime(unix_timestamp('1900-01-01 00:00:00') + floor((unix_timestamp(${date}) - unix_timestamp('1900-01-01 00:00:00')) / (60 * 60 * 24 * 7)) * (60 * 60 * 24 * 7)), 'yyyy-MM-dd 00:00:00.000')`,
   hour: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd HH:00:00.000')`,
+  minute: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd HH:mm:00.000')`,
+  second: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd HH:mm:ss.000')`,
   month: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-01 00:00:00.000')`,
   year: (date) => `DATE_FORMAT(${date}, 'yyyy-01-01 00:00:00.000')`
 };

packages/cubejs-schema-compiler/adapter/MssqlQuery.js

@@ -29,9 +29,11 @@ class MssqlParamAllocator extends ParamAllocator {
 }
 
 const GRANULARITY_TO_INTERVAL = {
-  date: (date) => `FORMAT(${date}, 'yyyy-MM-ddT00:00:00.000')`,
+  day: (date) => `FORMAT(${date}, 'yyyy-MM-ddT00:00:00.000')`,
   week: (date) => `FORMAT(dateadd(week, DATEDIFF(week, '1900-01-01', ${date}), '1900-01-01'), 'yyyy-MM-ddT00:00:00.000')`,
   hour: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:00:00.000')`,
+  minute: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:00.000')`,
+  second: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:ss.000')`,
   month: (date) => `FORMAT(${date}, 'yyyy-MM-01T00:00:00.000')`,
   year: (date) => `FORMAT(${date}, 'yyyy-01-01T00:00:00.000')`
 };

packages/cubejs-schema-compiler/adapter/MysqlQuery.js

@@ -4,9 +4,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT00:00:00.000')`,
+  day: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT00:00:00.000')`,
   week: (date) => `DATE_FORMAT(date_add('1900-01-01', interval TIMESTAMPDIFF(WEEK, '1900-01-01', ${date}) WEEK), '%Y-%m-%dT00:00:00.000')`,
   hour: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:00:00.000')`,
+  minute: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%M:00.000')`,
+  second: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT%H:%M:%S.000')`,
   month: (date) => `DATE_FORMAT(${date}, '%Y-%m-01T00:00:00.000')`,
   year: (date) => `DATE_FORMAT(${date}, '%Y-01-01T00:00:00.000')`
 };

packages/cubejs-schema-compiler/adapter/OracleQuery.js

@@ -2,9 +2,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_VALUE = {
-  date: 'DD',
+  day: 'DD',
   week: 'IW',
   hour: 'HH24',
+  minute: 'mm',
+  second: 'ss',
   month: 'MM',
   year: 'YYYY'
 };

packages/cubejs-schema-compiler/adapter/PostgresQuery.js

@@ -2,9 +2,11 @@ const BaseQuery = require('./BaseQuery');
 const ParamAllocator = require('./ParamAllocator');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'day',
+  day: 'day',
   week: 'week',
   hour: 'hour',
+  minute: 'minute',
+  second: 'second',
   month: 'month',
   year: 'year'
 };

packages/cubejs-schema-compiler/adapter/PreAggregations.js

@@ -131,7 +131,7 @@ class PreAggregations {
     function sortTimeDimensions(timeDimensions) {
       return timeDimensions && R.sortBy(
         R.prop(0),
-        timeDimensions.map(d => [d.dimension, d.granularity || 'date'])
+        timeDimensions.map(d => [d.dimension, d.granularity || 'day'])
       ) || [];
     }
 
@@ -145,6 +145,7 @@ class PreAggregations {
 
     const isAdditive = R.all(m => m.isAdditive(), query.measures);
     const leafMeasureAdditive = R.all(path => query.newMeasure(path).isAdditive(), leafMeasurePaths);
+    const granularityHierarchies = query.granularityHierarchies();
 
     return {
       sortedDimensions,
@@ -154,7 +155,8 @@ class PreAggregations {
       leafMeasures: leafMeasurePaths,
       hasNoTimeDimensionsWithoutGranularity,
       allFiltersWithinSelectedDimensions,
-      isAdditive
+      isAdditive,
+      granularityHierarchies
     };
   }
 
@@ -198,23 +200,15 @@ class PreAggregations {
     function sortTimeDimensions(timeDimensions) {
       return timeDimensions && R.sortBy(
         d => d.join('.'),
-        timeDimensions.map(d => [d.dimension, d.granularity || 'date'])
+        timeDimensions.map(d => [d.dimension, d.granularity || 'day'])
       ) || [];
     }
     // TimeDimension :: [Dimension, Granularity]
     // TimeDimension -> [TimeDimension]
     function expandTimeDimension(timeDimension) {
       const [dimension, granularity] = timeDimension;
       const makeTimeDimension = newGranularity => [dimension, newGranularity];
-
-      const tds = [timeDimension];
-      const updateTds = (...granularitys) => tds.push(...granularitys.map(makeTimeDimension))
-
-      if (granularity === 'year') updateTds('hour', 'date', 'month');
-      if (['month', 'week'].includes(granularity)) updateTds('hour', 'date');
-      if (granularity === 'date') updateTds('hour');
-
-      return tds;
+      return (transformedQuery.granularityHierarchies[granularity] || [granularity]).map(makeTimeDimension);
     }
     // [[TimeDimension]]
     const queryTimeDimensionsList = transformedQuery.sortedTimeDimensions.map(expandTimeDimension);
@@ -371,10 +365,6 @@ class PreAggregations {
   }
 
   castGranularity(granularity) {
-    // TODO replace date granularity with day
-    if (granularity === 'day') {
-      return 'date';
-    }
     return granularity;
   }
 
@@ -486,7 +476,7 @@ class PreAggregations {
       this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation).measures
     );
 
-    const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'date';
+    const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'day';
 
     return this.query.evaluateSymbolSqlWithContext(
       () => `SELECT ${this.query.baseSelect()} FROM ${table} ${this.query.baseWhere(filters)}` +

packages/cubejs-schema-compiler/adapter/PrestodbQuery.js

@@ -2,9 +2,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'day',
+  day: 'day',
   week: 'week',
   hour: 'hour',
+  minute: 'minute',
+  second: 'second',
   month: 'month',
   year: 'year'
 };

packages/cubejs-schema-compiler/adapter/SnowflakeQuery.js

@@ -1,9 +1,11 @@
 const BaseQuery = require('./BaseQuery');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'DAY',
+  day: 'DAY',
   week: 'WEEK',
   hour: 'HOUR',
+  minute: 'MINUTE',
+  second: 'SECOND',
   month: 'MONTH',
   year: 'YEAR'
 };

packages/cubejs-schema-compiler/adapter/SqliteQuery.js

@@ -4,9 +4,11 @@ const BaseQuery = require('./BaseQuery');
 const BaseFilter = require('./BaseFilter');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: (date) => `strftime('%Y-%m-%dT00:00:00.000', ${date})`,
+  day: (date) => `strftime('%Y-%m-%dT00:00:00.000', ${date})`,
   week: (date) => `strftime('%Y-%m-%dT00:00:00.000', CASE WHEN date(${date}, 'weekday 1') = date(${date}) THEN date(${date}, 'weekday 1') ELSE date(${date}, 'weekday 1', '-7 days') END)`,
   hour: (date) => `strftime('%Y-%m-%dT%H:00:00.000', ${date})`,
+  minute: (date) => `strftime('%Y-%m-%dT%H:%M:00.000', ${date})`,
+  second: (date) => `strftime('%Y-%m-%dT%H:%M:%S.000', ${date})`,
   month: (date) => `strftime('%Y-%m-01T00:00:00.000', ${date})`,
   year: (date) => `strftime('%Y-01-01T00:00:00.000', ${date})`
 };

packages/cubejs-schema-compiler/adapter/VerticaQuery.js

@@ -1,9 +1,11 @@
 const BaseQuery = require('./BaseQuery');
 
 const GRANULARITY_TO_INTERVAL = {
-  date: 'DD',
+  day: 'DD',
   week: 'W',
   hour: 'HH24',
+  minute: 'mm',
+  second: 'ss',
   month: 'MM',
   year: 'YY'
 };

packages/cubejs-schema-compiler/compiler/CubeValidator.js

@@ -115,7 +115,7 @@ const cubeSchema = Joi.object().keys({
             ])
           })),
           else: Joi.object().keys({
-          label: Joi.alternatives([
+            label: Joi.alternatives([
               Joi.string(),
               Joi.object().keys({
                 sql: Joi.func().required()
@@ -167,7 +167,9 @@ const cubeSchema = Joi.object().keys({
       dimensionReferences: Joi.func(),
       segmentReferences: Joi.func(),
       timeDimensionReference: Joi.func().required(),
-      granularity: Joi.any().valid('hour', 'day', 'week', 'month', 'year').required()
+      granularity: Joi.any().valid(
+        'second', 'minute', 'hour', 'day', 'week', 'month', 'year'
+      ).required()
     }))
   ))
 });
@@ -179,8 +181,8 @@ class CubeValidator {
   }
 
   compile(cubes, errorReporter) {
-    return this.cubeSymbols.cubeList.map((v) =>
-        this.validate(this.cubeSymbols.getCubeDefinition(v.name), errorReporter.inContext(`${v.name} cube`))
+    return this.cubeSymbols.cubeList.map(
+      (v) => this.validate(this.cubeSymbols.getCubeDefinition(v.name), errorReporter.inContext(`${v.name} cube`))
     );
   }
 

packages/cubejs-schema-compiler/test/ClickHouseDataSchemaCompilerTest.js

@@ -150,7 +150,7 @@ describe('ClickHouse DataSchemaCompiler', function test() {
         measures: ['visitors.visitor_count'],
         timeDimensions: [{
           dimension: 'visitors.created_at',
-          granularity: 'date',
+          granularity: 'day',
           dateRange: ['2017-01-01', '2017-01-30']
         }],
         filters: [{
@@ -169,10 +169,10 @@ describe('ClickHouse DataSchemaCompiler', function test() {
       return dbRunner.testQuery(query.buildSqlAndParams()).then(res => {
         res.should.be.deepEqual(
           [
-            { "visitors__created_at_date": "2017-01-02T00:00:00.000", "visitors__visitor_count": "1" },
-            { "visitors__created_at_date": "2017-01-04T00:00:00.000", "visitors__visitor_count": "1" },
-            { "visitors__created_at_date": "2017-01-05T00:00:00.000", "visitors__visitor_count": "1" },
-            { "visitors__created_at_date": "2017-01-06T00:00:00.000", "visitors__visitor_count": "2" }
+            { "visitors__created_at_day": "2017-01-02T00:00:00.000", "visitors__visitor_count": "1" },
+            { "visitors__created_at_day": "2017-01-04T00:00:00.000", "visitors__visitor_count": "1" },
+            { "visitors__created_at_day": "2017-01-05T00:00:00.000", "visitors__visitor_count": "1" },
+            { "visitors__created_at_day": "2017-01-06T00:00:00.000", "visitors__visitor_count": "2" }
           ]
         );
       });