feat(schema-compiler): allowNonStrictDateRangeMatch flag support for …

…the pre-aggregations with time dimension (#4582)

packages/cubejs-schema-compiler/src/adapter/PreAggregations.js

@@ -338,7 +338,8 @@ export class PreAggregations {
 
   canUsePreAggregationFn(query, refs) {
     return PreAggregations.canUsePreAggregationForTransformedQueryFn(
-      PreAggregations.transformQueryToCanUseForm(query), refs
+      PreAggregations.transformQueryToCanUseForm(query),
+      refs,
     );
   }
 
@@ -362,82 +363,175 @@ export class PreAggregations {
     }
   }
 
+  /**
+   * Returns function to determine whether pre-aggregation can be used or not
+   * for specified query, or its value for `refs` if specified.
+   * @param {Object} transformedQuery transformed query
+   * @param {Object?} refs pre-aggs reference
+   * @returns {function(preagg: Object): boolean}
+   */
   static canUsePreAggregationForTransformedQueryFn(transformedQuery, refs) {
+    /**
+     * Returns an array of 2-elements arrays with the dimension and granularity
+     * sorted by the concatenated dimension + granularity key.
+     * @param {Array<{dimension: string, granularity: string}>} timeDimensions
+     * @returns {Array<Array<string>>}
+     */
+    const sortTimeDimensions = (timeDimensions) => (
+      timeDimensions &&
+      R.sortBy(
+        d => d.join('.'),
+        timeDimensions.map(
+          d => [
+            d.dimension,
+            d.granularity || 'day', // TODO granularity shouldn't be null?
+          ]
+        ),
+      ) || []
+    );
+
+    /**
+     * @type {Set<string>}
+     */
     const filterDimensionsSingleValueEqual =
       transformedQuery.filterDimensionsSingleValueEqual instanceof Set
         ? transformedQuery.filterDimensionsSingleValueEqual
         : new Set(
-          Object.keys(transformedQuery.filterDimensionsSingleValueEqual || {})
+          Object.keys(
+            transformedQuery.filterDimensionsSingleValueEqual || {},
+          )
         );
+
+    /**
+     * Determine whether pre-aggregation can be used or not.
+     * @param {*} references
+     * @returns {boolean}
+     */
+    const canUsePreAggregationNotAdditive = (references) => {
+      const refTimeDimensions =
+        references.sortedTimeDimensions ||
+        sortTimeDimensions(references.timeDimensions);
 
-    function sortTimeDimensions(timeDimensions) {
-      return timeDimensions && R.sortBy(
-        d => d.join('.'),
-        timeDimensions.map(d => [d.dimension, d.granularity || 'day']) // TODO granularity shouldn't be null?
-      ) || [];
-    }
-    const expandGranularity = (granularity) => transformedQuery.granularityHierarchies[granularity] || [granularity];
-    // TimeDimension :: [Dimension, Granularity]
-    // TimeDimension -> [TimeDimension]
-    function expandTimeDimension(timeDimension) {
-      const [dimension, granularity] = timeDimension;
-      const makeTimeDimension = newGranularity => [dimension, newGranularity];
-      return expandGranularity(granularity).map(makeTimeDimension);
-    }
-    // [[TimeDimension]]
-    const queryTimeDimensionsList = transformedQuery.sortedTimeDimensions.map(expandTimeDimension);
+      const qryTimeDimensions = references.allowNonStrictDateRangeMatch
+        ? transformedQuery.timeDimensions
+        : transformedQuery.sortedTimeDimensions;
+
+      return ((
+        transformedQuery.hasNoTimeDimensionsWithoutGranularity
+      ) && (
+        !transformedQuery.hasCumulativeMeasures
+      ) && (
+        R.equals(qryTimeDimensions, refTimeDimensions)
+      ) && (
+        transformedQuery.isAdditive ||
+        R.equals(transformedQuery.timeDimensions, refTimeDimensions)
+      ) && (
+        references.dimensions.length === filterDimensionsSingleValueEqual.size &&
+        R.all(d => filterDimensionsSingleValueEqual.has(d), references.dimensions)
+      ) && (
+        R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.measures) ||
+        R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures)
+      ));
+    };
+
+    /**
+     * Wrap granularity string into an array.
+     * @param {string} granularity
+     * @returns {Array<string>}
+     */
+    const expandGranularity = (granularity) => (
+      transformedQuery.granularityHierarchies[granularity] ||
+        [granularity]
+    );
 
+    /**
+     * Determine whether time dimensions match to the window granularity or not.
+     * @param {*} references
+     * @returns {boolean}
+     */
     const windowGranularityMatches = (references) => {
       if (!transformedQuery.windowGranularity) {
         return true;
       }
+      const sortedTimeDimensions =
+        references.sortedTimeDimensions ||
+        sortTimeDimensions(references.timeDimensions);
+
+      return expandGranularity(transformedQuery.windowGranularity)
+        .map(
+          windowGranularity => R.all(
+            td => td[1] === windowGranularity,
+            sortedTimeDimensions,
+          )
+        )
+        .filter(x => !!x)
+        .length > 0;
+    };
 
-      const sortedTimeDimensions = references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions);
-      return expandGranularity(transformedQuery.windowGranularity).map(windowGranularity => R.all(
-        td => td[1] === windowGranularity,
-        sortedTimeDimensions,
-      )).filter(x => !!x).length > 0;
+    /**
+     * Returns an array of 2-element arrays with dimension and granularity.
+     * @param {*} timeDimension
+     * @returns {Array<Array<string>>}
+     */
+    const expandTimeDimension = (timeDimension) => {
+      const [dimension, granularity] = timeDimension;
+      return expandGranularity(granularity)
+        .map((newGranularity) => [dimension, newGranularity]);
     };
 
-    const canUsePreAggregationNotAdditive = (references) => {
-      const sortedTimeDimensions = references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions);
-      return transformedQuery.hasNoTimeDimensionsWithoutGranularity &&
-      (
-        references.dimensions.length === filterDimensionsSingleValueEqual.size &&
-        R.all(d => filterDimensionsSingleValueEqual.has(d), references.dimensions)
-      ) &&
-      (
-        R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.measures) ||
-        R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures)
-      ) &&
-      R.equals(
-        transformedQuery.sortedTimeDimensions,
-        sortedTimeDimensions
-      ) &&
-      (transformedQuery.isAdditive || R.equals(
-        transformedQuery.timeDimensions,
-        sortedTimeDimensions
-      )) &&
-      !transformedQuery.hasCumulativeMeasures;
+    /**
+     * Determine whether pre-aggregation can be used or not.
+     * TODO: revisit cumulative leaf measure matches.
+     * @param {*} references
+     * @returns {boolean}
+     */
+    const canUsePreAggregationLeafMeasureAdditive = (references) => {
+      /**
+       * Array of 2-element arrays with dimension and granularity.
+       * @type {Array<Array<string>>}
+       */
+      const queryTimeDimensionsList = references.allowNonStrictDateRangeMatch
+        ? transformedQuery.timeDimensions.map(expandTimeDimension)
+        : transformedQuery.sortedTimeDimensions.map(expandTimeDimension);
+
+      return ((
+        windowGranularityMatches(references)
+      ) && (
+        R.all(
+          m => references.measures.indexOf(m) !== -1,
+          transformedQuery.leafMeasures,
+        )
+      ) && (
+        R.all(
+          d => (
+            references.sortedDimensions ||
+            references.dimensions
+          ).indexOf(d) !== -1,
+          transformedQuery.sortedDimensions
+        )
+      ) && (
+        R.allPass(
+          queryTimeDimensionsList.map(
+            tds => R.anyPass(tds.map(td => R.contains(td)))
+          )
+        )(
+          references.sortedTimeDimensions ||
+          sortTimeDimensions(references.timeDimensions)
+        )
+      ));
     };
 
-    // TODO revisit cumulative leaf measure matches
-    const canUsePreAggregationLeafMeasureAdditive = (references) => R.all(
-      d => (references.sortedDimensions || references.dimensions).indexOf(d) !== -1,
-      transformedQuery.sortedDimensions
-    ) &&
-      R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures) &&
-      R.allPass(
-        queryTimeDimensionsList.map(tds => R.anyPass(tds.map(td => R.contains(td))))
-      )(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions)) &&
-      windowGranularityMatches(references);
-
-    let canUseFn;
-    if (transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures) {
-      canUseFn = (r) => canUsePreAggregationLeafMeasureAdditive(r) || canUsePreAggregationNotAdditive(r);
-    } else {
-      canUseFn = canUsePreAggregationNotAdditive;
-    }
+    /**
+     * Determine whether pre-aggregation can be used or not.
+     * @returns {boolean}
+     */
+    const canUseFn =
+      transformedQuery.leafMeasureAdditive &&
+      !transformedQuery.hasMultipliedMeasures
+        ? (r) => canUsePreAggregationLeafMeasureAdditive(r) ||
+          canUsePreAggregationNotAdditive(r)
+        : canUsePreAggregationNotAdditive;
+
     if (refs) {
       return canUseFn(refs);
     } else {
@@ -452,13 +546,24 @@ export class PreAggregations {
   }
 
   // eslint-disable-next-line no-unused-vars
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   getCubeLattice(cube, preAggregationName, preAggregation) {
     throw new UserError('Auto rollups supported only in Enterprise version');
   }
 
+  /**
+   * Returns pre-agg which determined as applicable for the query (the first one
+   * from the list of potentially applicable pre-aggs). The order of the
+   * potentially applicable pre-aggs is the same as the order in which these
+   * pre-aggs appear in the schema file.
+   * @returns {Object}
+   */
   findPreAggregationForQuery() {
     if (!this.preAggregationForQuery) {
-      this.preAggregationForQuery = this.rollupMatchResults().find(p => p.canUsePreAggregation);
+      this.preAggregationForQuery =
+        this
+          .rollupMatchResults()
+          .find(p => p.canUsePreAggregation);
     }
     return this.preAggregationForQuery;
   }
@@ -472,6 +577,7 @@ export class PreAggregations {
       return R.pipe(
         R.toPairs,
         // eslint-disable-next-line no-unused-vars
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
         R.filter(([k, a]) => a.type === 'autoRollup'),
         R.map(([preAggregationName, preAggregation]) => {
           const cubeLattice = this.getCubeLattice(cube, preAggregationName, preAggregation);
@@ -492,19 +598,35 @@ export class PreAggregations {
     return [];
   }
 
+  /**
+   * Returns an array of potencially applicable for the query preaggs in the
+   * same order they appear in the schema file.
+   * @returns {Array<Object>}
+   */
   rollupMatchResults() {
     const { query } = this;
 
     const canUsePreAggregation = this.canUsePreAggregationFn(query);
 
     return R.pipe(
       R.map(cube => {
-        const preAggregations = this.query.cubeEvaluator.preAggregationsForCube(cube);
+        const preAggregations =
+          this.query.cubeEvaluator.preAggregationsForCube(cube);
+
         let rollupPreAggregations =
-          this.findRollupPreAggregationsForCube(cube, canUsePreAggregation, preAggregations);
+          this.findRollupPreAggregationsForCube(
+            cube,
+            canUsePreAggregation,
+            preAggregations,
+          );
+
         rollupPreAggregations = rollupPreAggregations.concat(
-          this.findAutoRollupPreAggregationsForCube(cube, preAggregations)
+          this.findAutoRollupPreAggregationsForCube(
+            cube,
+            preAggregations,
+          ),
         );
+
         return rollupPreAggregations;
       }),
       R.unnest
@@ -515,6 +637,7 @@ export class PreAggregations {
     return R.pipe(
       R.toPairs,
       // eslint-disable-next-line no-unused-vars
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       R.filter(([k, a]) => a.type === 'rollup' || a.type === 'rollupJoin'),
       R.map(([preAggregationName, preAggregation]) => {
         const preAggObj = this.evaluatedPreAggregationObj(

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js

@@ -277,6 +277,7 @@ export class CubeEvaluator extends CubeSymbols {
       granularity: aggregation.granularity
     }] : [];
     return {
+      allowNonStrictDateRangeMatch: aggregation.allowNonStrictDateRangeMatch,
       dimensions:
         (aggregation.dimensionReferences && this.evaluateReferences(cube, aggregation.dimensionReferences) || [])
           .concat(

packages/cubejs-schema-compiler/src/compiler/CubeValidator.js

@@ -199,11 +199,13 @@ const OriginalSqlSchema = condition(
       type: Joi.any().valid('originalSql').required(),
       partitionGranularity: BasePreAggregation.partitionGranularity.required(),
       timeDimensionReference: Joi.func().required(),
+      allowNonStrictDateRangeMatch: Joi.bool(),
     }),
     inherit(BasePreAggregation, {
       type: Joi.any().valid('originalSql').required(),
       partitionGranularity: BasePreAggregation.partitionGranularity.required(),
       timeDimension: Joi.func().required(),
+      allowNonStrictDateRangeMatch: Joi.bool(),
     })
   ),
   inherit(BasePreAggregationWithoutPartitionGranularity, {
@@ -239,6 +241,7 @@ const RollUpJoinSchema = condition(
       type: Joi.any().valid('rollupJoin').required(),
       scheduledRefresh: Joi.boolean().valid(false),
       granularity: GranularitySchema,
+      allowNonStrictDateRangeMatch: Joi.bool(),
       timeDimensionReference: Joi.func().required(),
       rollupReferences: Joi.func().required(),
       measureReferences: Joi.func(),
@@ -251,6 +254,7 @@ const RollUpJoinSchema = condition(
       scheduledRefresh: Joi.boolean().valid(false),
       granularity: GranularitySchema,
       timeDimension: Joi.func().required(),
+      allowNonStrictDateRangeMatch: Joi.bool(),
       rollups: Joi.func().required(),
       measures: Joi.func(),
       dimensions: Joi.func(),
@@ -291,6 +295,7 @@ const RollUpSchema = condition(
       type: Joi.any().valid('rollup').required(),
       timeDimensionReference: Joi.func().required(),
       granularity: GranularitySchema,
+      allowNonStrictDateRangeMatch: Joi.bool(),
       measureReferences: Joi.func(),
       dimensionReferences: Joi.func(),
       segmentReferences: Joi.func(),
@@ -299,6 +304,7 @@ const RollUpSchema = condition(
     inherit(BasePreAggregation, {
       type: Joi.any().valid('rollup').required(),
       timeDimension: Joi.func().required(),
+      allowNonStrictDateRangeMatch: Joi.bool(),
       granularity: GranularitySchema,
       measures: Joi.func(),
       dimensions: Joi.func(),