Skip to content

Commit ea78c84

Browse files
Justin-ZSpaveltiunov
Justin-ZS
authored and
paveltiunov
committed
feat: support for pre-aggregation time hierarchies (#258) Thanks to @Justin-ZS!
* feat: enhance pre-aggregation for time hierarchy fix: 'week' granularity should not work for 'year' fix(pre-aggregation): handle time hierarchy on query fix: keep context away from arguments * fix(test): update broken test for pre-aggregation * fix: incorrect granularity in filter clause * test: add simple tests for the pre-aggregation in time hierarchy Fixes #246
1 parent 458c0c9 commit ea78c84

File tree

4 files changed

+208
-28
lines changed

4 files changed

+208
-28
lines changed

‎packages/cubejs-schema-compiler/adapter/BaseQuery.js

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -891,8 +891,9 @@ class BaseQuery {
891891
}
892892

893893
dimensionSql(dimension) {
894-
if (this.safeEvaluateSymbolContext().rollupQuery) {
895-
return this.escapeColumnName(dimension.unescapedAliasName());
894+
const context = this.safeEvaluateSymbolContext();
895+
if (context.rollupQuery) {
896+
return this.escapeColumnName(dimension.unescapedAliasName(context.rollupGranularity));
896897
}
897898
return this.evaluateSymbolSql(dimension.path()[0], dimension.path()[1], dimension.dimensionDefinition());
898899
}

‎packages/cubejs-schema-compiler/adapter/BaseTimeDimension.js

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,10 @@ class BaseTimeDimension extends BaseFilter {
4848
return super.aliasName();
4949
}
5050

51-
unescapedAliasName() {
52-
return `${this.query.aliasName(this.dimension)}_${this.granularity || 'date'}`; // TODO date here for rollups
51+
unescapedAliasName(granularity) {
52+
const actualGranularity = granularity || this.granularity || 'date';
53+
54+
return `${this.query.aliasName(this.dimension)}_${actualGranularity}`; // TODO date here for rollups
5355
}
5456

5557
dateSeriesAliasName() {
@@ -64,16 +66,19 @@ class BaseTimeDimension extends BaseFilter {
6466
}
6567

6668
dimensionSql() {
67-
if (this.query.safeEvaluateSymbolContext().rollupQuery) {
68-
return super.dimensionSql();
69+
const context = this.query.safeEvaluateSymbolContext();
70+
const granularity = context.granularityOverride || this.granularity;
71+
72+
if (context.rollupQuery) {
73+
if (context.rollupGranularity === this.granularity) {
74+
return super.dimensionSql();
75+
}
76+
return this.query.timeGroupedColumn(granularity, this.query.dimensionSql(this));
6977
}
70-
if (this.query.safeEvaluateSymbolContext().ungrouped) {
78+
if (context.ungrouped) {
7179
return this.convertedToTz();
7280
}
73-
return this.query.timeGroupedColumn(
74-
this.query.safeEvaluateSymbolContext().granularityOverride || this.granularity,
75-
this.convertedToTz()
76-
);
81+
return this.query.timeGroupedColumn(granularity, this.convertedToTz());
7782
}
7883

7984
convertedToTz() {

‎packages/cubejs-schema-compiler/adapter/PreAggregations.js

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,23 @@ class PreAggregations {
201201
timeDimensions.map(d => [d.dimension, d.granularity || 'date'])
202202
) || [];
203203
}
204+
// TimeDimension :: [Dimension, Granularity]
205+
// TimeDimension -> [TimeDimension]
206+
function expandTimeDimension(timeDimension) {
207+
const [dimension, granularity] = timeDimension;
208+
const makeTimeDimension = newGranularity => [dimension, newGranularity];
209+
210+
const tds = [timeDimension];
211+
const updateTds = (...granularitys) => tds.push(...granularitys.map(makeTimeDimension))
212+
213+
if (granularity === 'year') updateTds('hour', 'date', 'month');
214+
if (['month', 'week'].includes(granularity)) updateTds('hour', 'date');
215+
if (granularity === 'date') updateTds('hour');
216+
217+
return tds;
218+
}
219+
// [[TimeDimension]]
220+
const queryTimeDimensionsList = transformedQuery.sortedTimeDimensions.map(expandTimeDimension);
204221

205222
const canUsePreAggregationNotAdditive = (references) =>
206223
transformedQuery.hasNoTimeDimensionsWithoutGranularity &&
@@ -222,7 +239,7 @@ class PreAggregations {
222239
) &&
223240
R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures) &&
224241
R.allPass(
225-
transformedQuery.sortedTimeDimensions.map(td => R.contains(td))
242+
queryTimeDimensionsList.map(tds => R.anyPass(tds.map(td => R.contains(td))))
226243
)(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions));
227244

228245
const canUsePreAggregationAdditive = (references) =>
@@ -235,7 +252,7 @@ class PreAggregations {
235252
R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures)
236253
) &&
237254
R.allPass(
238-
transformedQuery.sortedTimeDimensions.map(td => R.contains(td))
255+
queryTimeDimensionsList.map(tds => R.anyPass(tds.map(td => R.contains(td))))
239256
)(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions));
240257

241258

@@ -440,6 +457,8 @@ class PreAggregations {
440457
preAggregationForQuery.preAggregation.measures :
441458
this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation).measures
442459
);
460+
461+
const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'date';
443462

444463
return this.query.evaluateSymbolSqlWithContext(
445464
() => `SELECT ${this.query.baseSelect()} FROM ${table} ${this.query.baseWhere(filters)}` +
@@ -449,7 +468,8 @@ class PreAggregations {
449468
this.query.groupByDimensionLimit(),
450469
{
451470
renderedReference,
452-
rollupQuery: true
471+
rollupQuery: true,
472+
rollupGranularity,
453473
}
454474
);
455475
}

‎packages/cubejs-schema-compiler/test/PreAggregationsTest.js

Lines changed: 168 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,24 @@ describe('PreAggregations', function test() {
9494
measureReferences: [checkinsTotal],
9595
segmentReferences: [google],
9696
timeDimensionReference: createdAt,
97-
granularity: 'day',
97+
granularity: 'week',
9898
},
9999
approx: {
100100
type: 'rollup',
101101
measureReferences: [countDistinctApprox],
102102
timeDimensionReference: createdAt,
103103
granularity: 'day'
104104
},
105-
ratio: {
105+
multiStage: {
106+
useOriginalSqlPreAggregations: true,
106107
type: 'rollup',
107-
measureReferences: [checkinsTotal, uniqueSourceCount],
108+
measureReferences: [checkinsTotal],
108109
timeDimensionReference: createdAt,
109-
granularity: 'day'
110+
granularity: 'month',
111+
partitionGranularity: 'day',
112+
refreshKey: {
113+
sql: \`SELECT CASE WHEN \${FILTER_PARAMS.visitors.createdAt.filter((from, to) => \`\${to}::timestamp > now()\`)} THEN now() END\`
114+
}
110115
},
111116
partitioned: {
112117
type: 'rollup',
@@ -116,16 +121,11 @@ describe('PreAggregations', function test() {
116121
granularity: 'day',
117122
partitionGranularity: 'month'
118123
},
119-
multiStage: {
120-
useOriginalSqlPreAggregations: true,
124+
ratio: {
121125
type: 'rollup',
122-
measureReferences: [checkinsTotal],
126+
measureReferences: [checkinsTotal, uniqueSourceCount],
123127
timeDimensionReference: createdAt,
124-
granularity: 'month',
125-
partitionGranularity: 'day',
126-
refreshKey: {
127-
sql: \`SELECT CASE WHEN \${FILTER_PARAMS.visitors.createdAt.filter((from, to) => \`\${to}::timestamp > now()\`)} THEN now() END\`
128-
}
128+
granularity: 'day'
129129
}
130130
}
131131
})
@@ -518,7 +518,7 @@ describe('PreAggregations', function test() {
518518
preAggregationsSchema: '',
519519
timeDimensions: [{
520520
dimension: 'visitors.createdAt',
521-
granularity: 'date',
521+
granularity: 'week',
522522
dateRange: ['2016-12-30', '2017-01-05']
523523
}],
524524
order: [{
@@ -542,7 +542,7 @@ describe('PreAggregations', function test() {
542542
res.should.be.deepEqual(
543543
[
544544
{
545-
"visitors__created_at_date": "2017-01-05T00:00:00.000Z",
545+
"visitors__created_at_week": "2017-01-02T00:00:00.000Z",
546546
"visitors__checkins_total": "1"
547547
}
548548
]
@@ -590,3 +590,157 @@ describe('PreAggregations', function test() {
590590
});
591591
});
592592
});
593+
594+
595+
describe('PreAggregations in time hierarchy', function test() {
596+
this.timeout(20000);
597+
598+
after(async () => {
599+
await dbRunner.tearDown();
600+
});
601+
602+
const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(`
603+
cube(\`visitors\`, {
604+
sql: \`
605+
select * from visitors
606+
\`,
607+
608+
measures: {
609+
count: {
610+
type: 'count'
611+
}
612+
},
613+
614+
dimensions: {
615+
createdAt: {
616+
type: 'time',
617+
sql: 'created_at'
618+
},
619+
},
620+
621+
preAggregations: {
622+
month: {
623+
type: 'rollup',
624+
measureReferences: [count],
625+
timeDimensionReference: createdAt,
626+
granularity: 'month',
627+
},
628+
day: {
629+
type: 'rollup',
630+
measureReferences: [count],
631+
timeDimensionReference: createdAt,
632+
granularity: 'day',
633+
},
634+
}
635+
})
636+
`);
637+
638+
function replaceTableName(query, preAggregation, suffix) {
639+
const [toReplace, params] = query;
640+
console.log(toReplace);
641+
preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
642+
return [
643+
preAggregation.reduce((replacedQuery, desc) =>
644+
replacedQuery.replace(new RegExp(desc.tableName, 'g'), desc.tableName + '_' + suffix), toReplace
645+
),
646+
params
647+
];
648+
}
649+
650+
function tempTablePreAggregations(preAggregationsDescriptions) {
651+
return R.unnest(preAggregationsDescriptions.map(desc =>
652+
desc.invalidateKeyQueries.concat([
653+
[desc.loadSql[0].replace('CREATE TABLE', 'CREATE TEMP TABLE'), desc.loadSql[1]]
654+
])
655+
));
656+
}
657+
658+
it('query on year match to pre-agg on month', () => {
659+
return compiler.compile().then(() => {
660+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
661+
measures: [
662+
'visitors.count'
663+
],
664+
dimensions: [],
665+
timezone: 'America/Los_Angeles',
666+
timeDimensions: [{
667+
dimension: 'visitors.createdAt',
668+
granularity: 'year',
669+
dateRange: ['2016-12-30', '2018-12-30']
670+
}],
671+
preAggregationsSchema: '',
672+
order: [],
673+
});
674+
675+
const queryAndParams = query.buildSqlAndParams();
676+
677+
query.preAggregations.preAggregationForQuery.preAggregation.granularity.should.be.equal('month');
678+
679+
console.log(queryAndParams);
680+
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
681+
console.log(preAggregationsDescription);
682+
683+
const queries = tempTablePreAggregations(preAggregationsDescription);
684+
685+
console.log(JSON.stringify(queries.concat(queryAndParams)));
686+
687+
return dbRunner.testQueries(
688+
queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 1))
689+
).then(res => {
690+
console.log(JSON.stringify(res));
691+
res.should.be.deepEqual(
692+
[
693+
{
694+
"visitors__count": "5",
695+
"visitors__created_at_year": "2017-01-01T00:00:00.000Z"
696+
},
697+
]
698+
);
699+
});
700+
});
701+
});
702+
it('query on week match to pre-agg on day', () => {
703+
return compiler.compile().then(() => {
704+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
705+
measures: [
706+
'visitors.count'
707+
],
708+
dimensions: [],
709+
timezone: 'America/Los_Angeles',
710+
timeDimensions: [{
711+
dimension: 'visitors.createdAt',
712+
granularity: 'week',
713+
dateRange: ['2017-01-02', '2019-02-08']
714+
}],
715+
preAggregationsSchema: '',
716+
order: [],
717+
});
718+
719+
const queryAndParams = query.buildSqlAndParams();
720+
721+
query.preAggregations.preAggregationForQuery.preAggregation.granularity.should.be.equal('day');
722+
723+
console.log(queryAndParams);
724+
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
725+
console.log(preAggregationsDescription);
726+
727+
const queries = tempTablePreAggregations(preAggregationsDescription);
728+
729+
console.log(JSON.stringify(queries.concat(queryAndParams)));
730+
731+
return dbRunner.testQueries(
732+
queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 1))
733+
).then(res => {
734+
console.log(JSON.stringify(res));
735+
res.should.be.deepEqual(
736+
[
737+
{
738+
"visitors__count": "5",
739+
"visitors__created_at_week": "2017-01-02T00:00:00.000Z"
740+
},
741+
]
742+
);
743+
});
744+
});
745+
});
746+
});

0 commit comments

Comments
 (0)