Skip to content

Commit bef0626

Browse files
paveltiunovpaveltiunov
committed
feat: Default refreshKey implementations for mutable and immutable pre-aggregations.
1 parent ca8dab3 commit bef0626

File tree

3 files changed

+187
-33
lines changed

3 files changed

+187
-33
lines changed

‎packages/cubejs-schema-compiler/adapter/BaseQuery.js

Lines changed: 73 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,33 +1320,44 @@ class BaseQuery {
13201320
);
13211321
}
13221322

1323-
cacheKeyQueries() { // TODO collect sub queries
1324-
const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery();
1325-
if (preAggregationForQuery) {
1326-
return {
1327-
renewalThreshold: this.renewalThreshold(!!preAggregationForQuery.refreshKey),
1328-
queries: this.preAggregationInvalidateKeyQueries(preAggregationForQuery.cube, preAggregationForQuery)
1329-
};
1323+
cacheKeyQueries(transformFn) { // TODO collect sub queries
1324+
if (!this.safeEvaluateSymbolContext().preAggregationQuery) {
1325+
const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery();
1326+
if (preAggregationForQuery) {
1327+
return {
1328+
renewalThreshold: this.renewalThreshold(!!preAggregationForQuery.refreshKey),
1329+
queries: this.preAggregationInvalidateKeyQueries(preAggregationForQuery.cube, preAggregationForQuery)
1330+
};
1331+
}
13301332
}
13311333
let refreshKeyAllSetManually = true;
1332-
const queries = this.collectCubeNames()
1333-
.map(cube => {
1334-
const cubeFromPath = this.cubeEvaluator.cubeFromPath(cube);
1335-
if (cubeFromPath.refreshKey) {
1336-
return this.evaluateSql(cube, cubeFromPath.refreshKey.sql);
1337-
}
1338-
refreshKeyAllSetManually = false;
1339-
const timeDimensions = this.cubeEvaluator.timeDimensionPathsForCube(cube);
1340-
if (timeDimensions.length) {
1341-
const dimension = timeDimensions.filter(f => f.toLowerCase().indexOf('update') !== -1)[0] || timeDimensions[0];
1342-
const foundMainTimeDimension = this.newTimeDimension({ dimension });
1343-
const aggSelect = this.aggSelectForDimension(cube, foundMainTimeDimension, 'max');
1344-
if (aggSelect) {
1345-
return aggSelect;
1346-
}
1334+
const refreshKeyQueryByCube = cube => {
1335+
const cubeFromPath = this.cubeEvaluator.cubeFromPath(cube);
1336+
if (cubeFromPath.refreshKey && cubeFromPath.refreshKey.sql) {
1337+
return this.evaluateSql(cube, cubeFromPath.refreshKey.sql);
1338+
}
1339+
refreshKeyAllSetManually = false;
1340+
const timeDimensions =
1341+
!(cubeFromPath.refreshKey && cubeFromPath.refreshKey.immutable) ?
1342+
this.cubeEvaluator.timeDimensionPathsForCube(cube) :
1343+
[];
1344+
if (timeDimensions.length) {
1345+
const dimension = timeDimensions.filter(f => f.toLowerCase().indexOf('update') !== -1)[0] || timeDimensions[0];
1346+
const foundMainTimeDimension = this.newTimeDimension({ dimension });
1347+
const aggSelect = this.aggSelectForDimension(cube, foundMainTimeDimension, 'max');
1348+
if (aggSelect) {
1349+
return aggSelect;
13471350
}
1348-
return `select count(*) from ${this.cubeSql(cube)} ${this.asSyntaxTable} ${this.cubeAlias(cube)}`;
1349-
}).map(paramAnnotatedSql => this.paramAllocator.buildSqlAndParams(paramAnnotatedSql));
1351+
}
1352+
return this.evaluateSymbolSqlWithContext(
1353+
() => `select count(*) from ${this.cubeSql(cube)} ${this.asSyntaxTable} ${this.cubeAlias(cube)}`,
1354+
{ preAggregationQuery: true }
1355+
);
1356+
};
1357+
const queries = this.collectCubeNames()
1358+
.map(cube => [cube, refreshKeyQueryByCube(cube)])
1359+
.map(([cube, sql]) => (transformFn ? transformFn(sql, cube) : sql))
1360+
.map(paramAnnotatedSql => this.paramAllocator.buildSqlAndParams(paramAnnotatedSql));
13501361
return {
13511362
queries,
13521363
renewalThreshold: this.renewalThreshold(refreshKeyAllSetManually)
@@ -1423,14 +1434,46 @@ class BaseQuery {
14231434
}
14241435

14251436
preAggregationInvalidateKeyQueries(cube, preAggregation) {
1437+
const preAggregationQueryForSql = this.preAggregationQueryForSqlEvaluation(cube, preAggregation);
14261438
if (preAggregation.refreshKey) {
1427-
return [this.paramAllocator.buildSqlAndParams(
1428-
this.preAggregationQueryForSqlEvaluation(cube, preAggregation).evaluateSql(cube, preAggregation.refreshKey.sql)
1429-
)];
1439+
if (preAggregation.refreshKey.sql) {
1440+
return [this.paramAllocator.buildSqlAndParams(
1441+
preAggregationQueryForSql.evaluateSql(cube, preAggregation.refreshKey.sql)
1442+
)];
1443+
}
1444+
}
1445+
if (preAggregation.partitionGranularity) {
1446+
const cubeFromPath = this.cubeEvaluator.cubeFromPath(cube);
1447+
return preAggregationQueryForSql.evaluateSymbolSqlWithContext(
1448+
() => preAggregationQueryForSql.cacheKeyQueries(
1449+
(originalRefreshKey, refreshKeyCube) => {
1450+
if (cubeFromPath.refreshKey && cubeFromPath.refreshKey.immutable) {
1451+
return preAggregationQueryForSql.evaluateSql(
1452+
cube,
1453+
(FILTER_PARAMS) => `SELECT ${preAggregationQueryForSql.caseWhenStatement([{
1454+
sql: FILTER_PARAMS[
1455+
preAggregationQueryForSql.timeDimensions[0].path()[0]
1456+
][
1457+
preAggregationQueryForSql.timeDimensions[0].path()[1]
1458+
].filter((from, to) => `${preAggregationQueryForSql.nowTimestampSql()} < ${this.timeStampCast(to)}`),
1459+
label: `(${originalRefreshKey})`
1460+
}])}`
1461+
);
1462+
} else {
1463+
// TODO handle WHERE while generating originalRefreshKey
1464+
return refreshKeyCube === preAggregationQueryForSql.timeDimensions[0].path()[0] ?
1465+
`${originalRefreshKey} WHERE ${preAggregationQueryForSql.timeDimensions[0].filterToWhere()}` :
1466+
originalRefreshKey;
1467+
}
1468+
}
1469+
),
1470+
{ preAggregationQuery: true }
1471+
).queries;
14301472
}
1431-
return [this.paramAllocator.buildSqlAndParams(
1432-
`SELECT ${this.timeGroupedColumn('hour', this.convertTz(this.nowTimestampSql()))} as current_hour`
1433-
)];
1473+
return preAggregationQueryForSql.evaluateSymbolSqlWithContext(
1474+
() => preAggregationQueryForSql.cacheKeyQueries(),
1475+
{ preAggregationQuery: true }
1476+
).queries;
14341477
}
14351478

14361479
preAggregationStartEndQueries(cube, preAggregation) {

‎packages/cubejs-schema-compiler/compiler/CubeValidator.js

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,14 @@ const BasePreAggregation = {
7272
const cubeSchema = Joi.object().keys({
7373
name: identifier,
7474
sql: Joi.func().required(),
75-
refreshKey: Joi.object().keys({
76-
sql: Joi.func().required()
77-
}),
75+
refreshKey: Joi.alternatives().try(
76+
Joi.object().keys({
77+
sql: Joi.func().required()
78+
}),
79+
Joi.object().keys({
80+
immutable: Joi.boolean().required()
81+
})
82+
),
7883
fileName: Joi.string().required(),
7984
extends: Joi.func(),
8085
allDefinitions: Joi.func(),

‎packages/cubejs-schema-compiler/test/PreAggregationsTest.js

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ describe('PreAggregations', function test() {
174174
})
175175
176176
cube('GoogleVisitors', {
177+
refreshKey: {
178+
immutable: true,
179+
},
177180
extends: visitors,
178181
sql: \`select v.* from \${visitors.sql()} v where v.source = 'google'\`
179182
})
@@ -338,6 +341,109 @@ describe('PreAggregations', function test() {
338341
});
339342
});
340343

344+
it('immutable partition default refreshKey', () => {
345+
return compiler.compile().then(() => {
346+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
347+
measures: [
348+
'GoogleVisitors.checkinsTotal'
349+
],
350+
dimensions: [
351+
'GoogleVisitors.source'
352+
],
353+
timeDimensions: [{
354+
dimension: 'GoogleVisitors.createdAt',
355+
granularity: 'day',
356+
dateRange: ['2017-01-01', '2017-01-30']
357+
}],
358+
timezone: 'America/Los_Angeles',
359+
order: [{
360+
id: 'GoogleVisitors.createdAt'
361+
}],
362+
preAggregationsSchema: ''
363+
});
364+
365+
const queryAndParams = query.buildSqlAndParams();
366+
console.log(queryAndParams);
367+
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
368+
console.log(JSON.stringify(preAggregationsDescription, null, 2));
369+
370+
preAggregationsDescription[0].invalidateKeyQueries[0][0].should.match(/NOW\(\) </)
371+
372+
return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
373+
query.buildSqlAndParams()
374+
]).map(q => replaceTableName(q, preAggregationsDescription, 101))).then(res => {
375+
res.should.be.deepEqual(
376+
[
377+
{
378+
"google_visitors__source": "google",
379+
"google_visitors__created_at_day": "2017-01-05T00:00:00.000Z",
380+
"google_visitors__checkins_total": "1"
381+
}
382+
]
383+
);
384+
});
385+
});
386+
});
387+
388+
it('mutable partition default refreshKey', () => {
389+
return compiler.compile().then(() => {
390+
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
391+
measures: [
392+
'visitors.checkinsTotal'
393+
],
394+
dimensions: [
395+
'visitors.source'
396+
],
397+
timeDimensions: [{
398+
dimension: 'visitors.createdAt',
399+
granularity: 'day',
400+
dateRange: ['2017-01-01', '2017-01-30']
401+
}],
402+
timezone: 'America/Los_Angeles',
403+
order: [{
404+
id: 'visitors.createdAt'
405+
}],
406+
preAggregationsSchema: ''
407+
});
408+
409+
const queryAndParams = query.buildSqlAndParams();
410+
console.log(queryAndParams);
411+
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
412+
console.log(JSON.stringify(preAggregationsDescription, null, 2));
413+
414+
preAggregationsDescription[0].invalidateKeyQueries[0][0].should.match(/>=/);
415+
416+
return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
417+
query.buildSqlAndParams()
418+
]).map(q => replaceTableName(q, preAggregationsDescription, 102))).then(res => {
419+
res.should.be.deepEqual(
420+
[
421+
{
422+
visitors__source: 'some',
423+
visitors__created_at_day: '2017-01-02T00:00:00.000Z',
424+
visitors__checkins_total: '3'
425+
},
426+
{
427+
visitors__source: 'some',
428+
visitors__created_at_day: '2017-01-04T00:00:00.000Z',
429+
visitors__checkins_total: '2'
430+
},
431+
{
432+
visitors__source: 'google',
433+
visitors__created_at_day: '2017-01-05T00:00:00.000Z',
434+
visitors__checkins_total: '1'
435+
},
436+
{
437+
visitors__source: null,
438+
visitors__created_at_day: '2017-01-06T00:00:00.000Z',
439+
visitors__checkins_total: '0'
440+
}
441+
]
442+
);
443+
});
444+
});
445+
});
446+
341447
it('hll bigquery rollup', () => {
342448
return compiler.compile().then(() => {
343449
const query = new BigqueryQuery({ joinGraph, cubeEvaluator, compiler }, {

0 commit comments

Comments
 (0)