Skip to content

Commit

Permalink
feat: support for pre-aggregation time hierarchies (#258) Thanks to @…
Browse files Browse the repository at this point in the history 
…Justin-ZS!

* feat: enhance pre-aggregation for time hierarchy

fix: 'week' granularity should not work for 'year'

fix(pre-aggregation): handle time hierarchy on query

fix: keep context away from arguments

* fix(test): update broken test for pre-aggregation

* fix: incorrect granularity in filter clause

* test: add simple tests for the pre-aggregation in time hierarchy

Fixes #246
  • Loading branch information
@Justin-ZS @paveltiunov
Justin-ZS authored and paveltiunov committed Nov 18, 2019
1 parent 458c0c9 commit ea78c84
Show file tree
Hide file tree
Showing 4 changed files with 208 additions and 28 deletions.
5 changes: 3 additions & 2 deletions packages/cubejs-schema-compiler/adapter/BaseQuery.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -891,8 +891,9 @@ class BaseQuery {
}

dimensionSql(dimension) {
if (this.safeEvaluateSymbolContext().rollupQuery) {
return this.escapeColumnName(dimension.unescapedAliasName());
const context = this.safeEvaluateSymbolContext();
if (context.rollupQuery) {
return this.escapeColumnName(dimension.unescapedAliasName(context.rollupGranularity));
}
return this.evaluateSymbolSql(dimension.path()[0], dimension.path()[1], dimension.dimensionDefinition());
}
Expand Down
23 changes: 14 additions & 9 deletions packages/cubejs-schema-compiler/adapter/BaseTimeDimension.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@ class BaseTimeDimension extends BaseFilter {
return super.aliasName();
}

unescapedAliasName() {
return `${this.query.aliasName(this.dimension)}_${this.granularity || 'date'}`; // TODO date here for rollups
unescapedAliasName(granularity) {
const actualGranularity = granularity || this.granularity || 'date';

return `${this.query.aliasName(this.dimension)}_${actualGranularity}`; // TODO date here for rollups
}

dateSeriesAliasName() {
Expand All @@ -64,16 +66,19 @@ class BaseTimeDimension extends BaseFilter {
}

dimensionSql() {
if (this.query.safeEvaluateSymbolContext().rollupQuery) {
return super.dimensionSql();
const context = this.query.safeEvaluateSymbolContext();
const granularity = context.granularityOverride || this.granularity;

if (context.rollupQuery) {
if (context.rollupGranularity === this.granularity) {
return super.dimensionSql();
}
return this.query.timeGroupedColumn(granularity, this.query.dimensionSql(this));
}
if (this.query.safeEvaluateSymbolContext().ungrouped) {
if (context.ungrouped) {
return this.convertedToTz();
}
return this.query.timeGroupedColumn(
this.query.safeEvaluateSymbolContext().granularityOverride || this.granularity,
this.convertedToTz()
);
return this.query.timeGroupedColumn(granularity, this.convertedToTz());
}

convertedToTz() {
Expand Down
26 changes: 23 additions & 3 deletions packages/cubejs-schema-compiler/adapter/PreAggregations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,23 @@ class PreAggregations {
timeDimensions.map(d => [d.dimension, d.granularity || 'date'])
) || [];
}
// TimeDimension :: [Dimension, Granularity]
// TimeDimension -> [TimeDimension]
function expandTimeDimension(timeDimension) {
const [dimension, granularity] = timeDimension;
const makeTimeDimension = newGranularity => [dimension, newGranularity];

const tds = [timeDimension];
const updateTds = (...granularitys) => tds.push(...granularitys.map(makeTimeDimension))

if (granularity === 'year') updateTds('hour', 'date', 'month');
if (['month', 'week'].includes(granularity)) updateTds('hour', 'date');
if (granularity === 'date') updateTds('hour');

return tds;
}
// [[TimeDimension]]
const queryTimeDimensionsList = transformedQuery.sortedTimeDimensions.map(expandTimeDimension);

const canUsePreAggregationNotAdditive = (references) =>
transformedQuery.hasNoTimeDimensionsWithoutGranularity &&
Expand All @@ -222,7 +239,7 @@ class PreAggregations {
) &&
R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures) &&
R.allPass(
transformedQuery.sortedTimeDimensions.map(td => R.contains(td))
queryTimeDimensionsList.map(tds => R.anyPass(tds.map(td => R.contains(td))))
)(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions));

const canUsePreAggregationAdditive = (references) =>
Expand All @@ -235,7 +252,7 @@ class PreAggregations {
R.all(m => references.measures.indexOf(m) !== -1, transformedQuery.leafMeasures)
) &&
R.allPass(
transformedQuery.sortedTimeDimensions.map(td => R.contains(td))
queryTimeDimensionsList.map(tds => R.anyPass(tds.map(td => R.contains(td))))
)(references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions));


Expand Down Expand Up @@ -440,6 +457,8 @@ class PreAggregations {
preAggregationForQuery.preAggregation.measures :
this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation).measures
);

const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'date';

return this.query.evaluateSymbolSqlWithContext(
() => `SELECT ${this.query.baseSelect()} FROM ${table} ${this.query.baseWhere(filters)}` +
Expand All @@ -449,7 +468,8 @@ class PreAggregations {
this.query.groupByDimensionLimit(),
{
renderedReference,
rollupQuery: true
rollupQuery: true,
rollupGranularity,
}
);
}
Expand Down
182 changes: 168 additions & 14 deletions packages/cubejs-schema-compiler/test/PreAggregationsTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,19 +94,24 @@ describe('PreAggregations', function test() {
measureReferences: [checkinsTotal],
segmentReferences: [google],
timeDimensionReference: createdAt,
granularity: 'day',
granularity: 'week',
},
approx: {
type: 'rollup',
measureReferences: [countDistinctApprox],
timeDimensionReference: createdAt,
granularity: 'day'
},
ratio: {
multiStage: {
useOriginalSqlPreAggregations: true,
type: 'rollup',
measureReferences: [checkinsTotal, uniqueSourceCount],
measureReferences: [checkinsTotal],
timeDimensionReference: createdAt,
granularity: 'day'
granularity: 'month',
partitionGranularity: 'day',
refreshKey: {
sql: \`SELECT CASE WHEN \${FILTER_PARAMS.visitors.createdAt.filter((from, to) => \`\${to}::timestamp > now()\`)} THEN now() END\`
}
},
partitioned: {
type: 'rollup',
Expand All @@ -116,16 +121,11 @@ describe('PreAggregations', function test() {
granularity: 'day',
partitionGranularity: 'month'
},
multiStage: {
useOriginalSqlPreAggregations: true,
ratio: {
type: 'rollup',
measureReferences: [checkinsTotal],
measureReferences: [checkinsTotal, uniqueSourceCount],
timeDimensionReference: createdAt,
granularity: 'month',
partitionGranularity: 'day',
refreshKey: {
sql: \`SELECT CASE WHEN \${FILTER_PARAMS.visitors.createdAt.filter((from, to) => \`\${to}::timestamp > now()\`)} THEN now() END\`
}
granularity: 'day'
}
}
})
Expand Down Expand Up @@ -518,7 +518,7 @@ describe('PreAggregations', function test() {
preAggregationsSchema: '',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'date',
granularity: 'week',
dateRange: ['2016-12-30', '2017-01-05']
}],
order: [{
Expand All @@ -542,7 +542,7 @@ describe('PreAggregations', function test() {
res.should.be.deepEqual(
[
{
"visitors__created_at_date": "2017-01-05T00:00:00.000Z",
"visitors__created_at_week": "2017-01-02T00:00:00.000Z",
"visitors__checkins_total": "1"
}
]
Expand Down Expand Up @@ -590,3 +590,157 @@ describe('PreAggregations', function test() {
});
});
});


describe('PreAggregations in time hierarchy', function test() {
this.timeout(20000);

after(async () => {
await dbRunner.tearDown();
});

const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(`
cube(\`visitors\`, {
sql: \`
select * from visitors
\`,
measures: {
count: {
type: 'count'
}
},
dimensions: {
createdAt: {
type: 'time',
sql: 'created_at'
},
},
preAggregations: {
month: {
type: 'rollup',
measureReferences: [count],
timeDimensionReference: createdAt,
granularity: 'month',
},
day: {
type: 'rollup',
measureReferences: [count],
timeDimensionReference: createdAt,
granularity: 'day',
},
}
})
`);

function replaceTableName(query, preAggregation, suffix) {
const [toReplace, params] = query;
console.log(toReplace);
preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
return [
preAggregation.reduce((replacedQuery, desc) =>
replacedQuery.replace(new RegExp(desc.tableName, 'g'), desc.tableName + '_' + suffix), toReplace
),
params
];
}

function tempTablePreAggregations(preAggregationsDescriptions) {
return R.unnest(preAggregationsDescriptions.map(desc =>
desc.invalidateKeyQueries.concat([
[desc.loadSql[0].replace('CREATE TABLE', 'CREATE TEMP TABLE'), desc.loadSql[1]]
])
));
}

it('query on year match to pre-agg on month', () => {
return compiler.compile().then(() => {
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitors.count'
],
dimensions: [],
timezone: 'America/Los_Angeles',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'year',
dateRange: ['2016-12-30', '2018-12-30']
}],
preAggregationsSchema: '',
order: [],
});

const queryAndParams = query.buildSqlAndParams();

query.preAggregations.preAggregationForQuery.preAggregation.granularity.should.be.equal('month');

console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);

const queries = tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.testQueries(
queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 1))
).then(res => {
console.log(JSON.stringify(res));
res.should.be.deepEqual(
[
{
"visitors__count": "5",
"visitors__created_at_year": "2017-01-01T00:00:00.000Z"
},
]
);
});
});
});
it('query on week match to pre-agg on day', () => {
return compiler.compile().then(() => {
const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
measures: [
'visitors.count'
],
dimensions: [],
timezone: 'America/Los_Angeles',
timeDimensions: [{
dimension: 'visitors.createdAt',
granularity: 'week',
dateRange: ['2017-01-02', '2019-02-08']
}],
preAggregationsSchema: '',
order: [],
});

const queryAndParams = query.buildSqlAndParams();

query.preAggregations.preAggregationForQuery.preAggregation.granularity.should.be.equal('day');

console.log(queryAndParams);
const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
console.log(preAggregationsDescription);

const queries = tempTablePreAggregations(preAggregationsDescription);

console.log(JSON.stringify(queries.concat(queryAndParams)));

return dbRunner.testQueries(
queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 1))
).then(res => {
console.log(JSON.stringify(res));
res.should.be.deepEqual(
[
{
"visitors__count": "5",
"visitors__created_at_week": "2017-01-02T00:00:00.000Z"
},
]
);
});
});
});
});

0 comments on commit ea78c84

Please sign in to comment.