Skip to content

Commit

Permalink
feat!: Cube Views implementation (#5278)
Browse files Browse the repository at this point in the history 
BREAKING CHANGE: The logic of how cubes are included in joins has been changed. There are multiple member reference constructs that are now forbidden and should be rewritten. You can't reference foreign cubes anymore to define members inside other cubes: `${ForeignCube}.foo`. `foo` member should be defined in `ForeignCube` and referenced as `${ForeignCube.foo}`. You also can't mix references and members without `CUBE` self-reference. For example `${ForeignCube.foo} + bar` is invalid and `${ForeignCube.foo} + ${CUBE}.bar` should be used instead. If not fixed, it'll lead to missing tables in the `FROM` clause.
  • Loading branch information
@paveltiunov
paveltiunov committed Oct 2, 2022
1 parent 9423f46 commit 9937356
Show file tree
Hide file tree
Showing 15 changed files with 931 additions and 130 deletions.
5 changes: 4 additions & 1 deletion packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,10 @@ export class CubeStoreQuery extends BaseQuery {
wrapQuery: true,
wrappedGranularity: timeDimension?.granularity || rollupGranularity,
rollupGranularity: granularityOverride,
topLevelMerge: false
topLevelMerge: false,
renderedReference: {
[timeDimension.dimension]: timeDimension.cumulativeSelectColumns()[0]
}
}
);
}
Expand Down
10 changes: 7 additions & 3 deletions packages/cubejs-schema-compiler/src/adapter/BaseFilter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,13 @@ export class BaseFilter extends BaseDimension {
}

path() {
return this.measure ?
this.query.cubeEvaluator.parsePath('measures', this.measure) :
this.query.cubeEvaluator.parsePath('dimensions', this.dimension);
if (this.measure) {
return this.query.cubeEvaluator.parsePath('measures', this.measure);
} else if (this.query.cubeEvaluator.isInstanceOfType('segments', this.dimension)) {
return this.query.cubeEvaluator.parsePath('segments', this.dimension);
} else {
return this.query.cubeEvaluator.parsePath('dimensions', this.dimension);
}
}

cube() {
Expand Down
131 changes: 99 additions & 32 deletions packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ class BaseQuery {

this.timeDimensions = (this.options.timeDimensions || []).map(dimension => {
if (!dimension.dimension) {
const join = this.joinGraph.buildJoin(this.collectCubeNames(true));
const join = this.joinGraph.buildJoin(this.collectJoinHints(true));
if (!join) {
return undefined;
}
Expand All @@ -235,7 +235,7 @@ class BaseQuery {
}).filter(R.identity).map(this.newTimeDimension.bind(this));
this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);

this.join = this.joinGraph.buildJoin(this.allCubeNames);
this.join = this.joinGraph.buildJoin(this.allJoinHints);
this.cubeAliasPrefix = this.options.cubeAliasPrefix;
this.preAggregationsSchemaOption =
this.options.preAggregationsSchema != null ? this.options.preAggregationsSchema : DEFAULT_PREAGGREGATIONS_SCHEMA;
Expand Down Expand Up @@ -291,6 +291,13 @@ class BaseQuery {
return this.collectedCubeNames;
}

get allJoinHints() {
if (!this.collectedJoinHints) {
this.collectedJoinHints = this.collectJoinHints();
}
return this.collectedJoinHints;
}

get dataSource() {
const dataSources = R.uniq(this.allCubeNames.map(c => this.cubeDataSource(c)));
if (dataSources.length > 1 && !this.externalPreAggregationQuery()) {
Expand Down Expand Up @@ -1264,8 +1271,8 @@ class BaseQuery {
);

if (shouldBuildJoinForMeasureSelect) {
const cubes = this.collectFrom(measures, this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor');
const measuresJoin = this.joinGraph.buildJoin(cubes);
const joinHints = this.collectFrom(measures, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFor');
const measuresJoin = this.joinGraph.buildJoin(joinHints);
if (measuresJoin.multiplicationFactor[keyCubeName]) {
throw new UserError(
`'${measures.map(m => m.measure).join(', ')}' reference cubes that lead to row multiplication.`
Expand Down Expand Up @@ -1323,9 +1330,10 @@ class BaseQuery {

checkShouldBuildJoinForMeasureSelect(measures, keyCubeName) {
return measures.map(measure => {
const cubeNames = this.collectFrom([measure], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor');
if (R.any(cubeName => keyCubeName !== cubeName, cubeNames)) {
const measuresJoin = this.joinGraph.buildJoin(cubeNames);
const cubes = this.collectFrom([measure], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor');
const joinHints = this.collectFrom([measure], this.collectJoinHintsFor.bind(this), 'collectJoinHintsFor');
if (R.any(cubeName => keyCubeName !== cubeName, cubes)) {
const measuresJoin = this.joinGraph.buildJoin(joinHints);
if (measuresJoin.multiplicationFactor[keyCubeName]) {
throw new UserError(
`'${measure.measure}' references cubes that lead to row multiplication. Please rewrite it using sub query.`
Expand Down Expand Up @@ -1422,6 +1430,14 @@ class BaseQuery {
);
}

collectJoinHints(excludeTimeDimensions = false) {
return this.collectFromMembers(
excludeTimeDimensions,
this.collectJoinHintsFor.bind(this),
'collectJoinHintsFor'
);
}

collectFromMembers(excludeTimeDimensions, fn, methodName) {
const membersToCollectFrom = this.measures
.concat(this.dimensions)
Expand Down Expand Up @@ -1615,12 +1631,6 @@ class BaseQuery {
}

dimensionSql(dimension) {
const context = this.safeEvaluateSymbolContext();
if (context.rollupQuery) {
return this.escapeColumnName(dimension.unescapedAliasName(context.rollupGranularity));
} else if (context.wrapQuery) {
return this.escapeColumnName(dimension.unescapedAliasName(context.wrappedGranularity));
}
return this.evaluateSymbolSql(dimension.path()[0], dimension.path()[1], dimension.dimensionDefinition());
}

Expand Down Expand Up @@ -1649,11 +1659,30 @@ class BaseQuery {
}
}

pushJoinHints(joinHints) {
if (this.safeEvaluateSymbolContext().joinHints && joinHints) {
if (joinHints.length === 1) {
[joinHints] = joinHints;
}
this.safeEvaluateSymbolContext().joinHints.push(joinHints);
}
}

pushMemberNameForCollectionIfNecessary(cubeName, name) {
this.pushCubeNameForCollectionIfNecessary(cubeName);
const pathFromArray = this.cubeEvaluator.pathFromArray([cubeName, name]);
if (this.cubeEvaluator.byPathAnyType(pathFromArray).ownedByCube) {
const joinHints = this.cubeEvaluator.joinHints();
if (joinHints && joinHints.length) {
joinHints.forEach(cube => this.pushCubeNameForCollectionIfNecessary(cube));
this.pushJoinHints(joinHints);
} else {
this.pushCubeNameForCollectionIfNecessary(cubeName);
this.pushJoinHints(cubeName);
}
}
const context = this.safeEvaluateSymbolContext();
if (context.memberNames && name) {
context.memberNames.push(this.cubeEvaluator.pathFromArray([cubeName, name]));
context.memberNames.push(pathFromArray);
}
}

Expand All @@ -1663,7 +1692,9 @@ class BaseQuery {

evaluateSymbolSql(cubeName, name, symbol) {
this.pushMemberNameForCollectionIfNecessary(cubeName, name);
if (this.cubeEvaluator.isMeasure([cubeName, name])) {
const memberPathArray = [cubeName, name];
const memberPath = this.cubeEvaluator.pathFromArray(memberPathArray);
if (this.cubeEvaluator.isMeasure(memberPathArray)) {
let parentMeasure;
if (this.safeEvaluateSymbolContext().compositeCubeMeasures ||
this.safeEvaluateSymbolContext().leafMeasures) {
Expand All @@ -1672,13 +1703,13 @@ class BaseQuery {
if (parentMeasure &&
(
this.cubeEvaluator.cubeNameFromPath(parentMeasure) !== cubeName ||
this.newMeasure(this.cubeEvaluator.pathFromArray([cubeName, name])).isCumulative()
this.newMeasure(this.cubeEvaluator.pathFromArray(memberPathArray)).isCumulative()
)
) {
this.safeEvaluateSymbolContext().compositeCubeMeasures[parentMeasure] = true;
}
}
this.safeEvaluateSymbolContext().currentMeasure = this.cubeEvaluator.pathFromArray([cubeName, name]);
this.safeEvaluateSymbolContext().currentMeasure = this.cubeEvaluator.pathFromArray(memberPathArray);
if (this.safeEvaluateSymbolContext().leafMeasures) {
if (parentMeasure) {
this.safeEvaluateSymbolContext().leafMeasures[parentMeasure] = false;
Expand Down Expand Up @@ -1713,13 +1744,15 @@ class BaseQuery {
this.safeEvaluateSymbolContext().currentMeasure = parentMeasure;
}
return result;
} else if (this.cubeEvaluator.isDimension([cubeName, name])) {
} else if (this.cubeEvaluator.isDimension(memberPathArray)) {
if ((this.safeEvaluateSymbolContext().renderedReference || {})[memberPath]) {
return this.evaluateSymbolContext.renderedReference[memberPath];
}
if (symbol.subQuery) {
const dimensionPath = this.cubeEvaluator.pathFromArray([cubeName, name]);
if (this.safeEvaluateSymbolContext().subQueryDimensions) {
this.safeEvaluateSymbolContext().subQueryDimensions.push(dimensionPath);
this.safeEvaluateSymbolContext().subQueryDimensions.push(memberPath);
}
return this.escapeColumnName(this.aliasName(dimensionPath));
return this.escapeColumnName(this.aliasName(memberPath));
}
if (symbol.case) {
return this.renderDimensionCase(symbol, cubeName);
Expand All @@ -1732,7 +1765,10 @@ class BaseQuery {
} else {
return this.autoPrefixAndEvaluateSql(cubeName, symbol.sql);
}
} else if (this.cubeEvaluator.isSegment([cubeName, name])) {
} else if (this.cubeEvaluator.isSegment(memberPathArray)) {
if ((this.safeEvaluateSymbolContext().renderedReference || {})[memberPath]) {
return this.evaluateSymbolContext.renderedReference[memberPath];
}
return this.autoPrefixWithCubeName(cubeName, this.evaluateSql(cubeName, symbol.sql));
}
return this.evaluateSql(cubeName, symbol.sql);
Expand Down Expand Up @@ -1762,10 +1798,8 @@ class BaseQuery {
options = options || {};
const self = this;
const { cubeEvaluator } = this;
this.pushCubeNameForCollectionIfNecessary(cubeName);
return cubeEvaluator.resolveSymbolsCall(sql, (name) => {
const nextCubeName = cubeEvaluator.symbols[name] && name || cubeName;
this.pushCubeNameForCollectionIfNecessary(nextCubeName);
const resolvedSymbol =
cubeEvaluator.resolveSymbol(
cubeName,
Expand All @@ -1780,7 +1814,8 @@ class BaseQuery {
sqlResolveFn: options.sqlResolveFn || ((symbol, cube, n) => self.evaluateSymbolSql(cube, n, symbol)),
cubeAliasFn: self.cubeAlias.bind(self),
contextSymbols: this.parametrizedContextSymbols(),
query: this
query: this,
collectJoinHints: true,
});
}

Expand Down Expand Up @@ -1815,6 +1850,16 @@ class BaseQuery {
return R.uniq(context.cubeNames);
}

collectJoinHintsFor(fn) {
const context = { joinHints: [] };
this.evaluateSymbolSqlWithContext(
fn,
context
);

return context.joinHints;
}

collectMemberNamesFor(fn) {
const context = { memberNames: [] };
this.evaluateSymbolSqlWithContext(
Expand Down Expand Up @@ -2684,16 +2729,30 @@ class BaseQuery {
return this.parametrizedContextSymbolsValue;
}

static emptyParametrizedContextSymbols(cubeEvaluator, allocateParam) {
return {
filterParams: BaseQuery.filterProxyFromAllFilters({}, cubeEvaluator, allocateParam),
sqlUtils: {
convertTz: (field) => field,
},
securityContext: BaseQuery.contextSymbolsProxyFrom({}, allocateParam),
};
}

contextSymbolsProxy(symbols) {
return BaseQuery.contextSymbolsProxyFrom(symbols, this.paramAllocator.allocateParam.bind(this.paramAllocator));
}

static contextSymbolsProxyFrom(symbols, allocateParam) {
return new Proxy(symbols, {
get: (target, name) => {
const propValue = target[name];
const methods = (paramValue) => ({
filter: (column) => {
if (paramValue) {
const value = Array.isArray(paramValue) ?
paramValue.map(this.paramAllocator.allocateParam.bind(this.paramAllocator)) :
this.paramAllocator.allocateParam(paramValue);
paramValue.map(allocateParam) :
allocateParam(paramValue);
if (typeof column === 'function') {
return column(value);
} else {
Expand All @@ -2712,24 +2771,32 @@ class BaseQuery {
unsafeValue: () => paramValue
});
return methods(target)[name] ||
typeof propValue === 'object' && propValue !== null && this.contextSymbolsProxy(propValue) ||
typeof propValue === 'object' && propValue !== null && BaseQuery.contextSymbolsProxyFrom(propValue, allocateParam) ||
methods(propValue);
}
});
}

filtersProxy() {
const { allFilters } = this;
return BaseQuery.filterProxyFromAllFilters(
allFilters,
this.cubeEvaluator,
this.paramAllocator.allocateParam.bind(this.paramAllocator)
);
}

static filterProxyFromAllFilters(allFilters, cubeEvaluator, allocateParam) {
return new Proxy({}, {
get: (target, name) => {
if (name === '_objectWithResolvedProperties') {
return true;
}
const cubeName = this.cubeEvaluator.cubeNameFromPath(name);
const cubeName = cubeEvaluator.cubeNameFromPath(name);
return new Proxy({ cube: cubeName }, {
get: (cubeNameObj, propertyName) => {
const filters =
allFilters.filter(f => f.dimension === this.cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName]));
allFilters.filter(f => f.dimension === cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName]));
return {
filter: (column) => {
if (!filters.length) {
Expand All @@ -2745,7 +2812,7 @@ class BaseQuery {
// eslint-disable-next-line prefer-spread
return column.apply(
null,
filterParams.map(this.paramAllocator.allocateParam.bind(this.paramAllocator))
filterParams.map(allocateParam),
);
} else {
return filter.conditionSql(column);
Expand Down
Loading

0 comments on commit 9937356

Please sign in to comment.