From 2f372e4fbd2001627fedae415657bbae095987be Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sun, 11 Sep 2022 17:15:58 -0700 Subject: [PATCH 1/9] feat: Cube Views implementation --- .../src/adapter/BaseFilter.js | 10 +- .../src/adapter/BaseQuery.js | 36 +-- .../src/adapter/PreAggregations.js | 215 ++++++++++++------ .../src/compiler/CubeEvaluator.js | 32 +++ .../src/compiler/CubeSymbols.js | 7 +- .../postgres/pre-aggregations.test.ts | 128 +++++++++++ 6 files changed, 343 insertions(+), 85 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js index 6627990969f64..3bb2e6659f68d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.js @@ -66,9 +66,13 @@ export class BaseFilter extends BaseDimension { } path() { - return this.measure ? - this.query.cubeEvaluator.parsePath('measures', this.measure) : - this.query.cubeEvaluator.parsePath('dimensions', this.dimension); + if (this.measure) { + return this.query.cubeEvaluator.parsePath('measures', this.measure); + } else if (this.query.cubeEvaluator.isInstanceOfType('segments', this.dimension)) { + return this.query.cubeEvaluator.parsePath('segments', this.dimension); + } else { + return this.query.cubeEvaluator.parsePath('dimensions', this.dimension); + } } cube() { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index f02de2c69d011..c074e34c58f26 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1616,9 +1616,7 @@ class BaseQuery { dimensionSql(dimension) { const context = this.safeEvaluateSymbolContext(); - if (context.rollupQuery) { - return this.escapeColumnName(dimension.unescapedAliasName(context.rollupGranularity)); - } else if (context.wrapQuery) { + if (context.wrapQuery) { return this.escapeColumnName(dimension.unescapedAliasName(context.wrappedGranularity)); } return this.evaluateSymbolSql(dimension.path()[0], dimension.path()[1], dimension.dimensionDefinition()); @@ -1650,10 +1648,13 @@ class BaseQuery { } pushMemberNameForCollectionIfNecessary(cubeName, name) { - this.pushCubeNameForCollectionIfNecessary(cubeName); + const pathFromArray = this.cubeEvaluator.pathFromArray([cubeName, name]); + if (this.cubeEvaluator.byPathAnyType(pathFromArray).ownedByCube) { + this.pushCubeNameForCollectionIfNecessary(cubeName); + } const context = this.safeEvaluateSymbolContext(); if (context.memberNames && name) { - context.memberNames.push(this.cubeEvaluator.pathFromArray([cubeName, name])); + context.memberNames.push(pathFromArray); } } @@ -1663,7 +1664,9 @@ class BaseQuery { evaluateSymbolSql(cubeName, name, symbol) { this.pushMemberNameForCollectionIfNecessary(cubeName, name); - if (this.cubeEvaluator.isMeasure([cubeName, name])) { + const memberPathArray = [cubeName, name]; + const memberPath = this.cubeEvaluator.pathFromArray(memberPathArray); + if (this.cubeEvaluator.isMeasure(memberPathArray)) { let parentMeasure; if (this.safeEvaluateSymbolContext().compositeCubeMeasures || this.safeEvaluateSymbolContext().leafMeasures) { @@ -1672,13 +1675,13 @@ class BaseQuery { if (parentMeasure && ( this.cubeEvaluator.cubeNameFromPath(parentMeasure) !== cubeName || - this.newMeasure(this.cubeEvaluator.pathFromArray([cubeName, name])).isCumulative() + this.newMeasure(this.cubeEvaluator.pathFromArray(memberPathArray)).isCumulative() ) ) { this.safeEvaluateSymbolContext().compositeCubeMeasures[parentMeasure] = true; } } - this.safeEvaluateSymbolContext().currentMeasure = this.cubeEvaluator.pathFromArray([cubeName, name]); + this.safeEvaluateSymbolContext().currentMeasure = this.cubeEvaluator.pathFromArray(memberPathArray); if (this.safeEvaluateSymbolContext().leafMeasures) { if (parentMeasure) { this.safeEvaluateSymbolContext().leafMeasures[parentMeasure] = false; @@ -1713,13 +1716,15 @@ class BaseQuery { this.safeEvaluateSymbolContext().currentMeasure = parentMeasure; } return result; - } else if (this.cubeEvaluator.isDimension([cubeName, name])) { + } else if (this.cubeEvaluator.isDimension(memberPathArray)) { + if ((this.safeEvaluateSymbolContext().renderedReference || {})[memberPath]) { + return this.evaluateSymbolContext.renderedReference[memberPath]; + } if (symbol.subQuery) { - const dimensionPath = this.cubeEvaluator.pathFromArray([cubeName, name]); if (this.safeEvaluateSymbolContext().subQueryDimensions) { - this.safeEvaluateSymbolContext().subQueryDimensions.push(dimensionPath); + this.safeEvaluateSymbolContext().subQueryDimensions.push(memberPath); } - return this.escapeColumnName(this.aliasName(dimensionPath)); + return this.escapeColumnName(this.aliasName(memberPath)); } if (symbol.case) { return this.renderDimensionCase(symbol, cubeName); @@ -1732,7 +1737,10 @@ class BaseQuery { } else { return this.autoPrefixAndEvaluateSql(cubeName, symbol.sql); } - } else if (this.cubeEvaluator.isSegment([cubeName, name])) { + } else if (this.cubeEvaluator.isSegment(memberPathArray)) { + if ((this.safeEvaluateSymbolContext().renderedReference || {})[memberPath]) { + return this.evaluateSymbolContext.renderedReference[memberPath]; + } return this.autoPrefixWithCubeName(cubeName, this.evaluateSql(cubeName, symbol.sql)); } return this.evaluateSql(cubeName, symbol.sql); @@ -1762,10 +1770,8 @@ class BaseQuery { options = options || {}; const self = this; const { cubeEvaluator } = this; - this.pushCubeNameForCollectionIfNecessary(cubeName); return cubeEvaluator.resolveSymbolsCall(sql, (name) => { const nextCubeName = cubeEvaluator.symbols[name] && name || cubeName; - this.pushCubeNameForCollectionIfNecessary(nextCubeName); const resolvedSymbol = cubeEvaluator.resolveSymbol( cubeName, diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index ddf6e70095c77..7e0f4fa3de717 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -251,7 +251,7 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - references: this.evaluateAllReferences(cube, preAggregation) + references: this.evaluateAllReferences(cube, preAggregation, preAggregationName) }; } return null; @@ -264,7 +264,22 @@ export class PreAggregations { const collectLeafMeasures = query.collectLeafMeasures.bind(query); const dimensionsList = query.dimensions.map(dim => dim.dimension); const segmentsList = query.segments.map(s => s.segment); - + const ownedDimensions = PreAggregations.ownedMembers(query, PreAggregations.concatDimensionMembers(query)); + const ownedTimeDimensions = query.timeDimensions.map(td => { + const owned = PreAggregations.ownedMembers(query, [td]); + let { dimension } = td; + // TODO If there's more than one owned time dimension for the given input time dimension then it's some + // TODO kind of calculation which isn't supported yet + if (owned.length === 1) { + [dimension] = owned; + } + return query.newTimeDimension({ + dimension, + dateRange: td.dateRange, + granularity: td.granularity, + }); + }).map(d => query.newTimeDimension(d)); + const measureToLeafMeasures = {}; const leafMeasurePaths = @@ -287,13 +302,6 @@ export class PreAggregations { R.uniq )(measures); - function sortTimeDimensions(timeDimensions) { - return timeDimensions && R.sortBy( - R.prop(0), - timeDimensions.map(d => [d.dimension, d.rollupGranularity()]) - ) || []; - } - function allValuesEq1(map) { if (!map) return false; // eslint-disable-next-line no-restricted-syntax @@ -303,11 +311,10 @@ export class PreAggregations { return true; } - const sortedTimeDimensions = sortTimeDimensions(query.timeDimensions); - const timeDimensions = query.timeDimensions && R.sortBy( - R.prop(0), - query.timeDimensions.map(d => [d.dimension, d.granularity]) - ) || []; + const sortedTimeDimensions = PreAggregations.sortTimeDimensionsWithRollupGranularity(query.timeDimensions); + const timeDimensions = PreAggregations.timeDimensionsAsIs(query.timeDimensions); + const ownedTimeDimensionsWithRollupGranularity = PreAggregations.sortTimeDimensionsWithRollupGranularity(ownedTimeDimensions); + const ownedTimeDimensionsAsIs = PreAggregations.timeDimensionsAsIs(ownedTimeDimensions); const hasNoTimeDimensionsWithoutGranularity = !query.timeDimensions.filter(d => !d.granularity).length; @@ -352,10 +359,35 @@ export class PreAggregations { hasMultipliedMeasures, hasCumulativeMeasures, windowGranularity, - filterDimensionsSingleValueEqual + filterDimensionsSingleValueEqual, + ownedDimensions, + ownedTimeDimensionsWithRollupGranularity, + ownedTimeDimensionsAsIs }; } + static ownedMembers(query, members) { + return R.pipe(R.uniq, R.sortBy(R.identity))( + query + .collectFrom(members, query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor') + .filter(d => query.cubeEvaluator.byPathAnyType(d).ownedByCube) + ); + } + + static sortTimeDimensionsWithRollupGranularity(timeDimensions) { + return timeDimensions && R.sortBy( + R.prop(0), + timeDimensions.map(d => [d.dimension, d.rollupGranularity()]) + ) || []; + } + + static timeDimensionsAsIs(timeDimensions) { + return timeDimensions && R.sortBy( + R.prop(0), + timeDimensions.map(d => [d.dimension, d.granularity]), + ) || []; + } + static collectFilterDimensionsWithSingleValueEqual(filters, map) { // eslint-disable-next-line no-restricted-syntax for (const f of filters) { @@ -538,6 +570,27 @@ export class PreAggregations { ? transformedQuery.timeDimensions.map(expandTimeDimension) : transformedQuery.sortedTimeDimensions.map(expandTimeDimension); + const ownedQueryTimeDimensionsList = references.allowNonStrictDateRangeMatch + ? transformedQuery.ownedTimeDimensionsAsIs.map(expandTimeDimension) + : transformedQuery.ownedTimeDimensionsWithRollupGranularity.map(expandTimeDimension); + + const dimensionsMatch = (dimensions) => R.all( + d => ( + references.sortedDimensions || + references.dimensions + ).indexOf(d) !== -1, + dimensions + ); + + const timeDimensionsMatch = (timeDimensionsList) => R.allPass( + timeDimensionsList.map( + tds => R.anyPass(tds.map(td => R.contains(td))) + ) + )( + references.sortedTimeDimensions || + sortTimeDimensions(references.timeDimensions) + ); + return (( windowGranularityMatches(references) ) && ( @@ -546,22 +599,8 @@ export class PreAggregations { transformedQuery.leafMeasures, ) ) && ( - R.all( - d => ( - references.sortedDimensions || - references.dimensions - ).indexOf(d) !== -1, - transformedQuery.sortedDimensions - ) - ) && ( - R.allPass( - queryTimeDimensionsList.map( - tds => R.anyPass(tds.map(td => R.contains(td))) - ) - )( - references.sortedTimeDimensions || - sortTimeDimensions(references.timeDimensions) - ) + dimensionsMatch(transformedQuery.sortedDimensions) && timeDimensionsMatch(queryTimeDimensionsList) || + dimensionsMatch(transformedQuery.ownedDimensions) && timeDimensionsMatch(ownedQueryTimeDimensionsList) )); }; @@ -589,6 +628,10 @@ export class PreAggregations { ); } + static concatDimensionMembers(query) { + return query.dimensions.concat(query.filters).concat(query.segments); + } + // eslint-disable-next-line no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars getCubeLattice(cube, preAggregationName, preAggregation) { @@ -766,7 +809,7 @@ export class PreAggregations { } evaluatedPreAggregationObj(cube, preAggregationName, preAggregation, canUsePreAggregation) { - const references = this.evaluateAllReferences(cube, preAggregation); + const references = this.evaluateAllReferences(cube, preAggregation, preAggregationName); const preAggObj = { preAggregationName, preAggregation, @@ -955,24 +998,30 @@ export class PreAggregations { .toLowerCase(); } - evaluateAllReferences(cube, aggregation) { - const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); - if (aggregation.type === 'rollupLambda') { - if (references.rollups.length > 0) { - const [firstLambdaCube] = this.query.cubeEvaluator.parsePath('preAggregations', references.rollups[0]); - const firstLambdaPreAggregation = this.query.cubeEvaluator.byPath('preAggregations', references.rollups[0]); - const firstLambdaReferences = this.query.cubeEvaluator.evaluatePreAggregationReferences(firstLambdaCube, firstLambdaPreAggregation); - - if (references.measures.length === 0 && - references.dimensions.length === 0 && - references.timeDimensions.length === 0) { - return { ...firstLambdaReferences, rollups: references.rollups }; - } else { - return references; + evaluateAllReferences(cube, aggregation, preAggregationName ) { + const evaluateReferences = () => { + const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); + if (aggregation.type === 'rollupLambda') { + if (references.rollups.length > 0) { + const [firstLambdaCube] = this.query.cubeEvaluator.parsePath('preAggregations', references.rollups[0]); + const firstLambdaPreAggregation = this.query.cubeEvaluator.byPath('preAggregations', references.rollups[0]); + const firstLambdaReferences = this.query.cubeEvaluator.evaluatePreAggregationReferences(firstLambdaCube, firstLambdaPreAggregation); + + if (references.measures.length === 0 && + references.dimensions.length === 0 && + references.timeDimensions.length === 0) { + return { ...firstLambdaReferences, rollups: references.rollups }; + } else { + return references; + } } } + return references; + } + if (!preAggregationName) { + return evaluateReferences(); } - return references; + return this.query.cacheValue(['evaluateAllReferences', cube, preAggregationName], evaluateReferences); } originalSqlPreAggregationTable(preAggregationDescription) { @@ -1055,25 +1104,47 @@ export class PreAggregations { })) ).filter(f => !!f); - const renderedReference = R.pipe( - R.map(path => { - const measure = this.query.newMeasure(path); - return [ - path, - this.query.aggregateOnGroupedColumn( - measure.measureDefinition(), - measure.aliasName(), - !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate, - path - ) || `sum(${measure.aliasName()})` - ]; - }), - R.fromPairs - )(this.rollupMeasures(preAggregationForQuery)); - // TODO granularity shouldn't be null? const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'day'; + const renderedReference = { + ...R.pipe( + R.map(path => { + const measure = this.query.newMeasure(path); + return [ + path, + this.query.aggregateOnGroupedColumn( + measure.measureDefinition(), + measure.aliasName(), + !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate, + path + ) || `sum(${measure.aliasName()})` + ]; + }), + R.fromPairs + )(this.rollupMeasures(preAggregationForQuery)), + ...R.pipe( + R.map(path => { + const dimension = this.query.newDimension(path); + return [ + path, + this.query.escapeColumnName(dimension.unescapedAliasName()) + ]; + }), + R.fromPairs + )(this.rollupDimensions(preAggregationForQuery)), + ...R.pipe( + R.map((td) => { + const timeDimension = this.query.newTimeDimension(td); + return [ + td.dimension, + this.query.escapeColumnName(timeDimension.unescapedAliasName(rollupGranularity)) + ]; + }), + R.fromPairs + )(this.rollupTimeDimensions(preAggregationForQuery)), + }; + return this.query.evaluateSymbolSqlWithContext( // eslint-disable-next-line prefer-template () => `SELECT ${this.query.selectAllDimensionsAndMeasures(measures)} FROM ${from} ${this.query.baseWhere(replacedFilters)}` + @@ -1092,10 +1163,22 @@ export class PreAggregations { ); } - rollupMeasures(preAggregationForQuery) { + rollupMembers(preAggregationForQuery, type) { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? - preAggregationForQuery.preAggregation.measures : - this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation).measures; + preAggregationForQuery.preAggregation[type] : + this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation, preAggregationForQuery.preAggregationName)[type]; + } + + rollupMeasures(preAggregationForQuery) { + return this.rollupMembers(preAggregationForQuery, 'measures'); + } + + rollupDimensions(preAggregationForQuery) { + return this.rollupMembers(preAggregationForQuery, 'dimensions'); + } + + rollupTimeDimensions(preAggregationForQuery) { + return this.rollupMembers(preAggregationForQuery, 'timeDimensions'); } preAggregationId(preAggregation) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js index ecd26f109c4e4..bf9f6a19ccf32 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js @@ -87,6 +87,28 @@ export class CubeEvaluator extends CubeSymbols { } } } + this.transformMembers(cube.measures, cube); + this.transformMembers(cube.dimensions, cube); + this.transformMembers(cube.segments, cube); + } + + transformMembers(members, cube) { + members = members || {}; + for (const memberName of Object.keys(members)) { + const member = members[memberName]; + let ownedByCube = true; + if (member.sql && !member.subQuery) { + const funcArgs = this.funcArguments(member.sql); + // TODO case when foreign cube is referenced isn't covered + // TODO case when other dimension is referenced through CUBE ref is not covered: it'll be rendered as an owned + if (funcArgs.length > 0 && funcArgs.every( + ref => !cube.measures[ref] && !cube.dimensions[ref] && !cube.segments[ref] && !this.isCurrentCube(ref) && ref !== cube.name + )) { + ownedByCube = false; + } + } + members[memberName].ownedByCube = ownedByCube; + } } cubesByFileName(fileName) { @@ -214,6 +236,16 @@ export class CubeEvaluator extends CubeSymbols { this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]]; } + byPathAnyType(path) { + const type = ['measures', 'dimensions', 'segments'].find(t => this.isInstanceOfType(t, path)); + + if (!type) { + throw new UserError(`Can't resolve member '${path.join('.')}'`); + } + + return this.byPath(type, path); + } + byPath(type, path) { if (!type) { throw new Error(`Type can't be undefined for '${path}'`); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js index 71def1d19132c..31e18135e387a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js @@ -228,7 +228,12 @@ export class CubeSymbols { } const { sqlResolveFn, cubeAliasFn, query } = self.resolveSymbolsCallContext || {}; if (propertyName === 'toString') { - return () => cubeAliasFn && cubeAliasFn(cube.cubeName()) || cube.cubeName(); + return () => { + if (query) { + query.pushCubeNameForCollectionIfNecessary(cube.cubeName()); + } + return cubeAliasFn && cubeAliasFn(cube.cubeName()) || cube.cubeName(); + }; } if (propertyName === 'sql') { return () => query.cubeSql(cube.cubeName()); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 682af3e411bcc..093f40791c5df 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -387,6 +387,29 @@ describe('PreAggregations', () => { } } }) + + cube('VisitorView', { + sql: \`SELECT 1\`, + + measures: { + checkinsTotal: { + sql: \`\${visitors.checkinsTotal}\`, + type: 'number', + } + }, + + dimensions: { + source: { + sql: \`\${visitors.source}\`, + type: 'string', + }, + + createdAt: { + sql: \`\${visitors.createdAt}\`, + type: 'time' + } + }, + }); `); it('simple pre-aggregation', () => compiler.compile().then(() => { @@ -1501,4 +1524,109 @@ describe('PreAggregations', () => { ); }); })); + + it('simple view', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'VisitorView.checkinsTotal' + ], + dimensions: [ + 'VisitorView.source' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '', + timeDimensions: [{ + dimension: 'VisitorView.createdAt', + granularity: 'day', + dateRange: ['2016-12-30', '2017-01-05'] + }], + order: [{ + id: 'VisitorView.createdAt' + }], + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(JSON.stringify(preAggregationsDescription, null, 2)); + + expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/visitors_partitioned/); + + const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + + console.log(JSON.stringify(queries.concat(queryAndParams))); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + console.log(JSON.stringify(res)); + expect(res).toEqual( + [ + { + visitor_view__source: 'some', + visitor_view__created_at_day: '2017-01-02T00:00:00.000Z', + visitor_view__checkins_total: '3' + }, + { + visitor_view__source: 'some', + visitor_view__created_at_day: '2017-01-04T00:00:00.000Z', + visitor_view__checkins_total: '2' + }, + { + visitor_view__source: 'google', + visitor_view__created_at_day: '2017-01-05T00:00:00.000Z', + visitor_view__checkins_total: '1' + } + ] + ); + }); + })); + + it('simple view non matching time-dimension granularity', () => compiler.compile().then(() => { + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: [ + 'VisitorView.checkinsTotal' + ], + dimensions: [ + 'VisitorView.source' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '', + timeDimensions: [{ + dimension: 'VisitorView.createdAt', + granularity: 'month', + dateRange: ['2016-12-30', '2017-01-05'] + }], + order: [{ + id: 'VisitorView.createdAt' + }], + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(JSON.stringify(preAggregationsDescription, null, 2)); + + expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/visitors_partitioned/); + + const queries = dbRunner.tempTablePreAggregations(preAggregationsDescription); + + console.log(JSON.stringify(queries.concat(queryAndParams))); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + console.log(JSON.stringify(res)); + expect(res).toEqual( + [ + { + visitor_view__source: 'google', + visitor_view__created_at_month: '2017-01-01T00:00:00.000Z', + visitor_view__checkins_total: '1' + }, + { + visitor_view__source: 'some', + visitor_view__created_at_month: '2017-01-01T00:00:00.000Z', + visitor_view__checkins_total: '5' + } + ] + ); + }); + })); }); From 3c179bc52ddf96161d69869c55d0fbb6b12cc2ab Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sun, 11 Sep 2022 20:25:53 -0700 Subject: [PATCH 2/9] chore: Cube Views implementation -- fix Cube Store rolling window queries --- packages/cubejs-schema-compiler/src/adapter/BaseQuery.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c074e34c58f26..ac197149ce95d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1615,10 +1615,6 @@ class BaseQuery { } dimensionSql(dimension) { - const context = this.safeEvaluateSymbolContext(); - if (context.wrapQuery) { - return this.escapeColumnName(dimension.unescapedAliasName(context.wrappedGranularity)); - } return this.evaluateSymbolSql(dimension.path()[0], dimension.path()[1], dimension.dimensionDefinition()); } From 020251b5175dd74b68a896444cf3037a1e097416 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sun, 11 Sep 2022 21:22:02 -0700 Subject: [PATCH 3/9] chore: Cube Views implementation -- fix Cube Store rolling window queries --- .../src/CubeStoreQuery.ts | 2 +- .../src/adapter/PreAggregations.js | 82 +++++++++++-------- 2 files changed, 48 insertions(+), 36 deletions(-) diff --git a/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts b/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts index 85ab340f8a5e2..56ece1dfd9df1 100644 --- a/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts +++ b/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts @@ -155,7 +155,7 @@ export class CubeStoreQuery extends BaseQuery { public overTimeSeriesSelectRollup(cumulativeMeasures, otherMeasures, baseQuery, baseQueryAlias, timeDimension, preAggregationForQuery) { const cumulativeDimensions = this.dimensions.map(s => s.cumulativeSelectColumns()).filter(c => !!c).join(', '); const partitionByClause = this.dimensions.length ? `PARTITION BY ${cumulativeDimensions}` : ''; - const groupByDimensionClause = otherMeasures.length && timeDimension ? ` GROUP BY DIMENSION ${timeDimension.dimensionSql()}` : ''; + const groupByDimensionClause = otherMeasures.length && timeDimension ? ` GROUP BY DIMENSION ${timeDimension.cumulativeSelectColumns().join(', ')}` : ''; const rollingWindowOrGroupByClause = timeDimension ? ` ROLLING_WINDOW DIMENSION ${timeDimension.aliasName()}${partitionByClause}${groupByDimensionClause} FROM ${this.timeGroupedColumn(timeDimension.granularity, timeDimension.localDateTimeFromOrBuildRangeParam())} TO ${this.timeGroupedColumn(timeDimension.granularity, timeDimension.localDateTimeToOrBuildRangeParam())} EVERY INTERVAL '1 ${timeDimension.granularity}'` : this.groupByClause(); diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 7e0f4fa3de717..42ab054927759 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -1108,41 +1108,9 @@ export class PreAggregations { const rollupGranularity = this.castGranularity(preAggregationForQuery.preAggregation.granularity) || 'day'; const renderedReference = { - ...R.pipe( - R.map(path => { - const measure = this.query.newMeasure(path); - return [ - path, - this.query.aggregateOnGroupedColumn( - measure.measureDefinition(), - measure.aliasName(), - !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate, - path - ) || `sum(${measure.aliasName()})` - ]; - }), - R.fromPairs - )(this.rollupMeasures(preAggregationForQuery)), - ...R.pipe( - R.map(path => { - const dimension = this.query.newDimension(path); - return [ - path, - this.query.escapeColumnName(dimension.unescapedAliasName()) - ]; - }), - R.fromPairs - )(this.rollupDimensions(preAggregationForQuery)), - ...R.pipe( - R.map((td) => { - const timeDimension = this.query.newTimeDimension(td); - return [ - td.dimension, - this.query.escapeColumnName(timeDimension.unescapedAliasName(rollupGranularity)) - ]; - }), - R.fromPairs - )(this.rollupTimeDimensions(preAggregationForQuery)), + ...(this.measuresRenderedReference(preAggregationForQuery)), + ...(this.dimensionsRenderedReference(preAggregationForQuery)), + ...(this.timeDimensionsRenderedReference(rollupGranularity, preAggregationForQuery)), }; return this.query.evaluateSymbolSqlWithContext( @@ -1163,6 +1131,50 @@ export class PreAggregations { ); } + measuresRenderedReference(preAggregationForQuery) { + return R.pipe( + R.map(path => { + const measure = this.query.newMeasure(path); + return [ + path, + this.query.aggregateOnGroupedColumn( + measure.measureDefinition(), + measure.aliasName(), + !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate, + path, + ) || `sum(${measure.aliasName()})`, + ]; + }), + R.fromPairs, + )(this.rollupMeasures(preAggregationForQuery)); + } + + dimensionsRenderedReference(preAggregationForQuery) { + return R.pipe( + R.map(path => { + const dimension = this.query.newDimension(path); + return [ + path, + this.query.escapeColumnName(dimension.unescapedAliasName()), + ]; + }), + R.fromPairs, + )(this.rollupDimensions(preAggregationForQuery)); + } + + timeDimensionsRenderedReference(rollupGranularity, preAggregationForQuery) { + return R.pipe( + R.map((td) => { + const timeDimension = this.query.newTimeDimension(td); + return [ + td.dimension, + this.query.escapeColumnName(timeDimension.unescapedAliasName(rollupGranularity)), + ]; + }), + R.fromPairs, + )(this.rollupTimeDimensions(preAggregationForQuery)); + } + rollupMembers(preAggregationForQuery, type) { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? preAggregationForQuery.preAggregation[type] : From 5c52d5b4381b14a01317b1f4a19de7fadf7a91b6 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sun, 11 Sep 2022 22:02:48 -0700 Subject: [PATCH 4/9] chore: Cube Views implementation -- fix Cube Store rolling window queries --- packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts b/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts index 56ece1dfd9df1..6b16dbb553e2e 100644 --- a/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts +++ b/packages/cubejs-cubestore-driver/src/CubeStoreQuery.ts @@ -147,7 +147,10 @@ export class CubeStoreQuery extends BaseQuery { wrapQuery: true, wrappedGranularity: timeDimension?.granularity || rollupGranularity, rollupGranularity: granularityOverride, - topLevelMerge: false + topLevelMerge: false, + renderedReference: { + [timeDimension.dimension]: timeDimension.cumulativeSelectColumns()[0] + } } ); } @@ -155,7 +158,7 @@ export class CubeStoreQuery extends BaseQuery { public overTimeSeriesSelectRollup(cumulativeMeasures, otherMeasures, baseQuery, baseQueryAlias, timeDimension, preAggregationForQuery) { const cumulativeDimensions = this.dimensions.map(s => s.cumulativeSelectColumns()).filter(c => !!c).join(', '); const partitionByClause = this.dimensions.length ? `PARTITION BY ${cumulativeDimensions}` : ''; - const groupByDimensionClause = otherMeasures.length && timeDimension ? ` GROUP BY DIMENSION ${timeDimension.cumulativeSelectColumns().join(', ')}` : ''; + const groupByDimensionClause = otherMeasures.length && timeDimension ? ` GROUP BY DIMENSION ${timeDimension.dimensionSql()}` : ''; const rollingWindowOrGroupByClause = timeDimension ? ` ROLLING_WINDOW DIMENSION ${timeDimension.aliasName()}${partitionByClause}${groupByDimensionClause} FROM ${this.timeGroupedColumn(timeDimension.granularity, timeDimension.localDateTimeFromOrBuildRangeParam())} TO ${this.timeGroupedColumn(timeDimension.granularity, timeDimension.localDateTimeToOrBuildRangeParam())} EVERY INTERVAL '1 ${timeDimension.granularity}'` : this.groupByClause(); From 5f03a38cf3edd738567dfc31aacd502fb797b19c Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Thu, 22 Sep 2022 17:06:51 -0700 Subject: [PATCH 5/9] chore: Join hints POC --- .../src/adapter/BaseQuery.js | 62 ++++++++++++++++--- .../src/adapter/PreAggregations.js | 2 + .../src/compiler/CubeSymbols.js | 22 ++++++- .../src/compiler/JoinGraph.js | 22 ++++--- 4 files changed, 90 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index ac197149ce95d..5722283fbe96a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -220,7 +220,7 @@ class BaseQuery { this.timeDimensions = (this.options.timeDimensions || []).map(dimension => { if (!dimension.dimension) { - const join = this.joinGraph.buildJoin(this.collectCubeNames(true)); + const join = this.joinGraph.buildJoin(this.collectJoinHints(true)); if (!join) { return undefined; } @@ -235,7 +235,7 @@ class BaseQuery { }).filter(R.identity).map(this.newTimeDimension.bind(this)); this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters); - this.join = this.joinGraph.buildJoin(this.allCubeNames); + this.join = this.joinGraph.buildJoin(this.allJoinHints); this.cubeAliasPrefix = this.options.cubeAliasPrefix; this.preAggregationsSchemaOption = this.options.preAggregationsSchema != null ? this.options.preAggregationsSchema : DEFAULT_PREAGGREGATIONS_SCHEMA; @@ -291,6 +291,13 @@ class BaseQuery { return this.collectedCubeNames; } + get allJoinHints() { + if (!this.collectedJoinHints) { + this.collectedJoinHints = this.collectJoinHints(); + } + return this.collectedJoinHints; + } + get dataSource() { const dataSources = R.uniq(this.allCubeNames.map(c => this.cubeDataSource(c))); if (dataSources.length > 1 && !this.externalPreAggregationQuery()) { @@ -1264,8 +1271,8 @@ class BaseQuery { ); if (shouldBuildJoinForMeasureSelect) { - const cubes = this.collectFrom(measures, this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); - const measuresJoin = this.joinGraph.buildJoin(cubes); + const joinHints = this.collectFrom(measures, this.collectJoinHintsFor.bind(this), 'collectJoinHintsFor'); + const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { throw new UserError( `'${measures.map(m => m.measure).join(', ')}' reference cubes that lead to row multiplication.` @@ -1323,9 +1330,10 @@ class BaseQuery { checkShouldBuildJoinForMeasureSelect(measures, keyCubeName) { return measures.map(measure => { - const cubeNames = this.collectFrom([measure], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); - if (R.any(cubeName => keyCubeName !== cubeName, cubeNames)) { - const measuresJoin = this.joinGraph.buildJoin(cubeNames); + const cubes = this.collectFrom([measure], this.collectCubeNamesFor.bind(this), 'collectCubeNamesFor'); + const joinHints = this.collectFrom([measure], this.collectJoinHintsFor.bind(this), 'collectJoinHintsFor'); + if (R.any(cubeName => keyCubeName !== cubeName, cubes)) { + const measuresJoin = this.joinGraph.buildJoin(joinHints); if (measuresJoin.multiplicationFactor[keyCubeName]) { throw new UserError( `'${measure.measure}' references cubes that lead to row multiplication. Please rewrite it using sub query.` @@ -1422,6 +1430,14 @@ class BaseQuery { ); } + collectJoinHints(excludeTimeDimensions = false) { + return this.collectFromMembers( + excludeTimeDimensions, + this.collectJoinHintsFor.bind(this), + 'collectJoinHintsFor' + ); + } + collectFromMembers(excludeTimeDimensions, fn, methodName) { const membersToCollectFrom = this.measures .concat(this.dimensions) @@ -1643,10 +1659,26 @@ class BaseQuery { } } + pushJoinHints(joinHints) { + if (this.safeEvaluateSymbolContext().joinHints && joinHints) { + if (joinHints.length === 1) { + [joinHints] = joinHints; + } + this.safeEvaluateSymbolContext().joinHints.push(joinHints); + } + } + pushMemberNameForCollectionIfNecessary(cubeName, name) { const pathFromArray = this.cubeEvaluator.pathFromArray([cubeName, name]); if (this.cubeEvaluator.byPathAnyType(pathFromArray).ownedByCube) { - this.pushCubeNameForCollectionIfNecessary(cubeName); + const joinHints = this.cubeEvaluator.joinHints(); + if (joinHints && joinHints.length) { + joinHints.forEach(cube => this.pushCubeNameForCollectionIfNecessary(cube)); + this.pushJoinHints(joinHints); + } else { + this.pushCubeNameForCollectionIfNecessary(cubeName); + this.pushJoinHints(cubeName); + } } const context = this.safeEvaluateSymbolContext(); if (context.memberNames && name) { @@ -1766,6 +1798,7 @@ class BaseQuery { options = options || {}; const self = this; const { cubeEvaluator } = this; + const joinHints = []; return cubeEvaluator.resolveSymbolsCall(sql, (name) => { const nextCubeName = cubeEvaluator.symbols[name] && name || cubeName; const resolvedSymbol = @@ -1782,7 +1815,8 @@ class BaseQuery { sqlResolveFn: options.sqlResolveFn || ((symbol, cube, n) => self.evaluateSymbolSql(cube, n, symbol)), cubeAliasFn: self.cubeAlias.bind(self), contextSymbols: this.parametrizedContextSymbols(), - query: this + query: this, + joinHints }); } @@ -1817,6 +1851,16 @@ class BaseQuery { return R.uniq(context.cubeNames); } + collectJoinHintsFor(fn) { + const context = { joinHints: [] }; + this.evaluateSymbolSqlWithContext( + fn, + context + ); + + return context.joinHints; + } + collectMemberNamesFor(fn) { const context = { memberNames: [] }; this.evaluateSymbolSqlWithContext( diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 42ab054927759..9ad18740d28d1 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -736,9 +736,11 @@ export class PreAggregations { ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { const targetJoins = this.resolveJoinMembers( + // TODO join hints? this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj)) ); const existingJoins = R.unnest(preAggObjsToJoin.map( + // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) )); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js index 31e18135e387a..91f71fda232d2 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js @@ -193,6 +193,11 @@ export class CubeSymbols { return this.funcArgumentsValues[funcDefinition]; } + joinHints() { + const { joinHints } = this.resolveSymbolsCallContext || {}; + return joinHints; + } + resolveSymbol(cubeName, name) { const { sqlResolveFn, contextSymbols } = this.resolveSymbolsCallContext || {}; if (CONTEXT_SYMBOLS[name]) { @@ -226,11 +231,15 @@ export class CubeSymbols { } return undefined; } - const { sqlResolveFn, cubeAliasFn, query } = self.resolveSymbolsCallContext || {}; + const { sqlResolveFn, cubeAliasFn, query, joinHints } = self.resolveSymbolsCallContext || {}; if (propertyName === 'toString') { return () => { + if (joinHints && joinHints.length === 0) { + joinHints.push(cube.cubeName()); + } if (query) { query.pushCubeNameForCollectionIfNecessary(cube.cubeName()); + query.pushJoinHints(joinHints); } return cubeAliasFn && cubeAliasFn(cube.cubeName()) || cube.cubeName(); }; @@ -244,8 +253,17 @@ export class CubeSymbols { if (cube[propertyName]) { return { toString: () => sqlResolveFn(cube[propertyName], cubeName, propertyName) }; } + if (self.symbols[propertyName]) { + if (joinHints) { + if (joinHints.length === 0) { + joinHints.push(cubeName); + } + joinHints.push(propertyName); + } + return this.cubeReferenceProxy(propertyName); + } if (typeof propertyName === 'string') { - throw new UserError(`${cubeName}.${propertyName} cannot be resolved`); + throw new UserError(`${cubeName}.${propertyName} cannot be resolved. There's no such member or cube.`); } return undefined; } diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js index 85b0ea75a3846..1a3020ce66e4c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js @@ -98,6 +98,7 @@ export class JoinGraph { const key = JSON.stringify(cubesToJoin); if (!this.builtJoins[key]) { const join = R.pipe( + R.filter(cube => typeof cube === 'string'), R.map( cube => this.buildJoinTreeForRoot(cube, R.without([cube], cubesToJoin)) ), @@ -119,14 +120,21 @@ export class JoinGraph { buildJoinTreeForRoot(root, cubesToJoin) { const self = this; - const result = cubesToJoin.map(toJoin => { - const path = this.graph.path(root, toJoin); - if (!path) { - return null; + const result = cubesToJoin.map(joinHints => { + if (!Array.isArray(joinHints)) { + joinHints = [joinHints]; } - const foundJoins = self.joinsByPath(path); - return { cubes: path, joins: foundJoins }; - }).reduce((joined, res) => { + let prevNode = root; + return joinHints.map(toJoin => { + const path = this.graph.path(prevNode, toJoin); + if (!path) { + return null; + } + const foundJoins = self.joinsByPath(path); + prevNode = toJoin; + return { cubes: path, joins: foundJoins }; + }); + }).reduce((a, b) => a.concat(b), []).reduce((joined, res) => { if (!res || !joined) { return null; } From de9ee954c5c0a84012aee634d1397586b3edf385 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Thu, 22 Sep 2022 18:43:54 -0700 Subject: [PATCH 6/9] chore: Fix linter --- .../cubejs-schema-compiler/src/adapter/PreAggregations.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 9ad18740d28d1..f67ff7ff7fff0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -1000,7 +1000,7 @@ export class PreAggregations { .toLowerCase(); } - evaluateAllReferences(cube, aggregation, preAggregationName ) { + evaluateAllReferences(cube, aggregation, preAggregationName) { const evaluateReferences = () => { const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); if (aggregation.type === 'rollupLambda') { @@ -1019,7 +1019,7 @@ export class PreAggregations { } } return references; - } + }; if (!preAggregationName) { return evaluateReferences(); } From 938f178b04ce7d86a16c4f0e45423fa8e947ab8b Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Tue, 27 Sep 2022 21:07:00 -0700 Subject: [PATCH 7/9] chore: Cube Views basic tests --- .../src/compiler/CubeValidator.js | 11 +- .../src/compiler/DataSchemaCompiler.js | 6 +- .../src/compiler/JoinGraph.js | 10 +- .../CubeCheckDuplicatePropTranspiler.ts | 2 +- .../transpilers/CubePropContextTranspiler.ts | 2 +- .../integration/postgres/cube-views.test.ts | 257 ++++++++++++++++++ 6 files changed, 282 insertions(+), 6 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js index e65498dc4664e..e0dc9d45658cb 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.js @@ -432,7 +432,15 @@ const MeasuresSchema = Joi.object().pattern(identifierRegex, Joi.alternatives(). const cubeSchema = Joi.object().keys({ name: identifier, - sql: Joi.func().required(), + sql: Joi.alternatives().conditional( + Joi.ref('..isView'), [ + { + is: true, + then: Joi.forbidden(), + otherwise: Joi.func().required() + } + ] + ), refreshKey: CubeRefreshKeySchema, fileName: Joi.string().required(), extends: Joi.func(), @@ -442,6 +450,7 @@ const cubeSchema = Joi.object().keys({ dataSource: Joi.string(), description: Joi.string(), rewriteQueries: Joi.boolean().strict(), + isView: Joi.boolean().strict(), joins: Joi.object().pattern(identifierRegex, Joi.object().keys({ sql: Joi.func().required(), relationship: Joi.any().valid('hasMany', 'belongsTo', 'hasOne').required() diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index b517f7ca96c36..6aa2d949d293e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -163,7 +163,11 @@ export class DataSchemaCompiler { } try { vm.runInNewContext(file.content, { - view: (name, cube) => cubes.push(Object.assign({}, cube, { name, fileName: file.fileName })), + view: (name, cube) => ( + !cube ? + this.cubeFactory({ ...name, fileName: file.fileName, isView: true }) : + cubes.push(Object.assign({}, cube, { name, fileName: file.fileName, isView: true })) + ), cube: (name, cube) => ( !cube ? diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js index 1a3020ce66e4c..ea28c1955b772 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js @@ -98,7 +98,6 @@ export class JoinGraph { const key = JSON.stringify(cubesToJoin); if (!this.builtJoins[key]) { const join = R.pipe( - R.filter(cube => typeof cube === 'string'), R.map( cube => this.buildJoinTreeForRoot(cube, R.without([cube], cubesToJoin)) ), @@ -120,12 +119,19 @@ export class JoinGraph { buildJoinTreeForRoot(root, cubesToJoin) { const self = this; + if (Array.isArray(root)) { + const [newRoot, ...additionalToJoin] = root; + if (additionalToJoin.length > 0) { + cubesToJoin = [additionalToJoin].concat(cubesToJoin); + } + root = newRoot; + } const result = cubesToJoin.map(joinHints => { if (!Array.isArray(joinHints)) { joinHints = [joinHints]; } let prevNode = root; - return joinHints.map(toJoin => { + return joinHints.filter(toJoin => toJoin !== prevNode).map(toJoin => { const path = this.graph.path(prevNode, toJoin); if (!path) { return null; diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubeCheckDuplicatePropTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubeCheckDuplicatePropTranspiler.ts index ffb4baa38fe8c..accc30b808063 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubeCheckDuplicatePropTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubeCheckDuplicatePropTranspiler.ts @@ -9,7 +9,7 @@ export class CubeCheckDuplicatePropTranspiler implements TranspilerInterface { return { CallExpression: path => { // @ts-ignore @todo Unsafely? - if (path.node.callee.name === 'cube') { + if (path.node.callee.name === 'cube' || path.node.callee.name === 'view') { path.node.arguments.forEach(arg => { if (arg && arg.type === 'ObjectExpression') { this.checkExpression(arg, reporter); diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index 0d3445cb2b8d6..34892c05bb096 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -35,7 +35,7 @@ export class CubePropContextTranspiler implements TranspilerInterface { CallExpression: (path) => { if (t.isIdentifier(path.node.callee)) { const args = path.get('arguments'); - if (path.node.callee.name === 'cube') { + if (path.node.callee.name === 'cube' || path.node.callee.name === 'view') { if (args?.[args.length - 1]) { const cubeName = args[0].node.type === 'StringLiteral' && args[0].node.value || args[0].node.type === 'TemplateLiteral' && diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts new file mode 100644 index 0000000000000..66ca5abdd1237 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -0,0 +1,257 @@ +import { BaseQuery, PostgresQuery } from '../../../src/adapter'; +import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { dbRunner } from './PostgresDBRunner'; + +describe('Cube Views', () => { + jest.setTimeout(200000); + + const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` +cube(\`Orders\`, { + sql: \` + SELECT 1 as id, 1 as product_id, 'completed' as status, '2022-01-01T00:00:00.000Z'::timestamptz as created_at + UNION ALL + SELECT 2 as id, 2 as product_id, 'completed' as status, '2022-01-02T00:00:00.000Z'::timestamptz as created_at + \`, + + preAggregations: { + countByProductName: { + measures: [CUBE.count], + dimensions: [Products.name], + timeDimension: CUBE.createdAt, + granularity: \`day\`, + partitionGranularity: \`month\`, + buildRangeStart: { sql: \`SELECT '2022-01-01'\` }, + buildRangeEnd: { sql: \`SELECT '2022-03-01'\` }, + } + }, + + joins: { + Products: { + sql: \`\${CUBE}.product_id = \${Products}.id\`, + relationship: \`belongsTo\` + }, + ProductsAlt: { + sql: \`\${CUBE}.product_id = \${ProductsAlt}.id\`, + relationship: \`belongsTo\` + } + }, + + measures: { + count: { + type: \`count\`, + //drillMembers: [id, createdAt] + }, + }, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\`, + primaryKey: true + }, + + status: { + sql: \`status\`, + type: \`string\` + }, + + createdAt: { + sql: \`created_at\`, + type: \`time\` + } + }, + + dataSource: \`default\` +}); + +cube(\`Products\`, { + sql: \` + SELECT 1 as id, 1 as product_category_id, 'Tomato' as name + UNION ALL + SELECT 2 as id, 1 as product_category_id, 'Potato' as name + \`, + + joins: { + ProductCategories: { + sql: \`\${CUBE}.product_category_id = \${ProductCategories}.id\`, + relationship: \`belongsTo\` + }, + }, + + measures: { + count: { + type: \`count\`, + } + }, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\`, + primaryKey: true + }, + + name: { + sql: \`name\`, + type: \`string\` + }, + } +}); + +cube(\`ProductsAlt\`, { + sql: \`SELECT * FROM \${Products.sql()} as p WHERE id = 1\`, + + joins: { + ProductCategories: { + sql: \`\${CUBE}.product_category_id = \${ProductCategories}.id\`, + relationship: \`belongsTo\` + }, + }, + + measures: { + count: { + type: \`count\`, + } + }, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\`, + primaryKey: true + }, + + name: { + sql: \`name\`, + type: \`string\` + }, + } +}); + +cube(\`ProductCategories\`, { + sql: \` + SELECT 1 as id, 'Groceries' as name + UNION ALL + SELECT 2 as id, 'Electronics' as name + \`, + + joins: { + + }, + + measures: { + count: { + type: \`count\`, + } + }, + + dimensions: { + id: { + sql: \`id\`, + type: \`number\`, + primaryKey: true + }, + + name: { + sql: \`name\`, + type: \`string\` + }, + } +}); + +view(\`OrdersView\`, { + measures: { + count: { + sql: \`\${Orders.count}\`, + type: \`number\` + }, + + productCategoryCount: { + sql: \`\${Orders.ProductsAlt.ProductCategories.count}\`, + type: \`number\` + } + }, + + dimensions: { + createdAt: { + sql: \`\${Orders.createdAt}\`, + type: \`time\` + }, + + productName: { + sql: \`\${Products.name}\`, + type: \`string\` + }, + + categoryName: { + sql: \`\${Orders.ProductsAlt.ProductCategories.name}\`, + type: \`string\` + } + } +}); + `); + + async function runQueryTest(q: any, expectedResult: any, additionalTest?: (query: BaseQuery) => any) { + await compiler.compile(); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { ...q, timezone: 'UTC', preAggregationsSchema: '' }); + + console.log(query.buildSqlAndParams()); + + const res = await dbRunner.evaluateQueryWithPreAggregations(query); + console.log(JSON.stringify(res)); + + if (additionalTest) { + additionalTest(query); + } + + expect(res).toEqual( + expectedResult + ); + } + + it('simple view', async () => runQueryTest({ + measures: ['OrdersView.count'], + dimensions: [ + 'OrdersView.categoryName' + ], + order: [{ id: 'OrdersView.categoryName' }] + }, [{ + orders_view__category_name: 'Groceries', + orders_view__count: '1', + }, { + orders_view__category_name: null, + orders_view__count: '1', + }])); + + it('join from two join hint paths', async () => runQueryTest({ + measures: ['OrdersView.productCategoryCount'], + dimensions: [ + 'OrdersView.categoryName' + ], + order: [{ id: 'OrdersView.productCategoryCount' }] + }, [{ + orders_view__category_name: null, + orders_view__product_category_count: '0', + }, { + orders_view__category_name: 'Groceries', + orders_view__product_category_count: '1', + }])); + + it('pre-aggregation', async () => runQueryTest({ + measures: ['OrdersView.count'], + dimensions: [ + 'OrdersView.productName' + ], + order: [{ id: 'OrdersView.productName' }], + }, [{ + orders_view__product_name: 'Potato', + orders_view__count: '1', + }, { + orders_view__product_name: 'Tomato', + orders_view__count: '1', + }], (query) => { + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/count_by_product_name/); + })); +}); From 808fcfc0b0e19db31bece876017af568cd7e7e44 Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sat, 1 Oct 2022 14:39:13 -0700 Subject: [PATCH 8/9] chore: Cube Views join hints fixes --- .../src/adapter/BaseQuery.js | 37 +++++++++--- .../src/compiler/CubeEvaluator.js | 49 ++++++++++++--- .../src/compiler/CubeSymbols.js | 45 +++++++++----- .../mysql/mysql-pre-aggregations.test.ts | 10 ---- .../integration/postgres/cube-views.test.ts | 59 ++++++++++++++++++- .../postgres/dataschema-compiler.test.ts | 53 +++++++++++++++++ 6 files changed, 210 insertions(+), 43 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 5722283fbe96a..8d293558f7cba 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1798,7 +1798,6 @@ class BaseQuery { options = options || {}; const self = this; const { cubeEvaluator } = this; - const joinHints = []; return cubeEvaluator.resolveSymbolsCall(sql, (name) => { const nextCubeName = cubeEvaluator.symbols[name] && name || cubeName; const resolvedSymbol = @@ -1816,7 +1815,7 @@ class BaseQuery { cubeAliasFn: self.cubeAlias.bind(self), contextSymbols: this.parametrizedContextSymbols(), query: this, - joinHints + collectJoinHints: true, }); } @@ -2730,7 +2729,21 @@ class BaseQuery { return this.parametrizedContextSymbolsValue; } + static emptyParametrizedContextSymbols(cubeEvaluator, allocateParam) { + return { + filterParams: BaseQuery.filterProxyFromAllFilters({}, cubeEvaluator, allocateParam), + sqlUtils: { + convertTz: (field) => field, + }, + securityContext: BaseQuery.contextSymbolsProxyFrom({}, allocateParam), + }; + } + contextSymbolsProxy(symbols) { + return BaseQuery.contextSymbolsProxyFrom(symbols, this.paramAllocator.allocateParam.bind(this.paramAllocator)); + } + + static contextSymbolsProxyFrom(symbols, allocateParam) { return new Proxy(symbols, { get: (target, name) => { const propValue = target[name]; @@ -2738,8 +2751,8 @@ class BaseQuery { filter: (column) => { if (paramValue) { const value = Array.isArray(paramValue) ? - paramValue.map(this.paramAllocator.allocateParam.bind(this.paramAllocator)) : - this.paramAllocator.allocateParam(paramValue); + paramValue.map(allocateParam) : + allocateParam(paramValue); if (typeof column === 'function') { return column(value); } else { @@ -2758,7 +2771,7 @@ class BaseQuery { unsafeValue: () => paramValue }); return methods(target)[name] || - typeof propValue === 'object' && propValue !== null && this.contextSymbolsProxy(propValue) || + typeof propValue === 'object' && propValue !== null && BaseQuery.contextSymbolsProxyFrom(propValue, allocateParam) || methods(propValue); } }); @@ -2766,16 +2779,24 @@ class BaseQuery { filtersProxy() { const { allFilters } = this; + return BaseQuery.filterProxyFromAllFilters( + allFilters, + this.cubeEvaluator, + this.paramAllocator.allocateParam.bind(this.paramAllocator) + ); + } + + static filterProxyFromAllFilters(allFilters, cubeEvaluator, allocateParam) { return new Proxy({}, { get: (target, name) => { if (name === '_objectWithResolvedProperties') { return true; } - const cubeName = this.cubeEvaluator.cubeNameFromPath(name); + const cubeName = cubeEvaluator.cubeNameFromPath(name); return new Proxy({ cube: cubeName }, { get: (cubeNameObj, propertyName) => { const filters = - allFilters.filter(f => f.dimension === this.cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName])); + allFilters.filter(f => f.dimension === cubeEvaluator.pathFromArray([cubeNameObj.cube, propertyName])); return { filter: (column) => { if (!filters.length) { @@ -2791,7 +2812,7 @@ class BaseQuery { // eslint-disable-next-line prefer-spread return column.apply( null, - filterParams.map(this.paramAllocator.allocateParam.bind(this.paramAllocator)) + filterParams.map(allocateParam), ); } else { return filter.conditionSql(column); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js index bf9f6a19ccf32..09b2073527cf3 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.js @@ -3,6 +3,7 @@ import R from 'ramda'; import { CubeSymbols } from './CubeSymbols'; import { UserError } from './UserError'; +import { BaseQuery } from '../adapter'; export class CubeEvaluator extends CubeSymbols { constructor(cubeValidator) { @@ -87,25 +88,29 @@ export class CubeEvaluator extends CubeSymbols { } } } - this.transformMembers(cube.measures, cube); - this.transformMembers(cube.dimensions, cube); - this.transformMembers(cube.segments, cube); + this.transformMembers(cube.measures, cube, errorReporter); + this.transformMembers(cube.dimensions, cube, errorReporter); + this.transformMembers(cube.segments, cube, errorReporter); } - transformMembers(members, cube) { + transformMembers(members, cube, errorReporter) { members = members || {}; for (const memberName of Object.keys(members)) { const member = members[memberName]; let ownedByCube = true; if (member.sql && !member.subQuery) { const funcArgs = this.funcArguments(member.sql); - // TODO case when foreign cube is referenced isn't covered - // TODO case when other dimension is referenced through CUBE ref is not covered: it'll be rendered as an owned - if (funcArgs.length > 0 && funcArgs.every( - ref => !cube.measures[ref] && !cube.dimensions[ref] && !cube.segments[ref] && !this.isCurrentCube(ref) && ref !== cube.name - )) { + const cubeReferences = this.collectUsedCubeReferences(cube.name, member.sql); + if (funcArgs.length > 0 && cubeReferences.length === 0) { ownedByCube = false; } + const foreignCubes = cubeReferences.filter(usedCube => usedCube !== cube.name); + if (foreignCubes.length > 0) { + errorReporter.error(`Member '${cube.name}.${memberName}' references foreign cubes: ${foreignCubes.join(', ')}. Please split and move this definition to corresponding cubes.`); + } + } + if (ownedByCube && cube.isView) { + errorReporter.error(`View '${cube.name}' defines own member '${cube.name}.${memberName}'. Please move this member definition to one of the cubes.`); } members[memberName].ownedByCube = ownedByCube; } @@ -277,6 +282,32 @@ export class CubeEvaluator extends CubeSymbols { return path.split('.'); } + collectUsedCubeReferences(cube, sqlFn) { + const cubeEvaluator = this; + + const cubeReferencesUsed = []; + + cubeEvaluator.resolveSymbolsCall(sqlFn, (name) => { + const referencedCube = cubeEvaluator.symbols[name] && name || cube; + const resolvedSymbol = + cubeEvaluator.resolveSymbol( + cube, + name + ); + // eslint-disable-next-line no-underscore-dangle + if (resolvedSymbol._objectWithResolvedProperties) { + return resolvedSymbol; + } + return cubeEvaluator.pathFromArray([referencedCube, name]); + }, { + // eslint-disable-next-line no-shadow + sqlResolveFn: (symbol, cube, n) => cubeEvaluator.pathFromArray([cube, n]), + contextSymbols: BaseQuery.emptyParametrizedContextSymbols(this, () => '$empty_param$'), + cubeReferencesUsed, + }); + return cubeReferencesUsed; + } + evaluateReferences(cube, referencesFn, options = {}) { const cubeEvaluator = this; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js index 91f71fda232d2..41bb825beadfc 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js @@ -178,6 +178,16 @@ export class CubeSymbols { } } + withSymbolsCallContext(func, context) { + const oldContext = this.resolveSymbolsCallContext; + this.resolveSymbolsCallContext = context; + try { + return func(); + } finally { + this.resolveSymbolsCallContext = oldContext; + } + } + funcArguments(func) { const funcDefinition = func.toString(); if (!this.funcArgumentsValues[funcDefinition]) { @@ -199,7 +209,7 @@ export class CubeSymbols { } resolveSymbol(cubeName, name) { - const { sqlResolveFn, contextSymbols } = this.resolveSymbolsCallContext || {}; + const { sqlResolveFn, contextSymbols, collectJoinHints } = this.resolveSymbolsCallContext || {}; if (CONTEXT_SYMBOLS[name]) { // always resolves if contextSymbols aren't passed for transpile step const symbol = contextSymbols && contextSymbols[CONTEXT_SYMBOLS[name]] || {}; @@ -210,13 +220,19 @@ export class CubeSymbols { let cube = this.isCurrentCube(name) && this.symbols[cubeName] || this.symbols[name]; if (sqlResolveFn && cube) { - cube = this.cubeReferenceProxy(this.isCurrentCube(name) ? cubeName : name); + cube = this.cubeReferenceProxy( + this.isCurrentCube(name) ? cubeName : name, + collectJoinHints ? [] : undefined + ); } return cube || (this.symbols[cubeName] && this.symbols[cubeName][name]); } - cubeReferenceProxy(cubeName) { + cubeReferenceProxy(cubeName, joinHints) { + if (joinHints) { + joinHints = joinHints.concat(cubeName); + } const self = this; return new Proxy({}, { get: (v, propertyName) => { @@ -231,16 +247,16 @@ export class CubeSymbols { } return undefined; } - const { sqlResolveFn, cubeAliasFn, query, joinHints } = self.resolveSymbolsCallContext || {}; + const { sqlResolveFn, cubeAliasFn, query, cubeReferencesUsed } = self.resolveSymbolsCallContext || {}; if (propertyName === 'toString') { return () => { - if (joinHints && joinHints.length === 0) { - joinHints.push(cube.cubeName()); - } if (query) { query.pushCubeNameForCollectionIfNecessary(cube.cubeName()); query.pushJoinHints(joinHints); } + if (cubeReferencesUsed) { + cubeReferencesUsed.push(cube.cubeName()); + } return cubeAliasFn && cubeAliasFn(cube.cubeName()) || cube.cubeName(); }; } @@ -251,16 +267,15 @@ export class CubeSymbols { return true; } if (cube[propertyName]) { - return { toString: () => sqlResolveFn(cube[propertyName], cubeName, propertyName) }; + return { + toString: () => this.withSymbolsCallContext( + () => sqlResolveFn(cube[propertyName], cubeName, propertyName), + { ...this.resolveSymbolsCallContext, joinHints }, + ), + }; } if (self.symbols[propertyName]) { - if (joinHints) { - if (joinHints.length === 0) { - joinHints.push(cubeName); - } - joinHints.push(propertyName); - } - return this.cubeReferenceProxy(propertyName); + return this.cubeReferenceProxy(propertyName, joinHints); } if (typeof propertyName === 'string') { throw new UserError(`${cubeName}.${propertyName} cannot be resolved. There's no such member or cube.`); diff --git a/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts index a0201caa929cc..1ce395d6b9cf8 100644 --- a/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts @@ -23,11 +23,6 @@ describe('MySqlPreAggregations', () => { type: 'count' }, - checkinsTotal: { - sql: \`\${checkinsCount}\`, - type: 'sum' - }, - uniqueSourceCount: { sql: 'source', type: 'countDistinct' @@ -37,11 +32,6 @@ describe('MySqlPreAggregations', () => { sql: 'id', type: 'countDistinctApprox' }, - - ratio: { - sql: \`1.0 * \${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`, - type: 'number' - } }, dimensions: { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts index 66ca5abdd1237..79b5d98f2a3ee 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -55,6 +55,11 @@ cube(\`Orders\`, { type: \`string\` }, + statusProduct: { + sql: \`\${CUBE}.status || '_' || \${Products.name}\`, + type: \`string\` + }, + createdAt: { sql: \`created_at\`, type: \`time\` @@ -95,6 +100,11 @@ cube(\`Products\`, { sql: \`name\`, type: \`string\` }, + + proxyName: { + sql: \`\${name}\`, + type: \`string\`, + }, } }); @@ -186,7 +196,12 @@ view(\`OrdersView\`, { categoryName: { sql: \`\${Orders.ProductsAlt.ProductCategories.name}\`, type: \`string\` - } + }, + + productCategory: { + sql: \`\${Orders.ProductsAlt.name} || '_' || \${Orders.ProductsAlt.ProductCategories.name} || '_' || \${categoryName}\`, + type: \`string\` + }, } }); `); @@ -254,4 +269,46 @@ view(\`OrdersView\`, { console.log(preAggregationsDescription); expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/count_by_product_name/); })); + + it('proxy dimension', async () => runQueryTest({ + measures: ['OrdersView.count'], + dimensions: [ + 'Products.proxyName' + ], + order: [{ id: 'Products.proxyName' }], + }, [{ + products__proxy_name: 'Potato', + orders_view__count: '1', + }, { + products__proxy_name: 'Tomato', + orders_view__count: '1', + }], (query) => { + const preAggregationsDescription = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect((preAggregationsDescription)[0].loadSql[0]).toMatch(/count_by_product_name/); + })); + + it('compound dimension', async () => runQueryTest({ + measures: [], + dimensions: [ + 'Orders.statusProduct' + ], + order: [{ id: 'Orders.statusProduct' }], + }, [{ + orders__status_product: 'completed_Potato', + }, { + orders__status_product: 'completed_Tomato', + }])); + + it('view compound dimension', async () => runQueryTest({ + measures: [], + dimensions: [ + 'OrdersView.productCategory' + ], + order: [{ id: 'OrdersView.productCategory' }], + }, [{ + orders_view__product_category: 'Tomato_Groceries_Groceries', + }, { + orders_view__product_category: null, + }])); }); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts index 5abb2feaa9fe1..fb4cf4e385d12 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts @@ -465,4 +465,57 @@ describe('DataSchemaCompiler', () => { ['Marketing'] ); }); + + it('views should not contain own members', () => { + const { compiler } = prepareCompiler(` + view('Visitors', { + dimensions: { + id: { + type: 'number', + sql: 'id', + } + } + }) + `); + return compiler.compile().then(() => { + compiler.throwIfAnyErrors(); + throw new Error(); + }).catch((error) => { + console.log(error); + expect(error).toBeInstanceOf(CompileError); + }); + }); + + it('foreign cubes', () => { + const { compiler } = prepareCompiler(` + cube('Visitors', { + sql: 'select * from visitors', + + dimensions: { + foo: { + type: 'number', + sql: \`$\{Foreign}.bar\`, + } + } + }); + + cube('Foreign', { + sql: 'select * from foreign', + + dimensions: { + bar: { + type: 'number', + sql: 'id', + } + } + }) + `); + return compiler.compile().then(() => { + compiler.throwIfAnyErrors(); + throw new Error(); + }).catch((error) => { + console.log(error); + expect(error).toBeInstanceOf(CompileError); + }); + }); }); From 09315d1402672720e73241083368ee9c289696fd Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Sat, 1 Oct 2022 14:53:03 -0700 Subject: [PATCH 9/9] chore: Cube Views join hints fixes --- .../integration/postgres/cube-views.test.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts index 79b5d98f2a3ee..be92fa3b6119b 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -63,7 +63,12 @@ cube(\`Orders\`, { createdAt: { sql: \`created_at\`, type: \`time\` - } + }, + + productAndCategory: { + sql: \`\${Products.name} || '_' || \${Products.ProductCategories.name}\`, + type: \`string\` + }, }, dataSource: \`default\` @@ -300,6 +305,18 @@ view(\`OrdersView\`, { orders__status_product: 'completed_Tomato', }])); + it('compound dimension 2', async () => runQueryTest({ + measures: [], + dimensions: [ + 'Orders.productAndCategory' + ], + order: [{ id: 'Orders.productAndCategory' }], + }, [{ + orders__product_and_category: 'Potato_Groceries', + }, { + orders__product_and_category: 'Tomato_Groceries', + }])); + it('view compound dimension', async () => runQueryTest({ measures: [], dimensions: [