diff --git a/.github/actions/smoke.sh b/.github/actions/smoke.sh index 51402c993889f..b75ee3527de1d 100755 --- a/.github/actions/smoke.sh +++ b/.github/actions/smoke.sh @@ -14,6 +14,10 @@ yarn lerna run --concurrency 1 --stream --no-prefix integration:duckdb yarn lerna run --concurrency 1 --stream --no-prefix smoke:duckdb echo "::endgroup::" +echo "::group::View Groups" +yarn lerna run --concurrency 1 --stream --no-prefix smoke:view-groups +echo "::endgroup::" + echo "::group::Postgres" yarn lerna run --concurrency 1 --stream --no-prefix smoke:postgres echo "::endgroup::" diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 049c4e30391bc..84d017cb4956b 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -649,19 +649,30 @@ class ApiGateway { const compilerApi = await this.getCompilerApi(context); const metaConfig = await compilerApi.metaConfig(context, { requestId: context.requestId, - includeCompilerId: includeCompilerId || onlyCompilerId + includeCompilerId: includeCompilerId || onlyCompilerId, + includeViewGroups: true, }); if (onlyCompilerId) { - const response: { cubes: any[], compilerId?: string } = { + const response: { cubes: any[], viewGroups?: any[], compilerId?: string } = { cubes: [], compilerId: metaConfig.compilerId }; res(response); return; } - const cubesConfig = includeCompilerId ? metaConfig.cubes : metaConfig; + const cubesConfig = metaConfig.cubes; const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); - const response: { cubes: any[], compilerId?: string } = { cubes }; + const visibleCubeNames = new Set(cubes.map(c => c.name)); + const viewGroups = (metaConfig.viewGroups || []) + .map(group => ({ + ...group, + views: group.views.filter((v: string) => visibleCubeNames.has(v)), + })) + .filter(group => group.views.length > 0); + const response: { cubes: any[], viewGroups?: any[], compilerId?: string } = { cubes }; + if (viewGroups.length > 0) { + response.viewGroups = viewGroups; + } if (includeCompilerId) { response.compilerId = metaConfig.compilerId; } diff --git a/packages/cubejs-api-gateway/src/types/request.ts b/packages/cubejs-api-gateway/src/types/request.ts index c7804f84a5762..8478d008354a2 100644 --- a/packages/cubejs-api-gateway/src/types/request.ts +++ b/packages/cubejs-api-gateway/src/types/request.ts @@ -95,7 +95,7 @@ type ErrorResponse = { error: string, }; -type MetaResponse = { cubes: any[], compilerId?: string }; +type MetaResponse = { cubes: any[], viewGroups?: any[], compilerId?: string }; type MetaResponseResultFn = (message: MetaResponse | ErrorResponse) => void; /** diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 26c1c88a18ef0..1f150c2621b45 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -686,6 +686,35 @@ describe('API Gateway', () => { expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock'); }); + test('meta endpoint returns view groups', async () => { + const { app } = await createApiGateway(); + + const res = await request(app) + .get('/cubejs-api/v1/meta') + .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M') + .expect(200); + + expect(res.body).toHaveProperty('cubes'); + expect(res.body).toHaveProperty('viewGroups'); + + expect(res.body.viewGroups).toHaveLength(1); + expect(res.body.viewGroups[0]).toEqual({ + name: 'analytics', + title: 'Analytics', + description: 'Analytics related views', + views: ['FooView'], + }); + + const fooView = res.body.cubes.find(c => c.name === 'FooView'); + expect(fooView).toBeDefined(); + expect(fooView.viewGroups).toEqual(['analytics']); + expect(fooView.type).toBe('view'); + + const fooCube = res.body.cubes.find(c => c.name === 'Foo'); + expect(fooCube).toBeDefined(); + expect(fooCube.viewGroups).toBeUndefined(); + }); + test('meta endpoint extended to get schema information with additional data', async () => { const { app } = await createApiGateway(); diff --git a/packages/cubejs-api-gateway/test/mocks.ts b/packages/cubejs-api-gateway/test/mocks.ts index f2341b544c323..907d399ab9a4a 100644 --- a/packages/cubejs-api-gateway/test/mocks.ts +++ b/packages/cubejs-api-gateway/test/mocks.ts @@ -80,11 +80,12 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ return { query, denied: false }; }, - async metaConfig() { - return [ + async metaConfig(_ctx, options: any = {}) { + const cubes = [ { config: { name: 'Foo', + type: 'cube', description: 'cube from compilerApi mock', measures: [ { @@ -125,7 +126,48 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({ ], }, }, + { + config: { + name: 'FooView', + type: 'view', + description: 'view from compilerApi mock', + viewGroups: ['analytics'], + measures: [ + { + name: 'FooView.bar', + isVisible: true, + }, + ], + dimensions: [ + { + name: 'FooView.id', + isVisible: true, + }, + ], + segments: [], + }, + }, ]; + + if (options.includeCompilerId || options.includeViewGroups) { + const result: any = { cubes }; + if (options.includeCompilerId) { + result.compilerId = 'mock-compiler-id'; + } + if (options.includeViewGroups) { + result.viewGroups = [ + { + name: 'analytics', + title: 'Analytics', + description: 'Analytics related views', + views: ['FooView'], + }, + ]; + } + return result; + } + + return cubes; }, async metaConfigExtended() { diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 3ca32aa335dc8..876533d97d471 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -527,6 +527,7 @@ export type Cube = { isVisible?: boolean; public?: boolean; meta?: any; + viewGroups?: string[]; }; export type CubeMap = { @@ -540,8 +541,16 @@ export type CubesMap = Record< CubeMap >; +export type ViewGroup = { + name: string; + title?: string; + description?: string; + views: string[]; +}; + export type MetaResponse = { cubes: Cube[]; + viewGroups?: ViewGroup[]; }; export type FilterOperator = { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index caaa9005de6a1..e34ac15082c42 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -214,6 +214,8 @@ export interface CubeDefinition { excludes?: any; cubes?: any; isView?: boolean; + viewGroup?: string | ((...args: any[]) => any); + viewGroups?: string[] | ((...args: any[]) => any); calendar?: boolean; isSplitView?: boolean; includedMembers?: ViewIncludedMember[]; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 1c5f92d3cc169..0d8aa6551dc02 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -15,6 +15,7 @@ import type { CubeDefinitionExtended } from './CubeSymbols'; import type { CubeValidator } from './CubeValidator'; import type { CubeEvaluator } from './CubeEvaluator'; import type { ContextEvaluator } from './ContextEvaluator'; +import type { ViewGroupEvaluator, CompiledViewGroup } from './ViewGroupEvaluator'; import type { JoinGraph } from './JoinGraph'; import type { ErrorReporter } from './ErrorReporter'; import { CompilerInterface } from './PrepareCompiler'; @@ -139,6 +140,7 @@ export type CubeConfig = { isVisible: boolean; public: boolean; description?: string; + viewGroups?: string[]; connectedComponent: number; meta?: any; measures: MeasureConfig[]; @@ -162,6 +164,8 @@ export class CubeToMetaTransformer implements CompilerInterface { private readonly contextEvaluator: ContextEvaluator; + private readonly viewGroupEvaluator: ViewGroupEvaluator; + private readonly joinGraph: JoinGraph; public cubes: TransformedCube[]; @@ -175,17 +179,23 @@ export class CubeToMetaTransformer implements CompilerInterface { cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator, contextEvaluator: ContextEvaluator, + viewGroupEvaluator: ViewGroupEvaluator, joinGraph: JoinGraph ) { this.cubeValidator = cubeValidator; this.cubeSymbols = cubeEvaluator; this.cubeEvaluator = cubeEvaluator; this.contextEvaluator = contextEvaluator; + this.viewGroupEvaluator = viewGroupEvaluator; this.joinGraph = joinGraph; this.cubes = []; this.queries = []; } + public get viewGroups(): CompiledViewGroup[] { + return this.viewGroupEvaluator.compiledViewGroups; + } + public compile(_cubes: any[], errorReporter: ErrorReporter): void { this.cubes = this.cubeSymbols.cubeList .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) @@ -239,6 +249,10 @@ export class CubeToMetaTransformer implements CompilerInterface { const nestedFolders: NestedFolder[] = (extendedCube.folders || []).map((f: Folder) => processFolder(f)); + const viewGroupNames = extendedCube.isView + ? this.viewGroupEvaluator.viewGroupsForView(cubeName) + : []; + return { config: { name: cubeName, @@ -247,6 +261,7 @@ export class CubeToMetaTransformer implements CompilerInterface { isVisible: isCubeVisible, public: isCubeVisible, description: extendedCube.description, + ...(viewGroupNames.length > 0 ? { viewGroups: viewGroupNames } : {}), connectedComponent: this.joinGraph.connectedComponents()[cubeName], meta: extendedCube.meta, measures: Object.entries(extendedCube.measures || {}).map((nameToMetric: [string, any]) => { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 30bb915e6448d..b4d5b90b8572a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -1132,6 +1132,8 @@ const folderSchema = Joi.object().keys({ const viewSchema = inherit(baseSchema, { isView: Joi.boolean().strict(), + viewGroup: Joi.alternatives([Joi.string(), Joi.func()]), + viewGroups: Joi.alternatives([Joi.array().items(Joi.string().required()), Joi.func()]), cubes: Joi.array().items( Joi.object().keys({ joinPath: Joi.func().required(), diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts index ddc5c2204d5f8..500f35a23e457 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts @@ -74,6 +74,8 @@ export type DataSchemaCompilerOptions = { cubeAndViewSymbols: CubeSymbols; cubeCompilers?: CompilerInterface[]; contextCompilers?: CompilerInterface[]; + viewGroupCompilers?: CompilerInterface[]; + metaCompilers?: CompilerInterface[]; transpilers?: TranspilerInterface[]; viewCompilers?: CompilerInterface[]; viewCompilationGate: ViewCompilationGate; @@ -105,6 +107,8 @@ export type CompileStage = 0 | 1 | 2 | 3; type CompileCubeFilesCompilers = { cubeCompilers?: CompilerInterface[]; contextCompilers?: CompilerInterface[]; + viewGroupCompilers?: CompilerInterface[]; + metaCompilers?: CompilerInterface[]; }; export type CompileContext = any; @@ -116,6 +120,10 @@ export class DataSchemaCompiler { private readonly contextCompilers: CompilerInterface[]; + private readonly viewGroupCompilers: CompilerInterface[]; + + private readonly metaCompilers: CompilerInterface[]; + private readonly transpilers: TranspilerInterface[]; private readonly viewCompilers: CompilerInterface[]; @@ -181,6 +189,8 @@ export class DataSchemaCompiler { this.repository = repository; this.cubeCompilers = options.cubeCompilers || []; this.contextCompilers = options.contextCompilers || []; + this.viewGroupCompilers = options.viewGroupCompilers || []; + this.metaCompilers = options.metaCompilers || []; this.transpilers = options.transpilers || []; this.viewCompilers = options.viewCompilers || []; this.preTranspileCubeCompilers = options.preTranspileCubeCompilers || []; @@ -366,6 +376,7 @@ export class DataSchemaCompiler { let cubes: CubeDefinition[] = []; let exports: Record> = {}; let contexts: Record[] = []; + let viewGroups: Record[] = []; let compiledFiles: Record = {}; let asyncModules: CallableFunction[] = []; let transpiledFiles: FileContent[] = []; @@ -374,6 +385,7 @@ export class DataSchemaCompiler { cubes = []; exports = {}; contexts = []; + viewGroups = []; compiledFiles = {}; asyncModules = []; }; @@ -404,6 +416,15 @@ export class DataSchemaCompiler { } return contexts.push({ ...context, name, fileName: file.fileName }); }, + view_group: (name: string, viewGroup) => { + const file = ctxFileStorage.getStore(); + if (!file) { + throw new Error('No file stored in context'); + } + viewGroups.push({ ...viewGroup, name, fileName: file.fileName }); + this.compileV8ContextCache![name] = name; + return name; + }, addExport: (obj) => { const file = ctxFileStorage.getStore(); if (!file) { @@ -473,7 +494,7 @@ export class DataSchemaCompiler { const convertedToJsFiles = transpiledFiles.filter(f => !f.fileName.endsWith('.js')); toCompile = [...originalJsFiles, ...convertedToJsFiles]; - return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport); + return this.compileCubeFiles(cubes, contexts, viewGroups, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport); }; const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => { @@ -481,7 +502,7 @@ export class DataSchemaCompiler { cleanup(); transpiledFiles = await transpilePhase(stage); - return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport); + return this.compileCubeFiles(cubes, contexts, viewGroups, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport); }; return compilePhaseFirst({ cubeCompilers: this.cubeNameCompilers }, 0) @@ -492,6 +513,8 @@ export class DataSchemaCompiler { .then(() => compilePhase({ cubeCompilers: this.cubeCompilers, contextCompilers: this.contextCompilers, + viewGroupCompilers: this.viewGroupCompilers, + metaCompilers: this.metaCompilers, }, 3)) .then(() => { // Free unneeded resources @@ -820,6 +843,7 @@ export class DataSchemaCompiler { private async compileCubeFiles( cubes: CubeDefinition[], contexts: Record[], + viewGroups: Record[], compiledFiles: Record, asyncModules: CallableFunction[], compilers: CompileCubeFilesCompilers, @@ -836,7 +860,9 @@ export class DataSchemaCompiler { }); await asyncModules.reduce((a: Promise, b: CallableFunction) => a.then(() => b()), Promise.resolve()); return this.compileObjects(compilers.cubeCompilers || [], cubes, errorsReport) - .then(() => this.compileObjects(compilers.contextCompilers || [], contexts, errorsReport)); + .then(() => this.compileObjects(compilers.contextCompilers || [], contexts, errorsReport)) + .then(() => this.compileObjects(compilers.viewGroupCompilers || [], viewGroups, errorsReport)) + .then(() => this.compileObjects(compilers.metaCompilers || [], cubes, errorsReport)); } public throwIfAnyErrors() { diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index b2f7a4f5604b8..cd0e0c52fc7e8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -19,6 +19,7 @@ import { CubeSymbols } from './CubeSymbols'; import { CubeDictionary } from './CubeDictionary'; import { CubeEvaluator } from './CubeEvaluator'; import { ContextEvaluator } from './ContextEvaluator'; +import { ViewGroupEvaluator } from './ViewGroupEvaluator'; import { JoinGraph } from './JoinGraph'; import { CubeToMetaTransformer } from './CubeToMetaTransformer'; import { CompilerCache } from './CompilerCache'; @@ -50,6 +51,7 @@ export type Compiler = { metaTransformer: CubeToMetaTransformer; cubeEvaluator: CubeEvaluator; contextEvaluator: ContextEvaluator; + viewGroupEvaluator: ViewGroupEvaluator; joinGraph: JoinGraph; compilerCache: CompilerCache; headCommitId?: string; @@ -65,8 +67,9 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp const cubeValidator = new CubeValidator(cubeSymbols); const cubeEvaluator = new CubeEvaluator(cubeValidator); const contextEvaluator = new ContextEvaluator(cubeEvaluator); + const viewGroupEvaluator = new ViewGroupEvaluator(cubeEvaluator); const joinGraph = new JoinGraph(cubeValidator, cubeEvaluator); - const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, joinGraph); + const metaTransformer = new CubeToMetaTransformer(cubeValidator, cubeEvaluator, contextEvaluator, viewGroupEvaluator, joinGraph); const { maxQueryCacheSize, maxQueryCacheAge } = options; const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge }); const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler); @@ -97,8 +100,10 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp compiledYamlCache, compiledJinjaCache, viewCompilers: [viewCompiler], - cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer], + cubeCompilers: [cubeEvaluator, joinGraph], contextCompilers: [contextEvaluator], + viewGroupCompilers: [viewGroupEvaluator], + metaCompilers: [metaTransformer], cubeFactory: cubeSymbols.createCube.bind(cubeSymbols), compilerCache, cubeDictionary, @@ -122,6 +127,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp metaTransformer, cubeEvaluator, contextEvaluator, + viewGroupEvaluator, joinGraph, compilerCache, headCommitId: options.headCommitId, diff --git a/packages/cubejs-schema-compiler/src/compiler/ViewGroupEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/ViewGroupEvaluator.ts new file mode 100644 index 0000000000000..89734fa39e03d --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/ViewGroupEvaluator.ts @@ -0,0 +1,159 @@ +import type { CubeEvaluator } from './CubeEvaluator'; +import type { ErrorReporter } from './ErrorReporter'; +import { CompilerInterface } from './PrepareCompiler'; + +export interface ViewGroupInput { + name: string; + title?: string; + description?: string; + views?: string[] | (() => string[]); + fileName?: string; +} + +export interface CompiledViewGroup { + name: string; + title?: string; + description?: string; + views: string[]; +} + +export class ViewGroupEvaluator implements CompilerInterface { + private readonly cubeEvaluator: CubeEvaluator; + + private viewGroupDefinitions: Map; + + private resolvedViewGroups: CompiledViewGroup[]; + + private viewToGroups: Map; + + public constructor(cubeEvaluator: CubeEvaluator) { + this.cubeEvaluator = cubeEvaluator; + this.viewGroupDefinitions = new Map(); + this.resolvedViewGroups = []; + this.viewToGroups = new Map(); + } + + public compile(viewGroups: ViewGroupInput[], errorReporter?: ErrorReporter): void { + this.viewGroupDefinitions = new Map(); + + for (const viewGroup of viewGroups) { + if (errorReporter && this.viewGroupDefinitions.has(viewGroup.name)) { + errorReporter.error(`View group "${viewGroup.name}" already exists!`); + } else { + this.viewGroupDefinitions.set(viewGroup.name, this.compileViewGroup(viewGroup)); + } + } + + this.resolve(errorReporter); + } + + private compileViewGroup(viewGroup: ViewGroupInput): CompiledViewGroup { + let views: string[] = []; + if (viewGroup.views) { + if (typeof viewGroup.views === 'function') { + const evaluated = this.cubeEvaluator.evaluateReferences(null, viewGroup.views, { originalSorting: true }); + views = Array.isArray(evaluated) ? evaluated : [evaluated]; + } else if (Array.isArray(viewGroup.views)) { + views = viewGroup.views; + } + } + + return { + name: viewGroup.name, + title: viewGroup.title, + description: viewGroup.description, + views, + }; + } + + private resolve(errorReporter?: ErrorReporter): void { + const viewGroupMap = new Map(); + const validViewNames = new Set(); + + for (const cube of this.cubeEvaluator.cubeList) { + if (cube.isView) { + validViewNames.add(cube.name); + } + } + + for (const [name, def] of this.viewGroupDefinitions) { + viewGroupMap.set(name, { + name: def.name, + title: def.title, + description: def.description, + views: def.views.filter(v => validViewNames.has(v)), + }); + } + + for (const cube of this.cubeEvaluator.cubeList) { + if (!cube.isView) { + // eslint-disable-next-line no-continue + continue; + } + + const groupNames: string[] = []; + if (cube.viewGroup) { + const resolved = typeof cube.viewGroup === 'function' + ? this.cubeEvaluator.evaluateReferences(null, cube.viewGroup) + : cube.viewGroup; + const names = Array.isArray(resolved) ? resolved : [resolved]; + for (const n of names) { + if (!groupNames.includes(n)) { + groupNames.push(n); + } + } + } + if (cube.viewGroups) { + let resolved: string[]; + if (typeof cube.viewGroups === 'function') { + const evaluated = this.cubeEvaluator.evaluateReferences(null, cube.viewGroups, { originalSorting: true }); + resolved = Array.isArray(evaluated) ? evaluated : [evaluated]; + } else { + resolved = cube.viewGroups; + } + for (const n of resolved) { + if (!groupNames.includes(n)) { + groupNames.push(n); + } + } + } + + for (const groupName of groupNames) { + const group = viewGroupMap.get(groupName); + if (!group) { + if (errorReporter) { + errorReporter.error(`View "${cube.name}" references view group "${groupName}" which is not defined. Define it using view_group('${groupName}', { ... }).`); + } + } else if (!group.views.includes(cube.name)) { + group.views.push(cube.name); + } + } + } + + this.resolvedViewGroups = Array.from(viewGroupMap.values()); + + this.viewToGroups = new Map(); + for (const group of this.resolvedViewGroups) { + for (const viewName of group.views) { + let groups = this.viewToGroups.get(viewName); + if (!groups) { + groups = []; + this.viewToGroups.set(viewName, groups); + } + groups.push(group.name); + } + } + } + + public get viewGroupList(): string[] { + return Array.from(this.viewGroupDefinitions.keys()); + } + + public get compiledViewGroups(): CompiledViewGroup[] { + return this.resolvedViewGroups; + } + + public viewGroupsForView(viewName: string): string[] { + return this.viewToGroups.get(viewName) || []; + } +} diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 334aadb8057e0..a8f4b2e451d5e 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -109,8 +109,15 @@ export class YamlCompiler { const transpiledView = this.transpileAndPrepareJsFile('view', { name, ...cube }, errorsReport); transpiledFilesContent.push(transpiledView); }); + } else if (key === 'view_groups') { + this.checkDuplicateNames(yamlObj.view_groups || [], errorsReport, (name) => `Found duplicate view group name '${name}'.`); + + (yamlObj.view_groups || []).forEach(({ name, ...viewGroup }) => { + const transpiledViewGroup = this.transpileViewGroup({ name, ...viewGroup }); + transpiledFilesContent.push(transpiledViewGroup); + }); } else { - errorsReport.error(`Unexpected YAML key: ${key}. Only 'cubes' and 'views' are allowed here.`); + errorsReport.error(`Unexpected YAML key: ${key}. Only 'cubes', 'views', and 'view_groups' are allowed here.`); } } @@ -121,6 +128,32 @@ export class YamlCompiler { } as FileContent; } + private transpileViewGroup(viewGroupObj): string { + const properties: t.ObjectProperty[] = []; + + if (viewGroupObj.title) { + properties.push(t.objectProperty(t.stringLiteral('title'), t.stringLiteral(viewGroupObj.title))); + } + if (viewGroupObj.description) { + properties.push(t.objectProperty(t.stringLiteral('description'), t.stringLiteral(viewGroupObj.description))); + } + if (viewGroupObj.views && Array.isArray(viewGroupObj.views)) { + properties.push( + t.objectProperty( + t.stringLiteral('views'), + t.arrayExpression(viewGroupObj.views.map((v: string) => t.stringLiteral(v))) + ) + ); + } + + const viewGroupCall = t.callExpression( + t.identifier('view_group'), + [t.stringLiteral(viewGroupObj.name), t.objectExpression(properties)] + ); + + return babelGenerator(viewGroupCall, {}, '').code; + } + private transpileAndPrepareJsFile(methodFn: ('cube' | 'view'), cubeObj, errorsReport: ErrorReporter): string { const yamlAst = this.transformYamlCubeObj(cubeObj, errorsReport); diff --git a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts index d007111181876..518099362fb87 100644 --- a/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/transpilers/CubePropContextTranspiler.ts @@ -26,6 +26,9 @@ export const transpiledFieldsPatterns: Array = [ /^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(timeDimensions|time_dimensions)\.\d+\.dimension$/, /^(preAggregations|pre_aggregations)\.[_a-zA-Z][_a-zA-Z0-9]*\.(outputColumnTypes|output_column_types)\.\d+\.member$/, /^contextMembers$/, + /^views$/, + /^(viewGroup|view_group)$/, + /^(viewGroups|view_groups)$/, /^includes$/, /^excludes$/, /^hierarchies\.[_a-zA-Z][_a-zA-Z0-9]*\.levels$/, @@ -68,7 +71,7 @@ export class CubePropContextTranspiler implements TranspilerInterface { this.knownIdentifiersInjectVisitor('extends', name => this.cubeDictionary.resolveCube(name)) ); } - } else if (path.node.callee.name === 'context') { + } else if (path.node.callee.name === 'context' || path.node.callee.name === 'view_group') { args[args.length - 1].traverse(this.sqlAndReferencesFieldVisitor(null)); } } diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts index e1410703c36f6..bc115a1289233 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts @@ -1,6 +1,6 @@ import { CompileError } from '../../../src/compiler/CompileError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareJsCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler, prepareYamlCompiler } from '../../unit/PrepareCompiler'; import { prepareCompiler as originalPrepareCompiler } from '../../../src/compiler/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; @@ -476,4 +476,434 @@ describe('DataSchemaCompiler', () => { expect(error).toBeInstanceOf(CompileError); }); }); + + it('view_groups defined via standalone view_group()', async () => { + const { compiler, metaTransformer, viewGroupEvaluator } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + cube('Customers', { + sql: \`select * from customers\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view('customers_view', { + cubes: [{ + joinPath: Customers, + includes: '*' + }] + }) + + view_group('sales', { + title: 'Sales', + description: 'Sales related views', + views: [revenue, customers_view] + }); + `); + await compiler.compile(); + + expect(viewGroupEvaluator.viewGroupList).toEqual(['sales']); + expect(viewGroupEvaluator.compiledViewGroups).toEqual([{ + name: 'sales', + title: 'Sales', + description: 'Sales related views', + views: ['revenue', 'customers_view'], + }]); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + title: 'Sales', + description: 'Sales related views', + views: ['revenue', 'customers_view'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + + const customersView = metaTransformer.cubes.find(c => c.config.name === 'customers_view'); + expect(customersView?.config.viewGroups).toEqual(['sales']); + }); + + it('view_group with string references', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view_group('sales', { + title: 'Sales', + views: ['revenue'] + }); + `); + await compiler.compile(); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + title: 'Sales', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('view_group defined via view property', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + viewGroup: 'sales', + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view_group('sales', {}); + `); + await compiler.compile(); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('plural viewGroups property on view', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + viewGroups: ['sales', 'finance'], + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view_group('sales', { + title: 'Sales', + }); + + view_group('finance', { + title: 'Finance', + }); + `); + await compiler.compile(); + + const salesGroup = metaTransformer.viewGroups.find(g => g.name === 'sales'); + expect(salesGroup?.views).toContain('revenue'); + expect(salesGroup?.title).toBe('Sales'); + + const financeGroup = metaTransformer.viewGroups.find(g => g.name === 'finance'); + expect(financeGroup?.views).toContain('revenue'); + expect(financeGroup?.title).toBe('Finance'); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales', 'finance']); + }); + + it('singular viewGroup and plural viewGroups are merged', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + viewGroup: 'sales', + viewGroups: ['finance'], + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view_group('sales', {}); + view_group('finance', {}); + `); + await compiler.compile(); + + expect(metaTransformer.viewGroups).toHaveLength(2); + expect(metaTransformer.viewGroups.find(g => g.name === 'sales')?.views).toContain('revenue'); + expect(metaTransformer.viewGroups.find(g => g.name === 'finance')?.views).toContain('revenue'); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales', 'finance']); + }); + + it('view-level viewGroup as bare reference to view_group', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view_group('sales', { + title: 'Sales', + }); + + view('revenue', { + viewGroup: sales, + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + `); + await compiler.compile(); + + expect(metaTransformer.viewGroups.find(g => g.name === 'sales')?.views).toContain('revenue'); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('view-level viewGroups as bare references to view_group', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view_group('sales', { title: 'Sales' }); + view_group('finance', { title: 'Finance' }); + + view('revenue', { + viewGroups: [sales, finance], + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + `); + await compiler.compile(); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales', 'finance']); + }); + + it('view_group merges standalone and view-level definitions', async () => { + const { compiler, metaTransformer } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + cube('Customers', { + sql: \`select * from customers\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + viewGroup: 'sales', + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + + view('customers_view', { + cubes: [{ + joinPath: Customers, + includes: '*' + }] + }) + + view_group('sales', { + title: 'Sales', + description: 'Sales related views', + views: ['customers_view'] + }); + `); + await compiler.compile(); + + const salesGroup = metaTransformer.viewGroups.find(g => g.name === 'sales'); + expect(salesGroup).toBeDefined(); + expect(salesGroup?.title).toBe('Sales'); + expect(salesGroup?.description).toBe('Sales related views'); + expect(salesGroup?.views).toContain('customers_view'); + expect(salesGroup?.views).toContain('revenue'); + }); + + it('view_groups in YAML format', async () => { + const { compiler, metaTransformer, viewGroupEvaluator } = prepareYamlCompiler(` +cubes: + - name: Orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + type: number + sql: id + primary_key: true + +views: + - name: revenue + cubes: + - join_path: Orders + includes: '*' + +view_groups: + - name: sales + title: Sales + description: Sales related views + views: + - revenue + `); + await compiler.compile(); + + expect(viewGroupEvaluator.viewGroupList).toEqual(['sales']); + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + title: 'Sales', + description: 'Sales related views', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('view_group via view property in YAML', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: Orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + type: number + sql: id + primary_key: true + +views: + - name: revenue + view_group: sales + cubes: + - join_path: Orders + includes: '*' + +view_groups: + - name: sales + `); + await compiler.compile(); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('fails when view references non-existent view group', async () => { + const { compiler } = prepareJsCompiler(` + cube('Orders', { + sql: \`select * from orders\`, + measures: { + count: { type: 'count' }, + }, + dimensions: { + id: { type: 'number', sql: 'id', primaryKey: true }, + } + }) + + view('revenue', { + viewGroup: 'nonexistent', + cubes: [{ + joinPath: Orders, + includes: '*' + }] + }) + `); + + try { + await compiler.compile(); + throw new Error('compile must return an error'); + } catch (e: any) { + expect(e.message).toContain('View "revenue" references view group "nonexistent" which is not defined'); + } + }); }); diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 0a735d8183b79..bef4e2b69b76d 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -1200,6 +1200,431 @@ cubes: }); }); + describe('View groups', () => { + it('standalone view_groups definition', async () => { + const { compiler, metaTransformer, viewGroupEvaluator } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + title: Sales + description: Sales related views + views: + - revenue + `); + + await compiler.compile(); + + expect(viewGroupEvaluator.viewGroupList).toEqual(['sales']); + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + title: 'Sales', + description: 'Sales related views', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('view_group property on individual views', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + view_group: sales + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + `); + + await compiler.compile(); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + views: ['revenue'], + }]); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('merges standalone view_groups with view-level view_group', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: customers + sql_table: customers + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + view_group: sales + cubes: + - join_path: orders + includes: '*' + + - name: customers_view + cubes: + - join_path: customers + includes: '*' + +view_groups: + - name: sales + title: Sales + description: Sales related views + views: + - customers_view + `); + + await compiler.compile(); + + const salesGroup = metaTransformer.viewGroups.find(g => g.name === 'sales'); + expect(salesGroup).toBeDefined(); + expect(salesGroup?.title).toBe('Sales'); + expect(salesGroup?.description).toBe('Sales related views'); + expect(salesGroup?.views).toContain('customers_view'); + expect(salesGroup?.views).toContain('revenue'); + }); + + it('multiple view_groups', async () => { + const { compiler, metaTransformer, viewGroupEvaluator } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: users + sql_table: users + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + cubes: + - join_path: orders + includes: '*' + + - name: users_view + cubes: + - join_path: users + includes: '*' + +view_groups: + - name: sales + title: Sales + views: + - revenue + - name: people + title: People + description: People related views + views: + - users_view + `); + + await compiler.compile(); + + expect(viewGroupEvaluator.viewGroupList).toEqual(['sales', 'people']); + expect(metaTransformer.viewGroups).toHaveLength(2); + + const salesGroup = metaTransformer.viewGroups.find(g => g.name === 'sales'); + expect(salesGroup?.title).toBe('Sales'); + expect(salesGroup?.views).toEqual(['revenue']); + + const peopleGroup = metaTransformer.viewGroups.find(g => g.name === 'people'); + expect(peopleGroup?.title).toBe('People'); + expect(peopleGroup?.description).toBe('People related views'); + expect(peopleGroup?.views).toEqual(['users_view']); + }); + + it('detects duplicate view group names', async () => { + const { compiler } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + +view_groups: + - name: sales + views: + - some_view + - name: sales + views: + - other_view + `); + + try { + await compiler.compile(); + throw new Error('compile must return an error'); + } catch (e: any) { + expect(e.message).toContain("Found duplicate view group name 'sales'"); + } + }); + + it('empty view_groups section', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + +view_groups: + `); + + await compiler.compile(); + expect(metaTransformer.viewGroups).toEqual([]); + }); + + it('view_group without title or description', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + views: + - revenue + `); + + await compiler.compile(); + + expect(metaTransformer.viewGroups).toEqual([{ + name: 'sales', + views: ['revenue'], + }]); + }); + + it('cubes are not assigned to view groups', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + views: + - revenue + `); + + await compiler.compile(); + + const ordersCube = metaTransformer.cubes.find(c => c.config.name === 'orders'); + expect(ordersCube?.config.viewGroups).toBeUndefined(); + }); + + it('plural view_groups property on view in YAML', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + view_groups: + - sales + - finance + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + title: Sales + - name: finance + title: Finance + `); + + await compiler.compile(); + + const salesGroup = metaTransformer.viewGroups.find(g => g.name === 'sales'); + expect(salesGroup?.title).toBe('Sales'); + expect(salesGroup?.views).toContain('revenue'); + + const financeGroup = metaTransformer.viewGroups.find(g => g.name === 'finance'); + expect(financeGroup?.title).toBe('Finance'); + expect(financeGroup?.views).toContain('revenue'); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales', 'finance']); + }); + + it('singular view_group sets viewGroups', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + view_group: sales + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + `); + + await compiler.compile(); + + expect(metaTransformer.viewGroups).toHaveLength(1); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales']); + }); + + it('merges singular view_group and plural view_groups in YAML', async () => { + const { compiler, metaTransformer } = prepareYamlCompiler(` +cubes: + - name: orders + sql_table: orders + measures: + - name: count + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + +views: + - name: revenue + view_group: sales + view_groups: + - finance + cubes: + - join_path: orders + includes: '*' + +view_groups: + - name: sales + - name: finance + `); + + await compiler.compile(); + + expect(metaTransformer.viewGroups).toHaveLength(2); + + const revenueView = metaTransformer.cubes.find(c => c.config.name === 'revenue'); + expect(revenueView?.config.viewGroups).toEqual(['sales', 'finance']); + expect(metaTransformer.viewGroups.find(g => g.name === 'sales')?.views).toContain('revenue'); + expect(metaTransformer.viewGroups.find(g => g.name === 'finance')?.views).toContain('revenue'); + }); + }); + describe('Mask SQL with shorthand', () => { it('userAttributes shorthand in mask sql should compile and resolve', async () => { const compilers = prepareYamlCompiler(` diff --git a/packages/cubejs-server-core/src/core/CompilerApi.ts b/packages/cubejs-server-core/src/core/CompilerApi.ts index b0e2b1646efbe..0750a360b5cc9 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.ts +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -959,15 +959,19 @@ export class CompilerApi { public async metaConfig( requestContext: Context, - options: { includeCompilerId?: boolean; skipVisibilityPatch?: boolean; requestId?: string } = {} + options: { includeCompilerId?: boolean; includeViewGroups?: boolean; skipVisibilityPatch?: boolean; requestId?: string } = {} ): Promise { - const { includeCompilerId, skipVisibilityPatch, ...restOptions } = options; + const { includeCompilerId, includeViewGroups, skipVisibilityPatch, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); const { cubes } = compilers.metaTransformer; if (skipVisibilityPatch) { - if (includeCompilerId) { - return { cubes, compilerId: compilers.compilerId }; + if (includeCompilerId || includeViewGroups) { + const result: any = { cubes, compilerId: compilers.compilerId }; + if (includeViewGroups) { + result.viewGroups = compilers.metaTransformer.viewGroups; + } + return result; } return cubes; } @@ -977,15 +981,15 @@ export class CompilerApi { requestContext, cubes ); - if (includeCompilerId) { - return { + if (includeCompilerId || includeViewGroups) { + const result: any = { cubes: patchedCubes, - // This compilerId is primarily used by the cubejs-backend-native or caching purposes. - // By default, it doesn't account for member visibility changes introduced above by DAP. - // Here we're modifying the original compilerId in a way that it's distinct for - // distinct schema versions while still being a valid UUID. compilerId: visibilityMaskHash ? this.mixInVisibilityMaskHash(compilers.compilerId, visibilityMaskHash) : compilers.compilerId, }; + if (includeViewGroups) { + result.viewGroups = compilers.metaTransformer.viewGroups; + } + return result; } return patchedCubes; } diff --git a/packages/cubejs-server-core/test/unit/index.test.ts b/packages/cubejs-server-core/test/unit/index.test.ts index 1b7a6d772de2c..64c158e944ed9 100644 --- a/packages/cubejs-server-core/test/unit/index.test.ts +++ b/packages/cubejs-server-core/test/unit/index.test.ts @@ -369,6 +369,16 @@ describe('index.test', () => { metaConfigSpy.mockClear(); }); + test('CompilerApi metaConfig with includeViewGroups', async () => { + const metaConfig = await compilerApi.metaConfig({ securityContext: {} }, { requestId: 'XXX', includeViewGroups: true }); + expect(metaConfig).toHaveProperty('cubes'); + expect(metaConfig).toHaveProperty('viewGroups'); + expect((metaConfig.cubes)?.length).toBeGreaterThan(0); + expect(metaConfig.cubes[0]).toHaveProperty('config'); + expect(metaConfigSpy).toHaveBeenCalled(); + metaConfigSpy.mockClear(); + }); + test('CompilerApi metaConfigExtended', async () => { const metaConfigExtended = await compilerApi.metaConfigExtended({ securityContext: {} }, { requestId: 'XXX' }); expect(metaConfigExtended).toHaveProperty('metaConfig'); @@ -396,6 +406,16 @@ describe('index.test', () => { metaConfigSpy.mockClear(); }); + test('CompilerApi metaConfig with includeViewGroups', async () => { + const metaConfig = await compilerApi.metaConfig({ securityContext: {} }, { requestId: 'XXX', includeViewGroups: true }); + expect(metaConfig).toHaveProperty('cubes'); + expect(metaConfig).toHaveProperty('viewGroups'); + expect(metaConfig.cubes).toEqual([]); + expect(metaConfig.viewGroups).toEqual([]); + expect(metaConfigSpy).toHaveBeenCalled(); + metaConfigSpy.mockClear(); + }); + test('CompilerApi metaConfigExtended', async () => { const metaConfigExtended = await compilerApi.metaConfigExtended({ securityContext: {} }, { requestId: 'XXX' }); expect(metaConfigExtended).toHaveProperty('metaConfig'); diff --git a/packages/cubejs-testing/birdbox-fixtures/view-groups/schema/Models.js b/packages/cubejs-testing/birdbox-fixtures/view-groups/schema/Models.js new file mode 100644 index 0000000000000..442639890d908 --- /dev/null +++ b/packages/cubejs-testing/birdbox-fixtures/view-groups/schema/Models.js @@ -0,0 +1,115 @@ +cube(`Orders`, { + sql: ` + select 1 as id, 100 as amount, 'new' as status + UNION ALL + select 2 as id, 200 as amount, 'processed' as status + `, + + measures: { + count: { + type: `count`, + }, + totalAmount: { + sql: `amount`, + type: `sum`, + }, + }, + + dimensions: { + id: { + sql: `id`, + type: `number`, + primaryKey: true, + }, + status: { + sql: `status`, + type: `string`, + }, + }, +}); + +cube(`Customers`, { + sql: ` + select 1 as id, 'Alice' as name + UNION ALL + select 2 as id, 'Bob' as name + `, + + measures: { + count: { + type: `count`, + }, + }, + + dimensions: { + id: { + sql: `id`, + type: `number`, + primaryKey: true, + }, + name: { + sql: `name`, + type: `string`, + }, + }, +}); + +cube(`Products`, { + sql: ` + select 1 as id, 'Widget' as title + UNION ALL + select 2 as id, 'Gadget' as title + `, + + measures: { + count: { + type: `count`, + }, + }, + + dimensions: { + id: { + sql: `id`, + type: `number`, + primaryKey: true, + }, + title: { + sql: `title`, + type: `string`, + }, + }, +}); + +view(`RevenueView`, { + cubes: [{ + joinPath: Orders, + includes: `*`, + }], +}); + +view(`CustomersView`, { + viewGroup: `sales`, + cubes: [{ + joinPath: Customers, + includes: `*`, + }], +}); + +view(`CatalogView`, { + viewGroups: [`inventory`, `sales`], + cubes: [{ + joinPath: Products, + includes: `*`, + }], +}); + +view_group(`sales`, { + title: `Sales`, + description: `Sales related views`, + views: [`RevenueView`], +}); + +view_group(`inventory`, { + title: `Inventory`, + description: `Inventory related views`, +}); diff --git a/packages/cubejs-testing/package.json b/packages/cubejs-testing/package.json index 0b3c05f9f0262..c5c42ae258308 100644 --- a/packages/cubejs-testing/package.json +++ b/packages/cubejs-testing/package.json @@ -93,7 +93,8 @@ "smoke:mssql": "jest --verbose -i dist/test/smoke-mssql.test.js", "smoke:mssql:snapshot": "jest --verbose --forceExit --updateSnapshot -i dist/test/smoke-mssql.test.js", "smoke:duckdb": "jest --verbose -i dist/test/smoke-duckdb.test.js", - "smoke:duckdb:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-duckdb.test.js" + "smoke:duckdb:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-duckdb.test.js", + "smoke:view-groups": "jest --verbose --forceExit -i dist/test/smoke-view-groups.test.js" }, "files": [ "dist/src", diff --git a/packages/cubejs-testing/test/smoke-view-groups.test.ts b/packages/cubejs-testing/test/smoke-view-groups.test.ts new file mode 100644 index 0000000000000..0bb37b5f57b6a --- /dev/null +++ b/packages/cubejs-testing/test/smoke-view-groups.test.ts @@ -0,0 +1,116 @@ +import cubejs, { CubeApi } from '@cubejs-client/core'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { afterAll, beforeAll, describe, expect, jest, test } from '@jest/globals'; +import { BirdBox, getBirdbox } from '../src'; +import { + DEFAULT_API_TOKEN, + DEFAULT_CONFIG, + JEST_AFTER_ALL_DEFAULT_TIMEOUT, + JEST_BEFORE_ALL_DEFAULT_TIMEOUT, +} from './smoke-tests'; + +describe('view groups', () => { + jest.setTimeout(60 * 5 * 1000); + let birdbox: BirdBox; + let client: CubeApi; + + beforeAll(async () => { + birdbox = await getBirdbox( + 'duckdb', + { + CUBEJS_DB_TYPE: 'duckdb', + ...DEFAULT_CONFIG, + }, + { + schemaDir: 'view-groups/schema', + } + ); + client = cubejs(async () => DEFAULT_API_TOKEN, { + apiUrl: birdbox.configuration.apiUrl, + }); + }, JEST_BEFORE_ALL_DEFAULT_TIMEOUT); + + afterAll(async () => { + await birdbox.stop(); + }, JEST_AFTER_ALL_DEFAULT_TIMEOUT); + + test('meta response includes viewGroups', async () => { + const meta = await client.meta(); + + expect(meta.meta.viewGroups).toBeDefined(); + expect(meta.meta.viewGroups!.length).toBeGreaterThan(0); + }); + + test('standalone view_group definition is returned', async () => { + const meta = await client.meta(); + const viewGroups = meta.meta.viewGroups!; + + const salesGroup = viewGroups.find((g) => g.name === 'sales'); + expect(salesGroup).toBeDefined(); + expect(salesGroup!.title).toBe('Sales'); + expect(salesGroup!.description).toBe('Sales related views'); + expect(salesGroup!.views).toContain('RevenueView'); + }); + + test('view_group collects views from view-level viewGroup property', async () => { + const meta = await client.meta(); + const viewGroups = meta.meta.viewGroups!; + + const salesGroup = viewGroups.find((g) => g.name === 'sales'); + expect(salesGroup!.views).toContain('CustomersView'); + }); + + test('view_group collects views from plural viewGroups property', async () => { + const meta = await client.meta(); + const viewGroups = meta.meta.viewGroups!; + + const salesGroup = viewGroups.find((g) => g.name === 'sales'); + expect(salesGroup!.views).toContain('CatalogView'); + + const inventoryGroup = viewGroups.find((g) => g.name === 'inventory'); + expect(inventoryGroup).toBeDefined(); + expect(inventoryGroup!.title).toBe('Inventory'); + expect(inventoryGroup!.views).toContain('CatalogView'); + }); + + test('view cube config includes viewGroups reference', async () => { + const meta = await client.meta(); + + const revenueView = meta.cubes.find((c) => c.name === 'RevenueView'); + expect(revenueView).toBeDefined(); + expect(revenueView!.viewGroups).toEqual(expect.arrayContaining(['sales'])); + + const customersView = meta.cubes.find((c) => c.name === 'CustomersView'); + expect(customersView).toBeDefined(); + expect(customersView!.viewGroups).toEqual(['sales']); + }); + + test('cubes without view groups do not have viewGroups', async () => { + const meta = await client.meta(); + + const ordersCube = meta.cubes.find((c) => c.name === 'Orders'); + expect(ordersCube).toBeDefined(); + expect(ordersCube!.viewGroups).toBeUndefined(); + }); + + test('views can be queried normally', async () => { + const response = await client.load({ + measures: ['RevenueView.count'], + }); + expect(response.rawData()[0]['RevenueView.count']).toBe('2'); + }); + + test('view in a view group can be queried', async () => { + const response = await client.load({ + measures: ['CustomersView.count'], + }); + expect(response.rawData()[0]['CustomersView.count']).toBe('2'); + }); + + test('view in multiple view groups can be queried', async () => { + const response = await client.load({ + measures: ['CatalogView.count'], + }); + expect(response.rawData()[0]['CatalogView.count']).toBe('2'); + }); +});