[pull] master from cube-js:master

.github/actions/smoke.sh

@@ -14,6 +14,10 @@ yarn lerna run --concurrency 1 --stream --no-prefix integration:duckdb
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:duckdb
 echo "::endgroup::"
 
+echo "::group::View Groups"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:view-groups
+echo "::endgroup::"
+
 echo "::group::Postgres"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:postgres
 echo "::endgroup::"

packages/cubejs-api-gateway/src/gateway.ts

@@ -649,19 +649,30 @@ class ApiGateway {
       const compilerApi = await this.getCompilerApi(context);
       const metaConfig = await compilerApi.metaConfig(context, {
         requestId: context.requestId,
-        includeCompilerId: includeCompilerId || onlyCompilerId
+        includeCompilerId: includeCompilerId || onlyCompilerId,
+        includeViewGroups: true,
       });
       if (onlyCompilerId) {
-        const response: { cubes: any[], compilerId?: string } = {
+        const response: { cubes: any[], viewGroups?: any[], compilerId?: string } = {
           cubes: [],
           compilerId: metaConfig.compilerId
         };
         res(response);
         return;
       }
-      const cubesConfig = includeCompilerId ? metaConfig.cubes : metaConfig;
+      const cubesConfig = metaConfig.cubes;
       const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config);
-      const response: { cubes: any[], compilerId?: string } = { cubes };
+      const visibleCubeNames = new Set(cubes.map(c => c.name));
+      const viewGroups = (metaConfig.viewGroups || [])
+        .map(group => ({
+          ...group,
+          views: group.views.filter((v: string) => visibleCubeNames.has(v)),
+        }))
+        .filter(group => group.views.length > 0);
+      const response: { cubes: any[], viewGroups?: any[], compilerId?: string } = { cubes };
+      if (viewGroups.length > 0) {
+        response.viewGroups = viewGroups;
+      }
       if (includeCompilerId) {
         response.compilerId = metaConfig.compilerId;
       }

packages/cubejs-api-gateway/src/types/request.ts

@@ -95,7 +95,7 @@ type ErrorResponse = {
   error: string,
 };
 
-type MetaResponse = { cubes: any[], compilerId?: string };
+type MetaResponse = { cubes: any[], viewGroups?: any[], compilerId?: string };
 type MetaResponseResultFn = (message: MetaResponse | ErrorResponse) => void;
 
 /**

packages/cubejs-api-gateway/test/index.test.ts

@@ -686,6 +686,35 @@ describe('API Gateway', () => {
     expect(res.body.cubes[0]?.segments.find(segment => segment.name === 'Foo.quux').description).toBe('segment from compilerApi mock');
   });
 
+  test('meta endpoint returns view groups', async () => {
+    const { app } = await createApiGateway();
+
+    const res = await request(app)
+      .get('/cubejs-api/v1/meta')
+      .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M')
+      .expect(200);
+
+    expect(res.body).toHaveProperty('cubes');
+    expect(res.body).toHaveProperty('viewGroups');
+
+    expect(res.body.viewGroups).toHaveLength(1);
+    expect(res.body.viewGroups[0]).toEqual({
+      name: 'analytics',
+      title: 'Analytics',
+      description: 'Analytics related views',
+      views: ['FooView'],
+    });
+
+    const fooView = res.body.cubes.find(c => c.name === 'FooView');
+    expect(fooView).toBeDefined();
+    expect(fooView.viewGroups).toEqual(['analytics']);
+    expect(fooView.type).toBe('view');
+
+    const fooCube = res.body.cubes.find(c => c.name === 'Foo');
+    expect(fooCube).toBeDefined();
+    expect(fooCube.viewGroups).toBeUndefined();
+  });
+
   test('meta endpoint extended to get schema information with additional data', async () => {
     const { app } = await createApiGateway();
 

packages/cubejs-api-gateway/test/mocks.ts

@@ -80,11 +80,12 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
     return { query, denied: false };
   },
 
-  async metaConfig() {
-    return [
+  async metaConfig(_ctx, options: any = {}) {
+    const cubes = [
       {
         config: {
           name: 'Foo',
+          type: 'cube',
           description: 'cube from compilerApi mock',
           measures: [
             {
@@ -125,7 +126,48 @@ export const compilerApi = jest.fn().mockImplementation(async () => ({
           ],
         },
       },
+      {
+        config: {
+          name: 'FooView',
+          type: 'view',
+          description: 'view from compilerApi mock',
+          viewGroups: ['analytics'],
+          measures: [
+            {
+              name: 'FooView.bar',
+              isVisible: true,
+            },
+          ],
+          dimensions: [
+            {
+              name: 'FooView.id',
+              isVisible: true,
+            },
+          ],
+          segments: [],
+        },
+      },
     ];
+
+    if (options.includeCompilerId || options.includeViewGroups) {
+      const result: any = { cubes };
+      if (options.includeCompilerId) {
+        result.compilerId = 'mock-compiler-id';
+      }
+      if (options.includeViewGroups) {
+        result.viewGroups = [
+          {
+            name: 'analytics',
+            title: 'Analytics',
+            description: 'Analytics related views',
+            views: ['FooView'],
+          },
+        ];
+      }
+      return result;
+    }
+
+    return cubes;
   },
 
   async metaConfigExtended() {

packages/cubejs-client-core/src/types.ts

@@ -527,6 +527,7 @@ export type Cube = {
   isVisible?: boolean;
   public?: boolean;
   meta?: any;
+  viewGroups?: string[];
 };
 
 export type CubeMap = {
@@ -540,8 +541,16 @@ export type CubesMap = Record<
   CubeMap
 >;
 
+export type ViewGroup = {
+  name: string;
+  title?: string;
+  description?: string;
+  views: string[];
+};
+
 export type MetaResponse = {
   cubes: Cube[];
+  viewGroups?: ViewGroup[];
 };
 
 export type FilterOperator = {

packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts

@@ -214,6 +214,8 @@ export interface CubeDefinition {
   excludes?: any;
   cubes?: any;
   isView?: boolean;
+  viewGroup?: string | ((...args: any[]) => any);
+  viewGroups?: string[] | ((...args: any[]) => any);
   calendar?: boolean;
   isSplitView?: boolean;
   includedMembers?: ViewIncludedMember[];

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts

@@ -15,6 +15,7 @@ import type { CubeDefinitionExtended } from './CubeSymbols';
 import type { CubeValidator } from './CubeValidator';
 import type { CubeEvaluator } from './CubeEvaluator';
 import type { ContextEvaluator } from './ContextEvaluator';
+import type { ViewGroupEvaluator, CompiledViewGroup } from './ViewGroupEvaluator';
 import type { JoinGraph } from './JoinGraph';
 import type { ErrorReporter } from './ErrorReporter';
 import { CompilerInterface } from './PrepareCompiler';
@@ -139,6 +140,7 @@ export type CubeConfig = {
   isVisible: boolean;
   public: boolean;
   description?: string;
+  viewGroups?: string[];
   connectedComponent: number;
   meta?: any;
   measures: MeasureConfig[];
@@ -162,6 +164,8 @@ export class CubeToMetaTransformer implements CompilerInterface {
 
   private readonly contextEvaluator: ContextEvaluator;
 
+  private readonly viewGroupEvaluator: ViewGroupEvaluator;
+
   private readonly joinGraph: JoinGraph;
 
   public cubes: TransformedCube[];
@@ -175,17 +179,23 @@ export class CubeToMetaTransformer implements CompilerInterface {
     cubeValidator: CubeValidator,
     cubeEvaluator: CubeEvaluator,
     contextEvaluator: ContextEvaluator,
+    viewGroupEvaluator: ViewGroupEvaluator,
     joinGraph: JoinGraph
   ) {
     this.cubeValidator = cubeValidator;
     this.cubeSymbols = cubeEvaluator;
     this.cubeEvaluator = cubeEvaluator;
     this.contextEvaluator = contextEvaluator;
+    this.viewGroupEvaluator = viewGroupEvaluator;
     this.joinGraph = joinGraph;
     this.cubes = [];
     this.queries = [];
   }
 
+  public get viewGroups(): CompiledViewGroup[] {
+    return this.viewGroupEvaluator.compiledViewGroups;
+  }
+
   public compile(_cubes: any[], errorReporter: ErrorReporter): void {
     this.cubes = this.cubeSymbols.cubeList
       .filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator))
@@ -239,6 +249,10 @@ export class CubeToMetaTransformer implements CompilerInterface {
 
     const nestedFolders: NestedFolder[] = (extendedCube.folders || []).map((f: Folder) => processFolder(f));
 
+    const viewGroupNames = extendedCube.isView
+      ? this.viewGroupEvaluator.viewGroupsForView(cubeName)
+      : [];
+
     return {
       config: {
         name: cubeName,
@@ -247,6 +261,7 @@ export class CubeToMetaTransformer implements CompilerInterface {
         isVisible: isCubeVisible,
         public: isCubeVisible,
         description: extendedCube.description,
+        ...(viewGroupNames.length > 0 ? { viewGroups: viewGroupNames } : {}),
         connectedComponent: this.joinGraph.connectedComponents()[cubeName],
         meta: extendedCube.meta,
         measures: Object.entries(extendedCube.measures || {}).map((nameToMetric: [string, any]) => {

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -1132,6 +1132,8 @@ const folderSchema = Joi.object().keys({
 
 const viewSchema = inherit(baseSchema, {
   isView: Joi.boolean().strict(),
+  viewGroup: Joi.alternatives([Joi.string(), Joi.func()]),
+  viewGroups: Joi.alternatives([Joi.array().items(Joi.string().required()), Joi.func()]),
   cubes: Joi.array().items(
     Joi.object().keys({
       joinPath: Joi.func().required(),

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -74,6 +74,8 @@ export type DataSchemaCompilerOptions = {
   cubeAndViewSymbols: CubeSymbols;
   cubeCompilers?: CompilerInterface[];
   contextCompilers?: CompilerInterface[];
+  viewGroupCompilers?: CompilerInterface[];
+  metaCompilers?: CompilerInterface[];
   transpilers?: TranspilerInterface[];
   viewCompilers?: CompilerInterface[];
   viewCompilationGate: ViewCompilationGate;
@@ -105,6 +107,8 @@ export type CompileStage = 0 | 1 | 2 | 3;
 type CompileCubeFilesCompilers = {
   cubeCompilers?: CompilerInterface[];
   contextCompilers?: CompilerInterface[];
+  viewGroupCompilers?: CompilerInterface[];
+  metaCompilers?: CompilerInterface[];
 };
 
 export type CompileContext = any;
@@ -116,6 +120,10 @@ export class DataSchemaCompiler {
 
   private readonly contextCompilers: CompilerInterface[];
 
+  private readonly viewGroupCompilers: CompilerInterface[];
+
+  private readonly metaCompilers: CompilerInterface[];
+
   private readonly transpilers: TranspilerInterface[];
 
   private readonly viewCompilers: CompilerInterface[];
@@ -181,6 +189,8 @@ export class DataSchemaCompiler {
     this.repository = repository;
     this.cubeCompilers = options.cubeCompilers || [];
     this.contextCompilers = options.contextCompilers || [];
+    this.viewGroupCompilers = options.viewGroupCompilers || [];
+    this.metaCompilers = options.metaCompilers || [];
     this.transpilers = options.transpilers || [];
     this.viewCompilers = options.viewCompilers || [];
     this.preTranspileCubeCompilers = options.preTranspileCubeCompilers || [];
@@ -366,6 +376,7 @@ export class DataSchemaCompiler {
     let cubes: CubeDefinition[] = [];
     let exports: Record<string, Record<string, any>> = {};
     let contexts: Record<string, any>[] = [];
+    let viewGroups: Record<string, any>[] = [];
     let compiledFiles: Record<string, boolean> = {};
     let asyncModules: CallableFunction[] = [];
     let transpiledFiles: FileContent[] = [];
@@ -374,6 +385,7 @@ export class DataSchemaCompiler {
       cubes = [];
       exports = {};
       contexts = [];
+      viewGroups = [];
       compiledFiles = {};
       asyncModules = [];
     };
@@ -404,6 +416,15 @@ export class DataSchemaCompiler {
         }
         return contexts.push({ ...context, name, fileName: file.fileName });
       },
+      view_group: (name: string, viewGroup) => {
+        const file = ctxFileStorage.getStore();
+        if (!file) {
+          throw new Error('No file stored in context');
+        }
+        viewGroups.push({ ...viewGroup, name, fileName: file.fileName });
+        this.compileV8ContextCache![name] = name;
+        return name;
+      },
       addExport: (obj) => {
         const file = ctxFileStorage.getStore();
         if (!file) {
@@ -473,15 +494,15 @@ export class DataSchemaCompiler {
       const convertedToJsFiles = transpiledFiles.filter(f => !f.fileName.endsWith('.js'));
       toCompile = [...originalJsFiles, ...convertedToJsFiles];
 
-      return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
+      return this.compileCubeFiles(cubes, contexts, viewGroups, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
     };
 
     const compilePhase = async (compilers: CompileCubeFilesCompilers, stage: 0 | 1 | 2 | 3) => {
       // clear the objects for the next phase
       cleanup();
       transpiledFiles = await transpilePhase(stage);
 
-      return this.compileCubeFiles(cubes, contexts, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
+      return this.compileCubeFiles(cubes, contexts, viewGroups, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
     };
 
     return compilePhaseFirst({ cubeCompilers: this.cubeNameCompilers }, 0)
@@ -492,6 +513,8 @@ export class DataSchemaCompiler {
       .then(() => compilePhase({
         cubeCompilers: this.cubeCompilers,
         contextCompilers: this.contextCompilers,
+        viewGroupCompilers: this.viewGroupCompilers,
+        metaCompilers: this.metaCompilers,
       }, 3))
       .then(() => {
         // Free unneeded resources
@@ -820,6 +843,7 @@ export class DataSchemaCompiler {
   private async compileCubeFiles(
     cubes: CubeDefinition[],
     contexts: Record<string, any>[],
+    viewGroups: Record<string, any>[],
     compiledFiles: Record<string, boolean>,
     asyncModules: CallableFunction[],
     compilers: CompileCubeFilesCompilers,
@@ -836,7 +860,9 @@ export class DataSchemaCompiler {
       });
     await asyncModules.reduce((a: Promise<void>, b: CallableFunction) => a.then(() => b()), Promise.resolve());
     return this.compileObjects(compilers.cubeCompilers || [], cubes, errorsReport)
-      .then(() => this.compileObjects(compilers.contextCompilers || [], contexts, errorsReport));
+      .then(() => this.compileObjects(compilers.contextCompilers || [], contexts, errorsReport))
+      .then(() => this.compileObjects(compilers.viewGroupCompilers || [], viewGroups, errorsReport))
+      .then(() => this.compileObjects(compilers.metaCompilers || [], cubes, errorsReport));
   }
 
   public throwIfAnyErrors() {