feat(schema-compiler,api-gateway): Nested folders support

packages/cubejs-api-gateway/openspec.yml

@@ -175,6 +175,21 @@ components:
           type: array
           items:
             type: "string"
+    V1CubeMetaNestedFolder:
+      type: "object"
+      required:
+        - name
+        - members
+      properties:
+        name:
+          type: "string"
+        members:
+          type: array
+          items:
+            type: "string"
+            oneOf:
+              - type: string
+              - $ref: "#/components/schemas/V1CubeMetaNestedFolder"
     V1CubeMetaHierarchy:
       type: "object"
       required:
@@ -231,6 +246,10 @@ components:
           type: "array"
           items:
             $ref: "#/components/schemas/V1CubeMetaFolder"
+        nestedFolders:
+          type: "array"
+          items:
+            $ref: "#/components/schemas/V1CubeMetaNestedFolder"
         hierarchies:
           type: "array"
           items:

packages/cubejs-backend-shared/src/env.ts

@@ -240,6 +240,9 @@ const variables: Record<string, (...args: any) => any> = {
   transpilationNative: () => get('CUBEJS_TRANSPILATION_NATIVE')
     .default('false')
     .asBoolStrict(),
+  nestedFoldersDelimiter: () => get('CUBEJS_NESTED_FOLDERS_DELIMITER')
+    .default('')
+    .asString(),
 
   /** ****************************************************************
    * Common db options                                               *

packages/cubejs-client-core/src/types.ts

@@ -417,6 +417,11 @@ export type TCubeFolder = {
   members: string[];
 };
 
+export type TCubeNestedFolder = {
+  name: string;
+  members: (string | TCubeNestedFolder)[];
+};
+
 export type TCubeHierarchy = {
   name: string;
   title?: string;
@@ -451,6 +456,7 @@ export type Cube = {
   dimensions: TCubeDimension[];
   segments: TCubeSegment[];
   folders: TCubeFolder[];
+  nestedFolders: TCubeNestedFolder[];
   hierarchies: TCubeHierarchy[];
   connectedComponent?: number;
   type?: 'view' | 'cube';

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -259,34 +259,51 @@ export class CubeEvaluator extends CubeSymbols {
 
   private prepareFolders(cube: any, errorReporter: ErrorReporter) {
     const folders = cube.rawFolders();
-    if (folders.length) {
-      cube.folders = folders.map(it => {
-        const includedMembers = this.allMembersOrList(cube, it.includes);
-        const includes = includedMembers.map(memberName => {
-          if (memberName.includes('.')) {
-            errorReporter.error(
-              `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
-            );
-          }
+    if (!folders.length) return;
 
-          const member = cube.includedMembers.find(m => m.name === memberName);
-          if (!member) {
-            errorReporter.error(
-              `Member '${memberName}' included in folder '${it.name}' not found`
-            );
-            return null;
-          }
+    const checkMember = (memberName: string, folderName: string) => {
+      if (memberName.includes('.')) {
+        errorReporter.error(
+          `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
+        );
+      }
+
+      const member = cube.includedMembers.find(m => m.name === memberName);
+      if (!member) {
+        errorReporter.error(
+          `Member '${memberName}' included in folder '${folderName}' not found`
+        );
+        return null;
+      }
 
-          return member;
-        })
-          .filter(Boolean);
+      return member;
+    };
+
+    const processFolder = (folder: any): any => {
+      let includedMembers: string[];
+      let includes: any[] = [];
+
+      if (folder.includes === '*') {
+        includedMembers = this.allMembersOrList(cube, folder.includes);
+        includes = includedMembers.map(m => checkMember(m, folder.name)).filter(Boolean);
+      } else if (Array.isArray(folder.includes)) {
+        includes = folder.includes.map(item => {
+          if (typeof item === 'object' && item !== null) {
+            return processFolder(item);
+          }
 
-        return ({
-          ...it,
-          includes
+          return checkMember(item, folder.name);
         });
-      });
-    }
+      }
+
+      return {
+        ...folder,
+        type: 'folder',
+        includes: includes.filter(Boolean)
+      };
+    };
+
+    cube.folders = folders.map(processFolder);
   }
 
   private prepareHierarchies(cube: any, errorReporter: ErrorReporter): void {

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js

@@ -2,6 +2,7 @@ import inflection from 'inflection';
 import R from 'ramda';
 import camelCase from 'camelcase';
 
+import { getEnv } from '@cubejs-backend/shared';
 import { CubeSymbols } from './CubeSymbols';
 import { UserError } from './UserError';
 import { BaseMeasure } from '../adapter';
@@ -43,6 +44,43 @@ export class CubeToMetaTransformer {
 
     const isCubeVisible = this.isVisible(cube, true);
 
+    const flatFolderSeparator = getEnv('nestedFoldersDelimiter');
+    const flatFolders = [];
+
+    const processFolder = (folder, path = [], mergedMembers = []) => {
+      const flatMembers = [];
+      const nestedMembers = folder.includes.map(member => {
+        if (member.type === 'folder') {
+          return processFolder(member, [...path, folder.name], flatMembers);
+        }
+        const memberName = `${cube.name}.${member.name}`;
+        flatMembers.push(memberName);
+
+        return memberName;
+      });
+
+      if (flatFolderSeparator !== '') {
+        flatFolders.push({
+          name: [...path, folder.name].join(flatFolderSeparator),
+          members: flatMembers,
+        });
+      } else if (path.length > 0) {
+        mergedMembers.push(...flatMembers);
+      } else { // We're at the root level
+        flatFolders.push({
+          name: folder.name,
+          members: [...new Set(flatMembers)],
+        });
+      }
+
+      return {
+        name: folder.name,
+        members: nestedMembers,
+      };
+    };
+
+    const nestedFolders = (cube.folders || []).map(f => processFolder(f));
+
     return {
       config: {
         name: cube.name,
@@ -113,10 +151,8 @@ export class CubeToMetaTransformer {
           public: it.public ?? true,
           name: `${cube.name}.${it.name}`,
         })),
-        folders: (cube.folders || []).map((it) => ({
-          name: it.name,
-          members: it.includes.map(member => `${cube.name}.${member.name}`),
-        })),
+        folders: flatFolders,
+        nestedFolders,
       },
     };
   }

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -795,6 +795,19 @@ const cubeSchema = inherit(baseSchema, {
   'object.xor': 'You must use either sql or sqlTable within a model, but not both'
 });
 
+const folderSchema = Joi.object().keys({
+  name: Joi.string().required(),
+  includes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(
+      Joi.alternatives([
+        Joi.string().required(),
+        Joi.link('#folderSchema'), // Can contain nested folders
+      ]),
+    ),
+  ]).required(),
+}).id('folderSchema');
+
 const viewSchema = inherit(baseSchema, {
   isView: Joi.boolean().strict(),
   cubes: Joi.array().items(
@@ -822,13 +835,7 @@ const viewSchema = inherit(baseSchema, {
       'object.oxor': 'Using split together with prefix is not supported'
     })
   ),
-  folders: Joi.array().items(Joi.object().keys({
-    name: Joi.string().required(),
-    includes: Joi.alternatives([
-      Joi.string().valid('*'),
-      Joi.array().items(Joi.string().required())
-    ]).required(),
-  })),
+  folders: Joi.array().items(folderSchema),
 });
 
 function formatErrorMessageFromDetails(explain, d) {

packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap

@@ -1990,6 +1990,7 @@ Array [
       },
     ],
     "name": "folder1",
+    "type": "folder",
   },
   Object {
     "includes": Array [
@@ -2005,6 +2006,7 @@ Array [
       },
     ],
     "name": "folder2",
+    "type": "folder",
   },
 ]
 `;

packages/cubejs-schema-compiler/test/unit/__snapshots__/views.test.ts.snap

@@ -89,6 +89,7 @@ Object {
   ],
   "meta": undefined,
   "name": "simple_view",
+  "nestedFolders": Array [],
   "public": true,
   "segments": Array [],
   "title": "Simple View",
@@ -168,6 +169,7 @@ Object {
   ],
   "meta": undefined,
   "name": "simple_view",
+  "nestedFolders": Array [],
   "public": true,
   "segments": Array [],
   "title": "Simple View",
@@ -281,6 +283,7 @@ Object {
   ],
   "meta": undefined,
   "name": "simple_view",
+  "nestedFolders": Array [],
   "public": true,
   "segments": Array [],
   "title": "Simple View",
@@ -360,6 +363,7 @@ Object {
   ],
   "meta": undefined,
   "name": "simple_view",
+  "nestedFolders": Array [],
   "public": true,
   "segments": Array [],
   "title": "Simple View",

packages/cubejs-schema-compiler/test/unit/fixtures/folders.yml

@@ -105,6 +105,17 @@ views:
         includes:
           - users_city
           - users_renamed_in_view3_gender
+  - name: test_view4
+    extends: test_view3
+    folders:
+      - name: folder3
+        includes:
+          - users_city
+          - name: inner folder 4
+            includes:
+              - renamed_orders_status
+          - name: inner folder 5
+            includes: "*"
 
   # - name: empty_view
   #   cubes:

packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml

@@ -72,6 +72,9 @@ views:
           - age
           - renamed_gender
           - users.age
+          - name: inner folder
+            includes:
+              - users.renamed_gender
       - name: folder2
         includes: '*'
   - name: test_view2