diff --git a/packages/cubejs-api-gateway/src/interfaces.ts b/packages/cubejs-api-gateway/src/interfaces.ts index 681fa4507228c..0d1b5418fdbdd 100644 --- a/packages/cubejs-api-gateway/src/interfaces.ts +++ b/packages/cubejs-api-gateway/src/interfaces.ts @@ -21,6 +21,7 @@ import { Query, NormalizedQueryFilter, NormalizedQuery, + MemberExpression, } from './types/query'; import { @@ -69,6 +70,7 @@ export { Query, NormalizedQueryFilter, NormalizedQuery, + MemberExpression, JWTOptions, CheckAuthFn, CheckSQLAuthSuccessResponse, diff --git a/packages/cubejs-api-gateway/src/types/query.ts b/packages/cubejs-api-gateway/src/types/query.ts index 86c06c59ffce3..de9add137b8f5 100644 --- a/packages/cubejs-api-gateway/src/types/query.ts +++ b/packages/cubejs-api-gateway/src/types/query.ts @@ -146,11 +146,10 @@ interface Query { cache?: CacheMode; // Used in public interface ungrouped?: boolean; responseFormat?: ResultType; - // TODO incoming query, query with parsed exprs and query with evaluated exprs are all different types - subqueryJoins?: Array, - - joinHints?: Array + subqueryJoins?: Array; + joinHints?: Array; + requestId?: string; } /** diff --git a/packages/cubejs-schema-compiler/src/adapter/index.ts b/packages/cubejs-schema-compiler/src/adapter/index.ts index e1064c44437a4..7a4cb76cf5566 100644 --- a/packages/cubejs-schema-compiler/src/adapter/index.ts +++ b/packages/cubejs-schema-compiler/src/adapter/index.ts @@ -18,6 +18,8 @@ export * from './PostgresQuery'; export * from './MssqlQuery'; export * from './PrestodbQuery'; +export { PreAggregationReferences } from '../compiler/CubeEvaluator'; + // Candidates to move from this package to drivers packages // export * from './RedshiftQuery'; // export * from './SnowflakeQuery'; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 058b6fcb999a4..24c298f91947a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -9,7 +9,8 @@ import { JoinDefinition, PreAggregationDefinition, PreAggregationDefinitionRollup, - type ToString + type ToString, + ViewIncludedMember } from './CubeSymbols'; import { UserError } from './UserError'; import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; @@ -100,7 +101,7 @@ export type PreAggregationReferences = { export type PreAggregationInfo = { id: string, preAggregationName: string, - preAggregation: unknown, + preAggregation: any, cube: string, references: PreAggregationReferences, refreshKey: unknown, @@ -124,6 +125,7 @@ export type EvaluatedFolder = { }; export type EvaluatedCube = { + name: string; measures: Record; dimensions: Record; segments: Record; @@ -136,6 +138,8 @@ export type EvaluatedCube = { sql?: (...args: any[]) => string; sqlTable?: (...args: any[]) => string; accessPolicy?: AccessPolicyDefinition[]; + isView?: boolean; + includedMembers?: ViewIncludedMember[]; }; export class CubeEvaluator extends CubeSymbols { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index cac241dc0eaa3..fc13bad01feda 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -133,6 +133,9 @@ export type AccessPolicyDefinition = { includesMembers?: string[]; excludesMembers?: string[]; }; + conditions?: { + if: Function; + }[] }; export type ViewIncludedMember = { @@ -233,7 +236,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface private builtCubes: Record; - private cubeDefinitions: Record; + public cubeDefinitions: Record; private funcArgumentsValues: Record; @@ -925,7 +928,7 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface * resolveSymbolsCall are sync. Async support may be added later with deeper * refactoring. */ - protected evaluateContextFunction(cube: any, contextFn: any, context: any = {}) { + public evaluateContextFunction(cube: any, contextFn: any, context: any = {}) { return this.resolveSymbolsCall(contextFn, (name: string) => { const resolvedSymbol = this.resolveSymbol(cube, name); if (resolvedSymbol) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 036a102f680d8..eb9c786757f12 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -135,7 +135,7 @@ export class CubeToMetaTransformer implements CompilerInterface { private readonly cubeSymbols: CubeEvaluator; - private readonly cubeEvaluator: CubeEvaluator; + public readonly cubeEvaluator: CubeEvaluator; private readonly contextEvaluator: ContextEvaluator; diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts index 07774b48508cd..b2f7a4f5604b8 100644 --- a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts @@ -45,7 +45,18 @@ export interface CompilerInterface { compile: (cubes: any[], errorReporter: ErrorReporter) => void; } -export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}) => { +export type Compiler = { + compiler: DataSchemaCompiler; + metaTransformer: CubeToMetaTransformer; + cubeEvaluator: CubeEvaluator; + contextEvaluator: ContextEvaluator; + joinGraph: JoinGraph; + compilerCache: CompilerCache; + headCommitId?: string; + compilerId: string; +}; + +export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}): Compiler => { const nativeInstance = options.nativeInstance || new NativeInstance(); const cubeDictionary = new CubeDictionary(); const cubeSymbols = new CubeSymbols(); @@ -118,9 +129,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp }; }; -export const compile = (repo: SchemaFileRepository, options?: PrepareCompilerOptions) => { +export const compile = async (repo: SchemaFileRepository, options?: PrepareCompilerOptions): Promise => { const compilers = prepareCompiler(repo, options); - return compilers.compiler.compile().then( - () => compilers - ); + await compilers.compiler.compile(); + return compilers; }; diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index 3d29a8e8bfc9c..ff62cfaa59ddc 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -1,3 +1,13 @@ export * from './PrepareCompiler'; export * from './UserError'; export * from './converters'; +export { + CubeDefinition, + AccessPolicyDefinition, + ViewIncludedMember, +} from './CubeSymbols'; +export { + PreAggregationFilters, + PreAggregationInfo, + EvaluatedCube, +} from './CubeEvaluator'; diff --git a/packages/cubejs-server-core/index.js b/packages/cubejs-server-core/index.js index de215526e7df5..d28526756fea3 100644 --- a/packages/cubejs-server-core/index.js +++ b/packages/cubejs-server-core/index.js @@ -3,7 +3,7 @@ const { CubejsServerCore } = require('./dist/src/core/server'); /** * After 5 years working with TypeScript, now I know - * that commonjs and nodejs require is not compatibility with using export default + * that commonjs and Node.js require is not compatible with using export default */ const toExport = CubejsServerCore; diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index 4847dc3d3bd38..0285590e22d2c 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -41,6 +41,7 @@ "codesandbox-import-utils": "^2.1.12", "cross-spawn": "^7.0.1", "fs-extra": "^8.1.0", + "graphql": "^15.8.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-docker": "^2.1.1", diff --git a/packages/cubejs-server-core/src/core/CompilerApi.js b/packages/cubejs-server-core/src/core/CompilerApi.ts similarity index 67% rename from packages/cubejs-server-core/src/core/CompilerApi.js rename to packages/cubejs-server-core/src/core/CompilerApi.ts index dfe6027c5eecb..e5a565ea58adc 100644 --- a/packages/cubejs-server-core/src/core/CompilerApi.js +++ b/packages/cubejs-server-core/src/core/CompilerApi.ts @@ -1,31 +1,143 @@ import crypto from 'crypto'; +import vm from 'vm'; import { - createQuery, + AccessPolicyDefinition, + BaseQuery, + CanUsePreAggregationFn, compile, - queryClass, + Compiler, + createQuery, + CubeDefinition, + EvaluatedCube, + PreAggregationFilters, + PreAggregationInfo, + PreAggregationReferences, PreAggregations, + prepareCompiler, + queryClass, QueryFactory, - prepareCompiler + TransformedQuery, + ViewIncludedMember, } from '@cubejs-backend/schema-compiler'; -import { v4 as uuidv4, parse as uuidParse } from 'uuid'; +import { GraphQLSchema } from 'graphql'; +import { parse as uuidParse, v4 as uuidv4 } from 'uuid'; import { LRUCache } from 'lru-cache'; import { NativeInstance } from '@cubejs-backend/native'; +import type { SchemaFileRepository } from '@cubejs-backend/shared'; +import { NormalizedQuery, MemberExpression } from '@cubejs-backend/api-gateway'; +import { DbTypeAsyncFn, DialectClassFn, LoggerFn } from './types'; + +type Context = any; + +export interface CompilerApiOptions { + dialectClass?: DialectClassFn; + logger?: LoggerFn; + preAggregationsSchema?: string | ((context: Context) => string | Promise); + allowUngroupedWithoutPrimaryKey?: boolean; + convertTzForRawTimeDimension?: boolean; + schemaVersion?: () => string | object | Promise; + contextToRoles?: (context: Context) => string[] | Promise; + contextToGroups?: (context: Context) => string[] | Promise; + compileContext?: any; + allowJsDuplicatePropsInSchema?: boolean; + sqlCache?: boolean; + standalone?: boolean; + compilerCacheSize?: number; + maxCompilerCacheKeepAlive?: number; + updateCompilerCacheKeepAlive?: boolean; + externalDialectClass?: BaseQuery; + externalDbType?: string; + devServer?: boolean; + fastReload?: boolean; + allowNodeRequire?: boolean; +} + +export interface GetSqlOptions { + includeDebugInfo?: boolean; + exportAnnotatedSql?: boolean; + requestId?: string; +} + +export interface SqlResult { + external: any; + sql: any; + lambdaQueries: any; + timeDimensionAlias?: string; + timeDimensionField?: string; + order?: any; + cacheKeyQueries: any; + preAggregations: any; + dataSource: string; + aliasNameToMember: any; + rollupMatchResults?: any; + canUseTransformedQuery: boolean; + memberNames: string[]; +} + +export interface DataSourceInfo { + dataSource: string; + dbType: string; +} export class CompilerApi { - /** - * Class constructor. - * @param {SchemaFileRepository} repository - * @param {DbTypeAsyncFn} dbType - * @param {*} options - */ + protected readonly repository: SchemaFileRepository; + + protected readonly dbType: DbTypeAsyncFn; + + protected readonly dialectClass?: DialectClassFn; + + public readonly options: CompilerApiOptions; + + protected readonly allowNodeRequire: boolean; + + protected readonly logger: (msg: string, params: any) => void; + + protected readonly preAggregationsSchema?: string | ((context: Context) => string | Promise); + + protected readonly allowUngroupedWithoutPrimaryKey?: boolean; + + protected readonly convertTzForRawTimeDimension?: boolean; + + public schemaVersion?: () => string | object | Promise; + + protected readonly contextToRoles?: (context: Context) => string[] | Promise; + + protected readonly contextToGroups?: (context: Context) => string[] | Promise; + + protected readonly compileContext?: any; + + protected readonly allowJsDuplicatePropsInSchema?: boolean; - constructor(repository, dbType, options) { + protected readonly sqlCache?: boolean; + + protected readonly standalone?: boolean; + + protected readonly nativeInstance: NativeInstance; + + protected readonly compiledScriptCache: LRUCache; + + protected readonly compiledYamlCache: LRUCache; + + protected readonly compiledJinjaCache: LRUCache; + + protected compiledScriptCacheInterval?: NodeJS.Timeout; + + protected graphqlSchema?: GraphQLSchema; + + protected compilers?: Promise; + + protected compilerVersion?: string; + + protected queryFactory?: QueryFactory; + + public constructor(repository: SchemaFileRepository, dbType: DbTypeAsyncFn, options: CompilerApiOptions) { this.repository = repository; this.dbType = dbType; this.dialectClass = options.dialectClass; this.options = options || {}; this.allowNodeRequire = options.allowNodeRequire == null ? true : options.allowNodeRequire; - this.logger = this.options.logger; + // eslint-disable-next-line @typescript-eslint/no-empty-function + this.logger = this.options.logger || (() => {}); this.preAggregationsSchema = this.options.preAggregationsSchema; this.allowUngroupedWithoutPrimaryKey = this.options.allowUngroupedWithoutPrimaryKey; this.convertTzForRawTimeDimension = this.options.convertTzForRawTimeDimension; @@ -37,40 +149,57 @@ export class CompilerApi { this.sqlCache = options.sqlCache; this.standalone = options.standalone; this.nativeInstance = this.createNativeInstance(); + + // Caching stuff this.compiledScriptCache = new LRUCache({ max: options.compilerCacheSize || 250, ttl: options.maxCompilerCacheKeepAlive, updateAgeOnGet: options.updateCompilerCacheKeepAlive }); + this.compiledYamlCache = new LRUCache({ + max: options.compilerCacheSize || 250, + ttl: options.maxCompilerCacheKeepAlive, + updateAgeOnGet: options.updateCompilerCacheKeepAlive + }); + this.compiledJinjaCache = new LRUCache({ + max: options.compilerCacheSize || 250, + ttl: options.maxCompilerCacheKeepAlive, + updateAgeOnGet: options.updateCompilerCacheKeepAlive + }); // proactively free up old cache values occasionally if (this.options.maxCompilerCacheKeepAlive) { this.compiledScriptCacheInterval = setInterval( - () => this.compiledScriptCache.purgeStale(), + () => { + this.compiledScriptCache.purgeStale(); + this.compiledYamlCache.purgeStale(); + this.compiledJinjaCache.purgeStale(); + }, this.options.maxCompilerCacheKeepAlive ); } } - dispose() { + public dispose(): void { if (this.compiledScriptCacheInterval) { clearInterval(this.compiledScriptCacheInterval); + this.compiledScriptCacheInterval = null; } } - setGraphQLSchema(schema) { + public setGraphQLSchema(schema: GraphQLSchema): void { this.graphqlSchema = schema; } - getGraphQLSchema() { + public getGraphQLSchema(): GraphQLSchema { return this.graphqlSchema; } - createNativeInstance() { + public createNativeInstance(): NativeInstance { return new NativeInstance(); } - async getCompilers({ requestId } = {}) { + public async getCompilers(options: { requestId?: string } = {}): Promise { let compilerVersion = ( this.schemaVersion && await this.schemaVersion() || 'default_schema_version' @@ -86,7 +215,7 @@ export class CompilerApi { } if (!this.compilers || this.compilerVersion !== compilerVersion) { - this.compilers = this.compileSchema(compilerVersion, requestId).catch(e => { + this.compilers = this.compileSchema(compilerVersion, options.requestId).catch(e => { this.compilers = undefined; throw e; }); @@ -100,7 +229,7 @@ export class CompilerApi { * Creates the compilers instances without model compilation, * because it could fail and no compilers will be returned. */ - createCompilerInstances() { + public createCompilerInstances(): Compiler { return prepareCompiler(this.repository, { allowNodeRequire: this.allowNodeRequire, compileContext: this.compileContext, @@ -111,7 +240,7 @@ export class CompilerApi { }); } - async compileSchema(compilerVersion, requestId) { + public async compileSchema(compilerVersion: string, requestId?: string): Promise { const startCompilingTime = new Date().getTime(); try { this.logger(this.compilers ? 'Recompiling schema' : 'Compiling schema', { @@ -126,6 +255,8 @@ export class CompilerApi { standalone: this.standalone, nativeInstance: this.nativeInstance, compiledScriptCache: this.compiledScriptCache, + compiledJinjaCache: this.compiledJinjaCache, + compiledYamlCache: this.compiledYamlCache, }); this.queryFactory = await this.createQueryFactory(compilers); @@ -136,7 +267,7 @@ export class CompilerApi { }); return compilers; - } catch (e) { + } catch (e: any) { this.logger('Compiling schema error', { version: compilerVersion, requestId, @@ -147,7 +278,7 @@ export class CompilerApi { } } - async createQueryFactory(compilers) { + public async createQueryFactory(compilers: Compiler): Promise { const { cubeEvaluator } = compilers; const cubeToQueryClass = Object.fromEntries( @@ -163,15 +294,15 @@ export class CompilerApi { return new QueryFactory(cubeToQueryClass); } - async getDbType(dataSource = 'default') { - return this.dbType({ dataSource, }); + public async getDbType(dataSource: string = 'default'): Promise { + return this.dbType({ dataSource, securityContext: {}, requestId: '' }); } - getDialectClass(dataSource = 'default', dbType) { + public getDialectClass(dataSource: string = 'default', dbType: string): BaseQuery { return this.dialectClass?.({ dataSource, dbType }); } - async getSqlGenerator(query, dataSource) { + public async getSqlGenerator(query: NormalizedQuery, dataSource?: string): Promise<{ sqlGenerator: any; compilers: Compiler }> { const dbType = await this.getDbType(dataSource); const compilers = await this.getCompilers({ requestId: query.requestId }); let sqlGenerator = await this.createQueryByDataSource(compilers, query, dataSource, dbType); @@ -206,7 +337,7 @@ export class CompilerApi { return { sqlGenerator, compilers }; } - async getSql(query, options = {}) { + public async getSql(query: NormalizedQuery, options: GetSqlOptions = {}): Promise { const { includeDebugInfo, exportAnnotatedSql } = options; const { sqlGenerator, compilers } = await this.getSqlGenerator(query); @@ -237,32 +368,32 @@ export class CompilerApi { } } - async getRolesFromContext(context) { + protected async getRolesFromContext(context: Context): Promise> { if (!this.contextToRoles) { return new Set(); } return new Set(await this.contextToRoles(context)); } - async getGroupsFromContext(context) { + protected async getGroupsFromContext(context: Context): Promise> { if (!this.contextToGroups) { return new Set(); } return new Set(await this.contextToGroups(context)); } - userHasRole(userRoles, role) { + protected userHasRole(userRoles: Set, role: string): boolean { return userRoles.has(role) || role === '*'; } - userHasGroup(userGroups, group) { + protected userHasGroup(userGroups: Set, group: string | string[]): boolean { if (Array.isArray(group)) { return group.some(g => userGroups.has(g) || g === '*'); } return userGroups.has(group) || group === '*'; } - roleMeetsConditions(evaluatedConditions) { + protected roleMeetsConditions(evaluatedConditions?: any[]): boolean { if (evaluatedConditions?.length) { return evaluatedConditions.reduce((a, b) => { if (typeof b !== 'boolean') { @@ -274,25 +405,25 @@ export class CompilerApi { return true; } - async getCubesFromQuery(query, context) { - const sql = await this.getSql(query, { requestId: context.requestId }); + protected async getCubesFromQuery(query: NormalizedQuery, context?: Context): Promise> { + const sql = await this.getSql(query, { requestId: context?.requestId }); return new Set(sql.memberNames.map(memberName => memberName.split('.')[0])); } - hashRequestContext(context) { + protected hashRequestContext(context: Context): string { if (!context.__hash) { context.__hash = crypto.createHash('md5').update(JSON.stringify(context)).digest('hex'); } return context.__hash; } - async getApplicablePolicies(cube, context, compilers) { + protected async getApplicablePolicies(cube: EvaluatedCube, context: Context, compilers: Compiler): Promise { const cache = compilers.compilerCache.getRbacCacheInstance(); const cacheKey = `${cube.name}_${this.hashRequestContext(context)}`; if (!cache.has(cacheKey)) { const userRoles = await this.getRolesFromContext(context); const userGroups = await this.getGroupsFromContext(context); - const policies = cube.accessPolicy.filter(policy => { + const policies = cube.accessPolicy.filter((policy: AccessPolicyDefinition) => { // Validate that policy doesn't have both role and group/groups - this is invalid if (policy.role && (policy.group || policy.groups)) { const groupValue = policy.group || policy.groups; @@ -313,7 +444,7 @@ export class CompilerApi { } const evaluatedConditions = (policy.conditions || []).map( - condition => compilers.cubeEvaluator.evaluateContextFunction(cube, condition.if, context) + (condition: any) => compilers.cubeEvaluator.evaluateContextFunction(cube, condition.if, context) ); // Check if policy matches by role, group, or groups @@ -338,9 +469,8 @@ export class CompilerApi { return cache.get(cacheKey); } - evaluateNestedFilter(filter, cube, context, cubeEvaluator) { - const result = { - }; + protected evaluateNestedFilter(filter: any, cube: any, context: Context, cubeEvaluator: any): any { + const result: any = {}; if (filter.memberReference) { const evaluatedValues = cubeEvaluator.evaluateContextFunction( cube, @@ -352,10 +482,10 @@ export class CompilerApi { result.values = evaluatedValues; } if (filter.or) { - result.or = filter.or.map(f => this.evaluateNestedFilter(f, cube, context, cubeEvaluator)); + result.or = filter.or.map((f: any) => this.evaluateNestedFilter(f, cube, context, cubeEvaluator)); } if (filter.and) { - result.and = filter.and.map(f => this.evaluateNestedFilter(f, cube, context, cubeEvaluator)); + result.and = filter.and.map((f: any) => this.evaluateNestedFilter(f, cube, context, cubeEvaluator)); } return result; } @@ -365,12 +495,16 @@ export class CompilerApi { * * If RBAC is enabled, it looks at all the Cubes from the query with accessPolicy defined. * It extracts all policies applicable to for the current user context (contextToRoles() + conditions). - * It then generates an rls filter by + * It then generates a rls filter by * - combining all filters for the same role with AND * - combining all filters for different roles with OR * - combining cube and view filters with AND */ - async applyRowLevelSecurity(query, evaluatedQuery, context) { + public async applyRowLevelSecurity( + query: NormalizedQuery, + evaluatedQuery: NormalizedQuery, + context: Context + ): Promise<{ query: NormalizedQuery; denied: boolean }> { const compilers = await this.getCompilers({ requestId: context.requestId }); const { cubeEvaluator } = compilers; @@ -382,9 +516,9 @@ export class CompilerApi { // We collect Cube and View filters separately because they have to be // applied in "two layers": first Cube filters, then View filters on top - const cubeFiltersPerCubePerRole = {}; - const viewFiltersPerCubePerRole = {}; - const hasAllowAllForCube = {}; + const cubeFiltersPerCubePerRole: Record> = {}; + const viewFiltersPerCubePerRole: Record> = {}; + const hasAllowAllForCube: Record = {}; for (const cubeName of queryCubes) { const cube = cubeEvaluator.cubeFromPath(cubeName); @@ -396,7 +530,7 @@ export class CompilerApi { for (const policy of userPolicies) { hasAccessPermission = true; - (policy?.rowLevel?.filters || []).forEach(filter => { + (policy?.rowLevel?.filters || []).forEach((filter: any) => { filtersMap[cubeName] = filtersMap[cubeName] || {}; // Create a unique key for the policy (either role, group, or groups) const groupValue = policy.group || policy.groups; @@ -410,7 +544,7 @@ export class CompilerApi { }); if (!policy?.rowLevel || policy?.rowLevel?.allowAll) { hasAllowAllForCube[cubeName] = true; - // We don't have a way to add an "all alloed" filter like `WHERE 1 = 1` or something. + // We don't have a way to add an "all allowed" filter like `WHERE 1 = 1` or something. // Instead, we'll just mark that the user has "all" access to a given cube and remove // all filters later break; @@ -424,7 +558,7 @@ export class CompilerApi { expression: () => '1 = 0', cubeName: cube.name, name: 'rlsAccessDenied', - }); + } as unknown as MemberExpression); // If we hit this condition there's no need to evaluate the rest of the policy return { query, denied: true }; } @@ -443,26 +577,30 @@ export class CompilerApi { return { query, denied: false }; } - removeEmptyFilters(filter) { + protected removeEmptyFilters(filter: any): any { if (filter?.and) { - const and = filter.and.map(f => this.removeEmptyFilters(f)).filter(f => f); + const and = filter.and.map((f: any) => this.removeEmptyFilters(f)).filter((f: any) => f); return and.length > 1 ? { and } : and.at(0) || null; } if (filter?.or) { - const or = filter.or.map(f => this.removeEmptyFilters(f)).filter(f => f); + const or = filter.or.map((f: any) => this.removeEmptyFilters(f)).filter((f: any) => f); return or.length > 1 ? { or } : or.at(0) || null; } return filter; } - buildFinalRlsFilter(cubeFiltersPerCubePerRole, viewFiltersPerCubePerRole, hasAllowAllForCube) { + protected buildFinalRlsFilter( + cubeFiltersPerCubePerRole: Record>, + viewFiltersPerCubePerRole: Record>, + hasAllowAllForCube: Record + ): any { // - delete all filters for cubes where the user has allowAll // - combine the rest into per policy maps (policies can be role-based or group-based) // - join all filters for the same policy with AND // - join all filters for different policies with OR // - join cube and view filters with AND - const policyReducer = (filtersMap) => (acc, cubeName) => { + const policyReducer = (filtersMap: Record>) => (acc: Record, cubeName: string) => { if (!hasAllowAllForCube[cubeName]) { Object.keys(filtersMap[cubeName]).forEach(policyKey => { acc[policyKey] = (acc[policyKey] || []).concat(filtersMap[cubeName][policyKey]); @@ -493,7 +631,11 @@ export class CompilerApi { }); } - async compilerCacheFn(requestId, key, path) { + public async compilerCacheFn( + requestId: string, + key: any, + path: string[] + ): Promise<(subKey: string[], cacheFn: () => any) => any> { const compilers = await this.getCompilers({ requestId }); if (this.sqlCache) { return (subKey, cacheFn) => compilers.compilerCache.getQueryCache(key).cache(path.concat(subKey), cacheFn); @@ -502,22 +644,22 @@ export class CompilerApi { } } - /** - * - * @param {unknown|undefined} filter - * @returns {Promise>} - */ - async preAggregations(filter) { + public async preAggregations(filter?: PreAggregationFilters): Promise { const { cubeEvaluator } = await this.getCompilers(); return cubeEvaluator.preAggregations(filter); } - async scheduledPreAggregations() { + public async scheduledPreAggregations(): Promise { const { cubeEvaluator } = await this.getCompilers(); return cubeEvaluator.scheduledPreAggregations(); } - async createQueryByDataSource(compilers, query, dataSource, dbType) { + public async createQueryByDataSource( + compilers: Compiler, + query: NormalizedQuery | {}, + dataSource?: string, + dbType?: string + ): Promise { if (!dbType) { dbType = await this.getDbType(dataSource); } @@ -525,7 +667,7 @@ export class CompilerApi { return this.createQuery(compilers, dbType, this.getDialectClass(dataSource, dbType), query); } - createQuery(compilers, dbType, dialectClass, query) { + public createQuery(compilers: Compiler, dbType: string, dialectClass: BaseQuery, query: NormalizedQuery | {}): BaseQuery { return createQuery( compilers, dbType, @@ -546,8 +688,12 @@ export class CompilerApi { * if RBAC is enabled, this method is used to patch isVisible property of cube members * based on access policies. */ - async patchVisibilityByAccessPolicy(compilers, context, cubes) { - const isMemberVisibleInContext = {}; + protected async patchVisibilityByAccessPolicy( + compilers: Compiler, + context: Context, + cubes: any[] + ): Promise<{ cubes: any[]; visibilityMaskHash: string | null }> { + const isMemberVisibleInContext: Record = {}; const { cubeEvaluator } = compilers; if (!cubeEvaluator.isRbacEnabled()) { @@ -559,7 +705,7 @@ export class CompilerApi { if (cubeEvaluator.isRbacEnabledForCube(evaluatedCube)) { const applicablePolicies = await this.getApplicablePolicies(evaluatedCube, context, compilers); - const computeMemberVisibility = (item) => { + const computeMemberVisibility = (item: any): boolean => { for (const policy of applicablePolicies) { if (policy.memberLevel) { if (policy.memberLevel.includesMembers.includes(item.name) && @@ -592,22 +738,22 @@ export class CompilerApi { } } - const visibilityPatcherForCube = (cube) => { + const visibilityPatcherForCube = (cube: any) => { const evaluatedCube = cubeEvaluator.cubeFromPath(cube.config.name); if (!cubeEvaluator.isRbacEnabledForCube(evaluatedCube)) { - return (item) => item; + return (item: any) => item; } - return (item) => ({ + return (item: any) => ({ ...item, isVisible: item.isVisible && isMemberVisibleInContext[item.name], public: item.public && isMemberVisibleInContext[item.name] }); }; - const visibiliyMask = JSON.stringify(isMemberVisibleInContext, Object.keys(isMemberVisibleInContext).sort()); + const visibilityMask = JSON.stringify(isMemberVisibleInContext, Object.keys(isMemberVisibleInContext).sort()); // This hash will be returned along the modified meta config and can be used // to distinguish between different "schema versions" after DAP visibility is applied - const visibilityMaskHash = crypto.createHash('sha256').update(visibiliyMask).digest('hex'); + const visibilityMaskHash = crypto.createHash('sha256').update(visibilityMask).digest('hex'); return { cubes: cubes @@ -624,14 +770,17 @@ export class CompilerApi { }; } - mixInVisibilityMaskHash(compilerId, visibilityMaskHash) { - const uuidBytes = uuidParse(compilerId); + protected mixInVisibilityMaskHash(compilerId: string, visibilityMaskHash: string): string { + const uuidBytes = Buffer.from(uuidParse(compilerId)); const hashBytes = Buffer.from(visibilityMaskHash, 'hex'); return uuidv4({ random: crypto.createHash('sha256').update(uuidBytes).update(hashBytes).digest() - .subarray(0, 16) }); + .subarray(0, 16) as any }); } - async metaConfig(requestContext, options = {}) { + public async metaConfig( + requestContext: Context, + options: { includeCompilerId?: boolean; requestId?: string } = {} + ): Promise { const { includeCompilerId, ...restOptions } = options; const compilers = await this.getCompilers(restOptions); const { cubes } = compilers.metaTransformer; @@ -644,8 +793,8 @@ export class CompilerApi { return { cubes: patchedCubes, // This compilerId is primarily used by the cubejs-backend-native or caching purposes. - // By default it doesn't account for member visibility changes introduced above by DAP. - // Here we're modifying the originila compilerId in a way that it's distinct for + // By default, it doesn't account for member visibility changes introduced above by DAP. + // Here we're modifying the original compilerId in a way that it's distinct for // distinct schema versions while still being a valid UUID. compilerId: visibilityMaskHash ? this.mixInVisibilityMaskHash(compilers.compilerId, visibilityMaskHash) : compilers.compilerId, }; @@ -653,7 +802,10 @@ export class CompilerApi { return patchedCubes; } - async metaConfigExtended(requestContext, options) { + public async metaConfigExtended( + requestContext: Context, + options?: { requestId?: string } + ): Promise<{ metaConfig: any; cubeDefinitions: Record }> { const compilers = await this.getCompilers(options); const { cubes: patchedCubes } = await this.patchVisibilityByAccessPolicy( compilers, @@ -666,11 +818,11 @@ export class CompilerApi { }; } - async compilerId(options = {}) { + public async compilerId(options: { requestId?: string } = {}): Promise { return (await this.getCompilers(options)).compilerId; } - async cubeNameToDataSource(query) { + public async cubeNameToDataSource(query: { requestId?: string }): Promise> { const { cubeEvaluator } = await this.getCompilers({ requestId: query.requestId }); return cubeEvaluator .cubeNames() @@ -679,7 +831,7 @@ export class CompilerApi { ).reduce((a, b) => ({ ...a, ...b }), {}); } - async memberToDataSource(query) { + public async memberToDataSource(query: NormalizedQuery): Promise> { const { cubeEvaluator } = await this.getCompilers({ requestId: query.requestId }); const entries = cubeEvaluator @@ -688,7 +840,7 @@ export class CompilerApi { const cubeDef = cubeEvaluator.cubeFromPath(cube); if (cubeDef.isView) { const viewName = cubeDef.name; - return cubeDef.includedMembers?.map(included => { + return cubeDef.includedMembers?.map((included: ViewIncludedMember) => { const memberName = `${viewName}.${included.name}`; const refCubeDef = cubeEvaluator.cubeFromPath(included.memberPath); const dataSource = refCubeDef.dataSource ?? 'default'; @@ -707,14 +859,17 @@ export class CompilerApi { return Object.fromEntries(entries); } - async dataSources(orchestratorApi, query) { + public async dataSources( + orchestratorApi: any, + query?: NormalizedQuery + ): Promise<{ dataSources: DataSourceInfo[] }> { const cubeNameToDataSource = await this.cubeNameToDataSource(query || { requestId: `datasources-${uuidv4()}` }); let dataSources = Object.keys(cubeNameToDataSource).map(c => cubeNameToDataSource[c]); dataSources = [...new Set(dataSources)]; - dataSources = await Promise.all( + const dataSourcesInfo = await Promise.all( dataSources.map(async (dataSource) => { try { await orchestratorApi.driverFactory(dataSource); @@ -727,11 +882,11 @@ export class CompilerApi { ); return { - dataSources: dataSources.filter((source) => source), + dataSources: dataSourcesInfo.filter((source): source is DataSourceInfo => !!source), }; } - canUsePreAggregationForTransformedQuery(transformedQuery, refs) { + public canUsePreAggregationForTransformedQuery(transformedQuery: TransformedQuery, refs: PreAggregationReferences | null = null): CanUsePreAggregationFn { return PreAggregations.canUsePreAggregationForTransformedQueryFn(transformedQuery, refs); } } diff --git a/packages/cubejs-server-core/src/core/DevServer.ts b/packages/cubejs-server-core/src/core/DevServer.ts index 024a8c71f1a58..64e70e529eacb 100644 --- a/packages/cubejs-server-core/src/core/DevServer.ts +++ b/packages/cubejs-server-core/src/core/DevServer.ts @@ -326,11 +326,11 @@ export class DevServer { app.get('/playground/driver', catchErrors(async (req: Request, res: Response) => { const { driver } = req.query; - if (!driver || !DriverDependencies[driver]) { + if (!driver || typeof driver !== 'string' || !DriverDependencies[driver as keyof typeof DriverDependencies]) { return res.status(400).json('Wrong driver'); } - if (packageExists(DriverDependencies[driver])) { + if (packageExists(DriverDependencies[driver as keyof typeof DriverDependencies])) { return res.json({ status: 'installed' }); } else if (driverPromise) { return res.json({ status: 'installing' }); @@ -347,17 +347,19 @@ export class DevServer { app.post('/playground/driver', catchErrors((req, res) => { const { driver } = req.body; - if (!DriverDependencies[driver]) { + if (!driver || typeof driver !== 'string' || !DriverDependencies[driver as keyof typeof DriverDependencies]) { return res.status(400).json(`'${driver}' driver dependency not found`); } + const driverKey = driver as keyof typeof DriverDependencies; + async function installDriver() { driverError = null; try { await executeCommand( 'npm', - ['install', DriverDependencies[driver], '--save-dev'], + ['install', DriverDependencies[driverKey], '--save-dev'], { cwd: path.resolve('.') } ); } catch (error) { @@ -372,7 +374,7 @@ export class DevServer { } return res.json({ - dependency: DriverDependencies[driver] + dependency: DriverDependencies[driverKey] }); })); diff --git a/packages/cubejs-server-core/src/core/DriverDependencies.js b/packages/cubejs-server-core/src/core/DriverDependencies.ts similarity index 91% rename from packages/cubejs-server-core/src/core/DriverDependencies.js rename to packages/cubejs-server-core/src/core/DriverDependencies.ts index 2c500681aa4f4..04f42704048aa 100644 --- a/packages/cubejs-server-core/src/core/DriverDependencies.js +++ b/packages/cubejs-server-core/src/core/DriverDependencies.ts @@ -1,4 +1,6 @@ -module.exports = { +import { DatabaseType } from './types'; + +const DriverDependencies: Record = { postgres: '@cubejs-backend/postgres-driver', mysql: '@cubejs-backend/mysql-driver', mysqlauroraserverless: '@cubejs-backend/mysql-aurora-serverless-driver', @@ -31,3 +33,5 @@ module.exports = { // List for JDBC drivers 'databricks-jdbc': '@cubejs-backend/databricks-jdbc-driver', }; + +export default DriverDependencies; diff --git a/packages/cubejs-server-core/src/core/logger.js b/packages/cubejs-server-core/src/core/logger.js deleted file mode 100644 index 8f3a9852db5e5..0000000000000 --- a/packages/cubejs-server-core/src/core/logger.js +++ /dev/null @@ -1,118 +0,0 @@ -const SqlString = require('sqlstring'); -const R = require('ramda'); - -export const devLogger = (level) => (type, { error, warning, ...message }) => { - const colors = { - red: '31', // ERROR - green: '32', // INFO - yellow: '33', // WARNING - }; - - const withColor = (str, color = colors.green) => `\u001b[${color}m${str}\u001b[0m`; - const format = ({ requestId, duration, allSqlLines, query, values, showRestParams, ...json }) => { - const restParams = JSON.stringify(json, null, 2); - const durationStr = duration ? `(${duration}ms)` : ''; - const prefix = `${requestId} ${durationStr}`; - if (query && values) { - const queryMaxLines = 50; - query = query.replace(/\$(\d+)/g, '?'); - let formatted = SqlString.format(query, values).split('\n'); - if (formatted.length > queryMaxLines && !allSqlLines) { - formatted = R.take(queryMaxLines / 2, formatted) - .concat(['.....', '.....', '.....']) - .concat(R.takeLast(queryMaxLines / 2, formatted)); - } - return `${prefix}\n--\n ${formatted.join('\n')}\n--${showRestParams ? `\n${restParams}` : ''}`; - } else if (query) { - return `${prefix}\n--\n${JSON.stringify(query, null, 2)}\n--${showRestParams ? `\n${restParams}` : ''}`; - } - return `${prefix}${showRestParams ? `\n${restParams}` : ''}`; - }; - - const logWarning = () => console.log( - `${withColor(type, colors.yellow)}: ${format({ ...message, allSqlLines: true, showRestParams: true })} \n${withColor(warning, colors.yellow)}` - ); - const logError = () => console.log(`${withColor(type, colors.red)}: ${format({ ...message, allSqlLines: true, showRestParams: true })} \n${error}`); - const logDetails = (showRestParams) => console.log(`${withColor(type)}: ${format({ ...message, showRestParams })}`); - - if (error) { - logError(); - return; - } - - // eslint-disable-next-line default-case - switch ((level || 'info').toLowerCase()) { - case 'trace': { - if (!error && !warning) { - logDetails(true); - break; - } - } - // eslint-disable-next-line no-fallthrough - case 'info': { - if (!error && !warning && [ - 'Executing SQL', - 'Streaming SQL', - 'Executing Load Pre Aggregation SQL', - 'Load Request Success', - 'Performing query', - 'Performing query completed', - 'Streaming successfully completed', - ].includes(type)) { - logDetails(); - break; - } - } - // eslint-disable-next-line no-fallthrough - case 'warn': { - if (!error && warning) { - logWarning(); - break; - } - } - // eslint-disable-next-line no-fallthrough - case 'error': { - if (error) { - logError(); - break; - } - } - } -}; - -export const prodLogger = (level) => (msg, params) => { - const { error, warning } = params; - - const logMessage = () => console.log(JSON.stringify({ message: msg, ...params })); - // eslint-disable-next-line default-case - switch ((level || 'warn').toLowerCase()) { - case 'trace': { - if (!error && !warning) { - logMessage(); - break; - } - } - // eslint-disable-next-line no-fallthrough - case 'info': - if ([ - 'REST API Request', - ].includes(msg)) { - logMessage(); - break; - } - // eslint-disable-next-line no-fallthrough - case 'warn': { - if (!error && warning) { - logMessage(); - break; - } - } - // eslint-disable-next-line no-fallthrough - case 'error': { - if (error) { - logMessage(); - break; - } - } - } -}; diff --git a/packages/cubejs-server-core/src/core/logger.ts b/packages/cubejs-server-core/src/core/logger.ts new file mode 100644 index 0000000000000..c7c4844e2b0dc --- /dev/null +++ b/packages/cubejs-server-core/src/core/logger.ts @@ -0,0 +1,157 @@ +import SqlString from 'sqlstring'; +import R from 'ramda'; + +export type LogLevel = 'trace' | 'info' | 'warn' | 'error'; + +interface BaseLogMessage { + requestId?: string; + duration?: number; + query?: string | Record; + values?: any[]; + allSqlLines?: boolean; + showRestParams?: boolean; + error?: Error | string; + warning?: string; + [key: string]: any; +} + +type Color = '31' | '32' | '33'; + +const colors: Record<'red' | 'green' | 'yellow', Color> = { + red: '31', // ERROR + green: '32', // INFO + yellow: '33', // WARNING +}; + +const withColor = (str: string, color: Color = colors.green): string => `\u001b[${color}m${str}\u001b[0m`; + +interface FormatOptions { + requestId?: string; + duration?: number; + query?: string | Record; + values?: any[]; + allSqlLines?: boolean; + showRestParams?: boolean; + [key: string]: any; +} + +const format = ({ requestId, duration, allSqlLines, query, values, showRestParams, ...json }: FormatOptions): string => { + const restParams = JSON.stringify(json, null, 2); + const durationStr = duration ? `(${duration}ms)` : ''; + const prefix = `${requestId || ''} ${durationStr}`.trim(); + + if (query && values) { + const queryMaxLines = 50; + let queryStr = typeof query === 'string' ? query : JSON.stringify(query); + queryStr = queryStr.replaceAll(/\$(\d+)/g, '?'); + let formatted = SqlString.format(queryStr, values).split('\n'); + + if (formatted.length > queryMaxLines && !allSqlLines) { + formatted = R.take(queryMaxLines / 2, formatted) + .concat(['.....', '.....', '.....']) + .concat(R.takeLast(queryMaxLines / 2, formatted)); + } + + return `${prefix}\n--\n ${formatted.join('\n')}\n--${showRestParams ? `\n${restParams}` : ''}`; + } else if (query) { + return `${prefix}\n--\n${JSON.stringify(query, null, 2)}\n--${showRestParams ? `\n${restParams}` : ''}`; + } + + return `${prefix}${showRestParams ? `\n${restParams}` : ''}`; +}; + +export const devLogger = (level?: LogLevel) => (type: string, { error, warning, ...message }: BaseLogMessage): void => { + const logWarning = () => console.log( + `${withColor(type, colors.yellow)}: ${format({ ...message, allSqlLines: true, showRestParams: true })} \n${withColor(warning || '', colors.yellow)}` + ); + + const logError = () => console.log( + `${withColor(type, colors.red)}: ${format({ ...message, allSqlLines: true, showRestParams: true })} \n${error}` + ); + + const logDetails = (showRestParams?: boolean) => console.log( + `${withColor(type)}: ${format({ ...message, showRestParams })}` + ); + + if (error) { + logError(); + return; + } + + // eslint-disable-next-line default-case + switch ((level || 'info').toLowerCase()) { + case 'trace': { + if (!error && !warning) { + logDetails(true); + } + break; + } + case 'info': { + if (!error && !warning && [ + 'Executing SQL', + 'Streaming SQL', + 'Executing Load Pre Aggregation SQL', + 'Load Request Success', + 'Performing query', + 'Performing query completed', + 'Streaming successfully completed', + ].includes(type)) { + logDetails(); + } + break; + } + case 'warn': { + if (!error && warning) { + logWarning(); + } + break; + } + case 'error': { + if (error) { + logError(); + } + break; + } + } +}; + +interface ProdLogParams { + error?: Error | string; + warning?: string; + [key: string]: any; +} + +export const prodLogger = (level?: LogLevel) => (msg: string, params: ProdLogParams): void => { + const { error, warning } = params; + + const logMessage = () => console.log(JSON.stringify({ message: msg, ...params })); + + // eslint-disable-next-line default-case + switch ((level || 'warn').toLowerCase()) { + case 'trace': { + if (!error && !warning) { + logMessage(); + } + break; + } + case 'info': + if ([ + 'REST API Request', + ].includes(msg)) { + logMessage(); + } + break; + case 'warn': { + if (!error && warning) { + logMessage(); + } + break; + } + case 'error': { + if (error) { + logMessage(); + } + break; + } + } +}; diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 1db5966d5af31..47e08a640f416 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -183,8 +183,8 @@ export class CubejsServerCore { this.logger = opts.logger || ( process.env.NODE_ENV !== 'production' - ? devLogger(process.env.CUBEJS_LOG_LEVEL) - : prodLogger(process.env.CUBEJS_LOG_LEVEL) + ? devLogger(process.env.CUBEJS_LOG_LEVEL as any) + : prodLogger(process.env.CUBEJS_LOG_LEVEL as any) ); this.optsHandler = new OptsHandler(this, opts, systemOptions); diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 818f39e37445f..b1278f671fd2f 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -1,13 +1,13 @@ import { Required, SchemaFileRepository } from '@cubejs-backend/shared'; import { + CanSwitchSQLUserFn, CheckAuthFn, + CheckSQLAuthFn, + ContextToApiScopesFn, ExtendContextFn, JWTOptions, - UserBackgroundContext, QueryRewriteFn, - CheckSQLAuthFn, - CanSwitchSQLUserFn, - ContextToApiScopesFn, + UserBackgroundContext, } from '@cubejs-backend/api-gateway'; import { BaseDriver, CacheAndQueryDriverType } from '@cubejs-backend/query-orchestrator'; import { BaseQuery } from '@cubejs-backend/schema-compiler'; @@ -108,17 +108,24 @@ export type DatabaseType = | 'mongobi' | 'mssql' | 'mysql' + | 'mysqlauroraserverless' | 'elasticsearch' | 'awselasticsearch' | 'oracle' | 'postgres' | 'prestodb' + | 'trino' | 'redshift' | 'snowflake' | 'sqlite' | 'questdb' | 'materialize' - | 'pinot'; + | 'pinot' + | 'dremio' + | 'duckdb' + | 'ksql' + | 'vertica' + | 'databricks-jdbc'; export type ContextToAppIdFn = (context: RequestContext) => string | Promise; export type ContextToRolesFn = (context: RequestContext) => string[] | Promise; @@ -163,6 +170,8 @@ export type DriverFactoryAsyncFn = (context: DriverContext) => export type DialectFactoryFn = (context: DialectContext) => BaseQuery; +export type DialectClassFn = (options: { dataSource: string; dbType: string }) => BaseQuery; + // external export type ExternalDbTypeFn = (context: RequestContext) => DatabaseType; export type ExternalDriverFactoryFn = (context: RequestContext) => Promise | BaseDriver;