From 42368937f391ccc97bf1e885842e97f2760b7821 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Tue, 30 Sep 2025 14:03:59 +0400 Subject: [PATCH 1/3] fix(schema-compiler): Do not collect disabled external pre-aggregations Signed-off-by: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> --- .../cubejs-schema-compiler/src/adapter/BaseQuery.js | 6 ++++-- .../src/adapter/PreAggregations.ts | 3 ++- .../src/cube_bridge/base_query_options.rs | 2 ++ .../optimizers/pre_aggregation/optimizer.rs | 9 +++++++-- .../pre_aggregation/pre_aggregations_compiler.rs | 6 +++++- .../cubesqlplanner/src/planner/base_query.rs | 6 +++++- .../src/planner/planners/dimension_subquery_planner.rs | 1 + .../planners/multi_stage/member_query_planner.rs | 2 ++ .../cubesqlplanner/src/planner/query_properties.rs | 10 ++++++++++ 9 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index a7fdec383d8a5..32280f6d4e721 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -897,7 +897,8 @@ export class BaseQuery { preAggregationQuery: this.options.preAggregationQuery, totalQuery: this.options.totalQuery, joinHints: this.options.joinHints, - cubestoreSupportMultistage: this.options.cubestoreSupportMultistage ?? getEnv('cubeStoreRollingWindowJoin') + cubestoreSupportMultistage: this.options.cubestoreSupportMultistage ?? getEnv('cubeStoreRollingWindowJoin'), + disableExternalPreAggregations: !!this.options.disableExternalPreAggregations, }; try { @@ -945,7 +946,8 @@ export class BaseQuery { ungrouped: this.options.ungrouped, exportAnnotatedSql: false, preAggregationQuery: this.options.preAggregationQuery, - cubestoreSupportMultistage: this.options.cubestoreSupportMultistage ?? getEnv('cubeStoreRollingWindowJoin') + cubestoreSupportMultistage: this.options.cubestoreSupportMultistage ?? getEnv('cubeStoreRollingWindowJoin'), + disableExternalPreAggregations: !!this.options.disableExternalPreAggregations, }; const buildResult = nativeBuildSqlAndParams(queryParams); diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0d1e78cc0a267..2c8bf81e99053 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -104,6 +104,7 @@ export class PreAggregations { * It returns full pre-aggregation object (with keyQueries, previewSql, loadSql, and so on. */ public preAggregationsDescription(): FullPreAggregationDescription[] { + const disableExternalPreAggregations = this.query.options?.disableExternalPreAggregations; const preAggregations = [this.preAggregationsDescriptionLocal()].concat( this.query.subQueryDimensions.map(d => this.query.subQueryDescription(d).subQuery) .map(q => q.preAggregations.preAggregationsDescription()) @@ -114,7 +115,7 @@ export class PreAggregations { R.uniqBy(desc => desc.tableName) )( preAggregations - ); + ).filter(agg => !(disableExternalPreAggregations && agg.external)); } private preAggregationsDescriptionLocal(): FullPreAggregationDescription[] { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index 2777ff560cc69..be1b2a8c35625 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -68,6 +68,8 @@ pub struct BaseQueryOptionsStatic { pub total_query: Option, #[serde(rename = "cubestoreSupportMultistage")] pub cubestore_support_multistage: Option, + #[serde(rename = "disableExternalPreAggregations")] + pub disable_external_pre_aggregations: bool, } #[nativebridge::native_bridge(BaseQueryOptionsStatic)] diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs index e54410a735ebf..40b1fee63f8d7 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs @@ -24,11 +24,16 @@ impl PreAggregationOptimizer { } } - pub fn try_optimize(&mut self, plan: Rc) -> Result>, CubeError> { + pub fn try_optimize( + &mut self, + plan: Rc, + disable_external_pre_aggregations: bool, + ) -> Result>, CubeError> { let cube_names = collect_cube_names_from_node(&plan)?; let mut compiler = PreAggregationsCompiler::try_new(self.query_tools.clone(), &cube_names)?; - let compiled_pre_aggregations = compiler.compile_all_pre_aggregations()?; + let compiled_pre_aggregations = + compiler.compile_all_pre_aggregations(disable_external_pre_aggregations)?; for pre_aggregation in compiled_pre_aggregations.iter() { let new_query = self.try_rewrite_query(plan.clone(), pre_aggregation)?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs index 4d606cda130e7..2282aaa168a79 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs @@ -406,10 +406,14 @@ impl PreAggregationsCompiler { pub fn compile_all_pre_aggregations( &mut self, + disable_external_pre_aggregations: bool, ) -> Result>, CubeError> { let mut result = Vec::new(); for (name, _) in self.descriptions.clone().iter() { - result.push(self.compile_pre_aggregation(&name)?); + let pre_aggregation = self.compile_pre_aggregation(name)?; + if !(disable_external_pre_aggregations && pre_aggregation.external == Some(true)) { + result.push(pre_aggregation); + } } Ok(result) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index fe5d5a1b61e3b..63ebf56770673 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -124,7 +124,11 @@ impl BaseQuery { self.query_tools.clone(), self.cubestore_support_multistage, ); - if let Some(result) = pre_aggregation_optimizer.try_optimize(plan.clone())? { + let disable_external_pre_aggregations = + self.request.disable_external_pre_aggregations(); + if let Some(result) = pre_aggregation_optimizer + .try_optimize(plan.clone(), disable_external_pre_aggregations)? + { if pre_aggregation_optimizer.get_used_pre_aggregations().len() == 1 { ( result, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs index bf02ac3b1d528..9cec97be9e3fb 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs @@ -123,6 +123,7 @@ impl DimensionSubqueryPlanner { false, Rc::new(vec![]), true, + self.query_properties.disable_external_pre_aggregations(), )?; let query_planner = QueryPlanner::new(sub_query_properties, self.query_tools.clone()); let sub_query = query_planner.plan()?; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 46279af948084..20a74fe5720d4 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -76,6 +76,7 @@ impl MultiStageMemberQueryPlanner { false, Rc::new(vec![]), true, + self.query_properties.disable_external_pre_aggregations(), )?; let simple_query_planer = @@ -377,6 +378,7 @@ impl MultiStageMemberQueryPlanner { false, self.query_properties.query_join_hints().clone(), false, + self.query_properties.disable_external_pre_aggregations(), )?; let query_planner = diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index e3e823d76a301..ba79f19e86b2e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -107,6 +107,7 @@ pub struct QueryProperties { total_query: bool, query_join_hints: Rc>, allow_multi_stage: bool, + disable_external_pre_aggregations: bool, } impl QueryProperties { @@ -405,6 +406,8 @@ impl QueryProperties { let pre_aggregation_query = options.static_data().pre_aggregation_query.unwrap_or(false); let total_query = options.static_data().total_query.unwrap_or(false); + let disable_external_pre_aggregations = + options.static_data().disable_external_pre_aggregations; let mut res = Self { measures, @@ -425,6 +428,7 @@ impl QueryProperties { total_query, query_join_hints, allow_multi_stage: true, + disable_external_pre_aggregations, }; res.apply_static_filters()?; Ok(Rc::new(res)) @@ -448,6 +452,7 @@ impl QueryProperties { total_query: bool, query_join_hints: Rc>, allow_multi_stage: bool, + disable_external_pre_aggregations: bool, ) -> Result, CubeError> { let order_by = if order_by.is_empty() { Self::default_order(&dimensions, &time_dimensions, &measures) @@ -474,6 +479,7 @@ impl QueryProperties { total_query, query_join_hints, allow_multi_stage, + disable_external_pre_aggregations, }; res.apply_static_filters()?; @@ -721,6 +727,10 @@ impl QueryProperties { self.pre_aggregation_query } + pub fn disable_external_pre_aggregations(&self) -> bool { + self.disable_external_pre_aggregations + } + pub fn all_filters(&self) -> Option { let items = self .time_dimensions_filters From 9131d794080c4d6ec7517b4dbececc55d703d686 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 1 Oct 2025 20:42:02 +0300 Subject: [PATCH 2/3] code polish --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 2c8bf81e99053..ea8e70fd71140 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -112,10 +112,9 @@ export class PreAggregations { return R.pipe( R.unnest as (list: any[][]) => any[], + R.filter((agg: FullPreAggregationDescription) => !(disableExternalPreAggregations && agg.external)), R.uniqBy(desc => desc.tableName) - )( - preAggregations - ).filter(agg => !(disableExternalPreAggregations && agg.external)); + )(preAggregations); } private preAggregationsDescriptionLocal(): FullPreAggregationDescription[] { From 3889a4e2ba4e1a785bc6c544346292736cbefeb9 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 1 Oct 2025 20:52:45 +0300 Subject: [PATCH 3/3] add a comment --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index ea8e70fd71140..106319d1ed55f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -112,6 +112,8 @@ export class PreAggregations { return R.pipe( R.unnest as (list: any[][]) => any[], + // TODO: Move this to somewhere BEFORE pre-agg matching, possibly to rollupMatchResults() + // to avoid constly matching and then throwing it away. R.filter((agg: FullPreAggregationDescription) => !(disableExternalPreAggregations && agg.external)), R.uniqBy(desc => desc.tableName) )(preAggregations);