From d0208bc3a7c0a5c1e85c45902f70c62cca42a47f Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Tue, 11 Feb 2025 15:34:27 +0100 Subject: [PATCH 1/4] chore(tesseract): Fix issue with FILTER_PARAMS and issue with multi-stage behavior of not multi-stage members --- .../src/adapter/BaseQuery.js | 22 ++++ .../postgres/sql-generation.test.ts | 114 +++++++++++++++++- rust/cubenativeutils/src/wrappers/context.rs | 4 + .../src/wrappers/neon/context.rs | 7 ++ .../src/wrappers/serializer/serializer.rs | 2 +- .../src/cube_bridge/base_query_options.rs | 2 +- .../src/cube_bridge/base_tools.rs | 8 +- .../cubesqlplanner/src/plan/builder/select.rs | 4 +- .../src/planner/filter/compiler.rs | 2 +- .../src/planner/filter/filter_operator.rs | 28 +++++ .../planners/multi_stage/applied_state.rs | 27 ++++- .../planners/multi_stage_query_planner.rs | 23 +++- .../src/planner/sql_evaluator/sql_call.rs | 73 +++++++++-- .../src/planner/sql_evaluator/sql_visitor.rs | 13 +- .../sql_evaluator/symbols/member_symbol.rs | 9 ++ .../symbols/time_dimension_symbol.rs | 4 + .../src/planner/visitor_context.rs | 7 +- 17 files changed, 326 insertions(+), 23 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 9415f397af2c7..5247fe5d437f6 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -1855,6 +1855,7 @@ export class BaseQuery { 'collectSubQueryDimensionsFor' ) ), inlineWhereConditions); + return `SELECT ${this.selectAllDimensionsAndMeasures(measures)} FROM ${ query } ${this.baseWhere(filters.concat(inlineWhereConditions))}` + @@ -4001,6 +4002,23 @@ export class BaseQuery { ); } + filtersProxyForRust(usedFilters) { + const filters = this.extractFiltersAsTree(usedFilters || []); + const allFilters = filters.map(this.initFilter.bind(this)); + return BaseQuery.filterProxyFromAllFilters( + allFilters, + this.cubeEvaluator, + this.paramAllocator.allocateParam.bind(this.paramAllocator), + this.newGroupFilter.bind(this), + ); + } + + filterGroupFunctionForRust(usedFilters) { + const filters = this.extractFiltersAsTree(usedFilters || []); + const allFilters = filters.map(this.initFilter.bind(this)); + return this.filterGroupFunctionImpl(allFilters); + } + static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases) { if (!filter) { return BaseFilter.ALWAYS_TRUE; @@ -4046,6 +4064,10 @@ export class BaseQuery { filterGroupFunction() { const { allFilters } = this; + return this.filterGroupFunctionImpl(allFilters); + } + + filterGroupFunctionImpl(allFilters) { const allocateParam = this.paramAllocator.allocateParam.bind(this.paramAllocator); const newGroupFilter = this.newGroupFilter.bind(this); return (...filterParamArgs) => { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index 1fda04d9c0705..ac429c4cb0768 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -1,3 +1,4 @@ +import { getEnv } from '@cubejs-backend/shared'; import { UserError } from '../../../src/compiler/UserError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; import { BigqueryQuery } from '../../../src/adapter/BigqueryQuery'; @@ -567,6 +568,84 @@ describe('SQL Generation', () => { }, } }); + + cube('rollingWindowDates', { + sql: \` + SELECT cast('2024-01-13' AS timestamp) as time UNION ALL + SELECT cast('2024-02-13' AS timestamp) as time UNION ALL + SELECT cast('2024-03-13' AS timestamp) as time UNION ALL + SELECT cast('2024-04-13' AS timestamp) as time UNION ALL + SELECT cast('2024-05-13' AS timestamp) as time UNION ALL + SELECT cast('2024-06-13' AS timestamp) as time UNION ALL + SELECT cast('2024-07-13' AS timestamp) as time UNION ALL + SELECT cast('2024-08-13' AS timestamp) as time UNION ALL + SELECT cast('2024-09-13' AS timestamp) as time UNION ALL + SELECT cast('2024-10-13' AS timestamp) as time UNION ALL + SELECT cast('2024-11-13' AS timestamp) as time UNION ALL + SELECT cast('2024-12-13' AS timestamp) as time + \`, + + dimensions: { + time: { + type: 'time', + sql: 'time', + primaryKey: true + } + } + }); + + cube('rollingWindowTest', { + sql: \` +SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-02-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-03-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-04-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-05-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-06-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-07-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-08-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-09-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-10-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-11-01' AS timestamp) as time UNION ALL + SELECT 1 AS revenue, cast('2024-12-01' AS timestamp) as time + \`, + + dimensions: { + time: { + type: 'time', + sql: 'time', + primaryKey: true + } + }, + measures: { + revenue: { + sql: 'revenue', + type: 'sum', + filters: [{ + sql: \`\${rollingWindowDates.time} <= current_date\` + }] + }, + revenue_ytd: { + sql: \`\${CUBE.revenue}\`, + type: 'sum', + rolling_window: { + type: 'to_date', + granularity: 'year' + } + }, + revenue_ms: { + sql: \`\${CUBE.revenue}\`, + type: 'sum', + multi_stage: true, + }, + }, + joins: { + rollingWindowDates: { + relationship: 'manyToOne', + sql: \`\${CUBE}.time = date_trunc('month', \${rollingWindowDates.time})\` + } + } + }); `); it('simple join', async () => { @@ -1170,7 +1249,7 @@ describe('SQL Generation', () => { }); }); - it('filter join', async () => { + it('filter join 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -1790,7 +1869,7 @@ describe('SQL Generation', () => { }); }); - it('security context', async () => { + it('security context 1', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -3152,6 +3231,37 @@ describe('SQL Generation', () => { }] )); + if (getEnv('nativeSqlPlanner')) { + it('nested aggregations with filtered measures and rolling windows', async () => runQueryTest( + { + measures: ['rollingWindowTest.revenue_ms'], + }, + [{ + rolling_window_test__revenue_ms: '12' + }] + )); + } + + it('multiplied sum and count no dimensions through view', async () => runQueryTest( + { + measures: ['visitors_visitors_checkins_view.revenue', 'visitors_visitors_checkins_view.visitor_checkins_count'], + }, + [{ + visitors_visitors_checkins_view__revenue: '2000', + visitors_visitors_checkins_view__visitor_checkins_count: '6' + }] + )); + + it('multiplied sum no dimensions through view', async () => runQueryTest( + { + measures: ['visitors_visitors_checkins_view.revenue', 'visitors_visitors_checkins_view.id_sum'], + }, + [{ + visitors_visitors_checkins_view__revenue: '2000', + visitors_visitors_checkins_view__id_sum: '21' + }] + )); + // Subquery aggregation for multiplied measure (and any `keysSelect` for that matter) // should pick up all dimensions, even through member expressions it('multiplied sum with dimension member expressions', async () => runQueryTest( diff --git a/rust/cubenativeutils/src/wrappers/context.rs b/rust/cubenativeutils/src/wrappers/context.rs index 9695b30717ec3..805e7923ffa54 100644 --- a/rust/cubenativeutils/src/wrappers/context.rs +++ b/rust/cubenativeutils/src/wrappers/context.rs @@ -6,6 +6,7 @@ pub trait NativeContext: Clone { fn string(&self, v: String) -> Result; fn number(&self, v: f64) -> Result; fn undefined(&self) -> Result, CubeError>; + fn null(&self) -> Result, CubeError>; fn empty_array(&self) -> Result; fn empty_struct(&self) -> Result; //fn boxed(&self, value: T) -> impl NativeBox; @@ -36,6 +37,9 @@ impl NativeContextHolder { pub fn undefined(&self) -> Result, CubeError> { self.context.undefined() } + pub fn null(&self) -> Result, CubeError> { + self.context.null() + } pub fn empty_array(&self) -> Result { self.context.empty_array() } diff --git a/rust/cubenativeutils/src/wrappers/neon/context.rs b/rust/cubenativeutils/src/wrappers/neon/context.rs index aff6f55a693e3..10be1a53f98a2 100644 --- a/rust/cubenativeutils/src/wrappers/neon/context.rs +++ b/rust/cubenativeutils/src/wrappers/neon/context.rs @@ -144,6 +144,13 @@ impl + 'static> NativeContext> for Context ))) } + fn null(&self) -> Result>, CubeError> { + Ok(NativeObjectHandle::new(NeonObject::new( + self.clone(), + self.with_context(|cx| cx.null().upcast())?, + ))) + } + fn empty_array(&self) -> Result, CubeError> { let obj = NeonObject::new( self.clone(), diff --git a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs index 6570df1e3ff5d..f8ee638db7d8d 100644 --- a/rust/cubenativeutils/src/wrappers/serializer/serializer.rs +++ b/rust/cubenativeutils/src/wrappers/serializer/serializer.rs @@ -138,7 +138,7 @@ impl ser::Serializer for NativeSerdeSerializer { } fn serialize_none(self) -> Result { - Ok(self.context.undefined()?) + Ok(self.context.null()?) } fn serialize_some(self, value: &T) -> Result diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs index e5d75e4fab299..1e87da9363565 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs @@ -23,7 +23,7 @@ pub struct TimeDimension { pub struct FilterItem { pub or: Option>, pub and: Option>, - member: Option, + pub member: Option, pub dimension: Option, pub operator: Option, pub values: Option>>, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index 0ccc1567be747..e6f69768e03ac 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -1,3 +1,4 @@ +use super::base_query_options::FilterItem; use super::filter_group::{FilterGroup, NativeFilterGroup}; use super::filter_params::{FilterParams, NativeFilterParams}; use super::member_sql::{MemberSql, NativeMemberSql}; @@ -36,8 +37,11 @@ pub trait BaseTools { ) -> Result, CubeError>; fn security_context_for_rust(&self) -> Result, CubeError>; fn sql_utils_for_rust(&self) -> Result, CubeError>; - fn filters_proxy(&self) -> Result, CubeError>; - fn filter_group_function(&self) -> Result, CubeError>; + fn filters_proxy_for_rust( + &self, + used_filters: Option>, + ) -> Result, CubeError>; + fn filter_group_function_for_rust(&self, used_filters: Option>) -> Result, CubeError>; fn timestamp_precision(&self) -> Result; fn in_db_time_zone(&self, date: String) -> Result; fn generate_time_series( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs index 0631a868cec13..4312aa48fd867 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs @@ -193,11 +193,11 @@ impl SelectBuilder { Select { projection_columns: self.projection_columns, from: self.from, - filter: self.filter, + filter: self.filter.clone(), group_by: self.group_by, having: self.having, order_by: self.order_by, - context: Rc::new(VisitorContext::new(&nodes_factory)), + context: Rc::new(VisitorContext::new(&nodes_factory, self.filter)), ctes: self.ctes, is_distinct: self.is_distinct, limit: self.limit, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs index e918fbb8d1b46..43109bd5ffb6e 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs @@ -129,7 +129,7 @@ impl<'a> FilterCompiler<'a> { } else { Err(CubeError::user(format!( "Member and operator attributes is required for filter" - ))) //TODO pring condition + ))) //TODO print condition } } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs index 99d946e462b98..a713e1dee58e9 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs @@ -53,3 +53,31 @@ impl FromStr for FilterOperator { } } } + +impl ToString for FilterOperator { + fn to_string(&self) -> String { + let str = match self { + FilterOperator::Equal => "equals", + FilterOperator::NotEqual => "notEquals", + FilterOperator::InDateRange => "inDateRange", + FilterOperator::RegularRollingWindowDateRange => "inDateRange", + FilterOperator::ToDateRollingWindowDateRange => "inDateRange", + FilterOperator::In => "in", + FilterOperator::NotIn => "notIn", + FilterOperator::Set => "set", + FilterOperator::NotSet => "notSet", + FilterOperator::Gt => "gt", + FilterOperator::Gte => "gte", + FilterOperator::Lt => "lt", + FilterOperator::Lte => "lte", + FilterOperator::Contains => "contains", + FilterOperator::NotContains => "notContains", + FilterOperator::StartsWith => "startsWith", + FilterOperator::NotStartsWith => "notStartsWith", + FilterOperator::NotEndsWith => "notEndsWith", + FilterOperator::EndsWith => "endsWith", + FilterOperator::MeasureFilter => "measureFilter", + }; + str.to_string() + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs index a60b755293819..1401d33a08533 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs @@ -173,6 +173,7 @@ impl MultiStageAppliedState { &self.time_dimensions_filters, &operator, &values, + &None, ); } @@ -188,6 +189,24 @@ impl MultiStageAppliedState { &self.time_dimensions_filters, &operator, &values, + &None, + ); + } + + pub fn replace_range_in_date_filter( + &mut self, + member_name: &String, + new_from: String, + new_to: String, + ) { + let operator = FilterOperator::InDateRange; + let replacement_values = vec![Some(new_from), Some(new_to)]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + &vec![], + &Some(replacement_values), ); } @@ -197,6 +216,7 @@ impl MultiStageAppliedState { filters: &Vec, operator: &FilterOperator, additional_values: &Vec>, + replacement_values: &Option>>, ) -> Vec { let mut result = Vec::new(); for item in filters.iter() { @@ -209,6 +229,7 @@ impl MultiStageAppliedState { filters, operator, additional_values, + replacement_values, ), ))); result.push(new_group); @@ -217,7 +238,11 @@ impl MultiStageAppliedState { let itm = if &itm.member_name() == member_name && matches!(itm.filter_operator(), FilterOperator::InDateRange) { - let mut values = itm.values().clone(); + let mut values = if let Some(values) = replacement_values { + values.clone() + } else { + itm.values().clone() + }; values.extend(additional_values.iter().cloned()); itm.change_operator(operator.clone(), values) } else { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs index 0a9218832cc63..45e2bb2f24366 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs @@ -248,7 +248,26 @@ impl MultiStageQueryPlanner { &time_dimension.get_granularity(), )?; - new_state.change_time_dimension_granularity(&time_dimension_name, result_granularity); + if time_dimension.get_date_range().is_some() && result_granularity.is_some() { + let granularity = time_dimension.get_granularity().unwrap(); //FIXME remove this unwrap + let date_range = time_dimension.get_date_range().unwrap(); //FIXME remove this unwrap + let seria = self + .query_tools + .base_tools() + .generate_time_series(granularity, date_range.clone())?; + if !seria.is_empty() { + let new_from_date = seria.first().unwrap()[0].clone(); + let new_to_date = seria.last().unwrap()[1].clone(); + new_state.replace_range_in_date_filter( + &time_dimension_name, + new_from_date, + new_to_date, + ); + } + } + + new_state + .change_time_dimension_granularity(&time_dimension_name, result_granularity.clone()); if let Some(granularity) = self.get_to_date_rolling_granularity(rolling_window)? { new_state.replace_to_date_date_range_filter(&time_dimension_name, &granularity); @@ -376,7 +395,7 @@ impl MultiStageQueryPlanner { let childs = member_childs(&member)?; - let description = if childs.is_empty() { + let description = if childs.is_empty() || !has_multi_stage_members(&member, false)? { if has_multi_stage_members(&member, false)? { return Err(CubeError::internal(format!( "Leaf multi stage query cannot contain multi stage member" diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs index 63a9a906f4805..a084abbc6f790 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_call.rs @@ -1,7 +1,9 @@ use super::dependecy::{ContextSymbolDep, CubeDepProperty, CubeDependency, Dependency}; use super::sql_nodes::SqlNode; use super::{symbols::MemberSymbol, SqlEvaluatorVisitor}; +use crate::cube_bridge::base_query_options::FilterItem as NativeFilterItem; use crate::cube_bridge::member_sql::{ContextSymbolArg, MemberSql, MemberSqlArg, MemberSqlStruct}; +use crate::plan::{Filter, FilterItem}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; @@ -167,7 +169,7 @@ impl SqlCall { templates, ), Dependency::ContextDependency(contex_symbol) => { - self.apply_context_symbol(contex_symbol, query_tools.clone()) + self.apply_context_symbol(visitor, contex_symbol, query_tools.clone()) } } } @@ -208,6 +210,7 @@ impl SqlCall { pub fn apply_context_symbol( &self, + visitor: &SqlEvaluatorVisitor, context_symbol: &ContextSymbolDep, query_tools: Rc, ) -> Result { @@ -217,16 +220,72 @@ impl SqlCall { query_tools.base_tools().security_context_for_rust()?, )) } - ContextSymbolDep::FilterParams => MemberSqlArg::ContextSymbol( - ContextSymbolArg::FilterParams(query_tools.base_tools().filters_proxy()?), - ), - ContextSymbolDep::FilterGroup => MemberSqlArg::ContextSymbol( - ContextSymbolArg::FilterGroup(query_tools.base_tools().filter_group_function()?), - ), + ContextSymbolDep::FilterParams => { + let filters = visitor.all_filters(); + let native_filters = self.filters_to_native_filter_item(filters); + let r = query_tools + .base_tools() + .filters_proxy_for_rust(native_filters)?; + MemberSqlArg::ContextSymbol(ContextSymbolArg::FilterParams(r)) + } + ContextSymbolDep::FilterGroup => { + let filters = visitor.all_filters(); + let native_filters = self.filters_to_native_filter_item(filters); + let r = query_tools + .base_tools() + .filter_group_function_for_rust(native_filters)?; + MemberSqlArg::ContextSymbol(ContextSymbolArg::FilterGroup(r)) + } ContextSymbolDep::SqlUtils => MemberSqlArg::ContextSymbol(ContextSymbolArg::SqlUtils( query_tools.base_tools().sql_utils_for_rust()?, )), }; Ok(res) } + + fn filters_to_native_filter_item( + &self, + filter: Option, + ) -> Option> { + if let Some(filter) = filter { + let mut res = Vec::new(); + for item in filter.items.iter() { + res.push(self.filters_to_native_filter_item_impl(item)); + } + Some(res) + } else { + None + } + } + + fn filters_to_native_filter_item_impl(&self, filter_item: &FilterItem) -> NativeFilterItem { + match filter_item { + FilterItem::Group(group) => { + let mut native_items = Vec::new(); + for itm in group.items.iter() { + native_items.push(self.filters_to_native_filter_item_impl(itm)); + } + let (or, and) = match group.operator { + crate::plan::filter::FilterGroupOperator::Or => (Some(native_items), None), + crate::plan::filter::FilterGroupOperator::And => (None, Some(native_items)), + }; + NativeFilterItem { + or, + and, + member: None, + dimension: None, + operator: None, + values: None, + } + } + FilterItem::Item(filter) => NativeFilterItem { + or: None, + and: None, + member: Some(filter.member_name()), + dimension: None, + operator: Some(filter.filter_operator().to_string()), + values: Some(filter.values().clone()), + }, + } + } } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs index 2d9087c91c6e7..24d39e7326a10 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_visitor.rs @@ -1,5 +1,6 @@ use super::sql_nodes::SqlNode; use super::MemberSymbol; +use crate::plan::Filter; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; @@ -8,11 +9,19 @@ use std::rc::Rc; #[derive(Clone)] pub struct SqlEvaluatorVisitor { query_tools: Rc, + all_filters: Option, //To pass to FILTER_PARAMS and FILTER_GROUP } impl SqlEvaluatorVisitor { - pub fn new(query_tools: Rc) -> Self { - Self { query_tools } + pub fn new(query_tools: Rc, all_filters: Option) -> Self { + Self { + query_tools, + all_filters, + } + } + + pub fn all_filters(&self) -> Option { + self.all_filters.clone() } pub fn apply( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index 442e52312cb45..42e55ca6f1782 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -62,6 +62,15 @@ impl MemberSymbol { } } + pub fn is_multi_stage(&self) -> bool { + match self { + Self::Dimension(d) => d.is_multi_stage(), + Self::TimeDimension(d) => d.is_multi_stage(), + Self::Measure(m) => m.is_multi_stage(), + _ => false + } + } + pub fn is_measure(&self) -> bool { matches!(self, Self::Measure(_)) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs index 590584e872730..82a14a8c84d2a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/time_dimension_symbol.rs @@ -46,6 +46,10 @@ impl TimeDimensionSymbol { self.base_symbol.cube_name() } + pub fn is_multi_stage(&self) -> bool { + self.base_symbol.is_multi_stage() + } + pub fn name(&self) -> String { self.base_symbol.name() } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs index 0eba5e2f4592a..1e0e9986e29bd 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/visitor_context.rs @@ -1,6 +1,7 @@ use super::query_tools::QueryTools; use super::sql_evaluator::sql_nodes::{SqlNode, SqlNodesFactory}; use super::sql_evaluator::{MemberSymbol, SqlCall}; +use crate::plan::Filter; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; use crate::planner::sql_templates::PlanSqlTemplates; use cubenativeutils::CubeError; @@ -8,17 +9,19 @@ use std::rc::Rc; pub struct VisitorContext { node_processor: Rc, + all_filters: Option, //To pass to FILTER_PARAMS and FILTER_GROUP } impl VisitorContext { - pub fn new(nodes_factory: &SqlNodesFactory) -> Self { + pub fn new(nodes_factory: &SqlNodesFactory, all_filters: Option) -> Self { Self { node_processor: nodes_factory.default_node_processor(), + all_filters, } } pub fn make_visitor(&self, query_tools: Rc) -> SqlEvaluatorVisitor { - SqlEvaluatorVisitor::new(query_tools) + SqlEvaluatorVisitor::new(query_tools, self.all_filters.clone()) } pub fn node_processor(&self) -> Rc { From d7ca17d654be260ba64e8902c8b391b1a9735d3a Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Tue, 11 Feb 2025 18:45:47 +0100 Subject: [PATCH 2/4] update --- .../integration/postgres/sql-generation.test.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index ac429c4cb0768..e11b7f01b3af3 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -313,9 +313,12 @@ describe('SQL Generation', () => { cube('visitor_checkins', { sql: \` - select * from visitor_checkins WHERE - \${FILTER_PARAMS.visitor_checkins.created_at.filter('created_at')} AND - \${FILTER_GROUP(FILTER_PARAMS.visitor_checkins.created_at.filter("(created_at - INTERVAL '3 DAY')"), FILTER_PARAMS.visitor_checkins.source.filter('source'))} + select visitor_checkins.* from visitor_checkins left join visitors on visitor_checkins.visitor_id = visitors.id WHERE + \${FILTER_PARAMS.visitor_checkins.created_at.filter('visitor_checkins.created_at')} AND + \${FILTER_GROUP(FILTER_PARAMS.visitor_checkins.created_at.filter("(visitor_checkins.created_at - INTERVAL '3 DAY')"), FILTER_PARAMS.visitor_checkins.source.filter('visitor_checkins.source'))} + AND \${SECURITY_CONTEXT.source.filter('visitors.source')} AND + \${SECURITY_CONTEXT.sourceArray.filter(sourceArray => \`visitors.source in (\${sourceArray.join(',')})\`)} + \`, sql_alias: \`vc\`, @@ -1249,7 +1252,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL }); }); - it('filter join 1', async () => { + it('filter join', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -1869,7 +1872,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL }); }); - it('security context 1', async () => { + it('security context', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -2045,7 +2048,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL ])); it( - 'contains filter 1', + 'contains filter', () => runQueryTest({ measures: [], dimensions: [ @@ -2840,7 +2843,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL ] )); - it('rank measure 1', async () => runQueryTest( + it('rank measure', async () => runQueryTest( { measures: ['visitors.revenue_rank'], }, From 9adab8b958a0478a737135201324f517cec4c1b9 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Tue, 11 Feb 2025 21:50:24 +0100 Subject: [PATCH 3/4] fmt --- .../cubesqlplanner/src/cube_bridge/base_tools.rs | 5 ++++- .../src/planner/sql_evaluator/symbols/member_symbol.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs index e6f69768e03ac..94ec487c28637 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs @@ -41,7 +41,10 @@ pub trait BaseTools { &self, used_filters: Option>, ) -> Result, CubeError>; - fn filter_group_function_for_rust(&self, used_filters: Option>) -> Result, CubeError>; + fn filter_group_function_for_rust( + &self, + used_filters: Option>, + ) -> Result, CubeError>; fn timestamp_precision(&self) -> Result; fn in_db_time_zone(&self, date: String) -> Result; fn generate_time_series( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs index 42e55ca6f1782..c8eefc7e22ea3 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_symbol.rs @@ -67,7 +67,7 @@ impl MemberSymbol { Self::Dimension(d) => d.is_multi_stage(), Self::TimeDimension(d) => d.is_multi_stage(), Self::Measure(m) => m.is_multi_stage(), - _ => false + _ => false, } } From 4a80bc5eba399a9813be1282a8631ed8c8685640 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Sun, 2 Mar 2025 16:48:29 +0100 Subject: [PATCH 4/4] update --- .../planner/planners/multi_stage_query_planner.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs index 45e2bb2f24366..79529e600c4a6 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage_query_planner.rs @@ -251,13 +251,13 @@ impl MultiStageQueryPlanner { if time_dimension.get_date_range().is_some() && result_granularity.is_some() { let granularity = time_dimension.get_granularity().unwrap(); //FIXME remove this unwrap let date_range = time_dimension.get_date_range().unwrap(); //FIXME remove this unwrap - let seria = self + let series = self .query_tools .base_tools() .generate_time_series(granularity, date_range.clone())?; - if !seria.is_empty() { - let new_from_date = seria.first().unwrap()[0].clone(); - let new_to_date = seria.last().unwrap()[1].clone(); + if !series.is_empty() { + let new_from_date = series.first().unwrap()[0].clone(); + let new_to_date = series.last().unwrap()[1].clone(); new_state.replace_range_in_date_filter( &time_dimension_name, new_from_date, @@ -394,9 +394,9 @@ impl MultiStageQueryPlanner { } let childs = member_childs(&member)?; - - let description = if childs.is_empty() || !has_multi_stage_members(&member, false)? { - if has_multi_stage_members(&member, false)? { + let has_multi_stage_members = has_multi_stage_members(&member, false)?; + let description = if childs.is_empty() || !has_multi_stage_members { + if has_multi_stage_members { return Err(CubeError::internal(format!( "Leaf multi stage query cannot contain multi stage member" )));