Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .design_docs/CAPABILITY_MATCHING_DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,12 @@ this config serve this query type?" and "fetch this time window from the store"
separated.

### Q9: Where does the capability matching logic live?
**Decision**: `sketch_db_common` (the shared crate). Rationale: this logic is pure — it takes a
**Decision**: `asap_types` (the shared crate). Rationale: this logic is pure — it takes a
map of `AggregationConfig` values and a `QueryRequirements` and produces an `AggregationIdInfo`.
It has no dependency on query engine internals. Putting it in common means the planner and other
components can eventually reuse it.

`AggregationIdInfo` (previously defined in `simple_engine.rs`) was moved to `sketch_db_common` as
`AggregationIdInfo` (previously defined in `simple_engine.rs`) was moved to `asap_types` as
a prerequisite, since the common function needs to return it.

`StreamingConfig` (in `asap-query-engine`) gets a thin wrapper method that delegates to the
Expand Down Expand Up @@ -208,13 +208,13 @@ Try find_query_config / find_query_config_sql ← existing path (unchanged)
StreamingConfig::find_compatible_aggregation(&requirements)
sketch_db_common::find_compatible_aggregation(
asap_types::find_compatible_aggregation(
&self.aggregation_configs, requirements
)
→ Option<AggregationIdInfo>
```

The `find_compatible_aggregation` function in `sketch_db_common`:
The `find_compatible_aggregation` function in `asap_types`:
1. For each statistic, collects candidates from `StreamingConfig` passing all filters
(metric, type, sub-type, window, labels, spatial filter).
2. Sorts candidates by `aggregation_priority` (largest window first).
Expand Down
32 changes: 16 additions & 16 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ members = [
"asap-common/dependencies/rs/promql_utilities",
"asap-common/dependencies/rs/sql_utilities",
"asap-common/dependencies/rs/elastic_dsl_utilities",
"asap-common/dependencies/rs/sketch_db_common",
"asap-common/dependencies/rs/asap_types",
"asap-common/dependencies/rs/datafusion_summary_library",
"asap-common/tests/compare_matched_tokens/rust_tests",
"asap-common/tests/compare_patterns",
Expand Down Expand Up @@ -35,7 +35,7 @@ tokio = { version = "1.0", features = ["full"] }
sketch-core = { path = "asap-common/sketch-core" }
promql_utilities = { path = "asap-common/dependencies/rs/promql_utilities" }
sql_utilities = { path = "asap-common/dependencies/rs/sql_utilities" }
sketch_db_common = { path = "asap-common/dependencies/rs/sketch_db_common" }
asap_types = { path = "asap-common/dependencies/rs/asap_types" }
datafusion_summary_library = { path = "asap-common/dependencies/rs/datafusion_summary_library" }
elastic_dsl_utilities = { path = "asap-common/dependencies/rs/elastic_dsl_utilities" }
asap_planner = { path = "asap-planner-rs" }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[package]
name = "sketch_db_common"
name = "asap_types"
version.workspace = true
edition.workspace = true

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ impl StreamingConfig {

impl StreamingConfig {
/// Find a compatible aggregation for the given requirements using capability-based matching.
/// Delegates to `sketch_db_common::find_compatible_aggregation`.
/// Delegates to `asap_types::find_compatible_aggregation`.
pub fn find_compatible_aggregation(
&self,
requirements: &QueryRequirements,
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ name = "asap-planner"
path = "src/main.rs"

[dependencies]
sketch_db_common.workspace = true
asap_types.workspace = true
promql_utilities.workspace = true
sql_utilities.workspace = true
sqlparser = "0.59.0"
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/config/input.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use asap_types::PromQLSchema;
use promql_utilities::data_model::KeyByLabelNames;
use serde::Deserialize;
use sketch_db_common::PromQLSchema;

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
Expand Down
8 changes: 4 additions & 4 deletions asap-planner-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,21 @@ pub mod planner;
pub mod prometheus_client;
pub mod query_log;

use asap_types::enums::QueryLanguage;
use asap_types::inference_config::InferenceConfig;
use asap_types::streaming_config::StreamingConfig;
use promql_utilities::data_model::KeyByLabelNames;
use serde_yaml::Value as YamlValue;
use sketch_db_common::enums::QueryLanguage;
use sketch_db_common::inference_config::InferenceConfig;
use sketch_db_common::streaming_config::StreamingConfig;
use std::path::Path;
use tracing::debug;

pub use asap_types::PromQLSchema;
pub use config::input::ControllerConfig;
pub use config::input::SQLControllerConfig;
pub use error::ControllerError;
pub use output::generator::{GeneratorOutput, PuntedQuery};
pub use output::sql_generator::SQLRuntimeOptions;
pub use prometheus_client::build_schema_from_prometheus;
pub use sketch_db_common::PromQLSchema;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StreamingEngine {
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use asap_planner::{Controller, RuntimeOptions, SQLController, SQLRuntimeOptions, StreamingEngine};
use asap_types::enums::QueryLanguage;
use clap::Parser;
use sketch_db_common::enums::QueryLanguage;
use std::path::PathBuf;

#[derive(Parser, Debug)]
Expand Down
8 changes: 4 additions & 4 deletions asap-planner-rs/src/output/generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ use serde_json::Value as JsonValue;
use serde_yaml::Value as YamlValue;
use std::collections::HashMap;

use asap_types::enums::CleanupPolicy;
use asap_types::PromQLSchema;
use promql_utilities::data_model::KeyByLabelNames;
use sketch_db_common::enums::CleanupPolicy;
use sketch_db_common::PromQLSchema;

use crate::config::input::ControllerConfig;
use crate::error::ControllerError;
Expand Down Expand Up @@ -367,7 +367,7 @@ pub fn params_to_yaml(params: &HashMap<String, JsonValue>) -> YamlValue {
fn build_streaming_yaml(
dedup_map: &IndexMap<String, IntermediateAggConfig>,
id_map: &HashMap<String, u32>,
metric_schema: &sketch_db_common::PromQLSchema,
metric_schema: &asap_types::PromQLSchema,
) -> Result<YamlValue, ControllerError> {
let aggregations: Vec<YamlValue> = dedup_map
.iter()
Expand Down Expand Up @@ -401,7 +401,7 @@ fn build_inference_yaml(
cleanup_policy_str: &str,
query_keys_map: &IndexMap<String, Vec<(String, Option<u64>)>>,
id_map: &HashMap<String, u32>,
metric_schema: &sketch_db_common::PromQLSchema,
metric_schema: &asap_types::PromQLSchema,
) -> Result<YamlValue, ControllerError> {
let mut cleanup_map = serde_yaml::Mapping::new();
cleanup_map.insert(
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/output/sql_generator.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use asap_types::enums::CleanupPolicy;
use indexmap::IndexMap;
use serde_yaml::Value as YamlValue;
use sketch_db_common::enums::CleanupPolicy;
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};

Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/planner/logics.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use crate::config::input::SketchParameterOverrides;
use asap_types::enums::CleanupPolicy;
use promql_utilities::ast_matching::PromQLMatchResult;
use promql_utilities::data_model::KeyByLabelNames;
use promql_utilities::query_logics::enums::{QueryPatternType, Statistic};
use promql_utilities::query_logics::logics::does_precompute_operator_support_subpopulations;
use sketch_db_common::enums::CleanupPolicy;
use std::collections::HashMap;

// Default sketch parameters
Expand Down
4 changes: 2 additions & 2 deletions asap-planner-rs/src/planner/single_query.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use asap_types::enums::CleanupPolicy;
use asap_types::PromQLSchema;
use promql_utilities::ast_matching::PromQLMatchResult;
use promql_utilities::data_model::KeyByLabelNames;
use promql_utilities::query_logics::enums::{QueryPatternType, QueryTreatmentType, Statistic};
Expand All @@ -8,8 +10,6 @@ use promql_utilities::query_logics::parsing::{
get_metric_and_spatial_filter, get_spatial_aggregation_output_labels, get_statistics_to_compute,
};
use serde_json::Value;
use sketch_db_common::enums::CleanupPolicy;
use sketch_db_common::PromQLSchema;
use std::collections::HashMap;

use crate::config::input::SketchParameterOverrides;
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/planner/sql_single_query.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use std::collections::HashSet;

use asap_types::enums::CleanupPolicy;
use promql_utilities::data_model::KeyByLabelNames;
use promql_utilities::query_logics::enums::{QueryTreatmentType, Statistic};
use sketch_db_common::enums::CleanupPolicy;
use sql_utilities::ast_matching::sqlhelper::Table;
use sql_utilities::ast_matching::sqlpattern_matcher::{QueryType, SQLPatternMatcher};
use sql_utilities::ast_matching::sqlpattern_parser::SQLPatternParser;
Expand Down
2 changes: 1 addition & 1 deletion asap-planner-rs/src/prometheus_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ use std::collections::HashSet;
use std::thread;
use std::time::Duration;

use asap_types::PromQLSchema;
use promql_parser::parser::Expr;
use promql_utilities::data_model::KeyByLabelNames;
use sketch_db_common::PromQLSchema;
use tracing::{debug, warn};

use crate::error::ControllerError;
Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition.workspace = true
sketch-core.workspace = true
promql_utilities.workspace = true
sql_utilities.workspace = true
sketch_db_common.workspace = true
asap_types.workspace = true
datafusion_summary_library.workspace = true
asap_planner.workspace = true

Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/bin/bench_precompute_sketch.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use asap_types::aggregation_config::AggregationConfig;
use clap::Parser;
use prost::Message;
use query_engine_rust::data_model::{
Expand All @@ -10,7 +11,6 @@ use query_engine_rust::precompute_engine::config::{LateDataPolicy, PrecomputeEng
use query_engine_rust::precompute_engine::output_sink::OutputSink;
use query_engine_rust::precompute_engine::PrecomputeEngine;
use query_engine_rust::stores::{SimpleMapStore, Store};
use sketch_db_common::aggregation_config::AggregationConfig;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/bin/test_e2e_precompute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
//! Usage:
//! cargo run --bin test_e2e_precompute

use asap_types::aggregation_config::AggregationConfig;
use prost::Message;
use query_engine_rust::data_model::{LockStrategy, QueryLanguage, StreamingConfig};
use query_engine_rust::drivers::ingest::prometheus_remote_write::{
Expand All @@ -23,7 +24,6 @@ use query_engine_rust::precompute_engine::PrecomputeEngine;
use query_engine_rust::stores::SimpleMapStore;
use query_engine_rust::utils::file_io::{read_inference_config, read_streaming_config};
use query_engine_rust::{HttpServer, HttpServerConfig};
use sketch_db_common::aggregation_config::AggregationConfig;
use std::collections::HashMap;
use std::sync::Arc;

Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/aggregation_config.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::aggregation_config::*;
pub use asap_types::aggregation_config::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/aggregation_reference.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::aggregation_reference::*;
pub use asap_types::aggregation_reference::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/enums.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ pub enum StreamingEngine {
Arroyo,
}

pub use sketch_db_common::enums::{CleanupPolicy, QueryLanguage};
pub use asap_types::enums::{CleanupPolicy, QueryLanguage};

#[derive(clap::ValueEnum, Clone, Debug, PartialEq)]
pub enum QueryProtocol {
Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/inference_config.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::inference_config::*;
pub use asap_types::inference_config::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/promql_schema.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::promql_schema::*;
pub use asap_types::promql_schema::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/query_config.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::query_config::*;
pub use asap_types::query_config::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/streaming_config.rs
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pub use sketch_db_common::streaming_config::*;
pub use asap_types::streaming_config::*;
2 changes: 1 addition & 1 deletion asap-query-engine/src/data_model/traits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::collections::HashMap;

use promql_utilities::query_logics::enums::Statistic;

pub use sketch_db_common::traits::SerializableToSink;
pub use asap_types::traits::SerializableToSink;

/// Core trait for all aggregates containing shared functionality
/// This trait provides common operations like serialization, cloning, and type identification
Expand Down
4 changes: 2 additions & 2 deletions asap-query-engine/src/engines/simple_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ use tracing::{debug, warn};

use crate::AggregateCore;

use asap_types::query_requirements::QueryRequirements;
use asap_types::utils::normalize_spatial_filter;
use promql_utilities::ast_matching::{PromQLMatchResult, PromQLPattern, PromQLPatternBuilder};
use promql_utilities::data_model::KeyByLabelNames;
use promql_utilities::query_logics::enums::{QueryPatternType, Statistic};
use promql_utilities::query_logics::parsing::{
get_metric_and_spatial_filter, get_spatial_aggregation_output_labels, get_statistics_to_compute,
};
use sketch_db_common::query_requirements::QueryRequirements;
use sketch_db_common::utils::normalize_spatial_filter;

use sql_utilities::ast_matching::QueryType;
use sql_utilities::ast_matching::{SQLPatternMatcher, SQLPatternParser, SQLQuery};
Expand Down
6 changes: 3 additions & 3 deletions asap-query-engine/src/planner_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ use anyhow::Result;
use asap_planner::{
build_schema_from_prometheus, Controller, ControllerConfig, PlannerOutput, RuntimeOptions,
};
use sketch_db_common::enums::QueryLanguage;
use sketch_db_common::inference_config::InferenceConfig;
use sketch_db_common::streaming_config::StreamingConfig;
use asap_types::enums::QueryLanguage;
use asap_types::inference_config::InferenceConfig;
use asap_types::streaming_config::StreamingConfig;
use tracing::warn;

pub struct PlannerResult {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::precompute_operators::{
IncreaseAccumulator, MinMaxAccumulator, MultipleIncreaseAccumulator, MultipleMinMaxAccumulator,
MultipleSumAccumulator, SumAccumulator,
};
use sketch_db_common::aggregation_config::AggregationConfig;
use asap_types::aggregation_config::AggregationConfig;

/// Generate the two boilerplate clone-based `AccumulatorUpdater` methods
/// for updaters whose inner `acc` field implements `Clone + AggregateCore`.
Expand Down
2 changes: 1 addition & 1 deletion asap-query-engine/src/precompute_engine/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use crate::precompute_engine::ingest_handler::{
use crate::precompute_engine::output_sink::OutputSink;
use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage};
use crate::precompute_engine::worker::{Worker, WorkerRuntimeConfig};
use asap_types::aggregation_config::AggregationConfig;
use axum::{routing::post, Router};
use sketch_db_common::aggregation_config::AggregationConfig;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::net::TcpListener;
Expand Down
Loading
Loading