diff --git a/libs/@local/hashql/mir/src/builder/rvalue.rs b/libs/@local/hashql/mir/src/builder/rvalue.rs index 7abaf762b80..d1017f5451b 100644 --- a/libs/@local/hashql/mir/src/builder/rvalue.rs +++ b/libs/@local/hashql/mir/src/builder/rvalue.rs @@ -235,6 +235,17 @@ macro_rules! rvalue { rv.tuple(members) }; $payload; $($rest)*) }; + ($resume:path; $payload:tt; list; $($rest:tt)*) => { + $resume!(@rvalue |rv| { + rv.list([] as [!; 0]) + }; $payload; $($rest)*) + }; + ($resume:path; $payload:tt; list $($members:tt),+; $($rest:tt)*) => { + $resume!(@rvalue |rv| { + let members = [$($crate::builder::_private::operand!(rv; $members)),*]; + rv.list(members) + }; $payload; $($rest)*) + }; ($resume:path; $payload:tt; struct $($field:ident : $value:tt),+ $(,)?; $($rest:tt)*) => { $resume!(@rvalue |rv| { let fields = [$( diff --git a/libs/@local/hashql/mir/src/pass/analysis/mod.rs b/libs/@local/hashql/mir/src/pass/analysis/mod.rs index f1c6a27f99c..0dbc94ecce8 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/mod.rs +++ b/libs/@local/hashql/mir/src/pass/analysis/mod.rs @@ -1,7 +1,6 @@ mod callgraph; mod data_dependency; pub mod dataflow; -pub mod execution; pub mod size_estimation; pub use self::{ callgraph::{CallGraph, CallGraphAnalysis, CallKind, CallSite}, diff --git a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/estimate.rs b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/estimate.rs index f01cdaf4db9..e4b35dfccb5 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/estimate.rs +++ b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/estimate.rs @@ -97,6 +97,31 @@ impl Estimate { } } + pub(crate) fn eval(&self, monoid: &M, values: impl IntoIterator>) -> T + where + T: Clone, + M: AdditiveMonoid, + for<'a> &'a T: SaturatingMul, + { + match self { + Self::Constant(value) => value.clone(), + Self::Affine(AffineEquation { + coefficients, + constant, + }) => { + let mut result = constant.clone(); + + // in case values are not provided, we assume they are zero, therefore cannot be + // added. + for (&coefficient, value) in coefficients.iter().zip(values) { + monoid.plus(&mut result, &value.as_ref().saturating_mul(coefficient)); + } + + result + } + } + } + /// Returns mutable access to the constant term. pub(crate) const fn constant_mut(&mut self) -> &mut T { match self { diff --git a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/footprint.rs b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/footprint.rs index b3a3931a968..50a51b08c21 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/footprint.rs +++ b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/footprint.rs @@ -13,6 +13,7 @@ use core::{alloc::Allocator, fmt}; use hashql_core::heap::TryCloneIn; use super::{ + InformationUnit, affine::AffineEquation, estimate::Estimate, range::{Cardinality, InformationRange}, @@ -225,6 +226,29 @@ impl Footprint { } } + #[must_use] + pub fn average( + &self, + units: &[InformationRange], + cardinality: &[Cardinality], + ) -> Option { + let units = self.units.eval(&SaturatingSemiring, units); + let cardinality = self.cardinality.eval(&SaturatingSemiring, cardinality); + + if units.is_empty() || cardinality.is_empty() { + return Some(InformationUnit::new(0)); + } + + let max = units.inclusive_max()?; + let max = max.checked_mul(cardinality.inclusive_max()?)?; + + let min = units.min(); + let min = min.checked_mul(cardinality.min())?; + + let avg = min.midpoint(max); + Some(avg) + } + /// Adds `other * coefficient` to this footprint (component-wise). pub(crate) fn saturating_mul_add( &mut self, diff --git a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/range.rs b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/range.rs index ad09d675901..41aef1dafdb 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/range.rs +++ b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/range.rs @@ -133,12 +133,31 @@ macro_rules! range { Self { min: one, max: Bound::Included(one) } } + #[inline] + pub const fn zero() -> Self { + let zero = <$inner>::new(0); + Self { min: zero, max: Bound::Included(zero) } + } + #[inline] pub const fn full() -> Self { let zero = <$inner>::new(0); Self { min: zero, max: Bound::Unbounded } } + #[inline] + pub const fn min(self) -> $inner { + self.min + } + + pub fn inclusive_max(self) -> Option<$inner> { + match self.max { + Bound::Included(max) => Some(max), + Bound::Excluded(max) => max.raw.checked_sub(1).map(<$inner>::new), + Bound::Unbounded => None, + } + } + #[inline] pub const fn is_empty(&self) -> bool { match self.max { @@ -260,6 +279,12 @@ macro_rules! range { } forward_ref_binop!(impl SaturatingMul::saturating_mul for $name); + + impl AsRef<$name> for $name { + fn as_ref(&self) -> &$name { + self + } + } }; } diff --git a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/unit.rs b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/unit.rs index 6e20b61c89d..2b528c8e9cf 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/size_estimation/unit.rs +++ b/libs/@local/hashql/mir/src/pass/analysis/size_estimation/unit.rs @@ -36,6 +36,11 @@ macro_rules! unit { Self { raw: inner } } + #[inline] + pub const fn as_u32(self) -> u32 { + self.raw as u32 + } + #[inline] pub const fn checked_add(self, rhs: Self) -> Option { match self.raw.checked_add(rhs.raw) { @@ -157,6 +162,25 @@ unit!( pub struct InformationUnit(u32) ); +impl InformationUnit { + #[inline] + #[must_use] + pub const fn checked_mul(self, cardinal: Cardinal) -> Option { + let raw = self.raw.checked_mul(cardinal.raw); + + match raw { + Some(value) => Some(Self::new(value)), + None => None, + } + } + + #[inline] + #[must_use] + pub const fn midpoint(self, other: Self) -> Self { + Self::new(u32::midpoint(self.raw, other.raw)) + } +} + unit!( /// A unit of cardinality (element count). /// diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/cost.rs b/libs/@local/hashql/mir/src/pass/execution/cost.rs similarity index 86% rename from libs/@local/hashql/mir/src/pass/analysis/execution/cost.rs rename to libs/@local/hashql/mir/src/pass/execution/cost.rs index 222caf06cc9..1ccaea406c9 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/cost.rs +++ b/libs/@local/hashql/mir/src/pass/execution/cost.rs @@ -37,9 +37,29 @@ use crate::{ pub struct Cost(core::num::niche_types::U32NotAllOnes); impl Cost { + /// The maximum representable cost value (`u32::MAX - 1`). + /// + /// Used as a sentinel for "effectively infinite" cost when exact values overflow. + /// + /// ``` + /// # use hashql_mir::pass::execution::Cost; + /// assert_eq!(Cost::MAX, Cost::new(u32::MAX - 1).unwrap()); + /// ``` + pub const MAX: Self = match core::num::niche_types::U32NotAllOnes::new(0xFFFF_FFFE) { + Some(cost) => Self(cost), + None => unreachable!(), + }; + /// Creates a cost from a `u32` value, returning `None` if the value is `u32::MAX`. /// - /// The `u32::MAX` value is reserved as a niche for `Option` optimization. + /// The `u32::MAX` value is reserved as a niche for [`Option`] optimization. + /// + /// ``` + /// # use hashql_mir::pass::execution::Cost; + /// assert!(Cost::new(0).is_some()); + /// assert!(Cost::new(100).is_some()); + /// assert!(Cost::new(u32::MAX).is_none()); // Reserved for niche + /// ``` #[must_use] pub const fn new(value: u32) -> Option { match core::num::niche_types::U32NotAllOnes::new(value) { @@ -69,6 +89,25 @@ impl Cost { // SAFETY: The caller must ensure `value` is not `u32::MAX`. Self(unsafe { core::num::niche_types::U32NotAllOnes::new_unchecked(value) }) } + + /// Adds `other` to this cost, saturating at [`Cost::MAX`] on overflow. + /// + /// ``` + /// # use hashql_mir::pass::execution::Cost; + /// let cost = Cost::new(100).unwrap(); + /// assert_eq!(cost.saturating_add(50), Cost::new(150).unwrap()); + /// + /// // Saturates at MAX instead of overflowing + /// let large = Cost::new(u32::MAX - 10).unwrap(); + /// assert_eq!(large.saturating_add(100), Cost::MAX); + /// ``` + #[inline] + #[must_use] + pub fn saturating_add(self, other: u32) -> Self { + let raw = self.0.as_inner(); + + Self::new(raw.saturating_add(other)).unwrap_or(Self::MAX) + } } impl fmt::Display for Cost { @@ -109,6 +148,7 @@ impl TraversalCostVec { } } + /// Iterates over all (local, cost) pairs that have assigned costs. pub fn iter(&self) -> impl Iterator { self.costs .iter_enumerated() @@ -144,7 +184,6 @@ impl StatementCostVec { mut iter: impl ExactSizeIterator, alloc: A, ) -> (Box, A>, usize) { - // Try to reuse existing offsets if available and of correct length let mut offsets = Box::new_uninit_slice_in(iter.len() + 1, alloc); let mut offset = 0_u32; @@ -193,6 +232,10 @@ impl StatementCostVec { ) } + /// Rebuilds the offset table for a new block layout. + /// + /// Call after transforms that change statement counts per block. Does not resize or clear + /// the cost data — callers must ensure the total statement count remains unchanged. #[expect(clippy::cast_possible_truncation)] pub fn remap(&mut self, blocks: &BasicBlocks) where @@ -207,16 +250,21 @@ impl StatementCostVec { self.offsets = offsets; } + /// Returns `true` if no statements have assigned costs. pub fn is_empty(&self) -> bool { self.costs.iter().all(Option::is_none) } + /// Returns the cost slice for all statements in `block`. + /// + /// The returned slice is indexed by statement position (0-based within the block). pub fn of(&self, block: BasicBlockId) -> &[Option] { let range = (self.offsets[block] as usize)..(self.offsets[block.plus(1)] as usize); &self.costs[range] } + /// Returns a reference to the allocator used by this cost vector. pub fn allocator(&self) -> &A { Box::allocator(&self.offsets) } diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/mod.rs b/libs/@local/hashql/mir/src/pass/execution/mod.rs similarity index 65% rename from libs/@local/hashql/mir/src/pass/analysis/execution/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/mod.rs index 761b20586b4..7f1c3fbbeec 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/mod.rs +++ b/libs/@local/hashql/mir/src/pass/execution/mod.rs @@ -1,6 +1,6 @@ macro_rules! cost { ($value:expr) => { - const { $crate::pass::analysis::execution::cost::Cost::new_panic($value) } + const { $crate::pass::execution::cost::Cost::new_panic($value) } }; } @@ -8,5 +8,6 @@ mod cost; pub mod splitting; pub mod statement_placement; pub mod target; +pub mod terminator_placement; pub use self::cost::{Cost, StatementCostVec, TraversalCostVec}; diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/splitting/mod.rs b/libs/@local/hashql/mir/src/pass/execution/splitting/mod.rs similarity index 100% rename from libs/@local/hashql/mir/src/pass/analysis/execution/splitting/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/splitting/mod.rs diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/splitting/tests.rs b/libs/@local/hashql/mir/src/pass/execution/splitting/tests.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/splitting/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/splitting/tests.rs index bc934f15371..a1129257fd5 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/splitting/tests.rs +++ b/libs/@local/hashql/mir/src/pass/execution/splitting/tests.rs @@ -29,7 +29,7 @@ use crate::{ builder::body, context::MirContext, intern::Interner, - pass::analysis::execution::{ + pass::execution::{ StatementCostVec, cost::Cost, target::{TargetArray, TargetBitSet, TargetId}, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/common.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/common.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/common.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/common.rs index abf70d07c23..01c177acd73 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/common.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/common.rs @@ -17,8 +17,8 @@ use crate::{ terminator::TerminatorKind, }, context::MirContext, - pass::analysis::{ - dataflow::{ + pass::{ + analysis::dataflow::{ framework::{DataflowAnalysis, DataflowResults}, lattice::PowersetLattice, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/mod.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/mod.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/mod.rs index f565d72713d..7d7e071439c 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/mod.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/mod.rs @@ -14,7 +14,7 @@ use crate::{ body::{Body, Source, local::Local, operand::Operand, place::Place, rvalue::RValue}, context::MirContext, pass::{ - analysis::execution::{ + execution::{ Cost, StatementCostVec, cost::TraversalCostVec, statement_placement::lookup::{Access, entity_projection_access}, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/tests.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/tests.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/tests.rs index 1d113b7b33f..ac8f274387a 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/embedding/tests.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/embedding/tests.rs @@ -9,7 +9,7 @@ use crate::{ context::MirContext, def::DefId, intern::Interner, - pass::analysis::execution::statement_placement::{ + pass::execution::statement_placement::{ EmbeddingStatementPlacement, tests::{assert_placement, run_placement}, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/mod.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/mod.rs similarity index 98% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/mod.rs index 1640c1b9360..60d75e68e12 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/mod.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/mod.rs @@ -11,7 +11,7 @@ use crate::{ }, context::MirContext, pass::{ - analysis::execution::{ + execution::{ cost::{Cost, StatementCostVec, TraversalCostVec}, target::Interpreter, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/tests.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/tests.rs similarity index 98% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/tests.rs index 6cf1374802b..bd0b38fe614 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/interpret/tests.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/interpret/tests.rs @@ -9,7 +9,7 @@ use crate::{ context::MirContext, def::DefId, intern::Interner, - pass::analysis::execution::statement_placement::{ + pass::execution::statement_placement::{ InterpreterStatementPlacement, tests::{assert_placement, run_placement}, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/entity.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/entity.rs similarity index 100% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/entity.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/entity.rs diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/mod.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/mod.rs similarity index 100% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/mod.rs diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/tests.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/tests.rs similarity index 100% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/tests.rs diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/trie.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/trie.rs similarity index 100% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/lookup/trie.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/lookup/trie.rs diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/mod.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/mod.rs similarity index 96% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/mod.rs index c7ca3774426..8f613633a7c 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/mod.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/mod.rs @@ -31,7 +31,7 @@ use crate::{ body::Body, context::MirContext, pass::{ - analysis::execution::cost::{StatementCostVec, TraversalCostVec}, + execution::cost::{StatementCostVec, TraversalCostVec}, transform::Traversals, }, }; diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/mod.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/mod.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/mod.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/mod.rs index 5f3482347e9..7a64ded81d4 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/mod.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/mod.rs @@ -26,7 +26,7 @@ use crate::{ }, context::MirContext, pass::{ - analysis::execution::{ + execution::{ cost::{Cost, StatementCostVec, TraversalCostVec}, statement_placement::lookup::{Access, entity_projection_access}, target::Postgres, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/tests.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/tests.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/tests.rs index adcdcbc9163..74cf5422ca2 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/postgres/tests.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/postgres/tests.rs @@ -20,7 +20,7 @@ use crate::{ intern::Interner, op, pass::{ - analysis::execution::statement_placement::{ + execution::statement_placement::{ PostgresStatementPlacement, StatementPlacement as _, tests::{assert_placement, run_placement}, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/tests.rs b/libs/@local/hashql/mir/src/pass/execution/statement_placement/tests.rs similarity index 99% rename from libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/tests.rs rename to libs/@local/hashql/mir/src/pass/execution/statement_placement/tests.rs index a2f949c754c..f5a65103cf7 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/statement_placement/tests.rs +++ b/libs/@local/hashql/mir/src/pass/execution/statement_placement/tests.rs @@ -21,7 +21,7 @@ use crate::{ intern::Interner, pass::{ Changed, TransformPass as _, - analysis::execution::{ + execution::{ cost::{StatementCostVec, TraversalCostVec}, statement_placement::{EmbeddingStatementPlacement, PostgresStatementPlacement}, }, diff --git a/libs/@local/hashql/mir/src/pass/analysis/execution/target.rs b/libs/@local/hashql/mir/src/pass/execution/target.rs similarity index 97% rename from libs/@local/hashql/mir/src/pass/analysis/execution/target.rs rename to libs/@local/hashql/mir/src/pass/execution/target.rs index 124c9fe4af5..6c18772e901 100644 --- a/libs/@local/hashql/mir/src/pass/analysis/execution/target.rs +++ b/libs/@local/hashql/mir/src/pass/execution/target.rs @@ -62,6 +62,15 @@ impl TargetId { VARIANTS } + + #[must_use] + pub const fn abbreviation(self) -> &'static str { + match self { + Self::Interpreter => "I", + Self::Postgres => "P", + Self::Embedding => "E", + } + } } const _: () = { diff --git a/libs/@local/hashql/mir/src/pass/execution/terminator_placement/mod.rs b/libs/@local/hashql/mir/src/pass/execution/terminator_placement/mod.rs new file mode 100644 index 00000000000..77a15353f6f --- /dev/null +++ b/libs/@local/hashql/mir/src/pass/execution/terminator_placement/mod.rs @@ -0,0 +1,719 @@ +//! Terminator placement analysis for MIR execution planning. +//! +//! Complements [`StatementPlacement`] by analyzing control flow edges rather than individual +//! statements. For each terminator edge, produces a [`TransMatrix`] encoding which backend +//! transitions are valid and their associated transfer costs. +//! +//! The execution planner uses this to determine optimal points for backend switches during query +//! execution. +//! +//! # Main Types +//! +//! - [`TransMatrix`]: Per-edge transition costs indexed by (source, destination) target pairs +//! - [`TerminatorCostVec`]: Collection of transition matrices for all edges in a body +//! - [`TerminatorPlacement`]: Analysis driver that computes placement for a body +//! +//! # Transition Rules +//! +//! The analysis enforces these constraints on backend transitions: +//! +//! | Transition | Allowed? | Cost | +//! |------------|----------|------| +//! | Same backend (A → A) | Always | 0 | +//! | Any → Interpreter | Always | Transfer cost | +//! | Other → Postgres | Never | — | +//! | Any Postgres in loop | Never | — | +//! | `GraphRead` edge | Interpreter → Interpreter only | 0 | +//! | `Goto` edge | Any supported transition | Transfer cost | +//! | `SwitchInt` edge | Same-backend or → Interpreter only | Transfer cost | +//! +//! Transfer cost is computed from the estimated size of live locals that must cross the edge. +//! +//! [`StatementPlacement`]: super::statement_placement::StatementPlacement + +use alloc::alloc::Global; +use core::{ + alloc::Allocator, + iter, + ops::{Index, IndexMut}, +}; + +use hashql_core::{ + graph::algorithms::{ + Tarjan, + tarjan::{Metadata, SccId, StronglyConnectedComponents}, + }, + heap::Heap, + id::{ + Id as _, + bit_vec::{BitRelations as _, DenseBitSet}, + }, +}; + +use super::{ + Cost, + target::{TargetBitSet, TargetId}, +}; +use crate::{ + body::{ + Body, + basic_block::{BasicBlock, BasicBlockId, BasicBlockSlice, BasicBlockVec}, + basic_blocks::BasicBlocks, + local::Local, + terminator::TerminatorKind, + }, + context::MirContext, + pass::analysis::{ + dataflow::{ + LivenessAnalysis, + framework::{DataflowAnalysis as _, DataflowResults}, + }, + size_estimation::{BodyFootprint, Cardinality, InformationRange}, + }, +}; + +#[cfg(test)] +mod tests; + +/// Transition cost matrix for a single terminator edge. +/// +/// Maps each (source, destination) [`TargetId`] pair to either `Some(cost)` if the transition is +/// valid, or `None` if disallowed. Supports indexing with tuple syntax: `matrix[(from, to)]`. +/// +/// # Invariants +/// +/// - Same-backend transitions (`A → A`) always have cost 0, enforced by [`insert`](Self::insert) +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct TransMatrix { + matrix: [Option; TargetId::VARIANT_COUNT * TargetId::VARIANT_COUNT], +} + +impl TransMatrix { + /// Creates an empty matrix with all transitions disallowed. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// let matrix = TransMatrix::new(); + /// assert!( + /// matrix + /// .get(TargetId::Interpreter, TargetId::Postgres) + /// .is_none() + /// ); + /// ``` + #[must_use] + pub const fn new() -> Self { + Self { + matrix: [None; TargetId::VARIANT_COUNT * TargetId::VARIANT_COUNT], + } + } + + #[inline] + fn offset(from: TargetId, to: TargetId) -> usize { + from.as_usize() * TargetId::VARIANT_COUNT + to.as_usize() + } + + #[inline] + #[expect(clippy::integer_division, clippy::integer_division_remainder_used)] + fn from_offset(offset: usize) -> (TargetId, TargetId) { + let from = TargetId::from_usize(offset / TargetId::VARIANT_COUNT); + let to = TargetId::from_usize(offset % TargetId::VARIANT_COUNT); + (from, to) + } + + /// Returns the cost for transitioning from `from` to `to`, or `None` if disallowed. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Interpreter, + /// Cost::new(100).unwrap(), + /// ); + /// + /// assert_eq!( + /// matrix.get(TargetId::Postgres, TargetId::Interpreter), + /// Some(Cost::new(100).unwrap()) + /// ); + /// assert_eq!(matrix.get(TargetId::Interpreter, TargetId::Postgres), None); + /// ``` + #[inline] + #[must_use] + pub fn get(&self, from: TargetId, to: TargetId) -> Option { + self.matrix[Self::offset(from, to)] + } + + /// Returns a mutable reference to the cost entry for the given transition. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// + /// *matrix.get_mut(TargetId::Postgres, TargetId::Interpreter) = Some(Cost::new(50).unwrap()); + /// assert_eq!( + /// matrix.get(TargetId::Postgres, TargetId::Interpreter), + /// Some(Cost::new(50).unwrap()) + /// ); + /// ``` + #[inline] + pub fn get_mut(&mut self, from: TargetId, to: TargetId) -> &mut Option { + &mut self.matrix[Self::offset(from, to)] + } + + /// Inserts a transition with the given cost. + /// + /// Same-backend transitions (where `from == to`) are always recorded with cost 0, + /// regardless of the `cost` argument. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// + /// // Cross-backend transition uses the provided cost + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Interpreter, + /// Cost::new(100).unwrap(), + /// ); + /// assert_eq!( + /// matrix.get(TargetId::Postgres, TargetId::Interpreter), + /// Some(Cost::new(100).unwrap()) + /// ); + /// + /// // Same-backend transition is always zero cost + /// matrix.insert( + /// TargetId::Interpreter, + /// TargetId::Interpreter, + /// Cost::new(100).unwrap(), + /// ); + /// assert_eq!( + /// matrix.get(TargetId::Interpreter, TargetId::Interpreter), + /// Some(Cost::new(0).unwrap()) + /// ); + /// ``` + #[inline] + pub fn insert(&mut self, from: TargetId, to: TargetId, mut cost: Cost) { + if from == to { + cost = cost!(0); + } + + self.matrix[Self::offset(from, to)] = Some(cost); + } + + /// Resets all transitions to disallowed. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Interpreter, + /// Cost::new(10).unwrap(), + /// ); + /// + /// matrix.clear(); + /// assert!( + /// matrix + /// .get(TargetId::Postgres, TargetId::Interpreter) + /// .is_none() + /// ); + /// ``` + #[inline] + pub fn clear(&mut self) { + self.matrix.fill(None); + } + + /// Removes all incoming transitions to `target` from other backends. + /// + /// Self-loops (`target` → `target`) are preserved. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// matrix.insert( + /// TargetId::Interpreter, + /// TargetId::Postgres, + /// Cost::new(10).unwrap(), + /// ); + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Postgres, + /// Cost::new(0).unwrap(), + /// ); + /// + /// matrix.remove_incoming(TargetId::Postgres); + /// + /// // Incoming from other backends removed + /// assert!( + /// matrix + /// .get(TargetId::Interpreter, TargetId::Postgres) + /// .is_none() + /// ); + /// // Self-loop preserved + /// assert!(matrix.get(TargetId::Postgres, TargetId::Postgres).is_some()); + /// ``` + #[inline] + pub fn remove_incoming(&mut self, target: TargetId) { + for source in TargetId::all() { + if source == target { + continue; + } + + self.matrix[Self::offset(source, target)] = None; + } + } + + /// Removes all transitions both to and from `target`. + /// + /// ``` + /// # use hashql_mir::pass::execution::terminator_placement::TransMatrix; + /// # use hashql_mir::pass::execution::target::TargetId; + /// # use hashql_mir::pass::execution::Cost; + /// let mut matrix = TransMatrix::new(); + /// matrix.insert( + /// TargetId::Interpreter, + /// TargetId::Postgres, + /// Cost::new(10).unwrap(), + /// ); + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Interpreter, + /// Cost::new(20).unwrap(), + /// ); + /// matrix.insert( + /// TargetId::Postgres, + /// TargetId::Postgres, + /// Cost::new(0).unwrap(), + /// ); + /// + /// matrix.remove_all(TargetId::Postgres); + /// + /// assert!( + /// matrix + /// .get(TargetId::Interpreter, TargetId::Postgres) + /// .is_none() + /// ); + /// assert!( + /// matrix + /// .get(TargetId::Postgres, TargetId::Interpreter) + /// .is_none() + /// ); + /// assert!(matrix.get(TargetId::Postgres, TargetId::Postgres).is_none()); + /// ``` + pub fn remove_all(&mut self, target: TargetId) { + for other in TargetId::all() { + self.matrix[Self::offset(other, target)] = None; + self.matrix[Self::offset(target, other)] = None; + } + } + + #[must_use] + pub fn iter(&self) -> impl ExactSizeIterator)> { + self.matrix.iter().enumerate().map(|(idx, cost)| { + let (from, to) = Self::from_offset(idx); + + (from, to, cost) + }) + } +} + +impl Default for TransMatrix { + fn default() -> Self { + Self::new() + } +} + +impl Index<(TargetId, TargetId)> for TransMatrix { + type Output = Option; + + fn index(&self, (from, to): (TargetId, TargetId)) -> &Self::Output { + &self.matrix[Self::offset(from, to)] + } +} + +impl IndexMut<(TargetId, TargetId)> for TransMatrix { + fn index_mut(&mut self, (from, to): (TargetId, TargetId)) -> &mut Self::Output { + &mut self.matrix[Self::offset(from, to)] + } +} + +/// Collection of [`TransMatrix`] entries for all terminator edges in a body. +/// +/// Indexed by [`BasicBlockId`] via [`of`](Self::of), returning a slice of matrices corresponding +/// to that block's successor edges. The slice length matches the terminator's successor count: +/// +/// | Terminator | Edges | +/// |------------|-------| +/// | [`Goto`] / [`GraphRead`] | 1 | +/// | [`SwitchInt`] | N (branch count) | +/// | [`Return`] / [`Unreachable`] | 0 | +/// +/// [`Goto`]: TerminatorKind::Goto +/// [`GraphRead`]: TerminatorKind::GraphRead +/// [`SwitchInt`]: TerminatorKind::SwitchInt +/// [`Return`]: TerminatorKind::Return +/// [`Unreachable`]: TerminatorKind::Unreachable +pub struct TerminatorCostVec { + offsets: Box, A>, + matrices: Vec, +} + +impl TerminatorCostVec { + #[expect(unsafe_code)] + fn compute_offsets( + mut iter: impl ExactSizeIterator, + alloc: A, + ) -> (Box, A>, usize) { + let mut offsets = Box::new_uninit_slice_in(iter.len() + 1, alloc); + let mut running_offset = 0_u32; + + offsets[0].write(0); + + let (_, rest) = offsets[1..].write_iter(iter::from_fn(|| { + let successor_count = iter.next()?; + running_offset += successor_count; + Some(running_offset) + })); + + debug_assert!(rest.is_empty()); + debug_assert_eq!(iter.len(), 0); + + // SAFETY: All elements initialized by write_iter loop. + let offsets = unsafe { offsets.assume_init() }; + let offsets = BasicBlockSlice::from_boxed_slice(offsets); + + (offsets, running_offset as usize) + } + + fn from_successor_counts(iter: impl ExactSizeIterator, alloc: A) -> Self + where + A: Clone, + { + let (offsets, total_edges) = Self::compute_offsets(iter, alloc.clone()); + let matrices = alloc::vec::from_elem_in(TransMatrix::new(), total_edges, alloc); + + Self { offsets, matrices } + } + + /// Creates a cost vector sized for `blocks`, with all transitions initially disallowed. + pub fn new(blocks: &BasicBlocks, alloc: A) -> Self + where + A: Clone, + { + Self::from_successor_counts(blocks.iter().map(Self::successor_count), alloc) + } + + #[expect(clippy::cast_possible_truncation)] + fn successor_count(block: &BasicBlock) -> u32 { + match &block.terminator.kind { + TerminatorKind::SwitchInt(switch) => switch.targets.targets().len() as u32, + TerminatorKind::Goto(_) | TerminatorKind::GraphRead(_) => 1, + TerminatorKind::Return(_) | TerminatorKind::Unreachable => 0, + } + } + + /// Returns the transition matrices for all successor edges of `block`. + pub fn of(&self, block: BasicBlockId) -> &[TransMatrix] { + let start = self.offsets[block] as usize; + let end = self.offsets[block.plus(1)] as usize; + + &self.matrices[start..end] + } + + /// Returns mutable transition matrices for all successor edges of `block`. + pub fn of_mut(&mut self, block: BasicBlockId) -> &mut [TransMatrix] { + let start = self.offsets[block] as usize; + let end = self.offsets[block.plus(1)] as usize; + + &mut self.matrices[start..end] + } +} + +/// Tarjan metadata that counts nodes per strongly connected component. +struct ComponentSizeMetadata; + +impl Metadata for ComponentSizeMetadata { + type Annotation = u32; + + fn annotate_node(&mut self, _: N) -> Self::Annotation { + 1 + } + + fn annotate_scc(&mut self, _: S, _: N) -> Self::Annotation { + 0 + } + + fn merge_into_scc(&mut self, lhs: &mut Self::Annotation, other: Self::Annotation) { + *lhs += other; + } + + fn merge_reachable(&mut self, _: &mut Self::Annotation, _: &Self::Annotation) {} +} + +/// Parameters for populating a single edge's [`TransMatrix`]. +struct PopulateEdgeMatrix { + /// Backends the source block can execute on. + source_targets: TargetBitSet, + /// Backends the destination block can execute on. + target_targets: TargetBitSet, + + /// Cost of transferring live data across this edge. + transfer_cost: Cost, + /// Whether this edge is part of a loop (disables Postgres transitions). + is_in_loop: bool, +} + +impl PopulateEdgeMatrix { + /// Populates a transition matrix for a single terminator edge. + fn populate(&self, matrix: &mut TransMatrix, terminator: &TerminatorKind) { + self.add_same_backend_transitions(matrix); + self.add_interpreter_fallback(matrix); + self.add_terminator_specific_transitions(matrix, terminator); + self.apply_postgres_restrictions(matrix); + } + + /// Adds zero-cost transitions for staying on the same backend. + fn add_same_backend_transitions(&self, matrix: &mut TransMatrix) { + let mut common = self.target_targets; + common.intersect(&self.source_targets); + + for target in &common { + matrix.insert(target, target, cost!(0)); + } + } + + /// Adds transitions to Interpreter (always allowed as fallback). + fn add_interpreter_fallback(&self, matrix: &mut TransMatrix) { + if !self.target_targets.contains(TargetId::Interpreter) { + // TODO: warning that this shouldn't happen + return; + } + + for source in &self.source_targets { + matrix.insert(source, TargetId::Interpreter, self.transfer_cost); + } + } + + /// Adds transitions based on terminator-specific rules. + fn add_terminator_specific_transitions( + &self, + matrix: &mut TransMatrix, + terminator: &TerminatorKind, + ) { + match terminator { + TerminatorKind::Goto(_) => { + self.add_goto_transitions(matrix); + } + TerminatorKind::SwitchInt(_) => { + // SwitchInt does not allow additional cross-backend transitions + // due to complexity of coordinating branches across backends. + } + TerminatorKind::GraphRead(_) => { + self.restrict_to_interpreter_only(matrix); + } + TerminatorKind::Return(_) | TerminatorKind::Unreachable => { + unreachable!("terminal blocks have no successor edges") + } + } + } + + /// Goto allows transitions to any supported target. + fn add_goto_transitions(&self, matrix: &mut TransMatrix) { + for source in &self.source_targets { + for target in &self.target_targets { + matrix.insert(source, target, self.transfer_cost); + } + } + } + + /// `GraphRead` requires Interpreter execution (graph operations need runtime). + fn restrict_to_interpreter_only(&self, matrix: &mut TransMatrix) { + matrix.clear(); + + if self.source_targets.contains(TargetId::Interpreter) + && self.target_targets.contains(TargetId::Interpreter) + { + matrix.insert(TargetId::Interpreter, TargetId::Interpreter, cost!(0)); + } + } + + /// Applies Postgres-specific restrictions. + /// + /// - Transitions *to* Postgres from other backends are not allowed (once data leaves the + /// database, it cannot return) + /// - In loops, all Postgres transitions are disabled (declarative SQL cannot model iteration) + fn apply_postgres_restrictions(&self, matrix: &mut TransMatrix) { + matrix.remove_incoming(TargetId::Postgres); + + if self.is_in_loop { + matrix.remove_all(TargetId::Postgres); + } + } +} + +/// Computes terminator placement for a [`Body`]. +/// +/// Analyzes control flow edges to determine valid backend transitions and their costs. The +/// resulting [`TerminatorCostVec`] is used by the execution planner alongside statement placement +/// to select optimal execution targets. +/// +/// # Usage +/// +/// ```ignore +/// let placement = TerminatorPlacement::new(entity_size, &alloc); +/// let costs = placement.terminator_placement(context, body, footprint, targets); +/// +/// // Query transitions for block 0's first successor edge +/// let matrices = costs.of(BasicBlockId::new(0)); +/// let can_transition = matrices[0].get(TargetId::Postgres, TargetId::Interpreter); +/// ``` +pub struct TerminatorPlacement { + alloc: A, + entity_size: InformationRange, +} + +impl TerminatorPlacement { + #[inline] + #[must_use] + pub const fn new(entity_size: InformationRange) -> Self { + Self::new_in(entity_size, Global) + } +} + +impl TerminatorPlacement { + /// Creates a new placement analyzer. + /// + /// The `entity_size` estimate is used when computing transfer costs — it represents the + /// expected size of entity data that may need to cross backend boundaries. + #[inline] + pub const fn new_in(entity_size: InformationRange, alloc: A) -> Self { + Self { alloc, entity_size } + } + + fn compute_liveness(&self, body: &Body) -> BasicBlockVec, &A> { + let DataflowResults { + analysis: _, + entry_states: live_in, + exit_states: _, + } = LivenessAnalysis.iterate_to_fixpoint_in(body, &self.alloc); + + live_in + } + + fn compute_scc<'a>( + &'a self, + body: &Body, + ) -> StronglyConnectedComponents { + Tarjan::new_with_metadata_in(&body.basic_blocks, ComponentSizeMetadata, &self.alloc).run() + } + + /// Computes transition costs for all terminator edges in `body`. + /// + /// For each edge, determines which (source → destination) backend transitions are valid and + /// their associated costs. The `targets` slice provides the set of backends each block can + /// execute on (from statement placement), and `footprint` provides size estimates for + /// computing transfer costs. + /// + /// The returned [`TerminatorCostVec`] can be indexed by block ID to get the transition + /// matrices for that block's successor edges. + pub fn terminator_placement<'heap>( + &self, + context: &MirContext<'_, 'heap>, + body: &Body<'heap>, + footprint: &BodyFootprint<&'heap Heap>, + targets: &BasicBlockSlice, + ) -> TerminatorCostVec<&'heap Heap> { + let live_in = self.compute_liveness(body); + let scc = self.compute_scc(body); + + let mut output = TerminatorCostVec::new(&body.basic_blocks, context.heap); + let mut required_locals = DenseBitSet::new_empty(body.local_decls.len()); + + for (block_id, block) in body.basic_blocks.iter_enumerated() { + let block_targets = targets[block_id]; + let is_in_loop = *scc.annotation(scc.scc(block_id)) > 1; + let matrices = output.of_mut(block_id); + + for (edge_index, successor_id) in block.terminator.kind.successor_blocks().enumerate() { + let is_in_loop = is_in_loop || (successor_id == block_id); + + let successor_targets = targets[successor_id]; + let transfer_cost = self.compute_transfer_cost( + &mut required_locals, + body, + footprint, + &live_in, + successor_id, + ); + + PopulateEdgeMatrix { + source_targets: block_targets, + target_targets: successor_targets, + transfer_cost, + is_in_loop, + } + .populate(&mut matrices[edge_index], &block.terminator.kind); + } + } + + output + } + + /// Computes the cost of transferring live data across an edge to `successor`. + /// + /// The cost is the sum of estimated sizes for all locals that are: + /// - Live at the successor's entry + /// - Passed as parameters to the successor block + fn compute_transfer_cost( + &self, + required_locals: &mut DenseBitSet, + body: &Body, + footprint: &BodyFootprint<&Heap>, + live_in: &BasicBlockSlice>, + successor: BasicBlockId, + ) -> Cost { + required_locals.clone_from(&live_in[successor]); + + for ¶m in body.basic_blocks[successor].params { + required_locals.insert(param); + } + + self.sum_local_sizes(footprint, required_locals) + } + + /// Sums the estimated sizes of all locals in the set. + /// + /// Uses conservative estimates: env size = 0 (amortized over many invocations), + /// entity cardinality = 1 (we operate on single entities in filter functions). + fn sum_local_sizes( + &self, + footprint: &BodyFootprint<&Heap>, + locals: &DenseBitSet, + ) -> Cost { + let mut total = cost!(0); + + for local in locals { + let Some(size_estimate) = footprint.locals[local].average( + &[InformationRange::zero(), self.entity_size], + &[Cardinality::one(), Cardinality::one()], + ) else { + return Cost::MAX; + }; + + total = total.saturating_add(size_estimate.as_u32()); + } + + total + } +} diff --git a/libs/@local/hashql/mir/src/pass/execution/terminator_placement/tests.rs b/libs/@local/hashql/mir/src/pass/execution/terminator_placement/tests.rs new file mode 100644 index 00000000000..ae8e4d90275 --- /dev/null +++ b/libs/@local/hashql/mir/src/pass/execution/terminator_placement/tests.rs @@ -0,0 +1,686 @@ +//! Tests for terminator placement analysis. +#![expect(clippy::min_ident_chars)] + +use alloc::alloc::Global; +use core::fmt::{self, Display}; +use std::{io::Write as _, path::PathBuf}; + +use hashql_core::{ + heap::Heap, + id::{Id as _, bit_vec::FiniteBitSet}, + pretty::Formatter, + r#type::{TypeFormatter, TypeFormatterOptions, builder::TypeBuilder, environment::Environment}, +}; +use hashql_diagnostics::DiagnosticIssues; +use insta::{Settings, assert_snapshot}; + +use super::{Cost, TerminatorCostVec, TerminatorPlacement}; +use crate::{ + body::{ + Body, + basic_block::{BasicBlockId, BasicBlockSlice}, + local::LocalVec, + operand::Operand, + terminator::{GraphRead, GraphReadHead, GraphReadTail, TerminatorKind}, + }, + builder::{BodyBuilder, body}, + context::MirContext, + intern::Interner, + pass::{ + analysis::size_estimation::{BodyFootprint, Footprint, InformationRange}, + execution::target::{TargetBitSet, TargetId}, + }, + pretty::TextFormatOptions, +}; + +#[expect(clippy::cast_possible_truncation)] +fn target_set(targets: &[TargetId]) -> TargetBitSet { + let mut set = FiniteBitSet::new_empty(TargetId::VARIANT_COUNT as u32); + for &target in targets { + set.insert(target); + } + set +} + +#[expect(clippy::cast_possible_truncation)] +fn all_targets() -> TargetBitSet { + let mut set = FiniteBitSet::new_empty(TargetId::VARIANT_COUNT as u32); + set.insert_range(TargetId::MIN..=TargetId::MAX); + set +} + +type TargetBitSetSlice = BasicBlockSlice; + +fn build_targets<'set>( + body: &Body<'_>, + per_block: &'set [TargetBitSet], +) -> &'set TargetBitSetSlice { + assert_eq!(body.basic_blocks.len(), per_block.len()); + + TargetBitSetSlice::from_raw(per_block) +} + +fn make_scalar_footprint<'heap>( + body: &Body<'heap>, + heap: &'heap Heap, +) -> BodyFootprint<&'heap Heap> { + BodyFootprint { + args: body.args, + locals: LocalVec::from_elem_in(Footprint::scalar(), body.local_decls.len(), heap), + returns: Footprint::scalar(), + } +} + +fn make_full_footprint<'heap>(body: &Body<'heap>, heap: &'heap Heap) -> BodyFootprint<&'heap Heap> { + BodyFootprint { + args: body.args, + locals: LocalVec::from_elem_in(Footprint::full(), body.local_decls.len(), heap), + returns: Footprint::full(), + } +} + +fn assert_snapshot<'heap>( + name: &'static str, + context: &MirContext<'_, 'heap>, + body: &Body<'heap>, + edges: &TerminatorCostVec<&'heap Heap>, +) { + let formatter = Formatter::new(context.heap); + let type_formatter = TypeFormatter::new(&formatter, context.env, TypeFormatterOptions::terse()); + + let mut text_format = TextFormatOptions { + writer: Vec::::new(), + indent: 4, + sources: (), + types: type_formatter, + annotations: (), + } + .build(); + + text_format.format_body(body).expect("formatting failed"); + + write!(text_format.writer, "\n\n{:=^50}\n\n", " Terminator Edges ").expect("infallible"); + + write!(text_format.writer, "{}", format_edge_summary(edges)).expect("formatting failed"); + + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let mut settings = Settings::clone_current(); + settings.set_snapshot_path(dir.join("tests/ui/pass/execution/terminator_placement")); + settings.set_prepend_module_to_snapshot(false); + + let _guard = settings.bind_to_scope(); + + let output = String::from_utf8_lossy(&text_format.writer); + assert_snapshot!(name, output); +} + +fn format_edge_summary( + edges: &TerminatorCostVec, +) -> impl Display + '_ { + fmt::from_fn(move |fmt| { + for block in 0..(edges.offsets.len() - 1) { + let block_id = BasicBlockId::from_usize(block); + let matrices = edges.of(block_id); + writeln!(fmt, "{block_id}:")?; + for (index, matrix) in matrices.iter().enumerate() { + write!(fmt, " edge[{index}]:")?; + + for (source, target, edge) in matrix.iter() { + let Some(cost) = edge else { continue }; + + write!( + fmt, + "{}->{}={}", + source.abbreviation(), + target.abbreviation(), + cost + )?; + } + + writeln!(fmt)?; + } + } + Ok(()) + }) +} + +#[test] +fn terminator_cost_vec_successor_counts() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl value: Int; + + bb0() { + goto bb1(); + }, + bb1() { + value = load 0; + switch value [0 => bb2(), 1 => bb2(), _ => bb2()]; + }, + bb2() { + return 0; + }, + bb3() { + unreachable; + } + }); + + let costs = TerminatorCostVec::new(&body.basic_blocks, &heap); + + assert_eq!(costs.of(BasicBlockId::new(0)).len(), 1); + assert_eq!(costs.of(BasicBlockId::new(1)).len(), 3); + assert_eq!(costs.of(BasicBlockId::new(2)).len(), 0); + assert_eq!(costs.of(BasicBlockId::new(3)).len(), 0); +} + +#[test] +fn goto_allows_cross_backend_non_postgres() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl param: Int; + + bb0() { + goto bb1(1); + }, + bb1(param) { + return param; + } + }); + + let targets = [ + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + ]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[0]; + assert_eq!( + matrix.get(TargetId::Interpreter, TargetId::Embedding), + Some(cost!(1)) + ); +} + +#[test] +fn switchint_blocks_cross_backend() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl selector: Int, param: Int, result: Int; + + bb0() { + selector = load 1; + switch selector [0 => bb1(1), _ => bb2()]; + }, + bb1(param) { + return param; + }, + bb2() { + result = load 10; + return result; + } + }); + + let targets = [ + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + ]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[0]; + assert_eq!(matrix.get(TargetId::Interpreter, TargetId::Embedding), None); + assert_eq!( + matrix.get(TargetId::Embedding, TargetId::Interpreter), + Some(cost!(1)) + ); +} + +#[test] +fn switchint_edge_targets_are_branch_specific() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl selector: Int; + + bb0() { + selector = load 1; + switch selector [0 => bb1(), _ => bb2()]; + }, + bb1() { + return 0; + }, + bb2() { + return 1; + } + }); + + let targets = [ + all_targets(), + target_set(&[TargetId::Interpreter]), + target_set(&[TargetId::Embedding]), + ]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let [first, second] = costs.of(BasicBlockId::new(0)) else { + unreachable!() + }; + + assert!( + first + .get(TargetId::Interpreter, TargetId::Interpreter) + .is_some() + ); + assert!( + first + .get(TargetId::Embedding, TargetId::Interpreter) + .is_some() + ); + assert!( + first + .get(TargetId::Interpreter, TargetId::Embedding) + .is_none() + ); + + assert!( + second + .get(TargetId::Embedding, TargetId::Embedding) + .is_some() + ); + assert!( + second + .get(TargetId::Interpreter, TargetId::Embedding) + .is_none() + ); + assert!( + second + .get(TargetId::Embedding, TargetId::Interpreter) + .is_none() + ); +} + +#[test] +fn graphread_interpreter_only() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let int_ty = TypeBuilder::synthetic(&env).integer(); + let unit_ty = TypeBuilder::synthetic(&env).tuple([] as [hashql_core::r#type::TypeId; 0]); + let entity_ty = TypeBuilder::synthetic(&env).opaque("Entity", int_ty); + + let mut builder = BodyBuilder::new(&interner); + let _env_local = builder.local("env", unit_ty); + let _vertex = builder.local("vertex", entity_ty); + let axis = builder.local("axis", int_ty); + + let const_0 = builder.const_int(0); + + let bb0 = builder.reserve_block([]); + let bb1 = builder.reserve_block([]); + + builder + .build_block(bb0) + .finish_with_terminator(TerminatorKind::GraphRead(GraphRead { + head: GraphReadHead::Entity { + axis: Operand::Place(axis), + }, + body: Vec::new_in(&heap), + tail: GraphReadTail::Collect, + target: bb1, + })); + + builder.build_block(bb1).ret(const_0); + + let body = builder.finish(2, int_ty); + + let targets = [all_targets(), all_targets()]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(bb0)[0]; + assert_eq!( + matrix.get(TargetId::Interpreter, TargetId::Interpreter), + Some(cost!(0)) + ); + assert_eq!(matrix.get(TargetId::Interpreter, TargetId::Embedding), None); + assert_eq!(matrix.get(TargetId::Embedding, TargetId::Interpreter), None); + assert_eq!(matrix.get(TargetId::Postgres, TargetId::Interpreter), None); +} + +#[test] +fn postgres_incoming_removed() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl value: Int; + + bb0() { + goto bb1(); + }, + bb1() { + value = load 10; + return value; + } + }); + + let targets = [all_targets(), all_targets()]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[0]; + assert_eq!(matrix.get(TargetId::Interpreter, TargetId::Postgres), None); + assert_eq!(matrix.get(TargetId::Embedding, TargetId::Postgres), None); + assert_eq!( + matrix.get(TargetId::Postgres, TargetId::Postgres), + Some(cost!(0)) + ); +} + +#[test] +fn postgres_removed_in_loops() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl value: Int; + + bb0() { + value = load 0; + goto bb1(); + }, + bb1() { + goto bb0(); + } + }); + + let targets = [all_targets(), all_targets()]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[0]; + assert_eq!(matrix.get(TargetId::Postgres, TargetId::Postgres), None); + assert_eq!(matrix.get(TargetId::Postgres, TargetId::Interpreter), None); + assert_eq!( + matrix.get(TargetId::Interpreter, TargetId::Interpreter), + Some(cost!(0)) + ); +} + +#[test] +fn postgres_removed_in_self_loops() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl value: Int; + + bb0() { + value = load 0; + goto bb0(); + } + }); + + let targets = [all_targets()]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[0]; + assert_eq!(matrix.get(TargetId::Postgres, TargetId::Postgres), None); + assert_eq!(matrix.get(TargetId::Postgres, TargetId::Interpreter), None); + assert_eq!( + matrix.get(TargetId::Interpreter, TargetId::Interpreter), + Some(cost!(0)) + ); +} + +#[test] +fn transfer_cost_counts_live_and_params() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl live: Int, param: Int; + + bb0() { + live = load 10; + if true then bb1(1) else bb2(); + }, + bb1(param) { + return live; + }, + bb2() { + return 0; + } + }); + + let targets = [ + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + ]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[1]; + assert_eq!( + matrix.get(TargetId::Postgres, TargetId::Interpreter), + Some(cost!(2)) + ); +} + +#[test] +fn transfer_cost_is_max_for_unbounded() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl arg: [List Int], param: [List Int]; + + bb0() { + arg = list 1, 2; + if true then bb1(arg) else bb2(); + }, + bb1(param) { + return 0; + }, + bb2() { + return 0; + } + }); + + let targets = [ + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + ]; + + let footprint = make_full_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + let matrix = costs.of(BasicBlockId::new(0))[1]; + assert_eq!( + matrix.get(TargetId::Postgres, TargetId::Interpreter), + Some(Cost::MAX) + ); +} + +#[test] +fn terminator_placement_snapshot() { + let heap = Heap::new(); + let interner = Interner::new(&heap); + let env = Environment::new(&heap); + + let body = body!(interner, env; fn@0/0 -> Int { + decl selector: Int, live: Int, param: Int; + + bb0() { + live = load 10; + selector = load 1; + switch selector [0 => bb1(param), _ => bb2()]; + }, + bb1(param) { + return live; + }, + bb2() { + return 0; + } + }); + + let targets = [ + all_targets(), + target_set(&[TargetId::Interpreter, TargetId::Postgres]), + target_set(&[TargetId::Interpreter, TargetId::Embedding]), + ]; + + let footprint = make_scalar_footprint(&body, &heap); + let placement = TerminatorPlacement::new_in(InformationRange::zero(), Global); + let costs = placement.terminator_placement( + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &footprint, + build_targets(&body, &targets), + ); + + assert_snapshot( + "terminator_placement_snapshot", + &MirContext { + heap: &heap, + env: &env, + interner: &interner, + diagnostics: DiagnosticIssues::new(), + }, + &body, + &costs, + ); +} diff --git a/libs/@local/hashql/mir/src/pass/mod.rs b/libs/@local/hashql/mir/src/pass/mod.rs index 870cd726875..a71f27bd006 100644 --- a/libs/@local/hashql/mir/src/pass/mod.rs +++ b/libs/@local/hashql/mir/src/pass/mod.rs @@ -31,6 +31,7 @@ use crate::{ }; pub mod analysis; +pub mod execution; pub mod transform; /// Extracts the simple type name from a fully qualified type path. diff --git a/libs/@local/hashql/mir/tests/ui/pass/execution/terminator_placement/terminator_placement_snapshot.snap b/libs/@local/hashql/mir/tests/ui/pass/execution/terminator_placement/terminator_placement_snapshot.snap new file mode 100644 index 00000000000..4996a0af2f9 --- /dev/null +++ b/libs/@local/hashql/mir/tests/ui/pass/execution/terminator_placement/terminator_placement_snapshot.snap @@ -0,0 +1,32 @@ +--- +source: libs/@local/hashql/mir/src/pass/execution/terminator_placement/tests.rs +expression: output +--- +fn {closure@4294967040}() -> Integer { + let %0: Integer + let %1: Integer + let %2: Integer + + bb0(): { + %1 = 10 + %0 = 1 + + switchInt(%0) -> [0: bb1(%2), otherwise: bb2()] + } + + bb1(%2): { + return %1 + } + + bb2(): { + return 0 + } +} + +================ Terminator Edges ================ + +bb0: + edge[0]:I->I=0P->I=2P->P=0E->I=2 + edge[1]:I->I=0P->I=0E->I=0E->E=0 +bb1: +bb2: