From 2aed0ada551e915d89f3678f48dc735c5be98217 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 13:18:43 +0800 Subject: [PATCH 1/7] ci(codeql): replay reviewed dismissals across re-raised alerts GitHub re-raises a dismissed CodeQL alert whenever an edit shifts its line number, and Rust has no inline suppression comment to pin it. Anchor known false positives to their sink text instead and replay the dismissal via the GitHub API after each CodeQL run on main. --- .github/codeql/suppressions.json | 44 +++++++++ .github/scripts/codeql_suppress.py | 134 ++++++++++++++++++++++++++ .github/workflows/codeql-suppress.yml | 48 +++++++++ 3 files changed, 226 insertions(+) create mode 100644 .github/codeql/suppressions.json create mode 100755 .github/scripts/codeql_suppress.py create mode 100644 .github/workflows/codeql-suppress.yml diff --git a/.github/codeql/suppressions.json b/.github/codeql/suppressions.json new file mode 100644 index 000000000..ea1f2937a --- /dev/null +++ b/.github/codeql/suppressions.json @@ -0,0 +1,44 @@ +{ + "note": [ + "Reviewed CodeQL findings that are false positives and are re-raised every time", + "the surrounding file is edited, because a shifted line number gives the alert a", + "new fingerprint and GitHub does not carry the dismissal across.", + "", + "Rust has no inline `// codeql[rule]` suppression: the AlertSuppression.ql query", + "that implements it exists for C/C++, C#, Go, Java, JS, Python, Ruby and Swift,", + "but not Rust. codeql-config.yml cannot help either \u2014 `paths-ignore` would drop", + "every rule on the file, and `query-filters` would drop the rule on every file.", + "So the dismissal is replayed by .github/workflows/codeql-suppress.yml instead.", + "", + "An entry is anchored to the SINK TEXT, never to a line number: `sink` must match", + "the source line the alert points at. A rule firing on a different expression in", + "the same file stays open and needs its own review. Keep `reason` under 280 chars", + "\u2014 that is the GitHub API's hard cap on a dismissal comment." + ], + "suppressions": [ + { + "rule": "rust/path-injection", + "path": "nodedb-wal/src/segment/atomic_io.rs", + "sink": "^\\s*(let mut f = fs::File::create\\(&tmp\\)|fs::rename\\(&(tmp, &dst|live, &backup|staged, &live)\\))", + "reason": "checked_name() runs is_plain_path_component on every name at the top of the function, before any path is built: rejects / \\ : NUL . .. leading/trailing dot-space and control chars, returning InvalidInput before any filesystem call. Both paths join the same caller dir." + }, + { + "rule": "rust/path-injection", + "path": "nodedb/src/control/server/shared/ddl/neutral/timeseries/rewrite.rs", + "sink": "^\\s*let _ = std::fs::remove_dir_all\\(ts_base\\.join\\(&backup_name\\)\\);", + "reason": "dir_name is validated with is_plain_path_component immediately above this join (rejects / \\ : NUL . .. leading/trailing dot-space and control chars); a failing name is skipped with a warning before any path is built. backup_name only appends .old to that validated component." + }, + { + "rule": "rust/path-injection", + "path": "nodedb-wal/src/segment/atomic_io.rs", + "sink": "^\\s*let dir_file = fs::File::open\\(dir\\)", + "reason": "fsync_directory opens the caller's own directory read-only and calls sync_all on the fd. It appends nothing to the path, so it reaches no resource the caller did not already hold. Every name joined onto that directory is validated by checked_name at its own entry point." + }, + { + "rule": "rust/path-injection", + "path": "nodedb/src/control/server/shared/ddl/neutral/timeseries/rewrite.rs", + "sink": "^\\s*let partition_dir = ts_base\\.join\\(dir_name\\);", + "reason": "dir_name is validated with is_plain_path_component immediately above this join (rejects / \\ : NUL . .. leading/trailing dot-space and control chars); a failing name is skipped with a warning before any path is built, so the join stays one level under ts_base." + } + ] +} diff --git a/.github/scripts/codeql_suppress.py b/.github/scripts/codeql_suppress.py new file mode 100755 index 000000000..d851dbe3b --- /dev/null +++ b/.github/scripts/codeql_suppress.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Replay reviewed CodeQL dismissals that GitHub drops when line numbers shift. + +An alert is dismissed only when its rule id, file path, and the *text* of the +line it points at all match an entry in the suppression list. Anchoring on the +sink text rather than the line number is the point: a shifted line is what +re-raises the alert, and a genuinely new sink in the same file will not match +an existing entry and stays open. + +Exit code is 0 when every open alert is either dismissed or reported; the +report goes to the workflow step summary so unreviewed alerts stay visible. +""" + +import json +import os +import re +import sys +import urllib.error +import urllib.request + +API = "https://api.github.com" +REASON_LIMIT = 280 # GitHub's cap on dismissed_comment. + + +def call(method, path, token, body=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(f"{API}{path}", data=data, method=method) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/vnd.github+json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + if data is not None: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read() or b"null") + + +def open_alerts(repo, token, ref): + out, page = [], 1 + while True: + batch = call( + "GET", + f"/repos/{repo}/code-scanning/alerts" + f"?state=open&ref={ref}&per_page=100&page={page}", + token, + ) + if not batch: + return out + out.extend(batch) + page += 1 + + +def source_line(path, line_no): + try: + with open(path, encoding="utf-8", errors="replace") as f: + for i, line in enumerate(f, 1): + if i == line_no: + return line.rstrip("\n") + except OSError: + return None + return None + + +def match(alert, rules): + loc = alert["most_recent_instance"]["location"] + path, line_no = loc["path"], loc["start_line"] + text = source_line(path, line_no) + if text is None: + return None + for rule in rules: + if rule["rule"] != alert["rule"]["id"] or rule["path"] != path: + continue + if re.search(rule["sink"], text): + return rule + return None + + +def main(): + token = os.environ["GITHUB_TOKEN"] + repo = os.environ["GITHUB_REPOSITORY"] + ref = os.environ.get("TARGET_REF", "refs/heads/main") + dry_run = os.environ.get("DRY_RUN") == "true" + + with open(".github/codeql/suppressions.json", encoding="utf-8") as f: + rules = json.load(f)["suppressions"] + + for rule in rules: + if len(rule["reason"]) > REASON_LIMIT: + sys.exit( + f"suppressions.json: reason for {rule['rule']} on {rule['path']} is " + f"{len(rule['reason'])} chars; the API rejects anything over {REASON_LIMIT}" + ) + re.compile(rule["sink"]) + + dismissed, unmatched = [], [] + for alert in open_alerts(repo, token, ref): + loc = alert["most_recent_instance"]["location"] + where = f"{alert['rule']['id']} {loc['path']}:{loc['start_line']}" + rule = match(alert, rules) + if rule is None: + unmatched.append(f"#{alert['number']} {where}") + continue + if not dry_run: + call( + "PATCH", + f"/repos/{repo}/code-scanning/alerts/{alert['number']}", + token, + { + "state": "dismissed", + "dismissed_reason": "false positive", + "dismissed_comment": rule["reason"], + }, + ) + dismissed.append(f"#{alert['number']} {where}") + + report = ["## CodeQL suppression replay", ""] + verb = "Would dismiss" if dry_run else "Dismissed" + report.append(f"{verb} {len(dismissed)} re-raised alert(s):") + report += [f"- `{d}`" for d in dismissed] or ["- none"] + report += ["", f"Open and unreviewed — {len(unmatched)} alert(s):"] + report += [f"- `{u}`" for u in unmatched] or ["- none"] + + text = "\n".join(report) + print(text) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write(text + "\n") + + +if __name__ == "__main__": + try: + main() + except urllib.error.HTTPError as e: + sys.exit(f"GitHub API {e.code}: {e.read().decode(errors='replace')}") diff --git a/.github/workflows/codeql-suppress.yml b/.github/workflows/codeql-suppress.yml new file mode 100644 index 000000000..47d61b223 --- /dev/null +++ b/.github/workflows/codeql-suppress.yml @@ -0,0 +1,48 @@ +# Replays reviewed CodeQL dismissals that GitHub drops when line numbers shift. +# +# GitHub fingerprints an alert partly by location, so editing a file re-raises +# every already-dismissed finding in it as a new alert. Rust has no inline +# `// codeql[rule]` suppression to pin them with, so the dismissal is replayed +# here from .github/codeql/suppressions.json, anchored to the sink text. +# +# Runs after CodeQL finishes on main, so the replay lands on the alerts that +# run just created. Alerts that match no entry are left open and listed in the +# step summary. + +name: CodeQL suppression replay + +on: + workflow_run: + workflows: ["CodeQL"] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + dry_run: + description: "Report what would be dismissed without dismissing it" + type: boolean + default: true + +concurrency: + group: codeql-suppress + cancel-in-progress: false + +permissions: + contents: read + security-events: write + +jobs: + replay: + name: Replay reviewed dismissals + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: main + - name: Replay + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + TARGET_REF: refs/heads/main + DRY_RUN: ${{ inputs.dry_run || 'false' }} + run: python3 .github/scripts/codeql_suppress.py From fe8f5eabce7b24bf19464842f6438f8cf6820c60 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:07 +0800 Subject: [PATCH 2/7] refactor(sql): split oversized planner and resolver modules Break the two largest files in the SQL crate into directories, one module per concern. No behavior change. - resolver/expr/convert.rs becomes convert/, split by expression family: entry, identifier, operators, predicates, literals, builtins - planner/subquery.rs becomes subquery/, split by subquery kind: extract, in_list, scalar - Move array table-valued-function resolution to resolver/array_tvf.rs - Move the comma-LATERAL branch to planner/select/comma_lateral.rs - Move CteCatalog to planner/select/cte_catalog.rs - Move LATERAL AST extraction to planner/lateral/subquery.rs --- nodedb-sql/src/planner/lateral/mod.rs | 1 + nodedb-sql/src/planner/lateral/subquery.rs | 166 ++++ .../src/planner/select/comma_lateral.rs | 86 ++ nodedb-sql/src/planner/select/cte_catalog.rs | 42 + nodedb-sql/src/planner/select/mod.rs | 4 + nodedb-sql/src/planner/select/order_by/mod.rs | 1 + nodedb-sql/src/planner/subquery.rs | 471 --------- nodedb-sql/src/planner/subquery/extract.rs | 168 ++++ nodedb-sql/src/planner/subquery/in_list.rs | 117 +++ nodedb-sql/src/planner/subquery/mod.rs | 11 + nodedb-sql/src/planner/subquery/scalar.rs | 137 +++ nodedb-sql/src/resolver/array_tvf.rs | 144 +++ nodedb-sql/src/resolver/expr/convert.rs | 934 ------------------ .../src/resolver/expr/convert/builtins.rs | 73 ++ nodedb-sql/src/resolver/expr/convert/entry.rs | 243 +++++ .../src/resolver/expr/convert/identifier.rs | 167 ++++ .../src/resolver/expr/convert/literals.rs | 206 ++++ nodedb-sql/src/resolver/expr/convert/mod.rs | 13 + .../src/resolver/expr/convert/operators.rs | 305 ++++++ .../src/resolver/expr/convert/predicates.rs | 169 ++++ 20 files changed, 2053 insertions(+), 1405 deletions(-) create mode 100644 nodedb-sql/src/planner/lateral/subquery.rs create mode 100644 nodedb-sql/src/planner/select/comma_lateral.rs create mode 100644 nodedb-sql/src/planner/select/cte_catalog.rs delete mode 100644 nodedb-sql/src/planner/subquery.rs create mode 100644 nodedb-sql/src/planner/subquery/extract.rs create mode 100644 nodedb-sql/src/planner/subquery/in_list.rs create mode 100644 nodedb-sql/src/planner/subquery/mod.rs create mode 100644 nodedb-sql/src/planner/subquery/scalar.rs create mode 100644 nodedb-sql/src/resolver/array_tvf.rs delete mode 100644 nodedb-sql/src/resolver/expr/convert.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/builtins.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/entry.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/identifier.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/literals.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/mod.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/operators.rs create mode 100644 nodedb-sql/src/resolver/expr/convert/predicates.rs diff --git a/nodedb-sql/src/planner/lateral/mod.rs b/nodedb-sql/src/planner/lateral/mod.rs index 069665b91..f9e247ce2 100644 --- a/nodedb-sql/src/planner/lateral/mod.rs +++ b/nodedb-sql/src/planner/lateral/mod.rs @@ -8,5 +8,6 @@ pub mod correlation; pub mod plan; +pub mod subquery; pub use plan::plan_lateral_join; diff --git a/nodedb-sql/src/planner/lateral/subquery.rs b/nodedb-sql/src/planner/lateral/subquery.rs new file mode 100644 index 000000000..457f69c13 --- /dev/null +++ b/nodedb-sql/src/planner/lateral/subquery.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Extraction over a LATERAL subquery's AST: its inner relation, its +//! non-correlated filters, and its row bounds. + +use sqlparser::ast; + +use crate::error::{Result, SqlError}; +use crate::parser::normalize::normalize_ident; +use crate::reserved::check_ast_identifier; +use crate::resolver::columns::TableScope; +use crate::types::Filter; + +/// Extract the alias of the single-table inner SELECT, if present. +pub(super) fn extract_inner_alias(select: &sqlparser::ast::Select) -> Result> { + let Some(from) = select.from.first() else { + return Ok(None); + }; + match &from.relation { + ast::TableFactor::Table { alias, .. } => alias + .as_ref() + .map(|alias| check_ast_identifier(&alias.name)) + .transpose(), + _ => Ok(None), + } +} + +/// Extract the collection name from a single-table inner SELECT. +pub(super) fn extract_inner_collection(select: &sqlparser::ast::Select) -> Result { + let from = select.from.first().ok_or_else(|| SqlError::Unsupported { + detail: "LATERAL subquery must have a FROM clause".into(), + })?; + crate::parser::normalize::table_name_from_factor(&from.relation)? + .map(|(name, _)| name) + .ok_or_else(|| SqlError::Unsupported { + detail: "LATERAL LateralTopK subquery must reference a plain table".into(), + }) +} + +/// Extract filters from the inner SELECT that do NOT reference the outer alias. +pub(super) fn inner_non_correlated_filters( + select: &sqlparser::ast::Select, + outer_alias: &str, + scope: &TableScope, +) -> Result> { + let Some(where_expr) = &select.selection else { + return Ok(Vec::new()); + }; + let remaining = strip_outer_refs(where_expr, outer_alias); + match remaining { + Some(expr) => crate::planner::select::convert_where_to_filters(&expr, scope), + None => Ok(Vec::new()), + } +} + +/// Remove all predicates referencing `outer_alias` from a WHERE expression. +fn strip_outer_refs(expr: &ast::Expr, outer_alias: &str) -> Option { + match expr { + ast::Expr::BinaryOp { + left, + op: ast::BinaryOperator::And, + right, + } => { + let l = strip_outer_refs(left, outer_alias); + let r = strip_outer_refs(right, outer_alias); + match (l, r) { + (None, None) => None, + (Some(e), None) | (None, Some(e)) => Some(e), + (Some(l), Some(r)) => Some(ast::Expr::BinaryOp { + left: Box::new(l), + op: ast::BinaryOperator::And, + right: Box::new(r), + }), + } + } + ast::Expr::BinaryOp { left, right, .. } => { + if refs_outer(left, outer_alias) || refs_outer(right, outer_alias) { + None + } else { + Some(expr.clone()) + } + } + ast::Expr::Nested(inner) => strip_outer_refs(inner, outer_alias), + _ => Some(expr.clone()), + } +} + +fn refs_outer(expr: &ast::Expr, outer_alias: &str) -> bool { + match expr { + ast::Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + normalize_ident(&parts[0]).eq_ignore_ascii_case(outer_alias) + } + ast::Expr::BinaryOp { left, right, .. } => { + refs_outer(left, outer_alias) || refs_outer(right, outer_alias) + } + _ => false, + } +} + +/// Extract the LIMIT value from a query, or fail on a bound that does not +/// resolve to `[0, usize::MAX]`. `LIMIT NULL` / `LIMIT ALL` and an absent +/// clause all mean no bound, so both map to `None`. +pub(super) fn limit_from_query(query: &ast::Query) -> Result> { + match &query.limit_clause { + Some(ast::LimitClause::LimitOffset { + limit: Some(limit), .. + }) + | Some(ast::LimitClause::OffsetCommaLimit { limit, .. }) => { + Ok(crate::coerce::checked_row_bound("LIMIT", limit)?.limit()) + } + Some(ast::LimitClause::LimitOffset { limit: None, .. }) | None => Ok(None), + } +} + +/// Reject an inner OFFSET on a LATERAL subquery. +/// +/// `SqlPlan::LateralTopK` carries no offset field and `SqlPlan::LateralLoop` +/// carries neither limit nor offset. A per-outer-row OFFSET needs a new plan +/// field plus Data Plane execution that skips rows per outer row, so this +/// rejects rather than silently drops the clause. `OFFSET 0` and `OFFSET +/// NULL` skip nothing and plan cleanly; a resolved offset above zero fails +/// with `SqlError::Unsupported`. An offset literal outside `[0, usize::MAX]` +/// fails first, inside `checked_row_bound`, with `SqlError::InvalidLimitValue`. +pub(super) fn reject_lateral_offset(query: &ast::Query) -> Result<()> { + let offset_expr = match &query.limit_clause { + Some(ast::LimitClause::LimitOffset { + offset: Some(offset), + .. + }) => Some(&offset.value), + Some(ast::LimitClause::OffsetCommaLimit { offset, .. }) => Some(offset), + Some(ast::LimitClause::LimitOffset { offset: None, .. }) | None => None, + }; + let Some(expr) = offset_expr else { + return Ok(()); + }; + if crate::coerce::checked_row_bound("OFFSET", expr)?.offset() > 0 { + return Err(SqlError::Unsupported { + detail: "OFFSET inside a LATERAL subquery is not supported".into(), + }); + } + Ok(()) +} + +/// Extract and validate a LATERAL alias from a `TableFactor::Derived`. +pub fn lateral_alias_from_factor(factor: &ast::TableFactor) -> Result> { + match factor { + ast::TableFactor::Derived { alias, .. } => alias + .as_ref() + .map(|alias| check_ast_identifier(&alias.name)) + .transpose(), + _ => Ok(None), + } +} + +/// True when a `TableFactor` is a LATERAL derived subquery. +pub fn is_lateral_derived(factor: &ast::TableFactor) -> bool { + matches!(factor, ast::TableFactor::Derived { lateral: true, .. }) +} + +/// Extract the subquery from a `TableFactor::Derived`. +pub fn subquery_from_factor(factor: &ast::TableFactor) -> Option<&ast::Query> { + match factor { + ast::TableFactor::Derived { subquery, .. } => Some(subquery), + _ => None, + } +} diff --git a/nodedb-sql/src/planner/select/comma_lateral.rs b/nodedb-sql/src/planner/select/comma_lateral.rs new file mode 100644 index 000000000..ec7c7faf2 --- /dev/null +++ b/nodedb-sql/src/planner/select/comma_lateral.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Comma-LATERAL FROM planning: `FROM t, LATERAL (SELECT ...) x`. + +use nodedb_types::DatabaseId; +use sqlparser::ast::Select; + +use super::helpers::convert_projection; +use crate::error::{Result, SqlError}; +use crate::planner::lateral::plan::{LateralJoinArgs, plan_lateral_join}; +use crate::planner::lateral::subquery::{ + is_lateral_derived, lateral_alias_from_factor, subquery_from_factor, +}; +use crate::resolver::columns::TableScope; +use crate::temporal::TemporalScope; +use crate::types::*; + +/// Plan `FROM t, LATERAL (SELECT ...) x`. +/// +/// sqlparser represents this as two `TableWithJoins` elements in `select.from`, +/// where the second has an empty joins list and a `Derived { lateral: true }` +/// relation. Returns `Ok(None)` when the FROM clause has another shape. +pub(super) fn try_plan_comma_lateral( + select: &Select, + scope: &TableScope, + catalog: &dyn SqlCatalog, + temporal: TemporalScope, +) -> Result> { + if select.from.len() != 2 || !is_lateral_derived(&select.from[1].relation) { + return Ok(None); + } + let outer_twj = &select.from[0]; + let lateral_twj = &select.from[1]; + + let outer_alias = extract_table_alias_from_twj(outer_twj)?; + let outer_collection = crate::parser::normalize::table_name_from_factor(&outer_twj.relation)? + .map(|(n, _)| n) + .ok_or_else(|| SqlError::Unsupported { + detail: "LATERAL: outer side must be a plain table".into(), + })?; + let outer_info = catalog + .resolve_relation(DatabaseId::DEFAULT, &outer_collection)? + .ok_or_else(|| SqlError::UnknownTable { + name: outer_collection.clone(), + })?; + let outer_scan = SqlPlan::Scan { + collection: outer_collection, + alias: outer_alias.clone(), + engine: outer_info.engine, + filters: Vec::new(), + projection: Vec::new(), + sort_keys: Vec::new(), + limit: None, + offset: 0, + distinct: false, + window_functions: Vec::new(), + temporal, + }; + + let lateral_alias = + lateral_alias_from_factor(&lateral_twj.relation)?.ok_or_else(|| SqlError::Unsupported { + detail: "LATERAL subquery requires an alias (e.g. LATERAL (...) AS x)".into(), + })?; + let subquery = subquery_from_factor(&lateral_twj.relation) + .expect("is_lateral_derived guarantees Derived variant"); + let projection = convert_projection(&select.projection, scope)?; + plan_lateral_join(LateralJoinArgs { + outer_plan: outer_scan, + outer_alias, + subquery, + lateral_alias: &lateral_alias, + // Comma-LATERAL carries INNER semantics, never LEFT. + left_join: false, + outer_projection: projection, + outer_scope: scope, + catalog, + temporal, + }) + .map(Some) +} + +/// The alias of the first table in a `TableWithJoins`, defaulting to its name. +fn extract_table_alias_from_twj(twj: &sqlparser::ast::TableWithJoins) -> Result> { + crate::parser::normalize::table_name_from_factor(&twj.relation) + .map(|relation| relation.map(|(name, alias)| alias.unwrap_or(name))) +} diff --git a/nodedb-sql/src/planner/select/cte_catalog.rs b/nodedb-sql/src/planner/select/cte_catalog.rs new file mode 100644 index 000000000..1664ae0b9 --- /dev/null +++ b/nodedb-sql/src/planner/select/cte_catalog.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Catalog wrapper that resolves CTE and derived-alias names as relations. + +use nodedb_types::DatabaseId; + +use crate::resolver::derived::open_subquery_relation; +use crate::types::{CollectionInfo, SqlCatalog, SqlCatalogError}; + +/// Catalog wrapper that answers for a synthesized relation before delegating +/// to the stored catalog. +pub(crate) struct CteCatalog<'a> { + pub(crate) inner: &'a dyn SqlCatalog, + /// Each synthesized relation name paired with the shape its body exposes. + pub(crate) relations: Vec<(String, CollectionInfo)>, +} + +impl<'a> CteCatalog<'a> { + /// A catalog exposing one relation whose shape is not inferable. + /// + /// The recursive arm of a `WITH RECURSIVE` names the working table while + /// planning its own body, so that arm's shape is not known yet. + pub(crate) fn open(inner: &'a dyn SqlCatalog, name: &str) -> Self { + Self { + inner, + relations: vec![(name.to_string(), open_subquery_relation(name))], + } + } +} + +impl SqlCatalog for CteCatalog<'_> { + fn get_collection( + &self, + database_id: DatabaseId, + name: &str, + ) -> std::result::Result, SqlCatalogError> { + if let Some((_, info)) = self.relations.iter().find(|(key, _)| key == name) { + return Ok(Some(info.clone())); + } + self.inner.get_collection(database_id, name) + } +} diff --git a/nodedb-sql/src/planner/select/mod.rs b/nodedb-sql/src/planner/select/mod.rs index 702af491d..f40006b55 100644 --- a/nodedb-sql/src/planner/select/mod.rs +++ b/nodedb-sql/src/planner/select/mod.rs @@ -6,6 +6,8 @@ //! search patterns (vector, text, hybrid, spatial) directly from the AST //! instead of reverse-engineering an optimizer's output. +mod comma_lateral; +mod cte_catalog; mod derived_from; mod entry; mod entry_ann; @@ -17,8 +19,10 @@ mod query_tail; mod select_stmt; mod where_search; +pub(crate) use cte_catalog::CteCatalog; pub use entry::plan_query; pub use helpers::{ convert_projection, convert_where_to_filters, extract_float, extract_func_args, extract_string_literal, qualified_name, }; +pub(crate) use order_by::select_output_aliases; diff --git a/nodedb-sql/src/planner/select/order_by/mod.rs b/nodedb-sql/src/planner/select/order_by/mod.rs index 068d1ddd0..b6066d946 100644 --- a/nodedb-sql/src/planner/select/order_by/mod.rs +++ b/nodedb-sql/src/planner/select/order_by/mod.rs @@ -21,5 +21,6 @@ mod projection; mod triggers; mod vector_join; +pub(crate) use aliases::select_output_aliases; pub(super) use apply::apply_order_by; pub(super) use projection::try_hybrid_from_projection; diff --git a/nodedb-sql/src/planner/subquery.rs b/nodedb-sql/src/planner/subquery.rs deleted file mode 100644 index ccabe42f0..000000000 --- a/nodedb-sql/src/planner/subquery.rs +++ /dev/null @@ -1,471 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Subquery planning: IN (SELECT ...), NOT IN (SELECT ...), scalar subqueries. -//! -//! Rewrites WHERE-clause subqueries into semi/anti joins so the existing -//! hash-join executor handles them without a dedicated subquery engine. -//! -//! Supported patterns: -//! - `WHERE col IN (SELECT col2 FROM tbl ...)` → semi-join -//! - `WHERE col NOT IN (SELECT col2 FROM tbl ...)` → anti-join -//! - `WHERE col > (SELECT AGG(...) FROM tbl ...)` → scalar subquery (materialized) - -use sqlparser::ast::{self, Expr, SetExpr}; - -use crate::error::{Result, SqlError}; -use crate::functions::registry::FunctionRegistry; -use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; -use crate::types::*; - -/// Result of extracting subqueries from a WHERE clause. -pub struct SubqueryExtraction { - /// Semi/anti joins to wrap around the base scan. - pub joins: Vec, - /// Remaining WHERE expression with subqueries removed (None if nothing remains). - pub remaining_where: Option, -} - -/// A subquery that was rewritten as a join. -pub struct SubqueryJoin { - /// The column on the outer table to join on. - pub outer_column: String, - /// The planned inner SELECT. - pub inner_plan: SqlPlan, - /// The column from the inner SELECT to join on. - pub inner_column: String, - /// Semi (IN) or Anti (NOT IN). - pub join_type: JoinType, -} - -fn canonical_aggregate_key(function: &str, field: &str) -> String { - format!("{function}({field})") -} - -/// Extract `IN (SELECT ...)` and `NOT IN (SELECT ...)` patterns from a WHERE clause. -/// -/// Returns the extracted subquery joins and the remaining WHERE expression -/// (with subquery predicates removed). If the entire WHERE is a single -/// subquery predicate, `remaining_where` is `None`. -pub fn extract_subqueries( - expr: &Expr, - catalog: &dyn SqlCatalog, - functions: &FunctionRegistry, - temporal: crate::TemporalScope, -) -> Result { - let mut joins = Vec::new(); - let remaining = extract_recursive(expr, &mut joins, catalog, functions, temporal)?; - Ok(SubqueryExtraction { - joins, - remaining_where: remaining, - }) -} - -/// Recursively walk the WHERE expression, extracting subquery predicates. -/// -/// Returns `None` if the entire expression was consumed (subquery-only), -/// or `Some(expr)` with the remaining non-subquery predicates. -fn extract_recursive( - expr: &Expr, - joins: &mut Vec, - catalog: &dyn SqlCatalog, - functions: &FunctionRegistry, - temporal: crate::TemporalScope, -) -> Result> { - match expr { - // AND: recurse both sides, reconstruct with remaining parts. - Expr::BinaryOp { - left, - op: ast::BinaryOperator::And, - right, - } => { - let left_remaining = extract_recursive(left, joins, catalog, functions, temporal)?; - let right_remaining = extract_recursive(right, joins, catalog, functions, temporal)?; - match (left_remaining, right_remaining) { - (None, None) => Ok(None), - (Some(l), None) => Ok(Some(l)), - (None, Some(r)) => Ok(Some(r)), - (Some(l), Some(r)) => Ok(Some(Expr::BinaryOp { - left: Box::new(l), - op: ast::BinaryOperator::And, - right: Box::new(r), - })), - } - } - - // IN (SELECT ...): rewrite as semi-join. - Expr::InSubquery { - expr: outer_expr, - subquery, - negated, - } => { - if let Some(join) = - try_plan_in_subquery(outer_expr, subquery, *negated, catalog, functions, temporal)? - { - joins.push(join); - Ok(None) // This predicate is consumed. - } else { - // Cannot plan as join — return original expression. - Ok(Some(expr.clone())) - } - } - - // Scalar subquery comparison: `col > (SELECT AGG(...) FROM ...)` - Expr::BinaryOp { left, op, right } if is_comparison_op(op) => { - if let Expr::Subquery(subquery) = right.as_ref() { - if let Some(scalar) = - try_plan_scalar_subquery(subquery, catalog, functions, temporal)? - { - joins.push(scalar.join); - Ok(Some(Expr::BinaryOp { - left: left.clone(), - op: op.clone(), - right: Box::new(scalar.replacement_expr), - })) - } else { - Ok(Some(expr.clone())) - } - } else { - Ok(Some(expr.clone())) - } - } - - // EXISTS (SELECT ...): rewrite as semi-join. - // NOT EXISTS (SELECT ...): rewrite as anti-join. - Expr::Exists { subquery, negated } => { - if let Some(join) = - try_plan_exists_subquery(subquery, *negated, catalog, functions, temporal)? - { - joins.push(join); - Ok(None) - } else { - Ok(Some(expr.clone())) - } - } - - // Nested parentheses. - Expr::Nested(inner) => extract_recursive(inner, joins, catalog, functions, temporal), - - // Not a subquery pattern — return as-is. - _ => Ok(Some(expr.clone())), - } -} - -/// Try to plan `col IN (SELECT col2 FROM tbl ...)` as a semi/anti join. -fn try_plan_in_subquery( - outer_expr: &Expr, - subquery: &ast::Query, - negated: bool, - catalog: &dyn SqlCatalog, - functions: &FunctionRegistry, - temporal: crate::TemporalScope, -) -> Result> { - // Extract outer column name. - let outer_col = match outer_expr { - Expr::Identifier(ident) => normalize_ident(ident), - Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { - let qualified: String = parts - .iter() - .map(normalize_ident) - .collect::>() - .join("."); - return Err(SqlError::Unsupported { - detail: format!( - "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" - ), - }); - } - Expr::CompoundIdentifier(parts) if parts.len() == 2 => normalize_ident(&parts[1]), - _ => return Ok(None), // Complex expression, can't rewrite. - }; - - // Plan the inner SELECT. - let inner_plan = super::select::plan_query(subquery, catalog, functions, temporal)?; - - // Extract the projected column from the inner plan. - let inner_col = extract_single_projected_column(subquery)?; - - Ok(Some(SubqueryJoin { - outer_column: outer_col, - inner_plan, - inner_column: inner_col, - join_type: if negated { - JoinType::Anti - } else { - JoinType::Semi - }, - })) -} - -/// Extract the single column name from a subquery's SELECT list. -/// -/// For `SELECT user_id FROM orders`, returns `"user_id"`. -fn extract_single_projected_column(query: &ast::Query) -> Result { - let select = match &*query.body { - SetExpr::Select(s) => s, - _ => { - return Err(SqlError::Unsupported { - detail: "subquery must be a simple SELECT".into(), - }); - } - }; - - if select.projection.len() != 1 { - return Err(SqlError::Unsupported { - detail: format!( - "subquery must select exactly 1 column, got {}", - select.projection.len() - ), - }); - } - - match &select.projection[0] { - ast::SelectItem::UnnamedExpr(expr) => match expr { - Expr::Identifier(ident) => Ok(normalize_ident(ident)), - Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { - let qualified: String = parts - .iter() - .map(normalize_ident) - .collect::>() - .join("."); - Err(SqlError::Unsupported { - detail: format!( - "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" - ), - }) - } - Expr::CompoundIdentifier(parts) if parts.len() == 2 => Ok(normalize_ident(&parts[1])), - _ => Err(SqlError::Unsupported { - detail: "subquery projection must be a column reference".into(), - }), - }, - ast::SelectItem::ExprWithAlias { alias, .. } => Ok(normalize_ident(alias)), - _ => Err(SqlError::Unsupported { - detail: "subquery projection must be a column reference".into(), - }), - } -} - -/// Plan `EXISTS (SELECT 1 FROM tbl WHERE tbl.col = outer.col)` as a semi/anti join. -/// -/// Extracts the correlated column from the subquery's WHERE clause. -fn try_plan_exists_subquery( - subquery: &ast::Query, - negated: bool, - catalog: &dyn SqlCatalog, - functions: &FunctionRegistry, - temporal: crate::TemporalScope, -) -> Result> { - let select = match &*subquery.body { - SetExpr::Select(s) => s, - _ => return Ok(None), - }; - - // Look for a correlated predicate in the WHERE: inner.col = outer.col - let (outer_col, inner_col) = match &select.selection { - Some(expr) => match extract_correlated_eq(expr) { - Some(pair) => pair, - None => return Ok(None), - }, - None => return Ok(None), - }; - - // Build a simplified subquery without the correlated predicate for planning. - let inner_plan = super::select::plan_query(subquery, catalog, functions, temporal)?; - - Ok(Some(SubqueryJoin { - outer_column: outer_col, - inner_plan, - inner_column: inner_col, - join_type: if negated { - JoinType::Anti - } else { - JoinType::Semi - }, - })) -} - -/// Extract a correlated equality predicate from a WHERE clause. -/// -/// Looks for patterns like `o.user_id = u.id` and returns (outer_col, inner_col). -/// The "inner" column is the one qualified with the subquery's table alias; -/// the "outer" column is the one referencing the outer query's table. -fn extract_correlated_eq(expr: &Expr) -> Option<(String, String)> { - match expr { - Expr::BinaryOp { - left, - op: ast::BinaryOperator::Eq, - right, - } => { - let left_parts = extract_qualified_column(left); - let right_parts = extract_qualified_column(right); - match (left_parts, right_parts) { - (Some((_lt, lc)), Some((_rt, rc))) => { - // Convention: left is inner (subquery table), right is outer. - // But we can't distinguish without schema, so just return both. - Some((rc, lc)) - } - _ => None, - } - } - // For AND, try to find a correlated eq in either side. - Expr::BinaryOp { - left, - op: ast::BinaryOperator::And, - right, - } => extract_correlated_eq(left).or_else(|| extract_correlated_eq(right)), - Expr::Nested(inner) => extract_correlated_eq(inner), - _ => None, - } -} - -/// Extract table.column from a qualified identifier. -/// -/// Returns `None` for schema-qualified references (`schema.table.col`) — those -/// are rejected upstream by `convert_expr` when the expression is fully evaluated. -fn extract_qualified_column(expr: &Expr) -> Option<(String, String)> { - match expr { - Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { - // Schema-qualified: cannot extract table/column sensibly. - // convert_expr will reject this path with Unsupported. - None - } - Expr::CompoundIdentifier(parts) if parts.len() == 2 => { - Some((normalize_ident(&parts[0]), normalize_ident(&parts[1]))) - } - Expr::Identifier(ident) => Some((String::new(), normalize_ident(ident))), - _ => None, - } -} - -fn is_comparison_op(op: &ast::BinaryOperator) -> bool { - matches!( - op, - ast::BinaryOperator::Gt - | ast::BinaryOperator::GtEq - | ast::BinaryOperator::Lt - | ast::BinaryOperator::LtEq - | ast::BinaryOperator::Eq - | ast::BinaryOperator::NotEq - ) -} - -/// Result of planning a scalar subquery. -struct ScalarSubqueryResult { - join: SubqueryJoin, - replacement_expr: Expr, -} - -/// Plan a scalar subquery (e.g., `(SELECT AVG(amount) FROM orders)`). -/// -/// Rewrites `col > (SELECT AVG(amount) FROM orders)` as: -/// cross-join with the aggregate result (1 row), then filter `col > result_col`. -/// -/// The cross-join produces a cartesian product, but since the aggregate returns -/// exactly 1 row, every outer row gets paired with that single result row. -fn try_plan_scalar_subquery( - subquery: &ast::Query, - catalog: &dyn SqlCatalog, - functions: &FunctionRegistry, - temporal: crate::TemporalScope, -) -> Result> { - let inner_plan = super::select::plan_query(subquery, catalog, functions, temporal)?; - - // Extract the result column name from the subquery's SELECT list. - let result_col = match extract_scalar_column(subquery) { - Some(col) => col, - None => return Ok(None), - }; - - let replacement = Expr::Identifier(ast::Ident::new(&result_col)); - - Ok(Some(ScalarSubqueryResult { - join: SubqueryJoin { - outer_column: String::new(), - inner_plan, - inner_column: String::new(), - join_type: JoinType::Cross, - }, - replacement_expr: replacement, - })) -} - -/// Extract the projected column name from a scalar subquery. -/// -/// Handles aliased aggregates like `SELECT AVG(amount) AS avg_amount`. -/// For unaliased aggregates, returns the canonical aggregate key emitted by -/// the aggregate executor (e.g. `avg(amount)`, `count(*)`). -fn extract_scalar_column(query: &ast::Query) -> Option { - let select = match &*query.body { - SetExpr::Select(s) => s, - _ => return None, - }; - if select.projection.len() != 1 { - return None; - } - match &select.projection[0] { - ast::SelectItem::ExprWithAlias { alias, .. } => Some(normalize_ident(alias)), - ast::SelectItem::UnnamedExpr(expr) => match expr { - Expr::Identifier(ident) => Some(normalize_ident(ident)), - Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { - // Schema-qualified: return None to propagate "unsupported" through convert_expr. - None - } - Expr::CompoundIdentifier(parts) if parts.len() == 2 => Some(normalize_ident(&parts[1])), - Expr::Function(func) => { - let func_name = func - .name - .0 - .iter() - .map(|p| match p { - ast::ObjectNamePart::Identifier(ident) => normalize_ident(ident), - _ => String::new(), - }) - .collect::>() - .join(".") - .to_lowercase(); - let arg = match &func.args { - ast::FunctionArguments::List(arg_list) => arg_list - .args - .first() - .and_then(|a| match a { - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( - Expr::Identifier(ident), - )) => Some(normalize_ident(ident)), - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( - Expr::CompoundIdentifier(parts), - )) if parts.len() >= 3 => None, - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( - Expr::CompoundIdentifier(parts), - )) if parts.len() == 2 => Some(normalize_ident(&parts[1])), - ast::FunctionArg::Unnamed( - ast::FunctionArgExpr::Wildcard - | ast::FunctionArgExpr::QualifiedWildcard(_), - ) => Some("all".to_string()), - _ => None, - }) - .unwrap_or_else(|| "*".to_string()), - _ => "*".to_string(), - }; - Some(canonical_aggregate_key(&func_name, &arg)) - } - _ => None, - }, - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::extract_scalar_column; - use crate::parser::statement::parse_sql; - use sqlparser::ast::Statement; - - #[test] - fn unaliased_scalar_aggregate_uses_canonical_aggregate_key() { - let statements = parse_sql("SELECT AVG(amount) FROM orders").unwrap(); - let Statement::Query(query) = &statements[0] else { - panic!("expected query"); - }; - assert_eq!(extract_scalar_column(query), Some("avg(amount)".into())); - } -} diff --git a/nodedb-sql/src/planner/subquery/extract.rs b/nodedb-sql/src/planner/subquery/extract.rs new file mode 100644 index 000000000..66a3da22d --- /dev/null +++ b/nodedb-sql/src/planner/subquery/extract.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The WHERE-clause walk that pulls subquery predicates out into joins. +//! +//! Rewrites WHERE-clause subqueries into semi/anti joins so the existing +//! hash-join executor handles them without a dedicated subquery engine. +//! +//! Supported patterns: +//! - `WHERE col IN (SELECT col2 FROM tbl ...)` → semi-join +//! - `WHERE col NOT IN (SELECT col2 FROM tbl ...)` → anti-join +//! - `WHERE EXISTS (SELECT ... )` → semi-join +//! - `WHERE NOT EXISTS (SELECT ... )` → anti-join +//! - `WHERE col > (SELECT AGG(...) FROM tbl ...)` → scalar subquery (materialized) + +use sqlparser::ast::{self, Expr}; + +use crate::error::Result; +use crate::functions::registry::FunctionRegistry; +use crate::resolver::columns::TableScope; +use crate::types::*; + +/// Result of extracting subqueries from a WHERE clause. +pub struct SubqueryExtraction { + /// Semi/anti joins to wrap around the base scan. + pub joins: Vec, + /// Remaining WHERE expression with subqueries removed (None if nothing remains). + pub remaining_where: Option, +} + +/// A subquery that was rewritten as a join. +pub struct SubqueryJoin { + /// Equi-join keys as `(outer column, inner column)` pairs. Empty for an + /// uncorrelated subquery: the probe then treats every inner row as a + /// candidate, which is what `EXISTS` over an unrelated table means. + pub on: Vec<(String, String)>, + /// The planned inner SELECT. + pub inner_plan: SqlPlan, + /// Semi (IN / EXISTS) or Anti (NOT IN / NOT EXISTS). + pub join_type: JoinType, +} + +/// Extract subquery predicates from a WHERE clause. +/// +/// `outer` is the scope of the enclosing query, so a correlated reference +/// inside a subquery resolves against the relation that owns it. +/// +/// Returns the extracted subquery joins and the remaining WHERE expression +/// (with subquery predicates removed). If the entire WHERE is a single +/// subquery predicate, `remaining_where` is `None`. +pub fn extract_subqueries( + expr: &Expr, + outer: &TableScope, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result { + let mut joins = Vec::new(); + let remaining = extract_recursive(expr, &mut joins, outer, catalog, functions, temporal)?; + Ok(SubqueryExtraction { + joins, + remaining_where: remaining, + }) +} + +/// Recursively walk the WHERE expression, extracting subquery predicates. +/// +/// Returns `None` if the entire expression was consumed (subquery-only), +/// or `Some(expr)` with the remaining non-subquery predicates. +fn extract_recursive( + expr: &Expr, + joins: &mut Vec, + outer: &TableScope, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result> { + match expr { + // AND: recurse both sides, reconstruct with remaining parts. + Expr::BinaryOp { + left, + op: ast::BinaryOperator::And, + right, + } => { + let left_remaining = + extract_recursive(left, joins, outer, catalog, functions, temporal)?; + let right_remaining = + extract_recursive(right, joins, outer, catalog, functions, temporal)?; + match (left_remaining, right_remaining) { + (None, None) => Ok(None), + (Some(l), None) => Ok(Some(l)), + (None, Some(r)) => Ok(Some(r)), + (Some(l), Some(r)) => Ok(Some(Expr::BinaryOp { + left: Box::new(l), + op: ast::BinaryOperator::And, + right: Box::new(r), + })), + } + } + + // IN (SELECT ...): rewrite as semi-join. + Expr::InSubquery { + expr: outer_expr, + subquery, + negated, + } => { + if let Some(join) = super::in_list::try_plan_in_subquery( + outer_expr, subquery, *negated, outer, catalog, functions, temporal, + )? { + joins.push(join); + Ok(None) // This predicate is consumed. + } else { + // Cannot plan as join — return original expression. + Ok(Some(expr.clone())) + } + } + + // Scalar subquery comparison: `col > (SELECT AGG(...) FROM ...)` + Expr::BinaryOp { left, op, right } if is_comparison_op(op) => { + if let Expr::Subquery(subquery) = right.as_ref() { + if let Some(scalar) = + super::scalar::try_plan_scalar_subquery(subquery, catalog, functions, temporal)? + { + joins.push(scalar.join); + Ok(Some(Expr::BinaryOp { + left: left.clone(), + op: op.clone(), + right: Box::new(scalar.replacement_expr), + })) + } else { + Ok(Some(expr.clone())) + } + } else { + Ok(Some(expr.clone())) + } + } + + // EXISTS (SELECT ...): rewrite as semi-join. + // NOT EXISTS (SELECT ...): rewrite as anti-join. + // + // A shape the planner cannot lower raises a typed error naming that + // shape. Leaving the node in the residual WHERE would instead surface + // it as an unsupported *expression*, which says nothing actionable. + Expr::Exists { subquery, negated } => { + joins.push(super::exists::plan_exists_subquery( + subquery, *negated, outer, catalog, functions, temporal, + )?); + Ok(None) + } + + // Nested parentheses. + Expr::Nested(inner) => extract_recursive(inner, joins, outer, catalog, functions, temporal), + + // Not a subquery pattern — return as-is. + _ => Ok(Some(expr.clone())), + } +} + +fn is_comparison_op(op: &ast::BinaryOperator) -> bool { + matches!( + op, + ast::BinaryOperator::Gt + | ast::BinaryOperator::GtEq + | ast::BinaryOperator::Lt + | ast::BinaryOperator::LtEq + | ast::BinaryOperator::Eq + | ast::BinaryOperator::NotEq + ) +} diff --git a/nodedb-sql/src/planner/subquery/in_list.rs b/nodedb-sql/src/planner/subquery/in_list.rs new file mode 100644 index 000000000..b256da2fa --- /dev/null +++ b/nodedb-sql/src/planner/subquery/in_list.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `IN (SELECT ...)` and `NOT IN (SELECT ...)` lowered to semi / anti joins. + +use sqlparser::ast::{self, Expr, SetExpr}; + +use crate::error::{Result, SqlError}; +use crate::functions::registry::FunctionRegistry; +use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::types::*; + +use super::extract::SubqueryJoin; + +/// Try to plan `col IN (SELECT col2 FROM tbl ...)` as a semi/anti join. +pub(super) fn try_plan_in_subquery( + outer_expr: &Expr, + subquery: &ast::Query, + negated: bool, + outer: &crate::resolver::columns::TableScope, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result> { + // This rewrite consumes the whole predicate, so the outer operand never + // reaches the expression converter. It is checked here or nowhere. + let outer_col = match outer_expr { + Expr::Identifier(ident) => { + let col = normalize_ident(ident); + outer.check_name(None, &col)?; + col + } + Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { + let qualified: String = parts + .iter() + .map(normalize_ident) + .collect::>() + .join("."); + return Err(SqlError::Unsupported { + detail: format!( + "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" + ), + }); + } + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let qualifier = normalize_ident(&parts[0]); + let col = normalize_ident(&parts[1]); + outer.check_name(Some(&qualifier), &col)?; + col + } + _ => return Ok(None), // Complex expression, can't rewrite. + }; + + // Plan the inner SELECT. + let inner_plan = crate::planner::select::plan_query(subquery, catalog, functions, temporal)?; + + // Extract the projected column from the inner plan. + let inner_col = extract_single_projected_column(subquery)?; + + Ok(Some(SubqueryJoin { + on: vec![(outer_col, inner_col)], + inner_plan, + join_type: if negated { + JoinType::Anti + } else { + JoinType::Semi + }, + })) +} + +/// Extract the single column name from a subquery's SELECT list. +/// +/// For `SELECT user_id FROM orders`, returns `"user_id"`. +fn extract_single_projected_column(query: &ast::Query) -> Result { + let select = match &*query.body { + SetExpr::Select(s) => s, + _ => { + return Err(SqlError::Unsupported { + detail: "subquery must be a simple SELECT".into(), + }); + } + }; + + if select.projection.len() != 1 { + return Err(SqlError::Unsupported { + detail: format!( + "subquery must select exactly 1 column, got {}", + select.projection.len() + ), + }); + } + + match &select.projection[0] { + ast::SelectItem::UnnamedExpr(expr) => match expr { + Expr::Identifier(ident) => Ok(normalize_ident(ident)), + Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { + let qualified: String = parts + .iter() + .map(normalize_ident) + .collect::>() + .join("."); + Err(SqlError::Unsupported { + detail: format!( + "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" + ), + }) + } + Expr::CompoundIdentifier(parts) if parts.len() == 2 => Ok(normalize_ident(&parts[1])), + _ => Err(SqlError::Unsupported { + detail: "subquery projection must be a column reference".into(), + }), + }, + ast::SelectItem::ExprWithAlias { alias, .. } => Ok(normalize_ident(alias)), + _ => Err(SqlError::Unsupported { + detail: "subquery projection must be a column reference".into(), + }), + } +} diff --git a/nodedb-sql/src/planner/subquery/mod.rs b/nodedb-sql/src/planner/subquery/mod.rs new file mode 100644 index 000000000..bdce71b45 --- /dev/null +++ b/nodedb-sql/src/planner/subquery/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! WHERE-clause subquery planning: `IN`, `EXISTS`, and scalar subqueries +//! rewritten into semi / anti / cross joins. + +pub mod exists; +pub mod extract; +pub mod in_list; +pub mod scalar; + +pub use extract::{SubqueryExtraction, SubqueryJoin, extract_subqueries}; diff --git a/nodedb-sql/src/planner/subquery/scalar.rs b/nodedb-sql/src/planner/subquery/scalar.rs new file mode 100644 index 000000000..7b2867eb0 --- /dev/null +++ b/nodedb-sql/src/planner/subquery/scalar.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Scalar subqueries: `col > (SELECT AGG(...) FROM tbl ...)`. + +use sqlparser::ast::{self, Expr, SetExpr}; + +use crate::error::Result; +use crate::functions::registry::FunctionRegistry; +use crate::parser::normalize::normalize_ident; +use crate::types::*; + +use super::extract::SubqueryJoin; + +fn canonical_aggregate_key(function: &str, field: &str) -> String { + format!("{function}({field})") +} + +/// Result of planning a scalar subquery. +pub(super) struct ScalarSubqueryResult { + pub(super) join: SubqueryJoin, + pub(super) replacement_expr: Expr, +} + +/// Plan a scalar subquery (e.g., `(SELECT AVG(amount) FROM orders)`). +/// +/// Rewrites `col > (SELECT AVG(amount) FROM orders)` as: +/// cross-join with the aggregate result (1 row), then filter `col > result_col`. +/// +/// The cross-join produces a cartesian product, but since the aggregate returns +/// exactly 1 row, every outer row gets paired with that single result row. +pub(super) fn try_plan_scalar_subquery( + subquery: &ast::Query, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result> { + let inner_plan = crate::planner::select::plan_query(subquery, catalog, functions, temporal)?; + + // Extract the result column name from the subquery's SELECT list. + let result_col = match extract_scalar_column(subquery) { + Some(col) => col, + None => return Ok(None), + }; + + let replacement = Expr::Identifier(ast::Ident::new(&result_col)); + + Ok(Some(ScalarSubqueryResult { + join: SubqueryJoin { + // A cross join pairs every row with every row, so it has no key. + on: Vec::new(), + inner_plan, + join_type: JoinType::Cross, + }, + replacement_expr: replacement, + })) +} + +/// Extract the projected column name from a scalar subquery. +/// +/// Handles aliased aggregates like `SELECT AVG(amount) AS avg_amount`. +/// For unaliased aggregates, returns the canonical aggregate key emitted by +/// the aggregate executor (e.g. `avg(amount)`, `count(*)`). +fn extract_scalar_column(query: &ast::Query) -> Option { + let select = match &*query.body { + SetExpr::Select(s) => s, + _ => return None, + }; + if select.projection.len() != 1 { + return None; + } + match &select.projection[0] { + ast::SelectItem::ExprWithAlias { alias, .. } => Some(normalize_ident(alias)), + ast::SelectItem::UnnamedExpr(expr) => match expr { + Expr::Identifier(ident) => Some(normalize_ident(ident)), + Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { + // Schema-qualified: return None to propagate "unsupported" through convert_expr. + None + } + Expr::CompoundIdentifier(parts) if parts.len() == 2 => Some(normalize_ident(&parts[1])), + Expr::Function(func) => { + let func_name = func + .name + .0 + .iter() + .map(|p| match p { + ast::ObjectNamePart::Identifier(ident) => normalize_ident(ident), + _ => String::new(), + }) + .collect::>() + .join(".") + .to_lowercase(); + let arg = match &func.args { + ast::FunctionArguments::List(arg_list) => arg_list + .args + .first() + .and_then(|a| match a { + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( + Expr::Identifier(ident), + )) => Some(normalize_ident(ident)), + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( + Expr::CompoundIdentifier(parts), + )) if parts.len() >= 3 => None, + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr( + Expr::CompoundIdentifier(parts), + )) if parts.len() == 2 => Some(normalize_ident(&parts[1])), + ast::FunctionArg::Unnamed( + ast::FunctionArgExpr::Wildcard + | ast::FunctionArgExpr::QualifiedWildcard(_), + ) => Some("all".to_string()), + _ => None, + }) + .unwrap_or_else(|| "*".to_string()), + _ => "*".to_string(), + }; + Some(canonical_aggregate_key(&func_name, &arg)) + } + _ => None, + }, + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::extract_scalar_column; + use crate::parser::statement::parse_sql; + use sqlparser::ast::Statement; + + #[test] + fn unaliased_scalar_aggregate_uses_canonical_aggregate_key() { + let statements = parse_sql("SELECT AVG(amount) FROM orders").unwrap(); + let Statement::Query(query) = &statements[0] else { + panic!("expected query"); + }; + assert_eq!(extract_scalar_column(query), Some("avg(amount)".into())); + } +} diff --git a/nodedb-sql/src/resolver/array_tvf.rs b/nodedb-sql/src/resolver/array_tvf.rs new file mode 100644 index 000000000..7d74b3e60 --- /dev/null +++ b/nodedb-sql/src/resolver/array_tvf.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Array table-valued function resolution. +//! +//! An `ARRAY_*(...)` factor in a FROM clause names an array, not a +//! collection. This module synthesizes the relation its dims and attrs +//! expose so column references and equi-join keys against it resolve. + +use crate::error::{Result, SqlError}; +use crate::parser::normalize::normalize_object_name_checked; +use crate::resolver::columns::ResolvedTable; +use crate::types::{ + ArrayCatalogView, CollectionInfo, ColumnInfo, EngineType, SqlCatalog, SqlDataType, +}; +use crate::types_array::{ArrayAttrType, ArrayDimType}; + +/// If `factor` is `ARRAY_*(name, ...)`, look up the array via the +/// catalog and build a `ResolvedTable` whose columns mirror the array's +/// dims + attrs. Returns `Ok(None)` for any non-array-TVF factor. +pub(super) fn resolve_array_tvf( + catalog: &dyn SqlCatalog, + factor: &sqlparser::ast::TableFactor, +) -> Result> { + let (fn_name, args, alias) = match factor { + sqlparser::ast::TableFactor::Table { + name, + args: Some(args), + alias, + .. + } => ( + normalize_object_name_checked(name)?, + args, + alias + .as_ref() + .map(|alias| crate::reserved::check_ast_identifier(&alias.name)) + .transpose()?, + ), + _ => return Ok(None), + }; + if !matches!( + fn_name.as_str(), + "array_slice" | "array_project" | "array_agg" | "array_elementwise" + ) { + return Ok(None); + } + + // First positional arg is the array name as a string literal. + let first = args.args.first().ok_or_else(|| SqlError::Unsupported { + detail: format!("{fn_name}: missing array-name argument"), + })?; + let array_name = extract_string_literal_arg(first).ok_or_else(|| SqlError::Unsupported { + detail: format!("{fn_name}: array-name argument must be a string literal"), + })?; + let view = catalog + .lookup_array(&array_name) + .ok_or_else(|| SqlError::UnknownTable { + name: array_name.clone(), + })?; + + let info = CollectionInfo { + name: view.name.clone(), + engine: EngineType::Array, + columns: array_columns(&view), + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: false, + }; + Ok(Some(ResolvedTable { + name: view.name, + alias, + info, + })) +} + +fn array_columns(view: &ArrayCatalogView) -> Vec { + let mut cols = Vec::with_capacity(view.dims.len() + view.attrs.len()); + for d in &view.dims { + cols.push(ColumnInfo { + name: d.name.clone(), + data_type: dim_type_to_sql(d.dtype), + nullable: false, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }); + } + for a in &view.attrs { + cols.push(ColumnInfo { + name: a.name.clone(), + data_type: attr_type_to_sql(a.dtype), + nullable: a.nullable, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }); + } + cols +} + +fn dim_type_to_sql(t: ArrayDimType) -> SqlDataType { + match t { + ArrayDimType::Int64 => SqlDataType::Int64, + ArrayDimType::Float64 => SqlDataType::Float64, + ArrayDimType::TimestampMs => SqlDataType::Timestamp, + ArrayDimType::String => SqlDataType::String, + } +} + +fn attr_type_to_sql(t: ArrayAttrType) -> SqlDataType { + match t { + ArrayAttrType::Int64 => SqlDataType::Int64, + ArrayAttrType::Float64 => SqlDataType::Float64, + ArrayAttrType::String => SqlDataType::String, + ArrayAttrType::Bytes => SqlDataType::Bytes, + } +} + +fn extract_string_literal_arg(arg: &sqlparser::ast::FunctionArg) -> Option { + use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, Value}; + let expr = match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => e, + FunctionArg::Named { + arg: FunctionArgExpr::Expr(e), + .. + } => e, + _ => return None, + }; + match expr { + Expr::Value(v) => match &v.value { + Value::SingleQuotedString(s) => Some(s.clone()), + _ => None, + }, + _ => None, + } +} diff --git a/nodedb-sql/src/resolver/expr/convert.rs b/nodedb-sql/src/resolver/expr/convert.rs deleted file mode 100644 index 154ef2c0f..000000000 --- a/nodedb-sql/src/resolver/expr/convert.rs +++ /dev/null @@ -1,934 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Convert sqlparser AST expressions to our SqlExpr IR. - -use sqlparser::ast::{self, Expr, UnaryOperator, Value}; - -use crate::error::{Result, SqlError}; -use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; -use crate::types::*; - -use super::binary_ops::{convert_binary_op, convert_unary_op}; -use super::functions::convert_function_depth; -use super::value::{convert_value, parse_interval_to_micros}; - -/// Maximum AST nesting depth accepted by `convert_expr`. -/// Exceeding this limit returns `Err` instead of overflowing the stack. -const MAX_CONVERT_DEPTH: usize = 128; - -/// SQL-standard niladic functions: written without parentheses. Parsers -/// emit them as bare identifiers; we promote them to function calls so -/// they fold to a value at plan time instead of resolving to a column. -fn is_zero_arg_keyword_function(name: &str) -> bool { - matches!( - name, - "current_timestamp" - | "current_date" - | "current_time" - | "localtime" - | "localtimestamp" - | "current_user" - | "current_role" - | "current_schema" - | "session_user" - | "user" - | "version" - ) -} - -/// Convert a sqlparser `Expr` to our `SqlExpr`. -pub fn convert_expr(expr: &Expr) -> Result { - convert_expr_depth(expr, &mut 0) -} - -/// Internal recursive helper that carries a depth counter to enforce -/// `MAX_CONVERT_DEPTH` and prevent stack overflow on malformed ASTs. -pub(super) fn convert_expr_depth(expr: &Expr, depth: &mut usize) -> Result { - *depth += 1; - if *depth > MAX_CONVERT_DEPTH { - return Err(SqlError::Unsupported { - detail: format!("expression nesting depth exceeds maximum of {MAX_CONVERT_DEPTH}"), - }); - } - let result = convert_expr_inner(expr, depth); - *depth -= 1; - result -} - -fn convert_expr_inner(expr: &Expr, depth: &mut usize) -> Result { - match expr { - Expr::Identifier(ident) => { - let name = normalize_ident(ident); - // SQL-standard zero-arg keyword functions parse as bare - // identifiers (no parentheses): `SELECT current_timestamp`, - // `SELECT current_user`, etc. Promote them to function calls - // so const folding evaluates them like the parenthesised form. - if is_zero_arg_keyword_function(&name) { - return Ok(SqlExpr::Function { - name, - args: vec![], - distinct: false, - }); - } - Ok(SqlExpr::Column { table: None, name }) - } - Expr::CompoundIdentifier(parts) if parts.len() >= 3 => { - let qualified: String = parts - .iter() - .map(normalize_ident) - .collect::>() - .join("."); - Err(SqlError::Unsupported { - detail: format!( - "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" - ), - }) - } - Expr::CompoundIdentifier(parts) if parts.len() == 2 => Ok(SqlExpr::Column { - table: Some(normalize_ident(&parts[0])), - name: normalize_ident(&parts[1]), - }), - Expr::Value(val) => Ok(SqlExpr::Literal(convert_value(&val.value)?)), - Expr::BinaryOp { left, op, right } => { - // JSON and FTS operators are lowered to function calls before the - // generic binary-op path so they are never passed to - // convert_binary_op. - use ast::BinaryOperator; - let json_fn: Option<&str> = match op { - BinaryOperator::Arrow => Some("pg_json_get"), - BinaryOperator::LongArrow => Some("pg_json_get_text"), - BinaryOperator::HashArrow => Some("pg_json_path_get"), - BinaryOperator::HashLongArrow => Some("pg_json_path_get_text"), - BinaryOperator::AtArrow => Some("pg_json_contains"), - BinaryOperator::ArrowAt => Some("pg_json_contained_by"), - BinaryOperator::Question => Some("pg_json_has_key"), - BinaryOperator::QuestionAnd => Some("pg_json_has_all_keys"), - BinaryOperator::QuestionPipe => Some("pg_json_has_any_key"), - _ => None, - }; - if let Some(name) = json_fn { - return Ok(SqlExpr::Function { - name: name.into(), - args: vec![ - convert_expr_depth(left, depth)?, - convert_expr_depth(right, depth)?, - ], - distinct: false, - }); - } - // `col @@ query` → pg_fts_match(col, query) - if matches!(op, BinaryOperator::AtAt) { - let col_expr = convert_expr_depth(left, depth)?; - let query_expr = convert_expr_depth(right, depth)?; - return Ok(crate::functions::fts_ops::pg_fts_funcs::lower_pg_fts_match( - col_expr, query_expr, - )); - } - Ok(SqlExpr::BinaryOp { - left: Box::new(convert_expr_depth(left, depth)?), - op: convert_binary_op(op)?, - right: Box::new(convert_expr_depth(right, depth)?), - }) - } - // A negative integer literal reaches sqlparser as unary minus applied - // to a *positive* number, so the most negative `BIGINT` arrives as - // `-(9223372036854775808)` — and that operand does not fit an `i64`. - // Converting the operand on its own therefore falls back to `Float` - // and silently turns an exact integer into an approximate one. Folding - // the sign into the literal before parsing keeps the whole `i64` range - // exact; anything that still does not fit falls through to the general - // path below and is handled as before. - Expr::UnaryOp { - op: UnaryOperator::Minus, - expr: inner, - } if matches!( - inner.as_ref(), - Expr::Value(v) if matches!(&v.value, Value::Number(..)) - ) => - { - let Expr::Value(v) = inner.as_ref() else { - unreachable!("guarded by the `matches!` above") - }; - let Value::Number(n, _) = &v.value else { - unreachable!("guarded by the `matches!` above") - }; - match format!("-{n}").parse::() { - Ok(i) => Ok(SqlExpr::Literal(SqlValue::Int(i))), - Err(_) => Ok(SqlExpr::UnaryOp { - op: UnaryOp::Neg, - expr: Box::new(convert_expr_depth(inner, depth)?), - }), - } - } - Expr::UnaryOp { op, expr } => Ok(SqlExpr::UnaryOp { - op: convert_unary_op(op)?, - expr: Box::new(convert_expr_depth(expr, depth)?), - }), - Expr::Function(func) => convert_function_depth(func, depth), - Expr::Nested(inner) => convert_expr_depth(inner, depth), - Expr::IsNull(inner) => Ok(SqlExpr::IsNull { - expr: Box::new(convert_expr_depth(inner, depth)?), - negated: false, - }), - Expr::IsNotNull(inner) => Ok(SqlExpr::IsNull { - expr: Box::new(convert_expr_depth(inner, depth)?), - negated: true, - }), - Expr::InList { - expr, - list, - negated, - } => Ok(SqlExpr::InList { - expr: Box::new(convert_expr_depth(expr, depth)?), - list: list - .iter() - .map(|e| convert_expr_depth(e, depth)) - .collect::>()?, - negated: *negated, - }), - Expr::Between { - expr, - low, - high, - negated, - } => Ok(SqlExpr::Between { - expr: Box::new(convert_expr_depth(expr, depth)?), - low: Box::new(convert_expr_depth(low, depth)?), - high: Box::new(convert_expr_depth(high, depth)?), - negated: *negated, - }), - Expr::Like { - expr, - pattern, - negated, - .. - } => Ok(SqlExpr::Like { - expr: Box::new(convert_expr_depth(expr, depth)?), - pattern: Box::new(convert_expr_depth(pattern, depth)?), - negated: *negated, - case_insensitive: false, - }), - Expr::ILike { - expr, - pattern, - negated, - .. - } => Ok(SqlExpr::Like { - expr: Box::new(convert_expr_depth(expr, depth)?), - pattern: Box::new(convert_expr_depth(pattern, depth)?), - negated: *negated, - case_insensitive: true, - }), - Expr::Case { - operand, - conditions, - else_result, - .. - } => { - let when_then = conditions - .iter() - .map(|cw| { - Ok(( - convert_expr_depth(&cw.condition, depth)?, - convert_expr_depth(&cw.result, depth)?, - )) - }) - .collect::>>()?; - Ok(SqlExpr::Case { - operand: operand - .as_ref() - .map(|e| convert_expr_depth(e, depth).map(Box::new)) - .transpose()?, - when_then, - else_expr: else_result - .as_ref() - .map(|e| convert_expr_depth(e, depth).map(Box::new)) - .transpose()?, - }) - } - Expr::TypedString(ts) => { - // TIMESTAMP '...' and TIMESTAMPTZ '...' typed string literals. - let type_str = format!("{}", ts.data_type).to_ascii_uppercase(); - let raw = match &ts.value.value { - Value::SingleQuotedString(s) => s.clone(), - other => { - return Err(SqlError::Unsupported { - detail: format!("typed string value: {other}"), - }); - } - }; - match type_str.as_str() { - "TIMESTAMP" => { - let dt = - nodedb_types::NdbDateTime::parse(&raw).ok_or_else(|| SqlError::Parse { - detail: format!("cannot parse TIMESTAMP literal: '{raw}'"), - })?; - return Ok(SqlExpr::Literal(SqlValue::Timestamp(dt))); - } - "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => { - let dt = - nodedb_types::NdbDateTime::parse(&raw).ok_or_else(|| SqlError::Parse { - detail: format!("cannot parse TIMESTAMPTZ literal: '{raw}'"), - })?; - return Ok(SqlExpr::Literal(SqlValue::Timestamptz(dt))); - } - _ => {} - } - // Fall through: return as a generic literal string. - Ok(SqlExpr::Literal(SqlValue::String(raw))) - } - Expr::Cast { - expr, data_type, .. - } => { - // `::tsvector` and `::tsquery` casts are PG surface notation; the - // inner expression is the actual text value. Elide the cast and - // return the inner expression directly — no runtime type change is - // needed since we operate on plain strings internally. - let type_str = format!("{data_type}").to_ascii_lowercase(); - if type_str == "tsvector" || type_str == "tsquery" { - return convert_expr_depth(expr, depth); - } - // `'...'::TIMESTAMP` and `'...'::TIMESTAMPTZ` — promote string literals - // to typed SqlValue when the inner expression is a string literal. - let upper = type_str.to_uppercase(); - if (upper == "TIMESTAMP" - || upper == "TIMESTAMPTZ" - || upper == "TIMESTAMP WITH TIME ZONE") - && let Expr::Value(v) = expr.as_ref() - && let Value::SingleQuotedString(s) = &v.value - { - let dt = nodedb_types::NdbDateTime::parse(s).ok_or_else(|| SqlError::Parse { - detail: format!("cannot parse timestamp cast: '{s}'"), - })?; - return Ok(SqlExpr::Literal(if upper == "TIMESTAMP" { - SqlValue::Timestamp(dt) - } else { - SqlValue::Timestamptz(dt) - })); - } - Ok(SqlExpr::Cast { - expr: Box::new(convert_expr_depth(expr, depth)?), - to_type: format!("{data_type}"), - }) - } - Expr::Array(ast::Array { elem, .. }) => { - let elems = elem - .iter() - .map(|e| convert_expr_depth(e, depth)) - .collect::>()?; - Ok(SqlExpr::ArrayLiteral(elems)) - } - Expr::Wildcard(_) => Ok(SqlExpr::Wildcard), - // TRIM([BOTH|LEADING|TRAILING] [what FROM] expr) - Expr::Trim { expr, .. } => Ok(SqlExpr::Function { - name: "trim".into(), - args: vec![convert_expr_depth(expr, depth)?], - distinct: false, - }), - // CEIL(expr) / FLOOR(expr) - Expr::Ceil { expr, .. } => Ok(SqlExpr::Function { - name: "ceil".into(), - args: vec![convert_expr_depth(expr, depth)?], - distinct: false, - }), - Expr::Floor { expr, .. } => Ok(SqlExpr::Function { - name: "floor".into(), - args: vec![convert_expr_depth(expr, depth)?], - distinct: false, - }), - // SUBSTRING(expr FROM start FOR len) - Expr::Substring { - expr, - substring_from, - substring_for, - .. - } => { - let mut args = vec![convert_expr_depth(expr, depth)?]; - if let Some(from) = substring_from { - args.push(convert_expr_depth(from, depth)?); - } - if let Some(len) = substring_for { - args.push(convert_expr_depth(len, depth)?); - } - Ok(SqlExpr::Function { - name: "substring".into(), - args, - distinct: false, - }) - } - Expr::Interval(interval) => { - // INTERVAL '1 hour' → microseconds as i64 literal. - // The interval value is typically a string literal. - let interval_str = match interval.value.as_ref() { - Expr::Value(v) => match &v.value { - Value::SingleQuotedString(s) => s.clone(), - Value::Number(n, _) => { - // INTERVAL 5 HOUR → combine number with leading_field. - if let Some(ref field) = interval.leading_field { - format!("{n} {field}") - } else { - n.clone() - } - } - _ => { - return Err(SqlError::Unsupported { - detail: format!("INTERVAL value: {}", interval.value), - }); - } - }, - _ => { - return Err(SqlError::Unsupported { - detail: format!("INTERVAL expression: {}", interval.value), - }); - } - }; - - // If leading_field is specified, append it: INTERVAL '5' HOUR → "5 HOUR" - let full_str = if interval_str.chars().all(|c| c.is_ascii_digit()) - && let Some(ref field) = interval.leading_field - { - format!("{interval_str} {field}") - } else { - interval_str - }; - - let micros = parse_interval_to_micros(&full_str).ok_or_else(|| SqlError::Parse { - detail: format!("cannot parse INTERVAL '{full_str}'"), - })?; - - Ok(SqlExpr::Literal(SqlValue::Int(micros))) - } - // `left = ANY(right)` — desugar into InList over array elements. - // When `right` resolves to an ArrayLiteral (or a function call that - // the bridge/evaluator will fold to an array), emit InList so the - // downstream scan filter path handles it natively. - Expr::AnyOp { - left, - compare_op, - right, - .. - } => { - // Only support `=` comparison for now; reject other operators - // with a clear, non-AST-leaking message. - use ast::BinaryOperator; - if !matches!(compare_op, BinaryOperator::Eq) { - return Err(SqlError::Unsupported { - detail: "ANY operator with non-equality comparison is not supported".into(), - }); - } - let left_expr = convert_expr_depth(left, depth)?; - let right_expr = convert_expr_depth(right, depth)?; - // Expand the right-hand side into a list if it is an array literal; - // otherwise wrap as a single-element list so InList still evaluates. - let list = match right_expr { - SqlExpr::ArrayLiteral(elems) => elems, - other => vec![other], - }; - Ok(SqlExpr::InList { - expr: Box::new(left_expr), - list, - negated: false, - }) - } - _ => Err(SqlError::Unsupported { - detail: format!("expression: {expr}"), - }), - } -} - -#[cfg(test)] -mod tests { - use sqlparser::ast::{Expr, SelectItem, Statement, Value}; - - use super::convert_expr; - use crate::error::SqlError; - use crate::parser::statement::parse_sql; - use crate::resolver::expr::value::convert_value; - use crate::types::*; - - /// Extract the first SELECT item expression from a simple `SELECT FROM `. - fn first_select_expr(sql: &str) -> Expr { - let stmts = parse_sql(sql).expect("parse failed"); - let Statement::Query(q) = &stmts[0] else { - panic!("expected query"); - }; - let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { - panic!("expected select body"); - }; - match &sel.projection[0] { - SelectItem::UnnamedExpr(e) => e.clone(), - SelectItem::ExprWithAlias { expr, .. } => expr.clone(), - other => panic!("unexpected projection item: {other:?}"), - } - } - - #[test] - fn compound_identifier_two_parts_is_column() { - let expr = first_select_expr("SELECT t.col FROM t"); - let result = convert_expr(&expr).expect("should succeed"); - match result { - SqlExpr::Column { - table: Some(t), - name, - } => { - assert_eq!(t, "t"); - assert_eq!(name, "col"); - } - other => panic!("expected Column with table, got {other:?}"), - } - } - - #[test] - fn compound_identifier_three_parts_rejected() { - // schema.table.col — should be rejected. - use sqlparser::ast::Ident; - let parts = vec![Ident::new("schema"), Ident::new("table"), Ident::new("col")]; - let expr = Expr::CompoundIdentifier(parts); - let err = convert_expr(&expr).unwrap_err(); - assert!( - matches!(err, SqlError::Unsupported { .. }), - "expected Unsupported, got {err:?}" - ); - let msg = format!("{err}"); - assert!( - msg.contains("schema.table.col") || msg.contains("schema-qualified"), - "error should mention the qualified name: {msg}" - ); - } - - #[test] - fn compound_identifier_four_parts_rejected() { - use sqlparser::ast::Ident; - let parts = vec![ - Ident::new("a"), - Ident::new("b"), - Ident::new("c"), - Ident::new("d"), - ]; - let expr = Expr::CompoundIdentifier(parts); - let err = convert_expr(&expr).unwrap_err(); - assert!( - matches!(err, SqlError::Unsupported { .. }), - "expected Unsupported, got {err:?}" - ); - } - - /// `"userId"` with the PostgreSQL dialect is an identifier (quoted, - /// case-preserved), not a string literal. - #[test] - fn double_quoted_is_identifier_not_literal() { - let expr = first_select_expr(r#"SELECT "userId" FROM users"#); - match expr { - Expr::Identifier(ident) => { - assert_eq!(ident.value, "userId"); - assert_eq!(ident.quote_style, Some('"')); - } - other => panic!("expected Identifier, got {other:?}"), - } - } - - /// `'userId'` is a single-quoted string literal. - #[test] - fn single_quoted_is_string_literal() { - let expr = first_select_expr("SELECT 'userId' FROM users"); - match &expr { - Expr::Value(v) => match &v.value { - Value::SingleQuotedString(s) => assert_eq!(s, "userId"), - other => panic!("expected SingleQuotedString, got {other:?}"), - }, - other => panic!("expected Value, got {other:?}"), - } - // And convert_value maps it to SqlValue::String. - let Expr::Value(v) = expr else { unreachable!() }; - assert!(matches!( - convert_value(&v.value), - Ok(SqlValue::String(s)) if s == "userId" - )); - } - - /// `Value::DoubleQuotedString` (non-Postgres dialect) falls through - /// `convert_value` to `SqlError::Unsupported`. With PostgreSQL dialect - /// this variant is never produced, but constructing it directly verifies - /// the arm is absent and not silently accepted. - #[test] - fn double_quoted_string_value_unsupported() { - // Construct the variant directly — it cannot be produced by parsing - // with PostgreSqlDialect, which is exactly why the arm was dead code. - let val = Value::DoubleQuotedString("userId".into()); - assert!( - matches!(convert_value(&val), Err(SqlError::Unsupported { .. })), - "DoubleQuotedString should be Unsupported, not silently accepted" - ); - } - - /// `"col" = 'literal'` — double-quoted identifier on the left, single-quoted - /// string literal on the right — must lower to `BinaryOp(Column("col"), Eq, - /// Literal(String("literal")))`. This is the canonical mixed-quotation form - /// used in WHERE clauses (e.g. WHERE "userId" = 'alice'). - #[test] - fn double_quoted_col_eq_single_quoted_literal() { - let expr = where_sql_expr(r#"SELECT * FROM t WHERE "col" = 'literal'"#); - match expr { - SqlExpr::BinaryOp { left, right, .. } => { - assert!( - matches!(*left, SqlExpr::Column { ref name, .. } if name == "col"), - "left should be Column(col), got {left:?}" - ); - assert!( - matches!(*right, SqlExpr::Literal(SqlValue::String(ref s)) if s == "literal"), - "right should be Literal(String(\"literal\")), got {right:?}" - ); - } - other => panic!("expected BinaryOp, got {other:?}"), - } - } - - /// `"colA" = "colB"` — both sides are double-quoted identifiers; both must - /// resolve as column references, not string literals. - #[test] - fn double_quoted_col_eq_double_quoted_col() { - let expr = where_sql_expr(r#"SELECT * FROM t WHERE "colA" = "colB""#); - match expr { - SqlExpr::BinaryOp { left, right, .. } => { - assert!( - matches!(*left, SqlExpr::Column { ref name, .. } if name == "colA"), - "left should be Column(colA), got {left:?}" - ); - assert!( - matches!(*right, SqlExpr::Column { ref name, .. } if name == "colB"), - "right should be Column(colB), got {right:?}" - ); - } - other => panic!("expected BinaryOp, got {other:?}"), - } - } - - /// A double-quoted identifier in the SELECT list resolves as `SqlExpr::Column` - /// with the exact case preserved (not lowercased, because it was quoted). - #[test] - fn double_quoted_select_col_case_preserved() { - let expr = first_select_expr(r#"SELECT "userId" FROM users"#); - let sql_expr = convert_expr(&expr).expect("convert_expr should succeed"); - match sql_expr { - SqlExpr::Column { name, table } => { - assert_eq!( - name, "userId", - "case must be preserved for quoted identifier" - ); - assert_eq!(table, None, "no table qualifier expected"); - } - other => panic!("expected Column, got {other:?}"), - } - } - - /// Extract and convert the WHERE predicate from a simple - /// `SELECT * FROM tbl WHERE ` statement. - fn where_sql_expr(sql: &str) -> SqlExpr { - let stmts = parse_sql(sql).expect("parse failed"); - let Statement::Query(q) = &stmts[0] else { - panic!("expected query"); - }; - let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { - panic!("expected select body"); - }; - let raw = sel.selection.as_ref().expect("expected WHERE clause"); - convert_expr(raw).expect("convert_expr failed") - } - - #[test] - fn like_is_case_sensitive() { - let expr = where_sql_expr("SELECT * FROM t WHERE name LIKE 'foo%'"); - match expr { - SqlExpr::Like { - negated, - case_insensitive, - .. - } => { - assert!(!negated, "LIKE should not be negated"); - assert!(!case_insensitive, "LIKE should be case-sensitive"); - } - other => panic!("expected SqlExpr::Like, got {other:?}"), - } - } - - #[test] - fn ilike_is_case_insensitive() { - let expr = where_sql_expr("SELECT * FROM t WHERE name ILIKE 'foo%'"); - match expr { - SqlExpr::Like { - negated, - case_insensitive, - .. - } => { - assert!(!negated, "ILIKE should not be negated"); - assert!(case_insensitive, "ILIKE should be case-insensitive"); - } - other => panic!("expected SqlExpr::Like, got {other:?}"), - } - } - - #[test] - fn not_like_is_negated_case_sensitive() { - let expr = where_sql_expr("SELECT * FROM t WHERE name NOT LIKE 'foo%'"); - match expr { - SqlExpr::Like { - negated, - case_insensitive, - .. - } => { - assert!(negated, "NOT LIKE should be negated"); - assert!(!case_insensitive, "NOT LIKE should be case-sensitive"); - } - other => panic!("expected SqlExpr::Like, got {other:?}"), - } - } - - #[test] - fn not_ilike_is_negated_case_insensitive() { - let expr = where_sql_expr("SELECT * FROM t WHERE name NOT ILIKE 'foo%'"); - match expr { - SqlExpr::Like { - negated, - case_insensitive, - .. - } => { - assert!(negated, "NOT ILIKE should be negated"); - assert!(case_insensitive, "NOT ILIKE should be case-insensitive"); - } - other => panic!("expected SqlExpr::Like, got {other:?}"), - } - } - - // ── JSON operator lowering tests ─────────────────────────────────────── - - /// Parses `SELECT FROM t` and returns the lowered `SqlExpr` for ``. - fn select_expr_lowered(sql: &str) -> SqlExpr { - let stmts = parse_sql(sql).expect("parse failed"); - let Statement::Query(q) = &stmts[0] else { - panic!("expected query"); - }; - let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { - panic!("expected select body"); - }; - let raw = &sel.projection[0]; - let raw_expr = match raw { - SelectItem::UnnamedExpr(e) => e, - SelectItem::ExprWithAlias { expr, .. } => expr, - other => panic!("unexpected projection: {other:?}"), - }; - convert_expr(raw_expr).expect("convert_expr failed") - } - - fn assert_json_fn(sql: &str, expected_fn: &str) { - let expr = select_expr_lowered(sql); - match expr { - SqlExpr::Function { name, args, .. } => { - assert_eq!(name, expected_fn, "wrong function name"); - assert_eq!(args.len(), 2, "expected 2 args"); - } - other => panic!("expected Function, got {other:?}"), - } - } - - #[test] - fn arrow_lowers_to_pg_json_get() { - assert_json_fn("SELECT data->'key' FROM t", "pg_json_get"); - } - - #[test] - fn long_arrow_lowers_to_pg_json_get_text() { - assert_json_fn("SELECT data->>'key' FROM t", "pg_json_get_text"); - } - - #[test] - fn hash_arrow_lowers_to_pg_json_path_get() { - assert_json_fn("SELECT data#>'{a,b}' FROM t", "pg_json_path_get"); - } - - #[test] - fn hash_long_arrow_lowers_to_pg_json_path_get_text() { - assert_json_fn("SELECT data#>>'{a,b}' FROM t", "pg_json_path_get_text"); - } - - #[test] - fn at_arrow_lowers_to_pg_json_contains() { - assert_json_fn("SELECT data @> '{\"a\":1}' FROM t", "pg_json_contains"); - } - - #[test] - fn arrow_at_lowers_to_pg_json_contained_by() { - assert_json_fn("SELECT '{\"a\":1}' <@ data FROM t", "pg_json_contained_by"); - } - - #[test] - fn question_lowers_to_pg_json_has_key() { - assert_json_fn("SELECT data ? 'key' FROM t", "pg_json_has_key"); - } - - #[test] - fn question_and_lowers_to_pg_json_has_all_keys() { - assert_json_fn( - "SELECT data ?& ARRAY['a','b'] FROM t", - "pg_json_has_all_keys", - ); - } - - #[test] - fn question_pipe_lowers_to_pg_json_has_any_key() { - assert_json_fn( - "SELECT data ?| ARRAY['a','b'] FROM t", - "pg_json_has_any_key", - ); - } - - #[test] - fn chained_arrow_lowers_nested() { - // data->'a'->'b' → pg_json_get(pg_json_get(data, 'a'), 'b') - let expr = select_expr_lowered("SELECT data->'a'->'b' FROM t"); - match expr { - SqlExpr::Function { name, ref args, .. } => { - assert_eq!(name, "pg_json_get", "outer fn should be pg_json_get"); - match &args[0] { - SqlExpr::Function { - name: inner_name, .. - } => { - assert_eq!(inner_name, "pg_json_get", "inner fn should be pg_json_get"); - } - other => panic!("expected inner pg_json_get, got {other:?}"), - } - } - other => panic!("expected outer pg_json_get, got {other:?}"), - } - } - - // ── FTS operator / function lowering tests ──────────────────────────────── - - fn where_fn(sql: &str) -> SqlExpr { - where_sql_expr(sql) - } - - #[test] - fn at_at_lowers_to_pg_fts_match() { - // col @@ to_tsquery('rust & lang') → pg_fts_match(col, pg_to_tsquery('rust & lang')) - let expr = where_fn("SELECT * FROM t WHERE body @@ to_tsquery('rust & lang')"); - match expr { - SqlExpr::Function { - ref name, ref args, .. - } => { - assert_eq!( - name, "pg_fts_match", - "operator @@ should lower to pg_fts_match" - ); - assert_eq!(args.len(), 2, "expected 2 args"); - match &args[1] { - SqlExpr::Function { name: inner, .. } => { - assert_eq!(inner, "pg_to_tsquery"); - } - other => panic!("expected pg_to_tsquery as right arg, got {other:?}"), - } - } - other => panic!("expected pg_fts_match Function, got {other:?}"), - } - } - - #[test] - fn at_at_with_plainto_tsquery() { - // col @@ plainto_tsquery('rust lang') → pg_fts_match(col, pg_plainto_tsquery(...)) - let expr = where_fn("SELECT * FROM t WHERE body @@ plainto_tsquery('rust lang')"); - match expr { - SqlExpr::Function { - ref name, ref args, .. - } => { - assert_eq!(name, "pg_fts_match"); - match &args[1] { - SqlExpr::Function { name: inner, .. } => { - assert_eq!(inner, "pg_plainto_tsquery"); - } - other => panic!("expected pg_plainto_tsquery, got {other:?}"), - } - } - other => panic!("expected pg_fts_match, got {other:?}"), - } - } - - #[test] - fn tsvector_cast_elided() { - // 'foo'::tsvector → Literal("foo") - let expr = select_expr_lowered("SELECT 'foo'::tsvector FROM t"); - assert!( - matches!(expr, SqlExpr::Literal(SqlValue::String(ref s)) if s == "foo"), - "expected Literal(\"foo\"), got {expr:?}" - ); - } - - #[test] - fn tsquery_cast_elided() { - // 'rust'::tsquery → Literal("rust") - let expr = select_expr_lowered("SELECT 'rust'::tsquery FROM t"); - assert!( - matches!(expr, SqlExpr::Literal(SqlValue::String(ref s)) if s == "rust"), - "expected Literal(\"rust\"), got {expr:?}" - ); - } - - #[test] - fn ts_rank_cd_is_unsupported() { - use crate::parser::statement::parse_sql; - let sql = "SELECT ts_rank_cd(body, to_tsquery('rust')) FROM t"; - let stmts = parse_sql(sql).expect("parse ok"); - let Statement::Query(q) = &stmts[0] else { - panic!("expected query"); - }; - let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { - panic!("expected select body"); - }; - let raw = match &sel.projection[0] { - SelectItem::UnnamedExpr(e) => e, - SelectItem::ExprWithAlias { expr, .. } => expr, - other => panic!("unexpected projection: {other:?}"), - }; - let err = convert_expr(raw).unwrap_err(); - assert!( - matches!(err, SqlError::Unsupported { .. }), - "ts_rank_cd should be Unsupported, got {err:?}" - ); - let msg = format!("{err}"); - assert!( - msg.contains("ts_rank_cd"), - "error should mention ts_rank_cd: {msg}" - ); - } - - #[test] - fn to_tsquery_lowers_to_pg_to_tsquery() { - let expr = select_expr_lowered("SELECT to_tsquery('rust & lang') FROM t"); - match expr { - SqlExpr::Function { ref name, .. } => { - assert_eq!(name, "pg_to_tsquery"); - } - other => panic!("expected pg_to_tsquery Function, got {other:?}"), - } - } - - #[test] - fn plainto_tsquery_lowers_correctly() { - let expr = select_expr_lowered("SELECT plainto_tsquery('rust lang') FROM t"); - match expr { - SqlExpr::Function { ref name, .. } => { - assert_eq!(name, "pg_plainto_tsquery"); - } - other => panic!("expected pg_plainto_tsquery, got {other:?}"), - } - } - - #[test] - fn ts_rank_lowers_to_pg_ts_rank() { - let expr = select_expr_lowered("SELECT ts_rank(body, to_tsquery('rust')) FROM t"); - match expr { - SqlExpr::Function { ref name, .. } => { - assert_eq!(name, "pg_ts_rank"); - } - other => panic!("expected pg_ts_rank, got {other:?}"), - } - } -} diff --git a/nodedb-sql/src/resolver/expr/convert/builtins.rs b/nodedb-sql/src/resolver/expr/convert/builtins.rs new file mode 100644 index 000000000..8390557b2 --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/builtins.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Built-in SQL constructs that parse to dedicated AST nodes and lower +//! to ordinary function calls. + +use sqlparser::ast::Expr; + +use crate::error::Result; +use crate::resolver::ColumnScope; +use crate::types::*; + +use super::entry::convert_expr_depth; + +/// TRIM([BOTH|LEADING|TRAILING] [what FROM] expr) +pub(super) fn convert_trim( + expr: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::Function { + name: "trim".into(), + args: vec![convert_expr_depth(expr, depth, scope)?], + distinct: false, + }) +} + +/// CEIL(expr) +pub(super) fn convert_ceil( + expr: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::Function { + name: "ceil".into(), + args: vec![convert_expr_depth(expr, depth, scope)?], + distinct: false, + }) +} + +/// FLOOR(expr) +pub(super) fn convert_floor( + expr: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::Function { + name: "floor".into(), + args: vec![convert_expr_depth(expr, depth, scope)?], + distinct: false, + }) +} + +/// SUBSTRING(expr FROM start FOR len) +pub(super) fn convert_substring( + expr: &Expr, + substring_from: Option<&Expr>, + substring_for: Option<&Expr>, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + let mut args = vec![convert_expr_depth(expr, depth, scope)?]; + if let Some(from) = substring_from { + args.push(convert_expr_depth(from, depth, scope)?); + } + if let Some(len) = substring_for { + args.push(convert_expr_depth(len, depth, scope)?); + } + Ok(SqlExpr::Function { + name: "substring".into(), + args, + distinct: false, + }) +} diff --git a/nodedb-sql/src/resolver/expr/convert/entry.rs b/nodedb-sql/src/resolver/expr/convert/entry.rs new file mode 100644 index 000000000..dc75e4b8d --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/entry.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Entry points and per-variant dispatch for AST expression conversion. + +use sqlparser::ast::Expr; + +use crate::error::{Result, SqlError}; +use crate::resolver::ColumnScope; +use crate::resolver::expr::functions::convert_function_depth; +use crate::types::*; + +use super::{builtins, identifier, literals, operators, predicates}; + +/// Maximum AST nesting depth accepted by `convert_expr`. +/// Exceeding this limit returns `Err` instead of overflowing the stack. +const MAX_CONVERT_DEPTH: usize = 128; + +/// Convert a sqlparser `Expr` to our `SqlExpr`. +pub fn convert_expr(expr: &Expr, scope: &ColumnScope<'_>) -> Result { + convert_expr_depth(expr, &mut 0, scope) +} + +/// Internal recursive helper that carries a depth counter to enforce +/// `MAX_CONVERT_DEPTH` and prevent stack overflow on malformed ASTs. +pub(in crate::resolver::expr) fn convert_expr_depth( + expr: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + *depth += 1; + if *depth > MAX_CONVERT_DEPTH { + return Err(SqlError::Unsupported { + detail: format!("expression nesting depth exceeds maximum of {MAX_CONVERT_DEPTH}"), + }); + } + let result = convert_expr_inner(expr, depth, scope); + *depth -= 1; + result +} + +fn convert_expr_inner(expr: &Expr, depth: &mut usize, scope: &ColumnScope<'_>) -> Result { + match expr { + Expr::Identifier(ident) => identifier::convert_identifier(ident, scope), + Expr::CompoundIdentifier(parts) if parts.len() >= 2 => { + identifier::convert_compound_identifier(parts, scope) + } + Expr::Value(val) => literals::convert_value_expr(val), + Expr::BinaryOp { left, op, right } => { + operators::convert_binary(left, op, right, depth, scope) + } + Expr::UnaryOp { op, expr } => operators::convert_unary(op, expr, depth, scope), + Expr::Function(func) => convert_function_depth(func, depth, scope), + Expr::Nested(inner) => convert_expr_depth(inner, depth, scope), + Expr::IsNull(inner) => predicates::convert_is_null(inner, false, depth, scope), + Expr::IsNotNull(inner) => predicates::convert_is_null(inner, true, depth, scope), + Expr::InList { + expr, + list, + negated, + } => predicates::convert_in_list(expr, list, *negated, depth, scope), + Expr::Between { + expr, + low, + high, + negated, + } => predicates::convert_between(expr, low, high, *negated, depth, scope), + Expr::Like { + expr, + pattern, + negated, + .. + } => predicates::convert_like(expr, pattern, *negated, false, depth, scope), + Expr::ILike { + expr, + pattern, + negated, + .. + } => predicates::convert_like(expr, pattern, *negated, true, depth, scope), + Expr::Case { + operand, + conditions, + else_result, + .. + } => predicates::convert_case( + operand.as_deref(), + conditions, + else_result.as_deref(), + depth, + scope, + ), + Expr::TypedString(ts) => literals::convert_typed_string(ts), + Expr::Cast { + expr, data_type, .. + } => literals::convert_cast(expr, data_type, depth, scope), + Expr::Array(array) => literals::convert_array(array, depth, scope), + Expr::Wildcard(_) => literals::convert_wildcard(), + Expr::Trim { expr, .. } => builtins::convert_trim(expr, depth, scope), + Expr::Ceil { expr, .. } => builtins::convert_ceil(expr, depth, scope), + Expr::Floor { expr, .. } => builtins::convert_floor(expr, depth, scope), + Expr::Substring { + expr, + substring_from, + substring_for, + .. + } => builtins::convert_substring( + expr, + substring_from.as_deref(), + substring_for.as_deref(), + depth, + scope, + ), + Expr::Interval(interval) => literals::convert_interval(interval), + Expr::AnyOp { + left, + compare_op, + right, + .. + } => operators::convert_any_op(left, compare_op, right, depth, scope), + _ => Err(SqlError::Unsupported { + detail: format!("expression: {expr}"), + }), + } +} + +#[cfg(test)] +pub(super) mod tests { + use sqlparser::ast::{Expr, SelectItem, Statement}; + + use super::convert_expr; + use crate::error::SqlError; + use crate::parser::statement::parse_sql; + use crate::resolver::ColumnScope; + use crate::types::*; + + /// Extract the first SELECT item expression from a simple `SELECT FROM `. + pub(in crate::resolver::expr::convert) fn first_select_expr(sql: &str) -> Expr { + let stmts = parse_sql(sql).expect("parse failed"); + let Statement::Query(q) = &stmts[0] else { + panic!("expected query"); + }; + let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { + panic!("expected select body"); + }; + match &sel.projection[0] { + SelectItem::UnnamedExpr(e) => e.clone(), + SelectItem::ExprWithAlias { expr, .. } => expr.clone(), + other => panic!("unexpected projection item: {other:?}"), + } + } + + /// Extract and convert the WHERE predicate from a simple + /// `SELECT * FROM tbl WHERE ` statement. + pub(in crate::resolver::expr::convert) fn where_sql_expr(sql: &str) -> SqlExpr { + let stmts = parse_sql(sql).expect("parse failed"); + let Statement::Query(q) = &stmts[0] else { + panic!("expected query"); + }; + let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { + panic!("expected select body"); + }; + let raw = sel.selection.as_ref().expect("expected WHERE clause"); + convert_expr(raw, &ColumnScope::Unchecked).expect("convert_expr failed") + } + + /// Parses `SELECT FROM t` and returns the lowered `SqlExpr` for ``. + pub(in crate::resolver::expr::convert) fn select_expr_lowered(sql: &str) -> SqlExpr { + let stmts = parse_sql(sql).expect("parse failed"); + let Statement::Query(q) = &stmts[0] else { + panic!("expected query"); + }; + let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { + panic!("expected select body"); + }; + let raw = &sel.projection[0]; + let raw_expr = match raw { + SelectItem::UnnamedExpr(e) => e, + SelectItem::ExprWithAlias { expr, .. } => expr, + other => panic!("unexpected projection: {other:?}"), + }; + convert_expr(raw_expr, &ColumnScope::Unchecked).expect("convert_expr failed") + } + + #[test] + fn ts_rank_cd_is_unsupported() { + use crate::parser::statement::parse_sql; + let sql = "SELECT ts_rank_cd(body, to_tsquery('rust')) FROM t"; + let stmts = parse_sql(sql).expect("parse ok"); + let Statement::Query(q) = &stmts[0] else { + panic!("expected query"); + }; + let sqlparser::ast::SetExpr::Select(sel) = q.body.as_ref() else { + panic!("expected select body"); + }; + let raw = match &sel.projection[0] { + SelectItem::UnnamedExpr(e) => e, + SelectItem::ExprWithAlias { expr, .. } => expr, + other => panic!("unexpected projection: {other:?}"), + }; + let err = convert_expr(raw, &ColumnScope::Unchecked).unwrap_err(); + assert!( + matches!(err, SqlError::Unsupported { .. }), + "ts_rank_cd should be Unsupported, got {err:?}" + ); + let msg = format!("{err}"); + assert!( + msg.contains("ts_rank_cd"), + "error should mention ts_rank_cd: {msg}" + ); + } + + #[test] + fn to_tsquery_lowers_to_pg_to_tsquery() { + let expr = select_expr_lowered("SELECT to_tsquery('rust & lang') FROM t"); + match expr { + SqlExpr::Function { ref name, .. } => { + assert_eq!(name, "pg_to_tsquery"); + } + other => panic!("expected pg_to_tsquery Function, got {other:?}"), + } + } + + #[test] + fn plainto_tsquery_lowers_correctly() { + let expr = select_expr_lowered("SELECT plainto_tsquery('rust lang') FROM t"); + match expr { + SqlExpr::Function { ref name, .. } => { + assert_eq!(name, "pg_plainto_tsquery"); + } + other => panic!("expected pg_plainto_tsquery, got {other:?}"), + } + } + + #[test] + fn ts_rank_lowers_to_pg_ts_rank() { + let expr = select_expr_lowered("SELECT ts_rank(body, to_tsquery('rust')) FROM t"); + match expr { + SqlExpr::Function { ref name, .. } => { + assert_eq!(name, "pg_ts_rank"); + } + other => panic!("expected pg_ts_rank, got {other:?}"), + } + } +} diff --git a/nodedb-sql/src/resolver/expr/convert/identifier.rs b/nodedb-sql/src/resolver/expr/convert/identifier.rs new file mode 100644 index 000000000..802890d44 --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/identifier.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Column references and bare-identifier conversion. + +use sqlparser::ast::Ident; + +use crate::error::{Result, SqlError}; +use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::resolver::ColumnScope; +use crate::types::*; + +/// SQL-standard niladic functions: written without parentheses. Parsers +/// emit them as bare identifiers; we promote them to function calls so +/// they fold to a value at plan time instead of resolving to a column. +fn is_zero_arg_keyword_function(name: &str) -> bool { + matches!( + name, + "current_timestamp" + | "current_date" + | "current_time" + | "localtime" + | "localtimestamp" + | "current_user" + | "current_role" + | "current_schema" + | "session_user" + | "user" + | "version" + ) +} + +pub(super) fn convert_identifier(ident: &Ident, scope: &ColumnScope<'_>) -> Result { + let name = normalize_ident(ident); + // SQL-standard zero-arg keyword functions parse as bare + // identifiers (no parentheses): `SELECT current_timestamp`, + // `SELECT current_user`, etc. Promote them to function calls + // so const folding evaluates them like the parenthesised form. + if is_zero_arg_keyword_function(&name) { + return Ok(SqlExpr::Function { + name, + args: vec![], + distinct: false, + }); + } + scope.check_column(None, &name)?; + Ok(SqlExpr::Column { table: None, name }) +} + +pub(super) fn convert_compound_identifier( + parts: &[Ident], + scope: &ColumnScope<'_>, +) -> Result { + if parts.len() >= 3 { + let qualified: String = parts + .iter() + .map(normalize_ident) + .collect::>() + .join("."); + return Err(SqlError::Unsupported { + detail: format!( + "schema-qualified column reference '{qualified}': {SCHEMA_QUALIFIED_MSG}" + ), + }); + } + let table = normalize_ident(&parts[0]); + let name = normalize_ident(&parts[1]); + scope.check_column(Some(&table), &name)?; + Ok(SqlExpr::Column { + table: Some(table), + name, + }) +} + +#[cfg(test)] +mod tests { + use sqlparser::ast::Expr; + + use crate::error::SqlError; + use crate::resolver::ColumnScope; + use crate::resolver::expr::convert::convert_expr; + use crate::resolver::expr::convert::entry::tests::first_select_expr; + use crate::types::*; + + #[test] + fn compound_identifier_two_parts_is_column() { + let expr = first_select_expr("SELECT t.col FROM t"); + let result = convert_expr(&expr, &ColumnScope::Unchecked).expect("should succeed"); + match result { + SqlExpr::Column { + table: Some(t), + name, + } => { + assert_eq!(t, "t"); + assert_eq!(name, "col"); + } + other => panic!("expected Column with table, got {other:?}"), + } + } + + #[test] + fn compound_identifier_three_parts_rejected() { + // schema.table.col — should be rejected. + use sqlparser::ast::Ident; + let parts = vec![Ident::new("schema"), Ident::new("table"), Ident::new("col")]; + let expr = Expr::CompoundIdentifier(parts); + let err = convert_expr(&expr, &ColumnScope::Unchecked).unwrap_err(); + assert!( + matches!(err, SqlError::Unsupported { .. }), + "expected Unsupported, got {err:?}" + ); + let msg = format!("{err}"); + assert!( + msg.contains("schema.table.col") || msg.contains("schema-qualified"), + "error should mention the qualified name: {msg}" + ); + } + + #[test] + fn compound_identifier_four_parts_rejected() { + use sqlparser::ast::Ident; + let parts = vec![ + Ident::new("a"), + Ident::new("b"), + Ident::new("c"), + Ident::new("d"), + ]; + let expr = Expr::CompoundIdentifier(parts); + let err = convert_expr(&expr, &ColumnScope::Unchecked).unwrap_err(); + assert!( + matches!(err, SqlError::Unsupported { .. }), + "expected Unsupported, got {err:?}" + ); + } + + /// `"userId"` with the PostgreSQL dialect is an identifier (quoted, + /// case-preserved), not a string literal. + #[test] + fn double_quoted_is_identifier_not_literal() { + let expr = first_select_expr(r#"SELECT "userId" FROM users"#); + match expr { + Expr::Identifier(ident) => { + assert_eq!(ident.value, "userId"); + assert_eq!(ident.quote_style, Some('"')); + } + other => panic!("expected Identifier, got {other:?}"), + } + } + + /// A double-quoted identifier in the SELECT list resolves as `SqlExpr::Column` + /// with the exact case preserved (not lowercased, because it was quoted). + #[test] + fn double_quoted_select_col_case_preserved() { + let expr = first_select_expr(r#"SELECT "userId" FROM users"#); + let sql_expr = + convert_expr(&expr, &ColumnScope::Unchecked).expect("convert_expr should succeed"); + match sql_expr { + SqlExpr::Column { name, table } => { + assert_eq!( + name, "userId", + "case must be preserved for quoted identifier" + ); + assert_eq!(table, None, "no table qualifier expected"); + } + other => panic!("expected Column, got {other:?}"), + } + } +} diff --git a/nodedb-sql/src/resolver/expr/convert/literals.rs b/nodedb-sql/src/resolver/expr/convert/literals.rs new file mode 100644 index 000000000..c3016682f --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/literals.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Literal, cast, array, and interval conversion. + +use sqlparser::ast::{Array, DataType, Expr, Interval, TypedString, Value, ValueWithSpan}; + +use crate::error::{Result, SqlError}; +use crate::resolver::ColumnScope; +use crate::resolver::expr::value::{convert_value, parse_interval_to_micros}; +use crate::types::*; + +use super::entry::convert_expr_depth; + +pub(super) fn convert_value_expr(val: &ValueWithSpan) -> Result { + Ok(SqlExpr::Literal(convert_value(&val.value)?)) +} + +pub(super) fn convert_typed_string(ts: &TypedString) -> Result { + // TIMESTAMP '...' and TIMESTAMPTZ '...' typed string literals. + let type_str = format!("{}", ts.data_type).to_ascii_uppercase(); + let raw = match &ts.value.value { + Value::SingleQuotedString(s) => s.clone(), + other => { + return Err(SqlError::Unsupported { + detail: format!("typed string value: {other}"), + }); + } + }; + match type_str.as_str() { + "TIMESTAMP" => { + let dt = nodedb_types::NdbDateTime::parse(&raw).ok_or_else(|| SqlError::Parse { + detail: format!("cannot parse TIMESTAMP literal: '{raw}'"), + })?; + return Ok(SqlExpr::Literal(SqlValue::Timestamp(dt))); + } + "TIMESTAMPTZ" | "TIMESTAMP WITH TIME ZONE" => { + let dt = nodedb_types::NdbDateTime::parse(&raw).ok_or_else(|| SqlError::Parse { + detail: format!("cannot parse TIMESTAMPTZ literal: '{raw}'"), + })?; + return Ok(SqlExpr::Literal(SqlValue::Timestamptz(dt))); + } + _ => {} + } + // Fall through: return as a generic literal string. + Ok(SqlExpr::Literal(SqlValue::String(raw))) +} + +pub(super) fn convert_cast( + expr: &Expr, + data_type: &DataType, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + // `::tsvector` and `::tsquery` casts are PG surface notation; the + // inner expression is the actual text value. Elide the cast and + // return the inner expression directly — no runtime type change is + // needed since we operate on plain strings internally. + let type_str = format!("{data_type}").to_ascii_lowercase(); + if type_str == "tsvector" || type_str == "tsquery" { + return convert_expr_depth(expr, depth, scope); + } + // `'...'::TIMESTAMP` and `'...'::TIMESTAMPTZ` — promote string literals + // to typed SqlValue when the inner expression is a string literal. + let upper = type_str.to_uppercase(); + if (upper == "TIMESTAMP" || upper == "TIMESTAMPTZ" || upper == "TIMESTAMP WITH TIME ZONE") + && let Expr::Value(v) = expr + && let Value::SingleQuotedString(s) = &v.value + { + let dt = nodedb_types::NdbDateTime::parse(s).ok_or_else(|| SqlError::Parse { + detail: format!("cannot parse timestamp cast: '{s}'"), + })?; + return Ok(SqlExpr::Literal(if upper == "TIMESTAMP" { + SqlValue::Timestamp(dt) + } else { + SqlValue::Timestamptz(dt) + })); + } + Ok(SqlExpr::Cast { + expr: Box::new(convert_expr_depth(expr, depth, scope)?), + to_type: format!("{data_type}"), + }) +} + +pub(super) fn convert_array( + array: &Array, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + let elems = array + .elem + .iter() + .map(|e| convert_expr_depth(e, depth, scope)) + .collect::>()?; + Ok(SqlExpr::ArrayLiteral(elems)) +} + +pub(super) fn convert_wildcard() -> Result { + Ok(SqlExpr::Wildcard) +} + +pub(super) fn convert_interval(interval: &Interval) -> Result { + // INTERVAL '1 hour' → microseconds as i64 literal. + // The interval value is typically a string literal. + let interval_str = match interval.value.as_ref() { + Expr::Value(v) => match &v.value { + Value::SingleQuotedString(s) => s.clone(), + Value::Number(n, _) => { + // INTERVAL 5 HOUR → combine number with leading_field. + if let Some(ref field) = interval.leading_field { + format!("{n} {field}") + } else { + n.clone() + } + } + _ => { + return Err(SqlError::Unsupported { + detail: format!("INTERVAL value: {}", interval.value), + }); + } + }, + _ => { + return Err(SqlError::Unsupported { + detail: format!("INTERVAL expression: {}", interval.value), + }); + } + }; + + // If leading_field is specified, append it: INTERVAL '5' HOUR → "5 HOUR" + let full_str = if interval_str.chars().all(|c| c.is_ascii_digit()) + && let Some(ref field) = interval.leading_field + { + format!("{interval_str} {field}") + } else { + interval_str + }; + + let micros = parse_interval_to_micros(&full_str).ok_or_else(|| SqlError::Parse { + detail: format!("cannot parse INTERVAL '{full_str}'"), + })?; + + Ok(SqlExpr::Literal(SqlValue::Int(micros))) +} + +#[cfg(test)] +mod tests { + use sqlparser::ast::{Expr, Value}; + + use crate::error::SqlError; + use crate::resolver::expr::convert::entry::tests::{first_select_expr, select_expr_lowered}; + use crate::resolver::expr::value::convert_value; + use crate::types::*; + + /// `'userId'` is a single-quoted string literal. + #[test] + fn single_quoted_is_string_literal() { + let expr = first_select_expr("SELECT 'userId' FROM users"); + match &expr { + Expr::Value(v) => match &v.value { + Value::SingleQuotedString(s) => assert_eq!(s, "userId"), + other => panic!("expected SingleQuotedString, got {other:?}"), + }, + other => panic!("expected Value, got {other:?}"), + } + // And convert_value maps it to SqlValue::String. + let Expr::Value(v) = expr else { unreachable!() }; + assert!(matches!( + convert_value(&v.value), + Ok(SqlValue::String(s)) if s == "userId" + )); + } + + /// `Value::DoubleQuotedString` (non-Postgres dialect) falls through + /// `convert_value` to `SqlError::Unsupported`. With PostgreSQL dialect + /// this variant is never produced, but constructing it directly verifies + /// the arm is absent and not silently accepted. + #[test] + fn double_quoted_string_value_unsupported() { + // Construct the variant directly — it cannot be produced by parsing + // with PostgreSqlDialect, which is exactly why the arm is dead code. + let val = Value::DoubleQuotedString("userId".into()); + assert!( + matches!(convert_value(&val), Err(SqlError::Unsupported { .. })), + "DoubleQuotedString should be Unsupported, not silently accepted" + ); + } + + #[test] + fn tsvector_cast_elided() { + // 'foo'::tsvector → Literal("foo") + let expr = select_expr_lowered("SELECT 'foo'::tsvector FROM t"); + assert!( + matches!(expr, SqlExpr::Literal(SqlValue::String(ref s)) if s == "foo"), + "expected Literal(\"foo\"), got {expr:?}" + ); + } + + #[test] + fn tsquery_cast_elided() { + // 'rust'::tsquery → Literal("rust") + let expr = select_expr_lowered("SELECT 'rust'::tsquery FROM t"); + assert!( + matches!(expr, SqlExpr::Literal(SqlValue::String(ref s)) if s == "rust"), + "expected Literal(\"rust\"), got {expr:?}" + ); + } +} diff --git a/nodedb-sql/src/resolver/expr/convert/mod.rs b/nodedb-sql/src/resolver/expr/convert/mod.rs new file mode 100644 index 000000000..ecf557321 --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Convert sqlparser AST expressions to our SqlExpr IR. + +pub mod builtins; +pub mod entry; +pub mod identifier; +pub mod literals; +pub mod operators; +pub mod predicates; + +pub use entry::convert_expr; +pub(in crate::resolver::expr) use entry::convert_expr_depth; diff --git a/nodedb-sql/src/resolver/expr/convert/operators.rs b/nodedb-sql/src/resolver/expr/convert/operators.rs new file mode 100644 index 000000000..e938d51e5 --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/operators.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Binary, unary, and `ANY` operator conversion. + +use sqlparser::ast::{BinaryOperator, Expr, UnaryOperator, Value}; + +use crate::error::{Result, SqlError}; +use crate::resolver::ColumnScope; +use crate::resolver::expr::binary_ops::{convert_binary_op, convert_unary_op}; +use crate::types::*; + +use super::entry::convert_expr_depth; + +pub(super) fn convert_binary( + left: &Expr, + op: &BinaryOperator, + right: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + // JSON and FTS operators are lowered to function calls before the + // generic binary-op path so they are never passed to + // convert_binary_op. + let json_fn: Option<&str> = match op { + BinaryOperator::Arrow => Some("pg_json_get"), + BinaryOperator::LongArrow => Some("pg_json_get_text"), + BinaryOperator::HashArrow => Some("pg_json_path_get"), + BinaryOperator::HashLongArrow => Some("pg_json_path_get_text"), + BinaryOperator::AtArrow => Some("pg_json_contains"), + BinaryOperator::ArrowAt => Some("pg_json_contained_by"), + BinaryOperator::Question => Some("pg_json_has_key"), + BinaryOperator::QuestionAnd => Some("pg_json_has_all_keys"), + BinaryOperator::QuestionPipe => Some("pg_json_has_any_key"), + _ => None, + }; + if let Some(name) = json_fn { + return Ok(SqlExpr::Function { + name: name.into(), + args: vec![ + convert_expr_depth(left, depth, scope)?, + convert_expr_depth(right, depth, scope)?, + ], + distinct: false, + }); + } + // `col @@ query` → pg_fts_match(col, query) + if matches!(op, BinaryOperator::AtAt) { + let col_expr = convert_expr_depth(left, depth, scope)?; + let query_expr = convert_expr_depth(right, depth, scope)?; + return Ok(crate::functions::fts_ops::pg_fts_funcs::lower_pg_fts_match( + col_expr, query_expr, + )); + } + Ok(SqlExpr::BinaryOp { + left: Box::new(convert_expr_depth(left, depth, scope)?), + op: convert_binary_op(op)?, + right: Box::new(convert_expr_depth(right, depth, scope)?), + }) +} + +/// A negative integer literal reaches sqlparser as unary minus applied +/// to a *positive* number, so the most negative `BIGINT` arrives as +/// `-(9223372036854775808)` — and that operand does not fit an `i64`. +/// Converting the operand on its own therefore falls back to `Float` +/// and silently turns an exact integer into an approximate one. Folding +/// the sign into the literal before parsing keeps the whole `i64` range +/// exact; anything that still does not fit takes the general path. +pub(super) fn convert_unary( + op: &UnaryOperator, + inner: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + if matches!(op, UnaryOperator::Minus) + && let Expr::Value(v) = inner + && let Value::Number(n, _) = &v.value + { + return match format!("-{n}").parse::() { + Ok(i) => Ok(SqlExpr::Literal(SqlValue::Int(i))), + Err(_) => Ok(SqlExpr::UnaryOp { + op: UnaryOp::Neg, + expr: Box::new(convert_expr_depth(inner, depth, scope)?), + }), + }; + } + Ok(SqlExpr::UnaryOp { + op: convert_unary_op(op)?, + expr: Box::new(convert_expr_depth(inner, depth, scope)?), + }) +} + +/// `left = ANY(right)` — desugar into InList over array elements. +/// When `right` resolves to an ArrayLiteral (or a function call that +/// the bridge/evaluator will fold to an array), emit InList so the +/// downstream scan filter path handles it natively. +pub(super) fn convert_any_op( + left: &Expr, + compare_op: &BinaryOperator, + right: &Expr, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + // Only support `=` comparison for now; reject other operators + // with a clear, non-AST-leaking message. + if !matches!(compare_op, BinaryOperator::Eq) { + return Err(SqlError::Unsupported { + detail: "ANY operator with non-equality comparison is not supported".into(), + }); + } + let left_expr = convert_expr_depth(left, depth, scope)?; + let right_expr = convert_expr_depth(right, depth, scope)?; + // Expand the right-hand side into a list if it is an array literal; + // otherwise wrap as a single-element list so InList still evaluates. + let list = match right_expr { + SqlExpr::ArrayLiteral(elems) => elems, + other => vec![other], + }; + Ok(SqlExpr::InList { + expr: Box::new(left_expr), + list, + negated: false, + }) +} + +#[cfg(test)] +mod tests { + use crate::resolver::expr::convert::entry::tests::{select_expr_lowered, where_sql_expr}; + use crate::types::*; + + /// `"col" = 'literal'` — double-quoted identifier on the left, single-quoted + /// string literal on the right — must lower to `BinaryOp(Column("col"), Eq, + /// Literal(String("literal")))`. This is the canonical mixed-quotation form + /// used in WHERE clauses (e.g. WHERE "userId" = 'alice'). + #[test] + fn double_quoted_col_eq_single_quoted_literal() { + let expr = where_sql_expr(r#"SELECT * FROM t WHERE "col" = 'literal'"#); + match expr { + SqlExpr::BinaryOp { left, right, .. } => { + assert!( + matches!(*left, SqlExpr::Column { ref name, .. } if name == "col"), + "left should be Column(col), got {left:?}" + ); + assert!( + matches!(*right, SqlExpr::Literal(SqlValue::String(ref s)) if s == "literal"), + "right should be Literal(String(\"literal\")), got {right:?}" + ); + } + other => panic!("expected BinaryOp, got {other:?}"), + } + } + + /// `"colA" = "colB"` — both sides are double-quoted identifiers; both must + /// resolve as column references, not string literals. + #[test] + fn double_quoted_col_eq_double_quoted_col() { + let expr = where_sql_expr(r#"SELECT * FROM t WHERE "colA" = "colB""#); + match expr { + SqlExpr::BinaryOp { left, right, .. } => { + assert!( + matches!(*left, SqlExpr::Column { ref name, .. } if name == "colA"), + "left should be Column(colA), got {left:?}" + ); + assert!( + matches!(*right, SqlExpr::Column { ref name, .. } if name == "colB"), + "right should be Column(colB), got {right:?}" + ); + } + other => panic!("expected BinaryOp, got {other:?}"), + } + } + + // ── JSON operator lowering tests ─────────────────────────────────────── + + fn assert_json_fn(sql: &str, expected_fn: &str) { + let expr = select_expr_lowered(sql); + match expr { + SqlExpr::Function { name, args, .. } => { + assert_eq!(name, expected_fn, "wrong function name"); + assert_eq!(args.len(), 2, "expected 2 args"); + } + other => panic!("expected Function, got {other:?}"), + } + } + + #[test] + fn arrow_lowers_to_pg_json_get() { + assert_json_fn("SELECT data->'key' FROM t", "pg_json_get"); + } + + #[test] + fn long_arrow_lowers_to_pg_json_get_text() { + assert_json_fn("SELECT data->>'key' FROM t", "pg_json_get_text"); + } + + #[test] + fn hash_arrow_lowers_to_pg_json_path_get() { + assert_json_fn("SELECT data#>'{a,b}' FROM t", "pg_json_path_get"); + } + + #[test] + fn hash_long_arrow_lowers_to_pg_json_path_get_text() { + assert_json_fn("SELECT data#>>'{a,b}' FROM t", "pg_json_path_get_text"); + } + + #[test] + fn at_arrow_lowers_to_pg_json_contains() { + assert_json_fn("SELECT data @> '{\"a\":1}' FROM t", "pg_json_contains"); + } + + #[test] + fn arrow_at_lowers_to_pg_json_contained_by() { + assert_json_fn("SELECT '{\"a\":1}' <@ data FROM t", "pg_json_contained_by"); + } + + #[test] + fn question_lowers_to_pg_json_has_key() { + assert_json_fn("SELECT data ? 'key' FROM t", "pg_json_has_key"); + } + + #[test] + fn question_and_lowers_to_pg_json_has_all_keys() { + assert_json_fn( + "SELECT data ?& ARRAY['a','b'] FROM t", + "pg_json_has_all_keys", + ); + } + + #[test] + fn question_pipe_lowers_to_pg_json_has_any_key() { + assert_json_fn( + "SELECT data ?| ARRAY['a','b'] FROM t", + "pg_json_has_any_key", + ); + } + + #[test] + fn chained_arrow_lowers_nested() { + // data->'a'->'b' → pg_json_get(pg_json_get(data, 'a'), 'b') + let expr = select_expr_lowered("SELECT data->'a'->'b' FROM t"); + match expr { + SqlExpr::Function { name, ref args, .. } => { + assert_eq!(name, "pg_json_get", "outer fn should be pg_json_get"); + match &args[0] { + SqlExpr::Function { + name: inner_name, .. + } => { + assert_eq!(inner_name, "pg_json_get", "inner fn should be pg_json_get"); + } + other => panic!("expected inner pg_json_get, got {other:?}"), + } + } + other => panic!("expected outer pg_json_get, got {other:?}"), + } + } + + // ── FTS operator / function lowering tests ──────────────────────────────── + + fn where_fn(sql: &str) -> SqlExpr { + where_sql_expr(sql) + } + + #[test] + fn at_at_lowers_to_pg_fts_match() { + // col @@ to_tsquery('rust & lang') → pg_fts_match(col, pg_to_tsquery('rust & lang')) + let expr = where_fn("SELECT * FROM t WHERE body @@ to_tsquery('rust & lang')"); + match expr { + SqlExpr::Function { + ref name, ref args, .. + } => { + assert_eq!( + name, "pg_fts_match", + "operator @@ should lower to pg_fts_match" + ); + assert_eq!(args.len(), 2, "expected 2 args"); + match &args[1] { + SqlExpr::Function { name: inner, .. } => { + assert_eq!(inner, "pg_to_tsquery"); + } + other => panic!("expected pg_to_tsquery as right arg, got {other:?}"), + } + } + other => panic!("expected pg_fts_match Function, got {other:?}"), + } + } + + #[test] + fn at_at_with_plainto_tsquery() { + // col @@ plainto_tsquery('rust lang') → pg_fts_match(col, pg_plainto_tsquery(...)) + let expr = where_fn("SELECT * FROM t WHERE body @@ plainto_tsquery('rust lang')"); + match expr { + SqlExpr::Function { + ref name, ref args, .. + } => { + assert_eq!(name, "pg_fts_match"); + match &args[1] { + SqlExpr::Function { name: inner, .. } => { + assert_eq!(inner, "pg_plainto_tsquery"); + } + other => panic!("expected pg_plainto_tsquery, got {other:?}"), + } + } + other => panic!("expected pg_fts_match, got {other:?}"), + } + } +} diff --git a/nodedb-sql/src/resolver/expr/convert/predicates.rs b/nodedb-sql/src/resolver/expr/convert/predicates.rs new file mode 100644 index 000000000..60fb82a54 --- /dev/null +++ b/nodedb-sql/src/resolver/expr/convert/predicates.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Predicate and conditional expression conversion. + +use sqlparser::ast::{CaseWhen, Expr}; + +use crate::error::Result; +use crate::resolver::ColumnScope; +use crate::types::*; + +use super::entry::convert_expr_depth; + +pub(super) fn convert_is_null( + inner: &Expr, + negated: bool, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::IsNull { + expr: Box::new(convert_expr_depth(inner, depth, scope)?), + negated, + }) +} + +pub(super) fn convert_in_list( + expr: &Expr, + list: &[Expr], + negated: bool, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::InList { + expr: Box::new(convert_expr_depth(expr, depth, scope)?), + list: list + .iter() + .map(|e| convert_expr_depth(e, depth, scope)) + .collect::>()?, + negated, + }) +} + +pub(super) fn convert_between( + expr: &Expr, + low: &Expr, + high: &Expr, + negated: bool, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::Between { + expr: Box::new(convert_expr_depth(expr, depth, scope)?), + low: Box::new(convert_expr_depth(low, depth, scope)?), + high: Box::new(convert_expr_depth(high, depth, scope)?), + negated, + }) +} + +pub(super) fn convert_like( + expr: &Expr, + pattern: &Expr, + negated: bool, + case_insensitive: bool, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + Ok(SqlExpr::Like { + expr: Box::new(convert_expr_depth(expr, depth, scope)?), + pattern: Box::new(convert_expr_depth(pattern, depth, scope)?), + negated, + case_insensitive, + }) +} + +pub(super) fn convert_case( + operand: Option<&Expr>, + conditions: &[CaseWhen], + else_result: Option<&Expr>, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { + let when_then = conditions + .iter() + .map(|cw| { + Ok(( + convert_expr_depth(&cw.condition, depth, scope)?, + convert_expr_depth(&cw.result, depth, scope)?, + )) + }) + .collect::>>()?; + Ok(SqlExpr::Case { + operand: operand + .map(|e| convert_expr_depth(e, depth, scope).map(Box::new)) + .transpose()?, + when_then, + else_expr: else_result + .map(|e| convert_expr_depth(e, depth, scope).map(Box::new)) + .transpose()?, + }) +} + +#[cfg(test)] +mod tests { + use crate::resolver::expr::convert::entry::tests::where_sql_expr; + use crate::types::*; + + #[test] + fn like_is_case_sensitive() { + let expr = where_sql_expr("SELECT * FROM t WHERE name LIKE 'foo%'"); + match expr { + SqlExpr::Like { + negated, + case_insensitive, + .. + } => { + assert!(!negated, "LIKE should not be negated"); + assert!(!case_insensitive, "LIKE should be case-sensitive"); + } + other => panic!("expected SqlExpr::Like, got {other:?}"), + } + } + + #[test] + fn ilike_is_case_insensitive() { + let expr = where_sql_expr("SELECT * FROM t WHERE name ILIKE 'foo%'"); + match expr { + SqlExpr::Like { + negated, + case_insensitive, + .. + } => { + assert!(!negated, "ILIKE should not be negated"); + assert!(case_insensitive, "ILIKE should be case-insensitive"); + } + other => panic!("expected SqlExpr::Like, got {other:?}"), + } + } + + #[test] + fn not_like_is_negated_case_sensitive() { + let expr = where_sql_expr("SELECT * FROM t WHERE name NOT LIKE 'foo%'"); + match expr { + SqlExpr::Like { + negated, + case_insensitive, + .. + } => { + assert!(negated, "NOT LIKE should be negated"); + assert!(!case_insensitive, "NOT LIKE should be case-sensitive"); + } + other => panic!("expected SqlExpr::Like, got {other:?}"), + } + } + + #[test] + fn not_ilike_is_negated_case_insensitive() { + let expr = where_sql_expr("SELECT * FROM t WHERE name NOT ILIKE 'foo%'"); + match expr { + SqlExpr::Like { + negated, + case_insensitive, + .. + } => { + assert!(negated, "NOT ILIKE should be negated"); + assert!(case_insensitive, "NOT ILIKE should be case-insensitive"); + } + other => panic!("expected SqlExpr::Like, got {other:?}"), + } + } +} From 4ae236563f2c172bc3432e9865c12328dc02279b Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:16 +0800 Subject: [PATCH 3/7] feat(sql): model per-relation column openness and an expression scope A collection's column set is open or closed depending on its engine. The resolver needs that distinction before it can reject an unknown name. - EngineRules gains accepts_undeclared_columns, implicit_columns, and value_column_name_is_free, answered by all seven engine impls. Only document_schemaless accepts undeclared fields. The kv engine carries an implicit key/value/ttl triple and leaves the value column name free until the DDL declares one. - CollectionInfo carries open_schema, derived from the engine for a stored collection and set independently for a synthesized relation. - TableScope gains check_name, outer-scope chaining for correlated subqueries, output-name widening for ORDER BY and HAVING aliases, and qualified-only relations for a MERGE source and ON CONFLICT excluded. - ColumnScope names the namespace an expression converts against. - resolver/derived.rs infers the output columns a derived, LATERAL, or CTE alias exposes, so those relations close instead of accepting any name. - SqlDataType::Unknown types a computed column with no declared type. --- nodedb-sql/src/engine_rules/array.rs | 5 + nodedb-sql/src/engine_rules/columnar.rs | 5 + .../src/engine_rules/document_schemaless.rs | 6 + .../src/engine_rules/document_strict.rs | 6 + nodedb-sql/src/engine_rules/kv.rs | 15 + nodedb-sql/src/engine_rules/rules.rs | 15 + nodedb-sql/src/engine_rules/spatial.rs | 5 + nodedb-sql/src/engine_rules/timeseries.rs | 5 + nodedb-sql/src/placeholder_types/infer.rs | 1 + .../src/planner/declared_type_coerce.rs | 3 +- nodedb-sql/src/planner/select/entry_ann.rs | 1 + nodedb-sql/src/resolver/columns.rs | 480 ++++++++---------- nodedb-sql/src/resolver/derived.rs | 323 ++++++++++++ nodedb-sql/src/resolver/mod.rs | 5 + nodedb-sql/src/resolver/scope.rs | 34 ++ nodedb-sql/src/types/collection.rs | 15 + nodedb-sql/src/types_expr.rs | 1 + .../catalog_adapter/sql_catalog_impl.rs | 1 + .../planner/sql_plan_convert/output_schema.rs | 3 + .../control/server/pgwire/catalog/schema.rs | 2 + .../server/pgwire/handler/prepared/parser.rs | 3 +- .../pgwire/handler/prepared/parser_schema.rs | 1 + .../control/server/response_shape/schema.rs | 2 + 23 files changed, 677 insertions(+), 260 deletions(-) create mode 100644 nodedb-sql/src/resolver/derived.rs create mode 100644 nodedb-sql/src/resolver/scope.rs diff --git a/nodedb-sql/src/engine_rules/array.rs b/nodedb-sql/src/engine_rules/array.rs index 172b916f8..1e9b7e8f7 100644 --- a/nodedb-sql/src/engine_rules/array.rs +++ b/nodedb-sql/src/engine_rules/array.rs @@ -74,6 +74,11 @@ impl EngineRules for ArrayRules { "use INSERT INTO ARRAY / DELETE FROM ARRAY for array engine mutations", )) } + + /// Columns are the array's declared dims and attrs. + fn accepts_undeclared_columns(&self) -> bool { + false + } } fn unsupported(op: &str, hint: &str) -> SqlError { diff --git a/nodedb-sql/src/engine_rules/columnar.rs b/nodedb-sql/src/engine_rules/columnar.rs index da89e922f..366b86840 100644 --- a/nodedb-sql/src/engine_rules/columnar.rs +++ b/nodedb-sql/src/engine_rules/columnar.rs @@ -150,4 +150,9 @@ impl EngineRules for ColumnarRules { ), }) } + + /// Every column is declared at creation and encoded per column. + fn accepts_undeclared_columns(&self) -> bool { + false + } } diff --git a/nodedb-sql/src/engine_rules/document_schemaless.rs b/nodedb-sql/src/engine_rules/document_schemaless.rs index e9d11be94..7b1af4059 100644 --- a/nodedb-sql/src/engine_rules/document_schemaless.rs +++ b/nodedb-sql/src/engine_rules/document_schemaless.rs @@ -146,4 +146,10 @@ impl EngineRules for SchemalessRules { sort_keys: Vec::new(), }) } + + /// Schemaless documents store fields the schema never declares, so an + /// undeclared name is a valid read. + fn accepts_undeclared_columns(&self) -> bool { + true + } } diff --git a/nodedb-sql/src/engine_rules/document_strict.rs b/nodedb-sql/src/engine_rules/document_strict.rs index 0ff9034fc..1bc621cbc 100644 --- a/nodedb-sql/src/engine_rules/document_strict.rs +++ b/nodedb-sql/src/engine_rules/document_strict.rs @@ -146,4 +146,10 @@ impl EngineRules for StrictRules { sort_keys: Vec::new(), }) } + + /// A Binary Tuple is positional over the declared schema, so an + /// undeclared field cannot exist. + fn accepts_undeclared_columns(&self) -> bool { + false + } } diff --git a/nodedb-sql/src/engine_rules/kv.rs b/nodedb-sql/src/engine_rules/kv.rs index 996527ee7..20d4ac5f3 100644 --- a/nodedb-sql/src/engine_rules/kv.rs +++ b/nodedb-sql/src/engine_rules/kv.rs @@ -132,4 +132,19 @@ impl EngineRules for KvRules { ), }) } + + /// The key/value/ttl shape is fixed. + fn accepts_undeclared_columns(&self) -> bool { + false + } + + /// The key/value/ttl triple is the engine's fixed shape, so a collection + /// carries all three even when the DDL names only the key. + fn implicit_columns(&self) -> &'static [&'static str] { + &["key", "value", "ttl"] + } + + fn value_column_name_is_free(&self) -> bool { + true + } } diff --git a/nodedb-sql/src/engine_rules/rules.rs b/nodedb-sql/src/engine_rules/rules.rs index 5a68b81db..1409cf150 100644 --- a/nodedb-sql/src/engine_rules/rules.rs +++ b/nodedb-sql/src/engine_rules/rules.rs @@ -45,4 +45,19 @@ pub trait EngineRules { /// MERGE semantics (everything except `document_schemaless` and /// `document_strict`). fn plan_merge(&self, params: MergeParams) -> Result>; + /// True when the engine accepts fields the schema never declares, so a + /// read of an undeclared name resolves to NULL instead of raising + /// `SqlError::UnknownColumn`. + fn accepts_undeclared_columns(&self) -> bool; + /// Columns every collection on this engine carries whether or not the + /// DDL declares them. A reference to one resolves like a declared column. + fn implicit_columns(&self) -> &'static [&'static str] { + &[] + } + /// True when the engine stores one value whose column name the statement + /// picks. Such a collection resolves any name while it declares no value + /// column, and closes to the declared set once it names one. + fn value_column_name_is_free(&self) -> bool { + false + } } diff --git a/nodedb-sql/src/engine_rules/spatial.rs b/nodedb-sql/src/engine_rules/spatial.rs index f05391187..23396d985 100644 --- a/nodedb-sql/src/engine_rules/spatial.rs +++ b/nodedb-sql/src/engine_rules/spatial.rs @@ -142,4 +142,9 @@ impl EngineRules for SpatialRules { ), }) } + + /// Every column is declared at creation and encoded per column. + fn accepts_undeclared_columns(&self) -> bool { + false + } } diff --git a/nodedb-sql/src/engine_rules/timeseries.rs b/nodedb-sql/src/engine_rules/timeseries.rs index 08dd91aa6..ab6ab210d 100644 --- a/nodedb-sql/src/engine_rules/timeseries.rs +++ b/nodedb-sql/src/engine_rules/timeseries.rs @@ -128,6 +128,11 @@ impl EngineRules for TimeseriesRules { ), }) } + + /// Every column is declared at creation and encoded per column. + fn accepts_undeclared_columns(&self) -> bool { + false + } } /// Default time range bounds for the SqlPlan IR. diff --git a/nodedb-sql/src/placeholder_types/infer.rs b/nodedb-sql/src/placeholder_types/infer.rs index b7043d63d..caf4cd72c 100644 --- a/nodedb-sql/src/placeholder_types/infer.rs +++ b/nodedb-sql/src/placeholder_types/infer.rs @@ -131,6 +131,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), } } diff --git a/nodedb-sql/src/planner/declared_type_coerce.rs b/nodedb-sql/src/planner/declared_type_coerce.rs index d9547ae13..cffdad297 100644 --- a/nodedb-sql/src/planner/declared_type_coerce.rs +++ b/nodedb-sql/src/planner/declared_type_coerce.rs @@ -154,7 +154,8 @@ fn coerce_value(column: &str, value: SqlValue, declared: &SqlDataType) -> Result | SqlDataType::Decimal | SqlDataType::Uuid | SqlDataType::Vector(_) - | SqlDataType::Geometry => Ok(value), + | SqlDataType::Geometry + | SqlDataType::Unknown => Ok(value), } } diff --git a/nodedb-sql/src/planner/select/entry_ann.rs b/nodedb-sql/src/planner/select/entry_ann.rs index a11e7df52..8422d9657 100644 --- a/nodedb-sql/src/planner/select/entry_ann.rs +++ b/nodedb-sql/src/planner/select/entry_ann.rs @@ -296,6 +296,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), })) } else { Ok(None) diff --git a/nodedb-sql/src/resolver/columns.rs b/nodedb-sql/src/resolver/columns.rs index 654933240..463859824 100644 --- a/nodedb-sql/src/resolver/columns.rs +++ b/nodedb-sql/src/resolver/columns.rs @@ -7,11 +7,13 @@ use std::collections::HashMap; use nodedb_types::DatabaseId; use crate::error::{Result, SqlError}; -use crate::parser::normalize::{normalize_object_name_checked, table_name_from_factor}; -use crate::types::{ - ArrayCatalogView, CollectionInfo, ColumnInfo, EngineType, SqlCatalog, SqlDataType, -}; -use crate::types_array::{ArrayAttrType, ArrayDimType}; +use crate::parser::normalize::table_name_from_factor; +use crate::types::{CollectionInfo, ColumnInfo, SqlCatalog}; + +/// Synthetic temporal columns an audit read injects into every version row. +/// They are not declared columns, so a bitemporal relation resolves them by +/// name. +const BITEMPORAL_AUDIT_COLUMNS: [&str; 3] = ["_ts_system", "_ts_valid_from", "_ts_valid_until"]; /// Resolved table reference: name, alias, and catalog info. #[derive(Debug, Clone)] @@ -29,12 +31,26 @@ impl ResolvedTable { } /// Context built during FROM clause resolution. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct TableScope { /// Tables by reference name (alias or table name). pub tables: HashMap, /// Insertion order for unambiguous column resolution. order: Vec, + /// Names resolvable here that belong to no relation: SELECT output + /// aliases visible to ORDER BY, GROUP BY, and HAVING, plus the output + /// names substituted for aggregate calls. + output_names: Vec, + /// The enclosing query's scope, for a correlated subquery. A qualifier + /// naming no relation here resolves there instead. Boxed rather than + /// borrowed: a lifetime on `TableScope` would ripple through every + /// planner signature that stores or returns one. + outer: Option>, + /// Relations a bare column name never resolves against, reachable only + /// through their qualifier. A MERGE source and the `excluded` relation of + /// `ON CONFLICT DO UPDATE` are both qualified-only, so a bare name in a + /// WHEN or SET clause names the target column. + qualified_only: Vec, } impl TableScope { @@ -55,74 +71,33 @@ impl TableScope { Ok(()) } - /// Resolve a column name, optionally qualified with a table reference. + /// Add a relation that only a qualified reference reaches. /// - /// For schemaless collections, any column is accepted (dynamic fields). - /// For typed collections, the column must exist in the schema. - pub fn resolve_column( - &self, - table_ref: Option<&str>, - column: &str, - ) -> Result<(String, String)> { - let col = column.to_lowercase(); + /// A bare column name skips it, so a name both this relation and a + /// bare-resolvable one declare is not ambiguous. + pub fn add_qualified_only(&mut self, table: ResolvedTable) -> Result<()> { + let key = table.ref_name().to_string(); + self.add(table)?; + self.qualified_only.push(key); + Ok(()) + } - if let Some(tref) = table_ref { - let tref_lower = tref.to_lowercase(); - let table = self - .tables - .get(&tref_lower) - .ok_or_else(|| SqlError::UnknownTable { - name: tref_lower.clone(), - })?; - self.validate_column(table, &col)?; - return Ok((table.name.clone(), col)); + fn column_exists(&self, table: &ResolvedTable, column: &str) -> bool { + if table.info.open_schema { + return true; } - - // Unqualified: search all tables. - let mut matches = Vec::new(); - for key in &self.order { - let table = &self.tables[key]; - if self.column_exists(table, &col) { - matches.push(table.name.clone()); - } + if table.info.bitemporal && BITEMPORAL_AUDIT_COLUMNS.contains(&column) { + return true; } - - match matches.len() { - 0 => { - // For single-table queries with schemaless, accept anything. - if self.tables.len() == 1 { - let table = self - .tables - .values() - .next() - .expect("invariant: self.tables.len() == 1 checked immediately above"); - if table.info.engine == EngineType::DocumentSchemaless { - return Ok((table.name.clone(), col)); - } - } - Err(SqlError::UnknownColumn { - table: self - .order - .first() - .cloned() - .unwrap_or_else(|| "".into()), - column: col, - }) - } - 1 => Ok(( - matches - .into_iter() - .next() - .expect("invariant: matches.len() == 1 guaranteed by this match arm"), - col, - )), - _ => Err(SqlError::AmbiguousColumn { column: col }), + let rules = crate::engine_rules::resolve_engine_rules(table.info.engine); + if rules.implicit_columns().contains(&column) { + return true; } - } - - fn column_exists(&self, table: &ResolvedTable, column: &str) -> bool { - // Schemaless accepts any column. - if table.info.engine == EngineType::DocumentSchemaless { + // A key-only collection on an engine with a free value-column name + // has not fixed that name yet, so any name resolves. + if rules.value_column_name_is_free() + && !table.info.columns.iter().any(|c| !c.is_primary_key) + { return true; } table.info.columns.iter().any(|c| c.name == column) @@ -148,6 +123,116 @@ impl TableScope { } } + /// A copy of this scope nested inside `outer`, for planning a correlated + /// subquery body. + pub fn nested_in(mut self, outer: TableScope) -> Self { + self.outer = Some(Box::new(outer)); + self + } + + /// A copy of this scope widened with output column names. + /// + /// An ORDER BY, GROUP BY, or HAVING identifier resolves against input + /// columns first and output names second, so this only widens: a name + /// that already resolves to a column keeps resolving to it. + pub fn with_output_names(&self, names: impl IntoIterator) -> Self { + let mut out = self.clone(); + out.output_names.extend(names); + out + } + + /// A single-relation scope, for the DML planners that resolve one + /// collection and build no FROM clause. + pub fn single(table: ResolvedTable) -> Result { + let mut scope = Self::new(); + scope.add(table)?; + Ok(scope) + } + + /// Reject a column reference that names nothing in scope. + pub fn check_name(&self, table_ref: Option<&str>, column: &str) -> Result<()> { + let col = column.to_lowercase(); + + if let Some(tref) = table_ref { + let tref_lower = tref.to_lowercase(); + return match self.tables.get(&tref_lower) { + Some(table) => self.validate_column(table, &col), + // A qualifier naming no relation here belongs to the outer + // query of a correlated subquery. With no outer scope it is + // an unknown relation, not an unknown column. + None => match &self.outer { + Some(outer) => outer.check_name(Some(&tref_lower), &col), + None => Err(SqlError::UnknownTable { name: tref_lower }), + }, + }; + } + + if self.output_names.iter().any(|n| n == &col) { + return Ok(()); + } + + // A relation that declares the column wins over one that merely + // accepts any name. An open-schema relation alongside a closed one + // that declares the column is not an ambiguity. + let bare: Vec<&ResolvedTable> = self + .order + .iter() + .filter(|key| !self.qualified_only.contains(*key)) + .map(|key| &self.tables[key]) + .collect(); + let declared = bare + .iter() + .filter(|table| table.info.columns.iter().any(|c| c.name == col)) + .count(); + // An open-schema relation contributes a maybe, never a yes. With no + // relation declaring the column, one that accepts any name resolves + // it: neither ambiguity nor absence is provable. + if declared == 0 && bare.iter().any(|table| self.column_exists(table, &col)) { + return Ok(()); + } + match declared { + 0 => match &self.outer { + Some(outer) => outer.check_name(None, &col), + None => Err(SqlError::UnknownColumn { + table: self + .order + .first() + .cloned() + .unwrap_or_else(|| "".into()), + column: col, + }), + }, + 1 => Ok(()), + _ => Err(SqlError::AmbiguousColumn { column: col }), + } + } + + /// The resolved tables, in the order the FROM clause introduced them. + pub fn tables_in_order(&self) -> impl Iterator { + self.order.iter().map(|key| &self.tables[key]) + } + + /// The relation registered under `ref_name`, alias or table name. + pub fn table_by_ref(&self, ref_name: &str) -> Option<&ResolvedTable> { + self.tables.get(&ref_name.to_lowercase()) + } + + /// The declared column a reference names, when the relation declares it. + pub fn declared_column(&self, table_ref: Option<&str>, column: &str) -> Option<&ColumnInfo> { + let col = column.to_lowercase(); + match table_ref { + Some(tref) => self + .table_by_ref(tref)? + .info + .columns + .iter() + .find(|c| c.name == col), + None => self + .tables_in_order() + .find_map(|t| t.info.columns.iter().find(|c| c.name == col)), + } + } + /// Resolve tables from a FROM clause. pub fn resolve_from( catalog: &dyn SqlCatalog, @@ -171,35 +256,32 @@ impl TableScope { // ARRAY_*(...) table-valued function: synthesize a ResolvedTable // from the array's dim+attr schema so equi-join keys against the // TVF's output rows resolve. - if let Some(resolved) = resolve_array_tvf(catalog, factor)? { + if let Some(resolved) = crate::resolver::array_tvf::resolve_array_tvf(catalog, factor)? { self.add(resolved)?; return Ok(()); } - // LATERAL derived subquery: register the alias as a schemaless - // collection so qualified column references (`alias.col`) resolve - // without a catalog lookup. The actual inner plan is built separately. + // Derived subquery, LATERAL or not: register the alias as the relation + // its projection list exposes, so a column reference on the alias + // resolves without a catalog lookup. The inner plan is built + // separately. if let sqlparser::ast::TableFactor::Derived { - lateral: true, + subquery, alias: Some(alias), .. } = factor { let alias_str = crate::reserved::check_ast_identifier(&alias.name)?; + let declared: Vec = alias + .columns + .iter() + .map(|column| crate::reserved::check_ast_identifier(&column.name)) + .collect::>()?; + let info = + crate::resolver::derived::infer_subquery_relation(catalog, &alias_str, subquery)?; self.add(ResolvedTable { name: alias_str.clone(), - alias: Some(alias_str.clone()), - info: CollectionInfo { - name: alias_str, - engine: EngineType::DocumentSchemaless, - columns: Vec::new(), - primary_key: None, - has_auto_tier: false, - indexes: Vec::new(), - bitemporal: false, - primary: nodedb_types::PrimaryEngine::Document, - vector_primary: None, - partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, - }, + alias: Some(alias_str), + info: crate::resolver::derived::rename_output_columns(info, &declared), })?; return Ok(()); } @@ -213,136 +295,39 @@ impl TableScope { } } -/// If `factor` is `ARRAY_*(name, ...)`, look up the array via the -/// catalog and build a `ResolvedTable` whose columns mirror the array's -/// dims + attrs. Returns `Ok(None)` for any non-array-TVF factor. -fn resolve_array_tvf( - catalog: &dyn SqlCatalog, - factor: &sqlparser::ast::TableFactor, -) -> Result> { - let (fn_name, args, alias) = match factor { - sqlparser::ast::TableFactor::Table { - name, - args: Some(args), - alias, - .. - } => ( - normalize_object_name_checked(name)?, - args, - alias - .as_ref() - .map(|alias| crate::reserved::check_ast_identifier(&alias.name)) - .transpose()?, - ), - _ => return Ok(None), - }; - if !matches!( - fn_name.as_str(), - "array_slice" | "array_project" | "array_agg" | "array_elementwise" - ) { - return Ok(None); - } - - // First positional arg is the array name as a string literal. - let first = args.args.first().ok_or_else(|| SqlError::Unsupported { - detail: format!("{fn_name}: missing array-name argument"), - })?; - let array_name = extract_string_literal_arg(first).ok_or_else(|| SqlError::Unsupported { - detail: format!("{fn_name}: array-name argument must be a string literal"), - })?; - let view = catalog - .lookup_array(&array_name) - .ok_or_else(|| SqlError::UnknownTable { - name: array_name.clone(), - })?; - - let info = CollectionInfo { - name: view.name.clone(), - engine: EngineType::Array, - columns: array_columns(&view), - primary_key: None, - has_auto_tier: false, - indexes: Vec::new(), - bitemporal: false, - primary: nodedb_types::PrimaryEngine::Document, - vector_primary: None, - partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, - }; - Ok(Some(ResolvedTable { - name: view.name, - alias, - info, - })) -} - -fn array_columns(view: &ArrayCatalogView) -> Vec { - let mut cols = Vec::with_capacity(view.dims.len() + view.attrs.len()); - for d in &view.dims { - cols.push(ColumnInfo { - name: d.name.clone(), - data_type: dim_type_to_sql(d.dtype), - nullable: false, - is_primary_key: false, - default: None, - raw_type: None, - int_width: None, - float_width: None, - }); - } - for a in &view.attrs { - cols.push(ColumnInfo { - name: a.name.clone(), - data_type: attr_type_to_sql(a.dtype), - nullable: a.nullable, - is_primary_key: false, - default: None, - raw_type: None, - int_width: None, - float_width: None, - }); - } - cols -} - -fn dim_type_to_sql(t: ArrayDimType) -> SqlDataType { - match t { - ArrayDimType::Int64 => SqlDataType::Int64, - ArrayDimType::Float64 => SqlDataType::Float64, - ArrayDimType::TimestampMs => SqlDataType::Timestamp, - ArrayDimType::String => SqlDataType::String, - } -} - -fn attr_type_to_sql(t: ArrayAttrType) -> SqlDataType { - match t { - ArrayAttrType::Int64 => SqlDataType::Int64, - ArrayAttrType::Float64 => SqlDataType::Float64, - ArrayAttrType::String => SqlDataType::String, - ArrayAttrType::Bytes => SqlDataType::Bytes, - } -} - -fn extract_string_literal_arg(arg: &sqlparser::ast::FunctionArg) -> Option { - use sqlparser::ast::{Expr, FunctionArg, FunctionArgExpr, Value}; - let expr = match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => e, - FunctionArg::Named { - arg: FunctionArgExpr::Expr(e), - .. - } => e, - _ => return None, - }; - match expr { - Expr::Value(v) => match &v.value { - Value::SingleQuotedString(s) => Some(s.clone()), - _ => None, - }, - _ => None, +/// Scope builders shared by the planner unit tests. +#[cfg(test)] +pub(crate) mod test_support { + use super::{ResolvedTable, TableScope}; + use crate::types::{CollectionInfo, EngineType}; + + /// A one-relation scope over `collection` that accepts any column name. + pub(crate) fn open_scope(collection: &str) -> TableScope { + let info = CollectionInfo { + name: collection.into(), + engine: EngineType::DocumentSchemaless, + columns: Vec::new(), + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), + }; + TableScope::single(ResolvedTable { + name: info.name.clone(), + alias: None, + info, + }) + .expect("single-relation scope") } } #[cfg(test)] mod tests { + use super::test_support::open_scope; use super::*; use crate::types::{CollectionInfo, ColumnInfo, EngineType, SqlDataType}; use nodedb_types::PrimaryEngine; @@ -371,21 +356,7 @@ mod tests { primary: PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, - } - } - - fn schemaless_collection(name: &str) -> CollectionInfo { - CollectionInfo { - name: name.into(), - engine: EngineType::DocumentSchemaless, - columns: Vec::new(), - primary_key: None, - has_auto_tier: false, - indexes: Vec::new(), - bitemporal: false, - primary: PrimaryEngine::Document, - vector_primary: None, - partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), } } @@ -401,39 +372,36 @@ mod tests { scope } - /// A double-quoted identifier resolves as a column name (case-preserved). + /// A double-quoted identifier resolves as a column name. /// `"userId"` is parsed by the SQL layer as `Expr::Identifier` with - /// `quote_style = Some('"')` and `value = "userId"`. At the - /// `TableScope` level the column name arrives lowercase (strict schema - /// columns are stored lowercase), so the resolved name is `"userid"`. + /// `quote_style = Some('"')` and `value = "userId"`. At the `TableScope` + /// level the column name arrives lowercase, because strict schema columns + /// are stored lowercase. /// - /// This test confirms the resolution path, not just `convert_expr`. + /// This test covers the resolution path, not just `convert_expr`. #[test] fn quoted_identifier_resolves_as_column() { let scope = scope_with(strict_collection("users", vec!["userid", "email"])); - let (table, col) = scope - .resolve_column(None, "userid") - .expect("should resolve"); - assert_eq!(table, "users"); - assert_eq!(col, "userid"); + scope.check_name(None, "userid").expect("must resolve"); } - /// An unrecognized column in a strict collection must yield - /// `SqlError::UnknownColumn`, NOT `SqlError::Unsupported`. - /// This verifies that a double-quoted identifier like `"ghost_col"` - /// that maps to `SqlExpr::Column { name: "ghost_col" }` surfaces the - /// right error variant when resolved against a strict schema. + /// An unrecognized column in a strict collection yields + /// `SqlError::UnknownColumn`, never `SqlError::Unsupported`. + /// + /// A double-quoted identifier like `"ghost_col"` maps to + /// `SqlExpr::Column { name: "ghost_col" }`. Resolving it against a strict + /// schema must name the missing column. #[test] fn unknown_column_in_strict_collection_yields_unknown_column_error() { let scope = scope_with(strict_collection("users", vec!["id", "email"])); let err = scope - .resolve_column(None, "ghost_col") - .expect_err("should fail for unknown column"); + .check_name(None, "ghost_col") + .expect_err("must reject an unknown column"); assert!( matches!(err, SqlError::UnknownColumn { ref column, .. } if column == "ghost_col"), "expected UnknownColumn(ghost_col), got {err:?}" ); - // Must NOT be Unsupported — that would be the wrong error variant. + // Unsupported is the wrong error variant here. assert!( !matches!(err, SqlError::Unsupported { .. }), "must not surface Unsupported for a missing column" @@ -444,23 +412,19 @@ mod tests { /// like they could be misidentified double-quoted identifiers. #[test] fn any_column_accepted_in_schemaless_collection() { - let scope = scope_with(schemaless_collection("events")); - let (table, col) = scope - .resolve_column(None, "ghost_col") - .expect("schemaless should accept any column"); - assert_eq!(table, "events"); - assert_eq!(col, "ghost_col"); + let scope = open_scope("events"); + scope + .check_name(None, "ghost_col") + .expect("a schemaless relation must accept any column"); } /// Qualified column reference: `"t"."col"` → table `t`, column `col`. #[test] fn qualified_column_resolves_correctly() { let scope = scope_with(strict_collection("t", vec!["col", "other"])); - let (table, col) = scope - .resolve_column(Some("t"), "col") - .expect("qualified column should resolve"); - assert_eq!(table, "t"); - assert_eq!(col, "col"); + scope + .check_name(Some("t"), "col") + .expect("a qualified column must resolve"); } /// Qualified reference to an unknown column in a strict collection must @@ -469,8 +433,8 @@ mod tests { fn qualified_unknown_column_in_strict_collection() { let scope = scope_with(strict_collection("t", vec!["id"])); let err = scope - .resolve_column(Some("t"), "missing") - .expect_err("should fail"); + .check_name(Some("t"), "missing") + .expect_err("must reject an unknown column"); assert!( matches!(err, SqlError::UnknownColumn { .. }), "expected UnknownColumn, got {err:?}" diff --git a/nodedb-sql/src/resolver/derived.rs b/nodedb-sql/src/resolver/derived.rs new file mode 100644 index 000000000..104c5e1b3 --- /dev/null +++ b/nodedb-sql/src/resolver/derived.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Output-column inference for derived, LATERAL, and CTE relations. + +use sqlparser::ast::{self, Expr, SelectItem, SelectItemQualifiedWildcardKind, SetExpr}; + +use crate::error::{Result, SqlError}; +use crate::parser::normalize::{normalize_ident, normalize_object_name_checked}; +use crate::resolver::columns::TableScope; +use crate::types::{CollectionInfo, ColumnInfo, EngineType, SqlCatalog, SqlDataType}; + +/// The relation a subquery alias exposes. +/// +/// A synthesized relation carries `EngineType::DocumentSchemaless`: it is a +/// MessagePack row stream, and the CTE lowering depends on that. Openness +/// rides on `CollectionInfo::open_schema`, not on the engine. +pub fn infer_subquery_relation( + catalog: &dyn SqlCatalog, + alias: &str, + query: &ast::Query, +) -> Result { + let (columns, open_schema) = infer_projection(catalog, query)?; + Ok(CollectionInfo { + name: alias.to_string(), + engine: EngineType::DocumentSchemaless, + columns, + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema, + }) +} + +/// A relation whose output shape is not inferable, named `alias`. +/// +/// Any column name resolves against it. The recursive arm of a `WITH +/// RECURSIVE` names the working table while planning its own body, so that +/// arm's shape is not known yet. +pub fn open_subquery_relation(alias: &str) -> CollectionInfo { + CollectionInfo { + name: alias.to_string(), + engine: EngineType::DocumentSchemaless, + columns: Vec::new(), + primary_key: Some("id".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: true, + } +} + +/// Rename an inferred relation's columns to an alias column list. +/// +/// `WITH c(a, b) AS (...)` and `FROM (...) AS t(a, b)` name the output +/// positionally. The declared names replace the inferred ones; a type the +/// inferred column carried survives at the same position. +pub fn rename_output_columns(mut info: CollectionInfo, names: &[String]) -> CollectionInfo { + if names.is_empty() { + return info; + } + let inferred = std::mem::take(&mut info.columns); + info.columns = names + .iter() + .enumerate() + .map(|(index, name)| match inferred.get(index) { + Some(column) => ColumnInfo { + name: name.clone(), + ..column.clone() + }, + None => synthetic_column(name), + }) + .collect(); + info +} + +/// The columns a query projects, and whether a name outside them resolves +/// against it. +fn infer_projection( + catalog: &dyn SqlCatalog, + query: &ast::Query, +) -> Result<(Vec, bool)> { + infer_body(catalog, &query.body) +} + +fn infer_body(catalog: &dyn SqlCatalog, body: &SetExpr) -> Result<(Vec, bool)> { + match body { + SetExpr::Select(select) => infer_select_projection(catalog, select), + SetExpr::Query(query) => infer_projection(catalog, query), + // A set operation takes its output names from the left arm. + SetExpr::SetOperation { left, .. } => infer_body(catalog, left), + // A row constructor, a `TABLE` command, and a DML body carry no + // projection list to read names from. + SetExpr::Values(_) + | SetExpr::Table(_) + | SetExpr::Insert(_) + | SetExpr::Update(_) + | SetExpr::Delete(_) + | SetExpr::Merge(_) => Ok((Vec::new(), true)), + } +} + +fn infer_select_projection( + catalog: &dyn SqlCatalog, + select: &ast::Select, +) -> Result<(Vec, bool)> { + let scope = TableScope::resolve_from(catalog, &select.from)?; + let mut columns = Vec::new(); + let mut open = false; + + for item in &select.projection { + match item { + SelectItem::Wildcard(_) => { + for table in scope.tables_in_order() { + columns.extend(table.info.columns.iter().cloned()); + open |= table.info.open_schema; + } + } + SelectItem::QualifiedWildcard(kind, _) => { + let table_ref = match kind { + SelectItemQualifiedWildcardKind::ObjectName(name) => { + normalize_object_name_checked(name)? + } + // `STRUCT<...>('x').*` expands a value, not a relation, so + // the names it yields are not readable from the FROM clause. + SelectItemQualifiedWildcardKind::Expr(_) => { + open = true; + continue; + } + }; + match scope.table_by_ref(&table_ref) { + Some(table) => { + columns.extend(table.info.columns.iter().cloned()); + open |= table.info.open_schema; + } + // The qualifier names a relation of an enclosing query. + None => open = true, + } + } + SelectItem::ExprWithAlias { alias, .. } => { + columns.push(synthetic_column(&normalize_ident(alias))); + } + SelectItem::UnnamedExpr(expr) => columns.push(unnamed_column(&scope, expr)), + SelectItem::ExprWithAliases { aliases, .. } => { + return Err(SqlError::Unsupported { + detail: format!( + "multi-alias projection ('AS' with {} names) is not supported; \ + give the expression a single alias", + aliases.len() + ), + }); + } + } + } + + Ok((columns, open)) +} + +/// The column an unaliased projection item exposes. +/// +/// A bare or two-part column reference keeps the source column's declared +/// type. Anything else takes its rendered form as its name, matching how the +/// projection converter names a computed output. +fn unnamed_column(scope: &TableScope, expr: &Expr) -> ColumnInfo { + let named = match expr { + Expr::Identifier(ident) => Some((None, normalize_ident(ident))), + Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + Some((Some(normalize_ident(&parts[0])), normalize_ident(&parts[1]))) + } + _ => None, + }; + match named { + Some((qualifier, name)) => match scope.declared_column(qualifier.as_deref(), &name) { + Some(column) => ColumnInfo { + name, + ..column.clone() + }, + None => synthetic_column(&name), + }, + None => synthetic_column(&format!("{expr}").to_lowercase()), + } +} + +/// A column that resolves by name and declares no type. +fn synthetic_column(name: &str) -> ColumnInfo { + ColumnInfo { + name: name.to_string(), + data_type: SqlDataType::Unknown, + nullable: true, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::SqlCatalogError; + use nodedb_types::DatabaseId; + use sqlparser::ast::Statement; + + struct TestCatalog; + + fn strict(name: &str, columns: &[&str]) -> CollectionInfo { + CollectionInfo { + name: name.into(), + engine: EngineType::DocumentStrict, + columns: columns + .iter() + .map(|c| ColumnInfo { + name: (*c).into(), + data_type: SqlDataType::Int64, + nullable: false, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }) + .collect(), + primary_key: Some("a".into()), + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), + } + } + + impl SqlCatalog for TestCatalog { + fn get_collection( + &self, + _database_id: DatabaseId, + name: &str, + ) -> std::result::Result, SqlCatalogError> { + Ok(match name { + "src" => Some(strict("src", &["a", "b"])), + "loose" => { + let mut info = strict("loose", &["x"]); + info.engine = EngineType::DocumentSchemaless; + info.open_schema = true; + Some(info) + } + _ => None, + }) + } + } + + fn parse_query(sql: &str) -> ast::Query { + let stmts = crate::parser::statement::parse_sql(sql).expect("parse failed"); + match &stmts[0] { + Statement::Query(query) => (**query).clone(), + other => panic!("expected a query, got {other:?}"), + } + } + + fn infer(sql: &str) -> CollectionInfo { + infer_subquery_relation(&TestCatalog, "t", &parse_query(sql)).expect("inference failed") + } + + #[test] + fn bare_identifier_keeps_the_declared_type() { + let info = infer("SELECT a FROM src"); + assert_eq!(info.columns.len(), 1); + assert_eq!(info.columns[0].name, "a"); + assert_eq!(info.columns[0].data_type, SqlDataType::Int64); + assert!(!info.open_schema); + } + + #[test] + fn qualified_identifier_exposes_the_unqualified_name() { + let info = infer("SELECT i.b FROM src AS i"); + assert_eq!(info.columns.len(), 1); + assert_eq!(info.columns[0].name, "b"); + } + + #[test] + fn alias_names_the_output_column() { + let info = infer("SELECT a + 1 AS total FROM src"); + assert_eq!(info.columns[0].name, "total"); + assert_eq!(info.columns[0].data_type, SqlDataType::Unknown); + } + + #[test] + fn star_over_a_closed_source_stays_closed() { + let info = infer("SELECT * FROM src"); + let names: Vec<&str> = info.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b"]); + assert!(!info.open_schema); + } + + #[test] + fn star_over_an_open_source_stays_open() { + let info = infer("SELECT * FROM loose"); + assert!(info.open_schema); + } + + #[test] + fn set_operation_takes_the_left_arm_shape() { + let info = infer("SELECT a FROM src UNION ALL SELECT b FROM src"); + assert_eq!(info.columns.len(), 1); + assert_eq!(info.columns[0].name, "a"); + } + + #[test] + fn alias_column_list_renames_positionally() { + let info = rename_output_columns(infer("SELECT a, b FROM src"), &["p".into(), "q".into()]); + let names: Vec<&str> = info.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["p", "q"]); + assert_eq!(info.columns[0].data_type, SqlDataType::Int64); + } +} diff --git a/nodedb-sql/src/resolver/mod.rs b/nodedb-sql/src/resolver/mod.rs index 99dcd9180..10ca28677 100644 --- a/nodedb-sql/src/resolver/mod.rs +++ b/nodedb-sql/src/resolver/mod.rs @@ -1,4 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 +pub mod array_tvf; pub mod columns; +pub mod derived; pub mod expr; +pub mod scope; + +pub use scope::ColumnScope; diff --git a/nodedb-sql/src/resolver/scope.rs b/nodedb-sql/src/resolver/scope.rs new file mode 100644 index 000000000..378bd6bc5 --- /dev/null +++ b/nodedb-sql/src/resolver/scope.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The column namespace an expression converts against. + +use crate::error::Result; +use crate::resolver::columns::TableScope; + +/// The relations an identifier in this expression can name. +#[derive(Debug, Clone, Copy)] +pub enum ColumnScope<'a> { + /// No relation is in scope, so no identifier is checkable here. + /// + /// Used where an expression has no FROM clause behind it: stored DEFAULT + /// expressions, partial-index predicates, and the constant folders. A + /// column reference reaching one of those fails in constant folding as + /// `SqlError::Unsupported`, which is the correct answer there — the value + /// is not row-dependent. + Unchecked, + /// Identifiers resolve against these relations. One that resolves against + /// no relation, output alias, or synthetic column raises + /// `SqlError::UnknownColumn`. + Relations(&'a TableScope), +} + +impl ColumnScope<'_> { + /// Reject `column`, optionally qualified by `table_ref`, when it names + /// nothing in this scope. + pub fn check_column(&self, table_ref: Option<&str>, column: &str) -> Result<()> { + match self { + Self::Unchecked => Ok(()), + Self::Relations(scope) => scope.check_name(table_ref, column), + } + } +} diff --git a/nodedb-sql/src/types/collection.rs b/nodedb-sql/src/types/collection.rs index 27282ba17..69be19055 100644 --- a/nodedb-sql/src/types/collection.rs +++ b/nodedb-sql/src/types/collection.rs @@ -33,6 +33,21 @@ pub struct CollectionInfo { /// Authoritative per-collection partition metadata. Future routing layers /// read this instead of inferring distribution from engine type. pub partition_strategy: nodedb_types::PartitionStrategy, + /// Whether a name outside `columns` resolves in this relation. + /// + /// A stored collection derives it from its engine. A synthesized relation + /// sets it independently: a derived, LATERAL, or CTE alias is closed when + /// its projection list is inferable, open only when it is `*` over an open + /// source. `pg_catalog` relations are open because NodeDB models a subset + /// of the columns clients ask for. + pub open_schema: bool, +} + +impl CollectionInfo { + /// The openness a stored collection on `engine` carries. + pub fn open_schema_for(engine: EngineType) -> bool { + crate::engine_rules::resolve_engine_rules(engine).accepts_undeclared_columns() + } } /// Secondary index metadata surfaced to the SQL planner. diff --git a/nodedb-sql/src/types_expr.rs b/nodedb-sql/src/types_expr.rs index 5b1c31bf0..f4d7ecca7 100644 --- a/nodedb-sql/src/types_expr.rs +++ b/nodedb-sql/src/types_expr.rs @@ -150,4 +150,5 @@ pub enum SqlDataType { Uuid, Vector(usize), Geometry, + Unknown, } diff --git a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs index 63d3a5e7c..e9c7db299 100644 --- a/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs +++ b/nodedb/src/control/planner/catalog_adapter/sql_catalog_impl.rs @@ -166,6 +166,7 @@ impl SqlCatalog for OriginCatalog { primary: stored.primary, vector_primary: stored.vector_primary, partition_strategy: stored.partition_strategy, + open_schema: CollectionInfo::open_schema_for(engine), })) } diff --git a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs b/nodedb/src/control/planner/sql_plan_convert/output_schema.rs index f44c6fb13..4e5df4633 100644 --- a/nodedb/src/control/planner/sql_plan_convert/output_schema.rs +++ b/nodedb/src/control/planner/sql_plan_convert/output_schema.rs @@ -824,6 +824,9 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: nodedb_sql::types::CollectionInfo::open_schema_for( + EngineType::DocumentStrict, + ), })) } } diff --git a/nodedb/src/control/server/pgwire/catalog/schema.rs b/nodedb/src/control/server/pgwire/catalog/schema.rs index 082af8c29..50c9f8126 100644 --- a/nodedb/src/control/server/pgwire/catalog/schema.rs +++ b/nodedb/src/control/server/pgwire/catalog/schema.rs @@ -235,5 +235,7 @@ pub fn catalog_collection_info(name: &str) -> Option { vector_primary: None, // Catalog relations are synthetic, read-only, and never sharded. partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + // NodeDB models a subset of the columns clients ask for. + open_schema: true, }) } diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs index 3718eea81..524fbb171 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser.rs @@ -98,7 +98,8 @@ fn inferred_param_type(inferred: &nodedb_sql::InferredParamType) -> Option SqlDataType::Decimal | SqlDataType::Uuid | SqlDataType::Vector(_) - | SqlDataType::Geometry => None, + | SqlDataType::Geometry + | SqlDataType::Unknown => None, } } diff --git a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs index a6d0b7e95..f812c04de 100644 --- a/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs +++ b/nodedb/src/control/server/pgwire/handler/prepared/parser_schema.rs @@ -154,6 +154,7 @@ pub(super) fn result_fields_for_returning( SqlDataType::Uuid => Type::TEXT, SqlDataType::Vector(_) => Type::BYTEA, SqlDataType::Geometry => Type::BYTEA, + SqlDataType::Unknown => Type::TEXT, } } diff --git a/nodedb/src/control/server/response_shape/schema.rs b/nodedb/src/control/server/response_shape/schema.rs index d92b5f14b..63a5b6784 100644 --- a/nodedb/src/control/server/response_shape/schema.rs +++ b/nodedb/src/control/server/response_shape/schema.rs @@ -60,6 +60,8 @@ pub fn sql_data_type_to_ddl_col_type( SqlDataType::Vector(_) => DdlColType::Text, // No dedicated wire type yet; falls back to Text (no regression). SqlDataType::Geometry => DdlColType::Text, + // No declared type at all; the column resolves by name only. + SqlDataType::Unknown => DdlColType::Text, } } From 24f1143eaeac8e6c88ac825852fae47933bfa5fb Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:27 +0800 Subject: [PATCH 4/7] fix(sql): reject unknown and ambiguous columns at plan time An identifier naming no column of a closed-schema collection resolved to NULL on every engine. Projections returned NULL-filled rows, a WHERE predicate matched nothing, IS NULL matched everything, ORDER BY no-opped, and UPDATE or DELETE touched zero rows while reporting success. Thread ColumnScope through convert_expr and gate both identifier arms, so every clause routing through the converter inherits the check: projection, WHERE, GROUP BY, HAVING, ORDER BY, window clauses, aggregate arguments, joins, and subqueries. Sites that build a column reference without the converter call TableScope::check_name directly: UPDATE and MERGE assignment targets, MERGE ON, INSERT column lists, UPDATE FROM join pairs, CTE join links, join equi-keys, and the IN-subquery outer operand. The document_schemaless engine stays open. It accepts undeclared fields on write, so a read of one resolves to NULL rather than raising. Map the two planner errors to their PostgreSQL SQLSTATEs: UnknownColumn to 42703 and AmbiguousColumn to 42702. Both previously fell through to PlanError and surfaced as 42601. --- nodedb-sql/src/aggregate_walk.rs | 102 ++++- nodedb-sql/src/error.rs | 6 + nodedb-sql/src/lib.rs | 2 +- nodedb-sql/src/parser/normalize.rs | 2 +- nodedb-sql/src/planner/agg_bind.rs | 31 +- nodedb-sql/src/planner/aggregate.rs | 32 +- nodedb-sql/src/planner/aggregate_order.rs | 9 +- nodedb-sql/src/planner/ast_helpers.rs | 3 +- nodedb-sql/src/planner/cte/join_link.rs | 23 +- nodedb-sql/src/planner/cte/recursive_scan.rs | 5 +- nodedb-sql/src/planner/dml.rs | 86 ++-- .../src/planner/dml_helpers/value_convert.rs | 3 +- nodedb-sql/src/planner/dml_update_delete.rs | 59 ++- .../src/planner/geometry_expr/resolve.rs | 3 +- nodedb-sql/src/planner/group_by.rs | 16 +- nodedb-sql/src/planner/grouping_sets.rs | 33 +- nodedb-sql/src/planner/having.rs | 16 +- nodedb-sql/src/planner/join/constraint.rs | 45 +- nodedb-sql/src/planner/join/plan.rs | 36 +- nodedb-sql/src/planner/merge.rs | 114 ++++- nodedb-sql/src/planner/select/derived_from.rs | 43 +- nodedb-sql/src/planner/select/entry.rs | 64 ++- nodedb-sql/src/planner/select/helpers.rs | 16 +- .../src/planner/select/order_by/aliases.rs | 14 + .../src/planner/select/order_by/apply.rs | 43 +- nodedb-sql/src/planner/select/query_tail.rs | 5 +- nodedb-sql/src/planner/select/select_stmt.rs | 139 +++--- nodedb-sql/src/planner/select/where_search.rs | 16 +- nodedb-sql/src/planner/sort.rs | 10 +- nodedb-sql/src/planner/window/extract.rs | 35 +- nodedb-sql/src/resolver/expr/functions.rs | 74 ++-- .../sql_suite/cases/limit_offset_bounds.rs | 17 +- .../cases/on_conflict_update_range_check.rs | 1 + .../cases/point_get_operand_order.rs | 12 +- .../cases/positional_insert_column_binding.rs | 3 + .../cases/schema_qualified_rejection.rs | 2 + nodedb-types/src/error/code.rs | 4 + nodedb-types/src/error/code_table.rs | 2 + .../src/error/ctors/read_query_auth.rs | 26 ++ nodedb-types/src/error/details.rs | 6 + nodedb-types/src/error/msgpack/constants.rs | 4 + .../error/msgpack/decode/from_messagepack.rs | 8 + nodedb-types/src/error/msgpack/encode.rs | 6 + nodedb-types/src/error/sqlstate.rs | 6 + nodedb-types/src/error/types.rs | 2 + nodedb/src/bridge/envelope/error_code.rs | 5 + .../control/planner/context/query/planning.rs | 9 + .../control/server/native/sqlstate_code.rs | 2 + .../control/server/pgwire/types/error_map.rs | 17 + .../src/control/server/shared/ddl/result.rs | 2 + .../control/server/shared/ddl/sql_parse.rs | 5 +- nodedb/src/error/types.rs | 19 + nodedb/src/error_classify.rs | 3 + nodedb/src/error_from_data_plane.rs | 4 +- .../executor_tests/test_group_by_alias.rs | 12 +- nodedb/tests/wire/cases/mod.rs | 3 + .../tests/wire/cases/sql_undefined_column.rs | 391 +++++++++++++++++ .../wire/cases/sql_undefined_column_dml.rs | 399 ++++++++++++++++++ 58 files changed, 1696 insertions(+), 359 deletions(-) create mode 100644 nodedb/tests/wire/cases/sql_undefined_column.rs create mode 100644 nodedb/tests/wire/cases/sql_undefined_column_dml.rs diff --git a/nodedb-sql/src/aggregate_walk.rs b/nodedb-sql/src/aggregate_walk.rs index f26cc5c37..ba0cf198e 100644 --- a/nodedb-sql/src/aggregate_walk.rs +++ b/nodedb-sql/src/aggregate_walk.rs @@ -30,6 +30,8 @@ use sqlparser::ast::{self, Expr, Visit, Visitor}; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::parser::normalize::normalize_ident; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::{AggregateExpr, SqlExpr}; @@ -48,14 +50,19 @@ pub fn contains_aggregate(expr: &Expr, functions: &FunctionRegistry) -> bool { /// the given output `alias`. Nested aggregates (e.g. `SUM(AVG(x))`, /// which is illegal SQL in Postgres and most other systems) are /// reported as a planner error rather than silently double-extracted. +/// +/// `scope` carries the relations the arguments resolve against, so a +/// column no relation declares is rejected at plan time. pub fn extract_aggregates( expr: &Expr, alias: &str, functions: &FunctionRegistry, + scope: &TableScope, ) -> Result> { let mut extractor = AggregateExtractor { functions, alias, + scope, inside_aggregate: 0, out: Vec::new(), error: None, @@ -97,6 +104,7 @@ impl Visitor for AggregateDetector<'_> { struct AggregateExtractor<'a> { functions: &'a FunctionRegistry, alias: &'a str, + scope: &'a TableScope, /// Depth counter: >0 means we're currently inside the argument /// subtree of an already-extracted aggregate. A second aggregate /// found in that subtree is an illegal nested aggregate. @@ -132,7 +140,13 @@ impl Visitor for AggregateExtractor<'_> { }); return ControlFlow::Break(()); } - let (args, distinct) = function_args_and_distinct(f); + let (args, distinct) = match function_args_and_distinct(f, self.scope) { + Ok(parsed) => parsed, + Err(e) => { + self.error = Some(e); + return ControlFlow::Break(()); + } + }; self.out.push(AggregateExpr { function: function_name(f), args, @@ -193,30 +207,44 @@ fn function_name(f: &ast::Function) -> String { .join(".") } -fn function_args_and_distinct(f: &ast::Function) -> (Vec, bool) { +/// Convert an aggregate call's argument list against `scope`. +/// +/// An argument the converter rejects is an error, never a dropped argument: +/// discarding one silently changes the aggregate's arity and hides an unknown +/// column behind a query that reports success. +fn function_args_and_distinct( + f: &ast::Function, + scope: &TableScope, +) -> Result<(Vec, bool)> { let ast::FunctionArguments::List(args) = &f.args else { - return (Vec::new(), false); + return Ok((Vec::new(), false)); }; - let parsed = args - .args - .iter() - .filter_map(|a| match a { - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)) => convert_expr(e).ok(), - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard) => Some(SqlExpr::Wildcard), - _ => None, - }) - .collect(); + let mut parsed = Vec::with_capacity(args.args.len()); + for a in &args.args { + match a { + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)) => { + parsed.push(convert_expr(e, &ColumnScope::Relations(scope))?); + } + // `COUNT(*)` carries no column to resolve. + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard) => { + parsed.push(SqlExpr::Wildcard); + } + _ => {} + } + } let distinct = matches!( args.duplicate_treatment, Some(ast::DuplicateTreatment::Distinct) ); - (parsed, distinct) + Ok((parsed, distinct)) } #[cfg(test)] mod tests { use super::*; use crate::parser::statement::parse_sql; + use crate::resolver::columns::ResolvedTable; + use crate::types::{CollectionInfo, EngineType}; fn first_select_projection(sql: &str) -> Vec { let stmts = parse_sql(sql).unwrap(); @@ -240,6 +268,29 @@ mod tests { FunctionRegistry::new() } + /// A one-relation scope that accepts any column name. + fn open_scope() -> TableScope { + let info = CollectionInfo { + name: "t".into(), + engine: EngineType::DocumentSchemaless, + columns: Vec::new(), + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), + }; + TableScope::single(ResolvedTable { + name: info.name.clone(), + alias: None, + info, + }) + .expect("single-relation scope") + } + // ── detection ── #[test] @@ -318,8 +369,13 @@ mod tests { #[test] fn extract_plain_aggregate() { - let aggs = - extract_aggregates(&first_expr("SELECT SUM(x) FROM t"), "total", &functions()).unwrap(); + let aggs = extract_aggregates( + &first_expr("SELECT SUM(x) FROM t"), + "total", + &functions(), + &open_scope(), + ) + .unwrap(); assert_eq!(aggs.len(), 1); assert_eq!(aggs[0].function, "sum"); assert_eq!(aggs[0].alias, "total"); @@ -331,6 +387,7 @@ mod tests { &first_expr("SELECT CAST(SUM(x) AS TEXT) AS n FROM t"), "n", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 1); @@ -343,6 +400,7 @@ mod tests { &first_expr("SELECT CASE WHEN x > 0 THEN SUM(y) ELSE 0 END FROM t"), "r", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 1); @@ -355,6 +413,7 @@ mod tests { &first_expr("SELECT COALESCE(SUM(x), 0) FROM t"), "r", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 1); @@ -367,6 +426,7 @@ mod tests { &first_expr("SELECT SUM(x) + COUNT(y) AS total FROM t"), "total", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 2); @@ -377,8 +437,13 @@ mod tests { #[test] fn nested_aggregate_directly_inside_aggregate_rejected() { - let err = extract_aggregates(&first_expr("SELECT SUM(AVG(x)) FROM t"), "r", &functions()) - .unwrap_err(); + let err = extract_aggregates( + &first_expr("SELECT SUM(AVG(x)) FROM t"), + "r", + &functions(), + &open_scope(), + ) + .unwrap_err(); let msg = format!("{err:?}"); assert!( msg.to_lowercase().contains("nested aggregate"), @@ -396,6 +461,7 @@ mod tests { &first_expr("SELECT SUM(CAST(AVG(x) AS BIGINT)) FROM t"), "r", &functions(), + &open_scope(), ) .unwrap_err(); assert!( @@ -414,6 +480,7 @@ mod tests { &first_expr("SELECT CAST(SUM(x) AS TEXT) || CAST(COUNT(y) AS TEXT) FROM t"), "r", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 2); @@ -425,6 +492,7 @@ mod tests { &first_expr("SELECT COUNT(DISTINCT x) FROM t"), "c", &functions(), + &open_scope(), ) .unwrap(); assert_eq!(aggs.len(), 1); diff --git a/nodedb-sql/src/error.rs b/nodedb-sql/src/error.rs index 4fd97a9d4..b31a34c5b 100644 --- a/nodedb-sql/src/error.rs +++ b/nodedb-sql/src/error.rs @@ -28,6 +28,12 @@ pub enum SqlError { #[error("type mismatch: {detail}")] TypeMismatch { detail: String }, + /// A statement lists a different number of targets than expressions, such + /// as an `INSERT` whose target column list does not match its `SELECT` + /// list. + #[error("{detail}")] + Arity { detail: String }, + /// A constant expression divided by zero at plan time. Distinct from the /// folder declining to fold: the expression *was* constant and evaluating /// it failed, so the statement must raise rather than yield NULL. diff --git a/nodedb-sql/src/lib.rs b/nodedb-sql/src/lib.rs index 58fb2a50f..7952b0505 100644 --- a/nodedb-sql/src/lib.rs +++ b/nodedb-sql/src/lib.rs @@ -62,7 +62,7 @@ pub fn parse_expr_string(expr_text: &str) -> Result { detail: e.to_string(), })?; - resolver::expr::convert_expr(&ast_expr) + resolver::expr::convert_expr(&ast_expr, &resolver::ColumnScope::Unchecked) } use functions::registry::FunctionRegistry; diff --git a/nodedb-sql/src/parser/normalize.rs b/nodedb-sql/src/parser/normalize.rs index 5160dc80e..66f0f6087 100644 --- a/nodedb-sql/src/parser/normalize.rs +++ b/nodedb-sql/src/parser/normalize.rs @@ -351,7 +351,7 @@ mod tests { #[test] fn table_aliases_enforce_ast_identifier_rules() { - use crate::planner::lateral::plan::lateral_alias_from_factor; + use crate::planner::lateral::subquery::lateral_alias_from_factor; let quoted = parse_table_factor("SELECT * FROM users AS \"MiXeD 雪\""); assert_eq!( diff --git a/nodedb-sql/src/planner/agg_bind.rs b/nodedb-sql/src/planner/agg_bind.rs index 06e2c4d9e..c32b15808 100644 --- a/nodedb-sql/src/planner/agg_bind.rs +++ b/nodedb-sql/src/planner/agg_bind.rs @@ -23,6 +23,7 @@ use crate::error::{Result, SqlError}; use crate::functions::registry::{FunctionCategory, FunctionRegistry}; use crate::parser::normalize::normalize_ident; use crate::planner::agg_naming::aggregate_output_key; +use crate::resolver::columns::TableScope; use crate::types::AggregateExpr; /// Which name an aggregate call is rewritten to. @@ -36,15 +37,19 @@ pub enum BindName { /// Rewrite every aggregate call in `expr` to a reference to its computed /// column, registering aggregates the projection did not already request. +/// +/// `scope` carries the input relations each aggregate argument resolves +/// against. pub fn bind_aggregate_calls( expr: &ast::Expr, projection: &[ast::SelectItem], aggregates: &mut Vec, functions: &FunctionRegistry, name: BindName, + scope: &TableScope, ) -> Result { let resolved = resolve_select_aliases(expr, projection); - bind(&resolved, aggregates, functions, name) + bind(&resolved, aggregates, functions, name, scope) } /// Substitute SELECT-list output aliases referenced by the expression. @@ -90,29 +95,30 @@ fn bind( aggregates: &mut Vec, functions: &FunctionRegistry, name: BindName, + scope: &TableScope, ) -> Result { match expr { ast::Expr::Function(func) if is_aggregate_call(func, functions) => { - let column = register_aggregate(expr, aggregates, functions, name)?; + let column = register_aggregate(expr, aggregates, functions, name, scope)?; Ok(ast::Expr::Identifier(ast::Ident::new(column))) } ast::Expr::BinaryOp { left, op, right } => Ok(ast::Expr::BinaryOp { - left: Box::new(bind(left, aggregates, functions, name)?), + left: Box::new(bind(left, aggregates, functions, name, scope)?), op: op.clone(), - right: Box::new(bind(right, aggregates, functions, name)?), + right: Box::new(bind(right, aggregates, functions, name, scope)?), }), ast::Expr::UnaryOp { op, expr } => Ok(ast::Expr::UnaryOp { op: *op, - expr: Box::new(bind(expr, aggregates, functions, name)?), + expr: Box::new(bind(expr, aggregates, functions, name, scope)?), }), ast::Expr::Nested(inner) => Ok(ast::Expr::Nested(Box::new(bind( - inner, aggregates, functions, name, + inner, aggregates, functions, name, scope, )?))), ast::Expr::IsNull(inner) => Ok(ast::Expr::IsNull(Box::new(bind( - inner, aggregates, functions, name, + inner, aggregates, functions, name, scope, )?))), ast::Expr::IsNotNull(inner) => Ok(ast::Expr::IsNotNull(Box::new(bind( - inner, aggregates, functions, name, + inner, aggregates, functions, name, scope, )?))), ast::Expr::Between { expr, @@ -120,10 +126,10 @@ fn bind( low, high, } => Ok(ast::Expr::Between { - expr: Box::new(bind(expr, aggregates, functions, name)?), + expr: Box::new(bind(expr, aggregates, functions, name, scope)?), negated: *negated, - low: Box::new(bind(low, aggregates, functions, name)?), - high: Box::new(bind(high, aggregates, functions, name)?), + low: Box::new(bind(low, aggregates, functions, name, scope)?), + high: Box::new(bind(high, aggregates, functions, name, scope)?), }), other => Ok(other.clone()), } @@ -153,10 +159,11 @@ fn register_aggregate( aggregates: &mut Vec, functions: &FunctionRegistry, name: BindName, + scope: &TableScope, ) -> Result { // The alias is replaced below for a newly registered aggregate, so the // placeholder is never observable. - let mut extracted = extract_aggregates(expr, "", functions)?; + let mut extracted = extract_aggregates(expr, "", functions, scope)?; let Some(mut agg) = extracted.pop() else { return Err(SqlError::Unsupported { detail: format!("aggregate `{expr}` could not be extracted"), diff --git a/nodedb-sql/src/planner/aggregate.rs b/nodedb-sql/src/planner/aggregate.rs index 316ca7ef0..10a1d8b7c 100644 --- a/nodedb-sql/src/planner/aggregate.rs +++ b/nodedb-sql/src/planner/aggregate.rs @@ -10,7 +10,8 @@ use crate::functions::registry::{FunctionRegistry, SearchTrigger}; use crate::parser::normalize::normalize_ident; use crate::planner::group_by::{convert_group_by_with_projection, group_by_output_aliases}; use crate::planner::grouping_sets::expand_group_by; -use crate::resolver::columns::ResolvedTable; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; use crate::resolver::expr::convert_expr; use crate::temporal::TemporalScope; use crate::types::*; @@ -20,12 +21,12 @@ pub fn plan_aggregate( select: &ast::Select, table: &ResolvedTable, filters: &[Filter], - _scope: &crate::resolver::columns::TableScope, + scope: &TableScope, functions: &FunctionRegistry, temporal: &TemporalScope, ) -> Result { // Detect ROLLUP / CUBE / GROUPING SETS before falling through to plain convert. - let grouping_expansion = expand_group_by(&select.group_by)?; + let grouping_expansion = expand_group_by(&select.group_by, scope)?; let (group_by_exprs, grouping_sets) = if let Some(exp) = grouping_expansion { (exp.canonical_keys, Some(exp.grouping_sets)) @@ -35,18 +36,19 @@ pub fn plan_aggregate( &select.group_by, &select.projection, &table.info.columns, + scope, )?, None, ) }; - let mut aggregates = extract_aggregates_from_projection(&select.projection, functions)?; + let mut aggregates = extract_aggregates_from_projection(&select.projection, functions, scope)?; // HAVING is bound to the aggregates' computed output columns, and any // aggregate it alone introduces is added to `aggregates` so it is actually // computed. let having = match &select.having { Some(expr) => { - super::having::plan_having(expr, &select.projection, &mut aggregates, functions)? + super::having::plan_having(expr, &select.projection, &mut aggregates, functions, scope)? } None => Vec::new(), }; @@ -54,7 +56,7 @@ pub fn plan_aggregate( // When grouping sets are present, detect GROUPING(col) in the projection and // synthesize AggregateExpr entries so the executor can compute them per-set. if grouping_sets.is_some() { - let grouping_aggs = extract_grouping_calls(&select.projection, &group_by_exprs)?; + let grouping_aggs = extract_grouping_calls(&select.projection, &group_by_exprs, scope)?; aggregates.extend(grouping_aggs); } @@ -66,6 +68,7 @@ pub fn plan_aggregate( &select.projection, &group_by_exprs, functions, + scope, )?; // Extract timeseries-specific params (bucket interval, group columns) if applicable. @@ -310,6 +313,7 @@ fn parse_interval_to_ms(s: &str) -> i64 { fn extract_grouping_calls( items: &[ast::SelectItem], canonical_keys: &[SqlExpr], + scope: &TableScope, ) -> Result> { let mut out = Vec::new(); for item in items { @@ -318,7 +322,7 @@ fn extract_grouping_calls( ast::SelectItem::ExprWithAlias { expr, alias } => (expr, normalize_ident(alias)), _ => continue, }; - collect_grouping_from_expr(expr, &alias, canonical_keys, &mut out)?; + collect_grouping_from_expr(expr, &alias, canonical_keys, &mut out, scope)?; } Ok(out) } @@ -329,6 +333,7 @@ fn collect_grouping_from_expr( alias: &str, canonical_keys: &[SqlExpr], out: &mut Vec, + scope: &TableScope, ) -> Result<()> { match expr { ast::Expr::Function(f) => { @@ -344,7 +349,7 @@ fn collect_grouping_from_expr( // Encode index in the field name; alias is user-visible output name. out.push(AggregateExpr { function: "grouping".into(), - args: vec![convert_expr(col_expr)?], + args: vec![convert_expr(col_expr, &ColumnScope::Relations(scope))?], alias: alias.to_string(), distinct: false, grouping_col_index: Some(canonical_idx), @@ -354,8 +359,8 @@ fn collect_grouping_from_expr( } // Recurse into binary ops and other wrappers. ast::Expr::BinaryOp { left, right, .. } => { - collect_grouping_from_expr(left, alias, canonical_keys, out)?; - collect_grouping_from_expr(right, alias, canonical_keys, out)?; + collect_grouping_from_expr(left, alias, canonical_keys, out, scope)?; + collect_grouping_from_expr(right, alias, canonical_keys, out, scope)?; } _ => {} } @@ -390,9 +395,13 @@ pub(super) fn normalize_function_name(f: &ast::Function) -> String { } /// Extract aggregate expressions from SELECT projection. +/// +/// `scope` gates each aggregate argument, so a column no relation declares +/// raises an undefined-column error instead of planning. pub fn extract_aggregates_from_projection( items: &[ast::SelectItem], functions: &FunctionRegistry, + scope: &TableScope, ) -> Result> { let mut aggregates = Vec::new(); for item in items { @@ -409,7 +418,8 @@ pub fn extract_aggregates_from_projection( ast::SelectItem::ExprWithAlias { expr, alias } => (expr, normalize_ident(alias)), _ => continue, }; - let mut extracted = crate::aggregate_walk::extract_aggregates(expr, &alias, functions)?; + let mut extracted = + crate::aggregate_walk::extract_aggregates(expr, &alias, functions, scope)?; aggregates.append(&mut extracted); } Ok(aggregates) diff --git a/nodedb-sql/src/planner/aggregate_order.rs b/nodedb-sql/src/planner/aggregate_order.rs index b30febd0c..6b89fe2d2 100644 --- a/nodedb-sql/src/planner/aggregate_order.rs +++ b/nodedb-sql/src/planner/aggregate_order.rs @@ -13,6 +13,8 @@ use crate::planner::aggregate::{ extract_aggregates_from_projection, function_args_exprs, normalize_function_name, }; use crate::planner::group_by::{expr_column_name, key_column_name}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::query::AggOutputSlot; use crate::types_expr::SqlExpr; @@ -31,8 +33,9 @@ pub fn compute_output_order( projection: &[ast::SelectItem], group_by: &[SqlExpr], functions: &FunctionRegistry, + scope: &TableScope, ) -> Result> { - let real_agg_count = extract_aggregates_from_projection(projection, functions)?.len(); + let real_agg_count = extract_aggregates_from_projection(projection, functions, scope)?.len(); let mut order = Vec::new(); let mut agg_cursor = 0usize; let mut grouping_cursor = 0usize; @@ -59,7 +62,7 @@ pub fn compute_output_order( // rendering — both the projection expr and the GROUP BY key pass // through the same `convert_expr`, so equal expressions render // identically. - if let Ok(converted) = convert_expr(expr) { + if let Ok(converted) = convert_expr(expr, &ColumnScope::Relations(scope)) { let rendered = format!("{converted:?}"); if let Some(index) = group_by .iter() @@ -87,7 +90,7 @@ pub fn compute_output_order( _ => format!("{expr}").to_lowercase(), }; let produced = - crate::aggregate_walk::extract_aggregates(expr, &alias, functions)?.len(); + crate::aggregate_walk::extract_aggregates(expr, &alias, functions, scope)?.len(); for _ in 0..produced { order.push(AggOutputSlot::Aggregate(agg_cursor)); agg_cursor += 1; diff --git a/nodedb-sql/src/planner/ast_helpers.rs b/nodedb-sql/src/planner/ast_helpers.rs index 3c9bc05e7..d6723e1ee 100644 --- a/nodedb-sql/src/planner/ast_helpers.rs +++ b/nodedb-sql/src/planner/ast_helpers.rs @@ -191,6 +191,7 @@ pub fn strip_single_table_qualifiers( pub fn strip_and_convert_filters( conjuncts: Vec, qualifier: &str, + scope: &crate::resolver::columns::TableScope, ) -> Result> { if conjuncts.is_empty() { return Ok(Vec::new()); @@ -200,5 +201,5 @@ pub fn strip_and_convert_filters( .map(|c| strip_table_qualifier(&c, qualifier)) .collect(); let rebuilt = rebuild_and_expr(stripped); - convert_where_to_filters(&rebuilt) + convert_where_to_filters(&rebuilt, scope) } diff --git a/nodedb-sql/src/planner/cte/join_link.rs b/nodedb-sql/src/planner/cte/join_link.rs index adc292f77..b95d00136 100644 --- a/nodedb-sql/src/planner/cte/join_link.rs +++ b/nodedb-sql/src/planner/cte/join_link.rs @@ -8,6 +8,8 @@ use sqlparser::ast::{self, SetExpr}; use crate::error::{Result, SqlError}; use crate::parser::normalize::{normalize_ident, table_name_from_factor}; +use crate::planner::select::CteCatalog; +use crate::resolver::columns::TableScope; use crate::types::*; /// Extract recursive info from the AST when normal planning fails @@ -18,7 +20,11 @@ use crate::types::*; /// hash-join. type RecursiveInfo = (Vec, Option<(String, String)>); -pub(super) fn extract_recursive_info(expr: &SetExpr, cte_name: &str) -> Result { +pub(super) fn extract_recursive_info( + expr: &SetExpr, + cte_name: &str, + catalog: &dyn SqlCatalog, +) -> Result { let select = match expr { SetExpr::Select(s) => s, _ => { @@ -60,6 +66,11 @@ pub(super) fn extract_recursive_info(expr: &SetExpr, cte_name: &str) -> Result Result, + catalog: &dyn SqlCatalog, ) -> Result { let RecursiveParts { left, @@ -195,7 +197,8 @@ fn plan_recursive_scan_from_parts( // intentionally absent from the ordinary catalog. Parse that supported // shape directly instead of attempting ordinary planning and swallowing // whichever error happens to occur first. - let (recursive_filters, join_link) = super::join_link::extract_recursive_info(right, cte_name)?; + let (recursive_filters, join_link) = + super::join_link::extract_recursive_info(right, cte_name, catalog)?; // The anchor plan carries the CTE's resolved output columns; propagate // them so the recursive scan self-describes its output schema. diff --git a/nodedb-sql/src/planner/dml.rs b/nodedb-sql/src/planner/dml.rs index 279ec773e..5ee92a914 100644 --- a/nodedb-sql/src/planner/dml.rs +++ b/nodedb-sql/src/planner/dml.rs @@ -6,7 +6,7 @@ use nodedb_types::DatabaseId; use sqlparser::ast::{self}; use super::dml_helpers::{ - build_kv_insert_plan, build_vector_primary_insert_plan, + bind_insert_select_columns, build_kv_insert_plan, build_vector_primary_insert_plan, check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, coerce_and_check_rows, convert_value_rows, resolve_insert_columns, }; @@ -14,6 +14,8 @@ use crate::engine_rules::{self, InsertParams}; use crate::error::{Result, SqlError}; use crate::parser::normalize::{normalize_insert_column, normalize_object_name_checked}; use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; use crate::resolver::expr::convert_expr; use crate::types::*; @@ -22,6 +24,39 @@ pub use dml_update_delete::{plan_delete, plan_truncate_stmt, plan_update}; #[path = "dml_update_delete.rs"] mod dml_update_delete; +/// The column namespace of an INSERT target. +fn target_scope(table_name: &str, info: &CollectionInfo) -> Result { + let mut scope = TableScope::single(ResolvedTable { + name: table_name.to_string(), + alias: None, + info: info.clone(), + })?; + // `ON CONFLICT DO UPDATE` addresses the proposed row as `excluded`. It + // carries the target's columns and is qualified-only, so a bare name in + // the SET clause names the stored row. + scope.add_qualified_only(ResolvedTable { + name: EXCLUDED_RELATION.to_string(), + alias: None, + info: info.clone(), + })?; + Ok(scope) +} + +/// The pseudo-relation `ON CONFLICT DO UPDATE` uses for the proposed row. +const EXCLUDED_RELATION: &str = "excluded"; + +/// Normalize an INSERT column list and reject a name the target does not have. +fn insert_columns(columns: &[ast::ObjectName], scope: &TableScope) -> Result> { + columns + .iter() + .map(|c| { + let col = normalize_insert_column(c)?; + scope.check_name(None, &col)?; + Ok(col) + }) + .collect() +} + /// Classification of an `ON CONFLICT` clause attached to an INSERT. enum OnConflict { /// No `ON CONFLICT` clause — plain INSERT (error on duplicate PK). @@ -33,7 +68,7 @@ enum OnConflict { DoUpdate(Vec<(String, SqlExpr)>), } -fn classify_on_conflict(ins: &ast::Insert) -> Result { +fn classify_on_conflict(ins: &ast::Insert, scope: &TableScope) -> Result { let Some(on) = ins.on.as_ref() else { return Ok(OnConflict::None); }; @@ -53,7 +88,8 @@ fn classify_on_conflict(ins: &ast::Insert) -> Result { }); } }; - let expr = convert_expr(&a.value)?; + scope.check_name(None, &name)?; + let expr = convert_expr(&a.value, &ColumnScope::Relations(scope))?; pairs.push((name, expr)); } Ok(OnConflict::DoUpdate(pairs)) @@ -63,16 +99,6 @@ fn classify_on_conflict(ins: &ast::Insert) -> Result { /// Plan an INSERT statement. pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result> { - // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path - // with the assignments carried through. `DO NOTHING` stays on the - // INSERT path with `if_absent=true`. - let if_absent = match classify_on_conflict(ins)? { - OnConflict::None => false, - OnConflict::DoNothing => true, - OnConflict::DoUpdate(updates) => { - return plan_upsert_with_on_conflict(ins, catalog, updates); - } - }; let table_name = match &ins.table { ast::TableObject::TableName(name) => normalize_object_name_checked(name)?, ast::TableObject::TableFunction(_) => { @@ -93,17 +119,26 @@ pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result = ins - .columns - .iter() - .map(normalize_insert_column) - .collect::>()?; + // `INSERT ... ON CONFLICT DO UPDATE SET` reroutes to the upsert path + // with the assignments carried through. `DO NOTHING` stays on the + // INSERT path with `if_absent=true`. + let if_absent = match classify_on_conflict(ins, &target_scope)? { + OnConflict::None => false, + OnConflict::DoNothing => true, + OnConflict::DoUpdate(updates) => { + return plan_upsert_with_on_conflict(ins, catalog, updates); + } + }; + + let columns = insert_columns(&ins.columns, &target_scope)?; // Check for INSERT...SELECT. if let Some(source) = &ins.source - && let ast::SetExpr::Select(_select) = &*source.body + && let ast::SetExpr::Select(select) = &*source.body { + let column_map = bind_insert_select_columns(catalog, &columns, select, &info)?; let source_plan = super::select::plan_query( source, catalog, @@ -114,6 +149,7 @@ pub fn plan_insert(ins: &ast::Insert, catalog: &dyn SqlCatalog) -> Result Result = ins - .columns - .iter() - .map(normalize_insert_column) - .collect::>()?; + let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { detail: "UPSERT requires VALUES".into(), @@ -304,11 +336,7 @@ fn plan_upsert_with_on_conflict( name: table_name.clone(), })?; - let columns: Vec = ins - .columns - .iter() - .map(normalize_insert_column) - .collect::>()?; + let columns = insert_columns(&ins.columns, &target_scope(&table_name, &info)?)?; let source = ins.source.as_ref().ok_or_else(|| SqlError::Parse { detail: "INSERT ... ON CONFLICT requires VALUES".into(), diff --git a/nodedb-sql/src/planner/dml_helpers/value_convert.rs b/nodedb-sql/src/planner/dml_helpers/value_convert.rs index 47deac3f5..9c55b9514 100644 --- a/nodedb-sql/src/planner/dml_helpers/value_convert.rs +++ b/nodedb-sql/src/planner/dml_helpers/value_convert.rs @@ -63,7 +63,8 @@ pub(crate) fn expr_to_sql_value(expr: &ast::Expr) -> Result { } fn fold_constant_value(expr: &ast::Expr) -> Result { - let sql_expr = crate::resolver::expr::convert_expr(expr)?; + let sql_expr = + crate::resolver::expr::convert_expr(expr, &crate::resolver::ColumnScope::Unchecked)?; crate::planner::const_fold::fold_constant_default(&sql_expr)?.ok_or_else(|| { SqlError::Unsupported { detail: format!("value expression: {expr}"), diff --git a/nodedb-sql/src/planner/dml_update_delete.rs b/nodedb-sql/src/planner/dml_update_delete.rs index b67f062e7..5973a88b8 100644 --- a/nodedb-sql/src/planner/dml_update_delete.rs +++ b/nodedb-sql/src/planner/dml_update_delete.rs @@ -18,6 +18,8 @@ use crate::parser::normalize::{ SCHEMA_QUALIFIED_MSG, normalize_ident, normalize_object_name_checked, }; use crate::planner::declared_type_coerce::coerce_assignments_to_declared_types; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; use crate::resolver::expr::convert_expr; use crate::types::*; @@ -50,7 +52,13 @@ pub fn plan_update(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Result super::super::select::convert_where_to_filters(expr)?, + Some(expr) => super::super::select::convert_where_to_filters(expr, &target_scope)?, None => Vec::new(), }; @@ -159,7 +167,26 @@ fn plan_update_from(update: &ast::Update, catalog: &dyn SqlCatalog) -> Result Result extract_join_predicate(expr, target_ref, source_ref)?, + Some(expr) => extract_join_predicate(expr, target_ref, source_ref, &join_scope)?, }; // Plan the source as a simple scan (no filters — all filtering is via join key). @@ -213,6 +240,7 @@ fn extract_join_predicate( expr: &ast::Expr, target_ref: &str, source_ref: &str, + scope: &TableScope, ) -> Result<(String, String, Vec)> { // Flatten the top-level AND chain. let mut conjuncts: Vec = Vec::new(); @@ -225,6 +253,8 @@ fn extract_join_predicate( for (i, conjunct) in conjuncts.iter().enumerate() { if let Some((tc, sc)) = try_equijoin_pair(conjunct, target_ref, source_ref) { + scope.check_name(Some(target_ref), &tc)?; + scope.check_name(Some(source_ref), &sc)?; target_col = tc; source_col = sc; join_idx = Some(i); @@ -243,7 +273,7 @@ fn extract_join_predicate( // Remaining conjuncts become target_filters. Strip table qualifier so // `uf_target.score` becomes `score` — documents store bare field names. - let target_filters = strip_and_convert_filters(conjuncts, target_ref)?; + let target_filters = strip_and_convert_filters(conjuncts, target_ref, scope)?; Ok((target_col, source_col, target_filters)) } @@ -298,7 +328,11 @@ fn try_equijoin_pair( } /// Convert `update.assignments` into `Vec<(col, SqlExpr)>`. -fn convert_assignments(assignments: &[ast::Assignment]) -> Result> { +fn convert_assignments( + assignments: &[ast::Assignment], + target_scope: &TableScope, + value_scope: &TableScope, +) -> Result> { assignments .iter() .map(|a| { @@ -319,7 +353,10 @@ fn convert_assignments(assignments: &[ast::Assignment]) -> Result>>()? .join(","), }; - let val = convert_expr(&a.value)?; + for name in col.split(',') { + target_scope.check_name(None, name)?; + } + let val = convert_expr(&a.value, &ColumnScope::Relations(value_scope))?; Ok((col, val)) }) .collect() @@ -358,8 +395,14 @@ pub fn plan_delete(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result super::super::select::convert_where_to_filters(expr)?, + Some(expr) => super::super::select::convert_where_to_filters(expr, &target_scope)?, None => Vec::new(), }; diff --git a/nodedb-sql/src/planner/geometry_expr/resolve.rs b/nodedb-sql/src/planner/geometry_expr/resolve.rs index 0b993f8cd..fc77861f4 100644 --- a/nodedb-sql/src/planner/geometry_expr/resolve.rs +++ b/nodedb-sql/src/planner/geometry_expr/resolve.rs @@ -63,7 +63,8 @@ pub(crate) fn resolve_geometry_expr(expr: &ast::Expr) -> Result { /// `Ok(None)` means the expression folded but is not a geometry, or could not /// be folded at plan time at all. fn resolve(expr: &ast::Expr) -> Result> { - let sql_expr = crate::resolver::expr::convert_expr(expr)?; + let sql_expr = + crate::resolver::expr::convert_expr(expr, &crate::resolver::ColumnScope::Unchecked)?; let Some(value) = crate::planner::const_fold::fold_constant_default(&sql_expr)? else { return Ok(None); }; diff --git a/nodedb-sql/src/planner/group_by.rs b/nodedb-sql/src/planner/group_by.rs index 3a9f94336..db75af66b 100644 --- a/nodedb-sql/src/planner/group_by.rs +++ b/nodedb-sql/src/planner/group_by.rs @@ -10,12 +10,14 @@ use sqlparser::ast::{self, GroupByExpr}; use crate::error::Result; use crate::parser::normalize::normalize_ident; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::{ColumnInfo, SqlExpr}; /// Convert GROUP BY clause to SqlExpr list. -pub fn convert_group_by(group_by: &GroupByExpr) -> Result> { - convert_group_by_with_projection(group_by, &[], &[]) +pub fn convert_group_by(group_by: &GroupByExpr, scope: &TableScope) -> Result> { + convert_group_by_with_projection(group_by, &[], &[], scope) } /// Convert a GROUP BY clause, resolving SELECT-list output aliases. @@ -36,15 +38,21 @@ pub fn convert_group_by_with_projection( group_by: &GroupByExpr, projection: &[ast::SelectItem], table_columns: &[ColumnInfo], + scope: &TableScope, ) -> Result> { + // A key written as an output alias is substituted for its SELECT-list + // expression above, so only the remaining names need the alias widening. + let key_scope = + scope.with_output_names(crate::planner::select::select_output_aliases(projection)); + let key_scope = ColumnScope::Relations(&key_scope); match group_by { GroupByExpr::All(_) => Ok(Vec::new()), GroupByExpr::Expressions(exprs, _) => exprs .iter() .map( |e| match resolve_output_alias(e, projection, table_columns) { - Some(aliased) => convert_expr(aliased), - None => convert_expr(e), + Some(aliased) => convert_expr(aliased, &key_scope), + None => convert_expr(e, &key_scope), }, ) .collect(), diff --git a/nodedb-sql/src/planner/grouping_sets.rs b/nodedb-sql/src/planner/grouping_sets.rs index dfb05637b..fb9b744d3 100644 --- a/nodedb-sql/src/planner/grouping_sets.rs +++ b/nodedb-sql/src/planner/grouping_sets.rs @@ -16,6 +16,8 @@ use sqlparser::ast::{self, GroupByExpr}; use crate::error::{Result, SqlError}; use crate::parser::normalize::normalize_ident; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::SqlExpr; @@ -33,7 +35,11 @@ pub struct GroupingSetsExpansion { /// /// Returns `None` when the GROUP BY is a plain expression list with no /// extensions — callers fall back to the existing single-set path. -pub fn expand_group_by(group_by: &GroupByExpr) -> Result> { +pub fn expand_group_by( + group_by: &GroupByExpr, + scope: &TableScope, +) -> Result> { + let scope = ColumnScope::Relations(scope); let exprs = match group_by { GroupByExpr::All(_) => return Ok(None), GroupByExpr::Expressions(exprs, _) => exprs, @@ -101,7 +107,7 @@ pub fn expand_group_by(group_by: &GroupByExpr) -> Result #[cfg(test)] mod tests { use super::*; + use crate::resolver::columns::test_support::open_scope; fn parse_group_by(sql: &str) -> GroupByExpr { use sqlparser::dialect::GenericDialect; @@ -233,7 +240,9 @@ mod tests { let gb = parse_group_by( "SELECT region, country, SUM(sales) FROM orders GROUP BY ROLLUP (region, country)", ); - let result = expand_group_by(&gb).unwrap().unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")) + .unwrap() + .unwrap(); // ROLLUP(region, country) → [[0,1], [0], []] assert_eq!(result.canonical_keys.len(), 2); assert_eq!(result.grouping_sets.len(), 3); @@ -247,7 +256,9 @@ mod tests { let gb = parse_group_by( "SELECT region, country, SUM(sales) FROM orders GROUP BY CUBE (region, country)", ); - let result = expand_group_by(&gb).unwrap().unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")) + .unwrap() + .unwrap(); // CUBE(region, country) → [[0,1], [0], [1], []] assert_eq!(result.canonical_keys.len(), 2); assert_eq!(result.grouping_sets.len(), 4); @@ -264,7 +275,9 @@ mod tests { "SELECT region, country, SUM(sales) FROM orders \ GROUP BY GROUPING SETS ((region, country), (region), ())", ); - let result = expand_group_by(&gb).unwrap().unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")) + .unwrap() + .unwrap(); assert_eq!(result.canonical_keys.len(), 2); assert_eq!(result.grouping_sets.len(), 3); assert_eq!(result.grouping_sets[0], vec![0, 1]); @@ -275,14 +288,16 @@ mod tests { #[test] fn plain_group_by_returns_none() { let gb = parse_group_by("SELECT region, COUNT(*) FROM orders GROUP BY region"); - let result = expand_group_by(&gb).unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")).unwrap(); assert!(result.is_none()); } #[test] fn mixed_plain_and_rollup() { let gb = parse_group_by("SELECT a, b, c, SUM(x) FROM t GROUP BY a, ROLLUP (b, c)"); - let result = expand_group_by(&gb).unwrap().unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")) + .unwrap() + .unwrap(); // Canonical: a(0), b(1), c(2). // Extension sets (from ROLLUP(b,c)): [[b,c], [b], []]. // Cross-product with plain [a]: @@ -299,7 +314,9 @@ mod tests { #[test] fn rollup_three_cols() { let gb = parse_group_by("SELECT a, b, c, SUM(x) FROM t GROUP BY ROLLUP (a, b, c)"); - let result = expand_group_by(&gb).unwrap().unwrap(); + let result = expand_group_by(&gb, &open_scope("orders")) + .unwrap() + .unwrap(); assert_eq!(result.grouping_sets.len(), 4); // (a,b,c),(a,b),(a),() } } diff --git a/nodedb-sql/src/planner/having.rs b/nodedb-sql/src/planner/having.rs index 3b32a3705..b9ed59be2 100644 --- a/nodedb-sql/src/planner/having.rs +++ b/nodedb-sql/src/planner/having.rs @@ -22,6 +22,9 @@ use crate::aggregate_walk::contains_aggregate; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::planner::agg_bind::{BindName, bind_aggregate_calls}; +use crate::planner::agg_naming::aggregate_output_key; +use crate::planner::select::select_output_aliases; +use crate::resolver::columns::TableScope; use crate::types::{AggregateExpr, Filter}; /// Convert a HAVING clause into filters over finalized group rows. @@ -33,6 +36,7 @@ pub fn plan_having( projection: &[ast::SelectItem], aggregates: &mut Vec, functions: &FunctionRegistry, + scope: &TableScope, ) -> Result> { let rewritten = bind_aggregate_calls( having, @@ -40,6 +44,7 @@ pub fn plan_having( aggregates, functions, BindName::Canonical, + scope, )?; // Any aggregate call still standing is one this rewrite did not reach. @@ -54,5 +59,14 @@ pub fn plan_having( }); } - crate::planner::select::convert_where_to_filters(&rewritten) + // The rewritten predicate addresses computed group columns: an + // aggregate's canonical key, its output alias, or a SELECT-list alias. + let having_scope = scope.with_output_names( + aggregates + .iter() + .map(aggregate_output_key) + .chain(aggregates.iter().map(|a| a.alias.clone())) + .chain(select_output_aliases(projection)), + ); + crate::planner::select::convert_where_to_filters(&rewritten, &having_scope) } diff --git a/nodedb-sql/src/planner/join/constraint.rs b/nodedb-sql/src/planner/join/constraint.rs index e2861574f..7b7c9a999 100644 --- a/nodedb-sql/src/planner/join/constraint.rs +++ b/nodedb-sql/src/planner/join/constraint.rs @@ -6,6 +6,8 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::parser::normalize::normalize_ident; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::*; @@ -13,34 +15,34 @@ use crate::types::*; pub(super) type JoinSpec = (JoinType, Vec<(String, String)>, Option); /// Extract join type, equi-join keys, and non-equi condition. -pub(super) fn extract_join_spec(op: &ast::JoinOperator) -> Result { +pub(super) fn extract_join_spec(op: &ast::JoinOperator, scope: &TableScope) -> Result { match op { ast::JoinOperator::Inner(constraint) | ast::JoinOperator::Join(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Inner, keys, cond)) } ast::JoinOperator::Left(constraint) | ast::JoinOperator::LeftOuter(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Left, keys, cond)) } ast::JoinOperator::Right(constraint) | ast::JoinOperator::RightOuter(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Right, keys, cond)) } ast::JoinOperator::FullOuter(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Full, keys, cond)) } ast::JoinOperator::CrossJoin(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Cross, keys, cond)) } ast::JoinOperator::Semi(constraint) | ast::JoinOperator::LeftSemi(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Semi, keys, cond)) } ast::JoinOperator::Anti(constraint) | ast::JoinOperator::LeftAnti(constraint) => { - let (keys, cond) = extract_join_constraint(constraint)?; + let (keys, cond) = extract_join_constraint(constraint, scope)?; Ok((JoinType::Anti, keys, cond)) } _ => Err(SqlError::Unsupported { @@ -52,21 +54,24 @@ pub(super) fn extract_join_spec(op: &ast::JoinOperator) -> Result { /// (equi_keys, non-equi condition) type JoinConstraintResult = (Vec<(String, String)>, Option); -fn extract_join_constraint(constraint: &ast::JoinConstraint) -> Result { +fn extract_join_constraint( + constraint: &ast::JoinConstraint, + scope: &TableScope, +) -> Result { match constraint { ast::JoinConstraint::On(expr) => { let mut keys = Vec::new(); let mut non_equi = Vec::new(); - extract_equi_keys(expr, &mut keys, &mut non_equi)?; + extract_equi_keys(expr, &mut keys, &mut non_equi, scope)?; let cond = if non_equi.is_empty() { None } else { - let mut combined = convert_expr(&non_equi[0])?; + let mut combined = convert_expr(&non_equi[0], &ColumnScope::Relations(scope))?; for pred in &non_equi[1..] { combined = SqlExpr::BinaryOp { left: Box::new(combined), op: crate::types::BinaryOp::And, - right: Box::new(convert_expr(pred)?), + right: Box::new(convert_expr(pred, &ColumnScope::Relations(scope))?), }; } Some(combined) @@ -78,6 +83,7 @@ fn extract_join_constraint(constraint: &ast::JoinConstraint) -> Result>>()?; @@ -96,6 +102,7 @@ fn extract_equi_keys( expr: &ast::Expr, keys: &mut Vec<(String, String)>, non_equi: &mut Vec, + scope: &TableScope, ) -> Result<()> { match expr { ast::Expr::BinaryOp { @@ -103,8 +110,8 @@ fn extract_equi_keys( op: ast::BinaryOperator::And, right, } => { - extract_equi_keys(left, keys, non_equi)?; - extract_equi_keys(right, keys, non_equi)?; + extract_equi_keys(left, keys, non_equi, scope)?; + extract_equi_keys(right, keys, non_equi, scope)?; } ast::Expr::BinaryOp { left, @@ -112,6 +119,8 @@ fn extract_equi_keys( right, } => { if let (Some(l), Some(r)) = (extract_col_ref(left), extract_col_ref(right)) { + check_join_key(scope, &l)?; + check_join_key(scope, &r)?; keys.push((l, r)); } else { non_equi.push(expr.clone()); @@ -154,6 +163,14 @@ pub(super) fn orient_keys_to_sides(keys: &mut [(String, String)], right_ids: &[S } } +/// Reject a join key, qualified or bare, that names nothing in `scope`. +fn check_join_key(scope: &TableScope, key: &str) -> Result<()> { + match key.rsplit_once('.') { + Some((table, column)) => scope.check_name(Some(table), column), + None => scope.check_name(None, key), + } +} + fn extract_col_ref(expr: &ast::Expr) -> Option { match expr { ast::Expr::Identifier(ident) => Some(normalize_ident(ident)), diff --git a/nodedb-sql/src/planner/join/plan.rs b/nodedb-sql/src/planner/join/plan.rs index 0d5e029f0..4d629f373 100644 --- a/nodedb-sql/src/planner/join/plan.rs +++ b/nodedb-sql/src/planner/join/plan.rs @@ -9,9 +9,9 @@ use super::array_arm; use super::constraint::extract_join_spec; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; -use crate::planner::lateral::plan::{ - LateralJoinArgs, is_lateral_derived, lateral_alias_from_factor, plan_lateral_join, - subquery_from_factor, +use crate::planner::lateral::plan::{LateralJoinArgs, plan_lateral_join}; +use crate::planner::lateral::subquery::{ + is_lateral_derived, lateral_alias_from_factor, subquery_from_factor, }; use crate::resolver::columns::TableScope; use crate::types::*; @@ -50,7 +50,7 @@ pub fn plan_join_from_select( let subquery = subquery_from_factor(&join_item.relation) .expect("is_lateral_derived guarantees Derived variant"); let left_join = is_left_join_operator(&join_item.join_operator); - let projection = super::super::select::convert_projection(&select.projection)?; + let projection = super::super::select::convert_projection(&select.projection, scope)?; return Ok(Some(plan_lateral_join(LateralJoinArgs { outer_plan: current_plan, outer_alias, @@ -58,6 +58,7 @@ pub fn plan_join_from_select( lateral_alias: &lateral_alias, left_join, outer_projection: projection, + outer_scope: scope, catalog, temporal, })?)); @@ -72,7 +73,8 @@ pub fn plan_join_from_select( scan_for_relation(&join_item.relation, scope)? }; - let (join_type, mut on_keys, condition) = extract_join_spec(&join_item.join_operator)?; + let (join_type, mut on_keys, condition) = + extract_join_spec(&join_item.join_operator, scope)?; // Orient equi-keys to FROM order: `on.0` must reference the left input // and `on.1` the right input. The ON clause may write the operands in @@ -96,15 +98,15 @@ pub fn plan_join_from_select( let (subquery_joins, effective_where) = if let Some(expr) = &select.selection { let extraction = - super::super::subquery::extract_subqueries(expr, catalog, functions, temporal)?; + super::super::subquery::extract_subqueries(expr, scope, catalog, functions, temporal)?; (extraction.joins, extraction.remaining_where) } else { (Vec::new(), None) }; - let projection = super::super::select::convert_projection(&select.projection)?; + let projection = super::super::select::convert_projection(&select.projection, scope)?; let filters = match &effective_where { - Some(expr) => super::super::select::convert_where_to_filters(expr)?, + Some(expr) => super::super::select::convert_where_to_filters(expr, scope)?, None => Vec::new(), }; @@ -112,7 +114,7 @@ pub fn plan_join_from_select( current_plan = SqlPlan::Join { left: Box::new(current_plan), right: Box::new(sq.inner_plan), - on: vec![(sq.outer_column, sq.inner_column)], + on: sq.on, join_type: sq.join_type, condition: None, limit: None, @@ -125,21 +127,31 @@ pub fn plan_join_from_select( ast::GroupByExpr::All(_) => true, ast::GroupByExpr::Expressions(exprs, _) => !exprs.is_empty(), }; - if super::super::select::convert_projection(&select.projection).is_ok() && group_by_non_empty { + if group_by_non_empty { let aggregates = super::super::aggregate::extract_aggregates_from_projection( &select.projection, functions, + scope, )?; - let group_by = super::super::group_by::convert_group_by(&select.group_by)?; + let group_by = super::super::group_by::convert_group_by(&select.group_by, scope)?; let group_by_aliases = super::super::group_by::group_by_output_aliases(&select.projection, &group_by); let output_order = super::super::aggregate_order::compute_output_order( &select.projection, &group_by, functions, + scope, )?; let having = match &select.having { - Some(expr) => super::super::select::convert_where_to_filters(expr)?, + Some(expr) => { + // A HAVING term addresses a computed group column, so the + // aggregate output names join the input columns here. + let having_scope = + scope.with_output_names(aggregates.iter().map(|a| a.alias.clone()).chain( + super::super::select::select_output_aliases(&select.projection), + )); + super::super::select::convert_where_to_filters(expr, &having_scope)? + } None => Vec::new(), }; return Ok(Some(SqlPlan::Aggregate { diff --git a/nodedb-sql/src/planner/merge.rs b/nodedb-sql/src/planner/merge.rs index 1f2922646..24087614f 100644 --- a/nodedb-sql/src/planner/merge.rs +++ b/nodedb-sql/src/planner/merge.rs @@ -13,6 +13,8 @@ use super::ast_helpers::{qualified_ident_pair, strip_and_convert_filters}; use crate::engine_rules::{self, MergeParams, ScanParams}; use crate::error::{Result, SqlError}; use crate::parser::normalize::{normalize_ident, normalize_object_name_checked}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; use crate::resolver::expr::convert_expr; use crate::temporal::TemporalScope; use crate::types::*; @@ -46,12 +48,35 @@ pub fn plan_merge(stmt: &ast::Statement, catalog: &dyn SqlCatalog) -> Result Res } } +/// The source relation as it appears in the MERGE column namespace. +/// +/// A named table resolves through the catalog. A derived subquery or VALUES +/// constructor has no declared schema, so it exposes whatever it projects. +fn merge_source_relation( + factor: &ast::TableFactor, + source_alias: &str, + catalog: &dyn SqlCatalog, +) -> Result { + if let ast::TableFactor::Table { name, .. } = factor { + let source_name = normalize_object_name_checked(name)?; + let info = catalog + .get_collection(DatabaseId::DEFAULT, &source_name)? + .ok_or_else(|| SqlError::UnknownTable { + name: source_name.clone(), + })?; + return Ok(ResolvedTable { + name: source_name, + alias: Some(source_alias.to_string()), + info, + }); + } + Ok(ResolvedTable { + name: source_alias.to_string(), + alias: None, + info: CollectionInfo { + name: source_alias.to_string(), + engine: EngineType::DocumentSchemaless, + columns: Vec::new(), + primary_key: None, + has_auto_tier: false, + indexes: Vec::new(), + bitemporal: false, + primary: nodedb_types::PrimaryEngine::Document, + vector_primary: None, + partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + // The alias exposes whatever the subquery projects. + open_schema: true, + }, + }) +} + /// Determine the alias used to qualify source-column references in WHEN arms. fn merge_source_alias(factor: &ast::TableFactor, source_plan: &SqlPlan) -> Result { match factor { @@ -175,6 +242,7 @@ fn extract_merge_equijoin( on: &ast::Expr, target_ref: &str, source_ref: &str, + scope: &TableScope, ) -> Result<(String, String)> { if let ast::Expr::BinaryOp { left, @@ -187,9 +255,13 @@ fn extract_merge_equijoin( match (lhs, rhs) { (Some((lt, lc)), Some((rt, rc))) => { if lt == target_ref && rt == source_ref { + scope.check_name(Some(<), &lc)?; + scope.check_name(Some(&rt), &rc)?; return Ok((lc, rc)); } if lt == source_ref && rt == target_ref { + scope.check_name(Some(<), &lc)?; + scope.check_name(Some(&rt), &rc)?; return Ok((rc, lc)); } } @@ -197,12 +269,18 @@ fn extract_merge_equijoin( // pattern when one side is unqualified. (Some((t, c)), None) if t == source_ref => { if let ast::Expr::Identifier(ident) = right.as_ref() { - return Ok((normalize_ident(ident), c)); + let target_col = normalize_ident(ident); + scope.check_name(Some(&t), &c)?; + scope.check_name(Some(target_ref), &target_col)?; + return Ok((target_col, c)); } } (None, Some((t, c))) if t == source_ref => { if let ast::Expr::Identifier(ident) = left.as_ref() { - return Ok((normalize_ident(ident), c)); + let target_col = normalize_ident(ident); + scope.check_name(Some(&t), &c)?; + scope.check_name(Some(target_ref), &target_col)?; + return Ok((target_col, c)); } } _ => {} @@ -223,10 +301,12 @@ fn convert_merge_clauses( clauses: &[ast::MergeClause], target_ref: &str, source_ref: &str, + scope: &TableScope, + target_scope: &TableScope, ) -> Result> { clauses .iter() - .map(|c| convert_one_clause(c, target_ref, source_ref)) + .map(|c| convert_one_clause(c, target_ref, source_ref, scope, target_scope)) .collect() } @@ -234,6 +314,8 @@ fn convert_one_clause( clause: &ast::MergeClause, target_ref: &str, source_ref: &str, + scope: &TableScope, + target_scope: &TableScope, ) -> Result { let kind = match clause.clause_kind { AstMergeClauseKind::Matched => MergeClauseKind::Matched, @@ -244,11 +326,11 @@ fn convert_one_clause( }; let extra_predicate = match &clause.predicate { - Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref)?, + Some(expr) => strip_and_convert_filters(vec![expr.clone()], target_ref, scope)?, None => Vec::new(), }; - let action = convert_merge_action(&clause.action, source_ref)?; + let action = convert_merge_action(&clause.action, source_ref, scope, target_scope)?; Ok(MergePlanClause { kind, @@ -257,7 +339,12 @@ fn convert_one_clause( }) } -fn convert_merge_action(action: &MergeAction, source_ref: &str) -> Result { +fn convert_merge_action( + action: &MergeAction, + source_ref: &str, + scope: &TableScope, + target_scope: &TableScope, +) -> Result { match action { MergeAction::Update(update_expr) => { let assignments = update_expr @@ -273,7 +360,8 @@ fn convert_merge_action(action: &MergeAction, source_ref: &str) -> Result>>()?; @@ -284,7 +372,11 @@ fn convert_merge_action(action: &MergeAction, source_ref: &str) -> Result = insert_expr .columns .iter() - .map(normalize_object_name_checked) + .map(|c| { + let col = normalize_object_name_checked(c)?; + target_scope.check_name(None, &col)?; + Ok(col) + }) .collect::>>()?; let values: Vec = match &insert_expr.kind { @@ -299,7 +391,7 @@ fn convert_merge_action(action: &MergeAction, source_ref: &str) -> Result>>()? } MergeInsertKind::Row => { diff --git a/nodedb-sql/src/planner/select/derived_from.rs b/nodedb-sql/src/planner/select/derived_from.rs index 1cf450274..a75398041 100644 --- a/nodedb-sql/src/planner/select/derived_from.rs +++ b/nodedb-sql/src/planner/select/derived_from.rs @@ -4,9 +4,10 @@ use sqlparser::ast::{self, Select}; -use super::entry::{CteCatalog, plan_query}; +use super::cte_catalog::CteCatalog; +use super::entry::plan_query; use super::query_tail::QueryTail; -use super::select_stmt::plan_select; +use super::select_stmt::{PlannedSelect, plan_select}; use crate::error::Result; use crate::functions::registry::FunctionRegistry; use crate::temporal::TemporalScope; @@ -16,9 +17,9 @@ use crate::types::*; /// /// Recognises the single-source, non-LATERAL derived-table pattern. The /// inner subquery is planned with the original catalog; the outer -/// SELECT is replanned with a `CteCatalog` that resolves the alias to -/// a schemaless source. The result is wrapped as `SqlPlan::Cte` so the -/// `convert_cte` lowering takes care of execution. +/// SELECT is replanned with a `CteCatalog` that resolves the alias to the +/// relation the subquery projects. The result is wrapped as `SqlPlan::Cte` +/// so the `convert_cte` lowering takes care of execution. /// /// Returns `Ok(None)` when the FROM clause is not a single derived /// table, so the caller falls through to the regular planning path. @@ -28,7 +29,7 @@ pub(in crate::planner::select) fn try_plan_derived_from( functions: &FunctionRegistry, temporal: TemporalScope, tail: &QueryTail<'_>, -) -> Result> { +) -> Result> { if select.from.len() != 1 { return Ok(None); } @@ -47,15 +48,24 @@ pub(in crate::planner::select) fn try_plan_derived_from( }; let alias_name = crate::reserved::check_ast_identifier(&alias_ident.name)?; + let declared: Vec = alias_ident + .columns + .iter() + .map(|column| crate::reserved::check_ast_identifier(&column.name)) + .collect::>()?; let inner_plan = plan_query(subquery, catalog, functions, temporal)?; - // Replan the outer SELECT against a catalog that resolves the alias - // as a schemaless source. The outer can reference `alias.col` - // qualified or unqualified — the resolver treats CTE rows as a - // schemaless document so any projected column flows through. + // Replan the outer SELECT against a catalog that resolves the alias to + // the columns the subquery projects. The outer can reference `alias.col` + // qualified or unqualified. + let relation = + crate::resolver::derived::infer_subquery_relation(catalog, &alias_name, subquery)?; let derived_catalog = CteCatalog { inner: catalog, - cte_names: vec![alias_name.clone()], + relations: vec![( + alias_name.clone(), + crate::resolver::derived::rename_output_columns(relation, &declared), + )], }; let mut outer_select = select.clone(); outer_select.from[0].relation = ast::TableFactor::Table { @@ -72,10 +82,13 @@ pub(in crate::planner::select) fn try_plan_derived_from( sample: None, index_hints: Vec::new(), }; - let outer_plan = plan_select(&outer_select, &derived_catalog, functions, temporal, tail)?; + let outer = plan_select(&outer_select, &derived_catalog, functions, temporal, tail)?; - Ok(Some(SqlPlan::Cte { - definitions: vec![(alias_name, inner_plan)], - outer: Box::new(outer_plan), + Ok(Some(PlannedSelect { + plan: SqlPlan::Cte { + definitions: vec![(alias_name, inner_plan)], + outer: Box::new(outer.plan), + }, + scope: outer.scope, })) } diff --git a/nodedb-sql/src/planner/select/entry.rs b/nodedb-sql/src/planner/select/entry.rs index ddbe3299b..48a41f8f6 100644 --- a/nodedb-sql/src/planner/select/entry.rs +++ b/nodedb-sql/src/planner/select/entry.rs @@ -7,6 +7,7 @@ use nodedb_types::DatabaseId; use sqlparser::ast::{Query, SetExpr}; +use super::cte_catalog::CteCatalog; use super::limit::apply_limit; use super::order_by::{apply_order_by, try_hybrid_from_projection}; use super::query_tail::QueryTail; @@ -14,6 +15,7 @@ use super::select_stmt::plan_select; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::reserved::check_ast_identifier; +use crate::resolver::derived::{infer_subquery_relation, rename_output_columns}; use crate::temporal::TemporalScope; use crate::types::{Projection, SqlExpr, *}; @@ -82,23 +84,27 @@ pub fn plan_query( pipe_operators: query.pipe_operators.clone(), }; - // Plan each CTE subquery. + // Plan each CTE subquery and infer the relation it exposes. let mut definitions = Vec::new(); - let mut cte_names = Vec::new(); + let mut relations = Vec::new(); for cte in &with.cte_tables { let name = check_ast_identifier(&cte.alias.name)?; - for column in &cte.alias.columns { - check_ast_identifier(&column.name)?; - } + let declared: Vec = cte + .alias + .columns + .iter() + .map(|column| check_ast_identifier(&column.name)) + .collect::>()?; let cte_plan = plan_query(&cte.query, catalog, functions, temporal)?; + let info = infer_subquery_relation(catalog, &name, &cte.query)?; definitions.push((name.clone(), cte_plan)); - cte_names.push(name); + relations.push((name, rename_output_columns(info, &declared))); } // Build CTE-aware catalog so the outer query can reference CTE names. let cte_catalog = CteCatalog { inner: catalog, - cte_names, + relations, }; let outer = plan_query(&inner_query, &cte_catalog, functions, temporal)?; @@ -119,7 +125,9 @@ pub fn plan_query( limit_clause: &query.limit_clause, fetch: query.fetch.as_ref(), }; - let mut plan = plan_select(select, catalog, functions, temporal, &tail)?; + let planned = plan_select(select, catalog, functions, temporal, &tail)?; + let scope = planned.scope; + let mut plan = planned.plan; // Snapshot the projection before ORDER BY transforms the plan, // in case `apply_order_by` converts a Scan into VectorSearch. let pre_order_by_projection: Option> = match &plan { @@ -131,7 +139,7 @@ pub fn plan_query( _ => None, }; if let Some(order_by) = &query.order_by { - plan = apply_order_by(&plan, order_by, functions, &select.projection)?; + plan = apply_order_by(&plan, order_by, functions, &select.projection, &scope)?; } // Fall back to a SELECT-projection scan for hybrid-search and // text-search triggers. The `SELECT id, rrf_score(...) AS score @@ -371,37 +379,6 @@ pub fn plan_query( } } -/// Catalog wrapper that resolves CTE names as schemaless document collections. -pub(crate) struct CteCatalog<'a> { - pub(crate) inner: &'a dyn SqlCatalog, - pub(crate) cte_names: Vec, -} - -impl SqlCatalog for CteCatalog<'_> { - fn get_collection( - &self, - database_id: DatabaseId, - name: &str, - ) -> std::result::Result, SqlCatalogError> { - // Check CTE names first. - if self.cte_names.iter().any(|n| n == name) { - return Ok(Some(CollectionInfo { - name: name.into(), - engine: EngineType::DocumentSchemaless, - columns: Vec::new(), - primary_key: Some("id".into()), - has_auto_tier: false, - indexes: Vec::new(), - bitemporal: false, - primary: nodedb_types::PrimaryEngine::Document, - vector_primary: None, - partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, - })); - } - self.inner.get_collection(database_id, name) - } -} - /// Unit tests for SELECT query planning. #[cfg(test)] mod tests { @@ -430,6 +407,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "users" => Some(CollectionInfo { name: "users".into(), @@ -442,6 +420,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "orders" => Some(CollectionInfo { name: "orders".into(), @@ -454,6 +433,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "docs" => Some(CollectionInfo { name: "docs".into(), @@ -466,6 +446,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "tags" => Some(CollectionInfo { name: "tags".into(), @@ -478,6 +459,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "user_prefs" => Some(CollectionInfo { name: "user_prefs".into(), @@ -490,6 +472,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::KeyValue), }), "embeddings" => Some(CollectionInfo { name: "embeddings".into(), @@ -502,6 +485,7 @@ mod tests { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), _ => None, }; diff --git a/nodedb-sql/src/planner/select/helpers.rs b/nodedb-sql/src/planner/select/helpers.rs index 378c5c4da..08631f839 100644 --- a/nodedb-sql/src/planner/select/helpers.rs +++ b/nodedb-sql/src/planner/select/helpers.rs @@ -8,6 +8,8 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::*; @@ -25,12 +27,16 @@ pub(super) fn source_projection(plan: &SqlPlan) -> Vec { } /// Convert SELECT projection items. -pub fn convert_projection(items: &[ast::SelectItem]) -> Result> { +pub fn convert_projection( + items: &[ast::SelectItem], + scope: &TableScope, +) -> Result> { + let scope = ColumnScope::Relations(scope); let mut result = Vec::new(); for item in items { match item { ast::SelectItem::UnnamedExpr(expr) => { - let sql_expr = convert_expr(expr)?; + let sql_expr = convert_expr(expr, &scope)?; match &sql_expr { SqlExpr::Column { table, name } => { result.push(Projection::Column(qualified_name(table.as_deref(), name))); @@ -47,7 +53,7 @@ pub fn convert_projection(items: &[ast::SelectItem]) -> Result> } } ast::SelectItem::ExprWithAlias { expr, alias } => { - let sql_expr = convert_expr(expr)?; + let sql_expr = convert_expr(expr, &scope)?; result.push(Projection::Computed { expr: sql_expr, alias: normalize_ident(alias), @@ -90,8 +96,8 @@ pub fn qualified_name(table: Option<&str>, name: &str) -> String { } /// Convert a WHERE expression into a list of Filter. -pub fn convert_where_to_filters(expr: &ast::Expr) -> Result> { - let sql_expr = canonicalize_predicate(convert_expr(expr)?); +pub fn convert_where_to_filters(expr: &ast::Expr, scope: &TableScope) -> Result> { + let sql_expr = canonicalize_predicate(convert_expr(expr, &ColumnScope::Relations(scope))?); Ok(vec![Filter { expr: FilterExpr::Expr(sql_expr), }]) diff --git a/nodedb-sql/src/planner/select/order_by/aliases.rs b/nodedb-sql/src/planner/select/order_by/aliases.rs index c53d48d28..f5ffadfc2 100644 --- a/nodedb-sql/src/planner/select/order_by/aliases.rs +++ b/nodedb-sql/src/planner/select/order_by/aliases.rs @@ -16,6 +16,20 @@ use sqlparser::ast; use crate::parser::normalize::normalize_ident; +/// The output names a SELECT list introduces via explicit `AS`. +pub(crate) fn select_output_aliases(items: &[ast::SelectItem]) -> Vec { + items + .iter() + .filter_map(|item| match item { + ast::SelectItem::ExprWithAlias { alias, .. } => Some(normalize_ident(alias)), + ast::SelectItem::UnnamedExpr(_) + | ast::SelectItem::ExprWithAliases { .. } + | ast::SelectItem::QualifiedWildcard(..) + | ast::SelectItem::Wildcard(_) => None, + }) + .collect() +} + /// Resolve a possibly-aliased ORDER BY expression against the SELECT list. /// /// Returns the expression to inspect for search-trigger detection plus the diff --git a/nodedb-sql/src/planner/select/order_by/apply.rs b/nodedb-sql/src/planner/select/order_by/apply.rs index d4a3ece7f..6789b353c 100644 --- a/nodedb-sql/src/planner/select/order_by/apply.rs +++ b/nodedb-sql/src/planner/select/order_by/apply.rs @@ -8,12 +8,14 @@ use sqlparser::ast; -use super::aliases::resolve_order_by_target; +use super::aliases::{resolve_order_by_target, select_output_aliases}; use super::triggers::try_extract_sort_search; use crate::error::Result; use crate::functions::registry::FunctionRegistry; use crate::planner::agg_bind::{BindName, bind_aggregate_calls}; use crate::planner::select::post_process::post_process; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::*; @@ -31,6 +33,7 @@ pub(in crate::planner::select) fn apply_order_by( order_by: &ast::OrderBy, functions: &FunctionRegistry, select_items: &[ast::SelectItem], + scope: &TableScope, ) -> Result { let exprs = match &order_by.kind { ast::OrderByKind::Expressions(exprs) => exprs, @@ -61,20 +64,35 @@ pub(in crate::planner::select) fn apply_order_by( let mut bound_aggregates: Option> = None; let sort_keys: Vec = if let SqlPlan::Aggregate { aggregates, .. } = plan { let mut extended = aggregates.clone(); - let keys = exprs + // ORDER BY sorts after aggregates are renamed to their user aliases, + // so each key must address the output name. Binding runs to + // completion first: a sort-only aggregate appends to `extended`, and + // the name it lands in has to be in scope when the key converts. + let bound: Vec = exprs .iter() .map(|o| { - // ORDER BY sorts after aggregates are renamed to their user - // aliases, so the key must address the output name. - let bound = bind_aggregate_calls( + bind_aggregate_calls( &o.expr, select_items, &mut extended, functions, BindName::Output, - )?; + scope, + ) + }) + .collect::>>()?; + let sort_scope = scope.with_output_names( + extended + .iter() + .map(|a| a.alias.clone()) + .chain(select_output_aliases(select_items)), + ); + let keys = exprs + .iter() + .zip(&bound) + .map(|(o, expr)| { Ok(SortKey { - expr: convert_expr(&bound)?, + expr: convert_expr(expr, &ColumnScope::Relations(&sort_scope))?, ascending: o.options.asc.unwrap_or(true), nulls_first: o .options @@ -86,11 +104,12 @@ pub(in crate::planner::select) fn apply_order_by( bound_aggregates = Some(extended); keys } else { + let sort_scope = scope.with_output_names(select_output_aliases(select_items)); exprs .iter() .map(|o| { Ok(SortKey { - expr: convert_expr(&o.expr)?, + expr: convert_expr(&o.expr, &ColumnScope::Relations(&sort_scope))?, ascending: o.options.asc.unwrap_or(true), nulls_first: o .options @@ -191,7 +210,13 @@ pub(in crate::planner::select) fn apply_order_by( // with the inner subquery plan; the sort_keys ride along. SqlPlan::Cte { definitions, outer } => Ok(SqlPlan::Cte { definitions: definitions.clone(), - outer: Box::new(apply_order_by(outer, order_by, functions, select_items)?), + outer: Box::new(apply_order_by( + outer, + order_by, + functions, + select_items, + scope, + )?), }), // The clause is non-empty here (`exprs` was checked above), so these // keys were asked for. A variant with no slot to hold them must not diff --git a/nodedb-sql/src/planner/select/query_tail.rs b/nodedb-sql/src/planner/select/query_tail.rs index 553b020e0..ad13b19ce 100644 --- a/nodedb-sql/src/planner/select/query_tail.rs +++ b/nodedb-sql/src/planner/select/query_tail.rs @@ -13,6 +13,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; +use crate::resolver::columns::TableScope; use crate::types::SortKey; /// The trailing clauses of the enclosing `Query`. @@ -31,10 +32,10 @@ impl QueryTail<'_> { /// This is the same conversion `apply_order_by` performs on the plan it /// receives, so a scan that already carries these keys is overwritten /// downstream with an identical list, never an appended one. - pub(in crate::planner::select) fn sort_keys(&self) -> Result> { + pub(in crate::planner::select) fn sort_keys(&self, scope: &TableScope) -> Result> { match self.order_by.map(|o| &o.kind) { Some(ast::OrderByKind::Expressions(exprs)) => { - crate::planner::sort::convert_sort_keys(exprs) + crate::planner::sort::convert_sort_keys(exprs, scope) } Some(ast::OrderByKind::All(_)) | None => Ok(Vec::new()), } diff --git a/nodedb-sql/src/planner/select/select_stmt.rs b/nodedb-sql/src/planner/select/select_stmt.rs index 5e0f00335..e6b6fa18e 100644 --- a/nodedb-sql/src/planner/select/select_stmt.rs +++ b/nodedb-sql/src/planner/select/select_stmt.rs @@ -2,9 +2,9 @@ //! Single SELECT statement planning (no UNION, no CTE wrapper). -use nodedb_types::DatabaseId; use sqlparser::ast::{self, Select}; +use super::comma_lateral::try_plan_comma_lateral; use super::derived_from::try_plan_derived_from; use super::helpers::{convert_projection, convert_where_to_filters}; use super::query_tail::QueryTail; @@ -12,14 +12,16 @@ use super::where_search::try_extract_where_search; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::planner::ast_helpers::strip_single_table_qualifiers; -use crate::planner::lateral::plan::{ - LateralJoinArgs, is_lateral_derived, lateral_alias_from_factor, plan_lateral_join, - subquery_from_factor, -}; use crate::resolver::columns::TableScope; use crate::temporal::TemporalScope; use crate::types::*; +/// A planned SELECT body and the column namespace it resolved against. +pub(in crate::planner::select) struct PlannedSelect { + pub plan: SqlPlan, + pub scope: TableScope, +} + /// Plan a single SELECT statement (no UNION, no CTE wrapper). /// /// `tail` carries the enclosing query's ORDER BY / LIMIT so the base scan can @@ -31,13 +33,18 @@ pub(super) fn plan_select( functions: &FunctionRegistry, temporal: TemporalScope, tail: &QueryTail<'_>, -) -> Result { +) -> Result { // 0. Intercept array table-valued functions before catalog resolution // so a name like `ARRAY_SLICE` is not looked up as a collection. if let Some(plan) = crate::planner::array_fn::try_plan_array_table_fn(&select.from, catalog, temporal)? { - return Ok(plan); + // `resolve_from` synthesizes a relation from the array's dims and + // attrs, so ORDER BY and the tail clauses resolve its columns. + return Ok(PlannedSelect { + plan, + scope: TableScope::resolve_from(catalog, &select.from)?, + }); } // 0.5. Derived FROM subquery: `FROM (SELECT ...) AS t`. @@ -49,8 +56,8 @@ pub(super) fn plan_select( // dropped non-LATERAL derived factors silently, the scope ended // up empty, and the planner errored with "multi-table FROM // without JOIN". - if let Some(plan) = try_plan_derived_from(select, catalog, functions, temporal, tail)? { - return Ok(plan); + if let Some(planned) = try_plan_derived_from(select, catalog, functions, temporal, tail)? { + return Ok(planned); } // 1. Resolve FROM tables. @@ -63,9 +70,9 @@ pub(super) fn plan_select( if let Some(plan) = crate::planner::array_fn::try_plan_array_maint_fn(&select.projection, catalog)? { - return Ok(plan); + return Ok(PlannedSelect { plan, scope }); } - let projection = convert_projection(&select.projection)?; + let projection = convert_projection(&select.projection, &scope)?; let mut columns = Vec::new(); let mut values = Vec::new(); for (i, proj) in projection.iter().enumerate() { @@ -86,73 +93,32 @@ pub(super) fn plan_select( } } } - return Ok(SqlPlan::ConstantResult { columns, values }); + return Ok(PlannedSelect { + plan: SqlPlan::ConstantResult { columns, values }, + scope, + }); } // 3. Check for JOINs (including LATERAL). if let Some(plan) = try_plan_join(select, &scope, catalog, functions, temporal)? { - return Ok(plan); + return Ok(PlannedSelect { plan, scope }); } // 3b. Comma-LATERAL syntax: `FROM t, LATERAL (SELECT ...) x`. - // sqlparser represents this as two TableWithJoins elements in `select.from`, - // where the second has an empty joins list and its relation is Derived{lateral:true}. - if select.from.len() == 2 && is_lateral_derived(&select.from[1].relation) { - let outer_twj = &select.from[0]; - let lateral_twj = &select.from[1]; - - // Build outer scan plan. - let outer_alias = extract_table_alias_from_twj(outer_twj)?; - let outer_collection = - crate::parser::normalize::table_name_from_factor(&outer_twj.relation)? - .map(|(n, _)| n) - .ok_or_else(|| SqlError::Unsupported { - detail: "LATERAL: outer side must be a plain table".into(), - })?; - let outer_info = catalog - .resolve_relation(DatabaseId::DEFAULT, &outer_collection)? - .ok_or_else(|| SqlError::UnknownTable { - name: outer_collection.clone(), - })?; - let outer_scan = SqlPlan::Scan { - collection: outer_collection, - alias: outer_alias.clone(), - engine: outer_info.engine, - filters: Vec::new(), - projection: Vec::new(), - sort_keys: Vec::new(), - limit: None, - offset: 0, - distinct: false, - window_functions: Vec::new(), - temporal, - }; - - let lateral_alias = lateral_alias_from_factor(&lateral_twj.relation)?.ok_or_else(|| { - SqlError::Unsupported { - detail: "LATERAL subquery requires an alias (e.g. LATERAL (...) AS x)".into(), - } - })?; - let subquery = subquery_from_factor(&lateral_twj.relation) - .expect("is_lateral_derived guarantees Derived variant"); - let projection = convert_projection(&select.projection)?; - return plan_lateral_join(LateralJoinArgs { - outer_plan: outer_scan, - outer_alias, - subquery, - lateral_alias: &lateral_alias, - left_join: false, // comma-LATERAL is INNER (no LEFT semantics) - outer_projection: projection, - catalog, - temporal, - }) - .map(Ok)?; + if let Some(plan) = try_plan_comma_lateral(select, &scope, catalog, temporal)? { + return Ok(PlannedSelect { plan, scope }); } // 4. Single-table query. - let table = scope.single_table().ok_or_else(|| SqlError::Unsupported { - detail: "multi-table FROM without JOIN".into(), - })?; + // Cloned rather than borrowed: `scope` moves into the returned + // `PlannedSelect` while this relation is still in use. + let single = scope + .single_table() + .cloned() + .ok_or_else(|| SqlError::Unsupported { + detail: "multi-table FROM without JOIN".into(), + })?; + let table = &single; // For a single table the column qualifier (`t.` or its alias) is always // redundant, so strip it from the projection, WHERE, and GROUP BY here — @@ -175,8 +141,9 @@ pub(super) fn plan_select( // 4. Extract subqueries from WHERE and rewrite as semi/anti joins. let (subquery_joins, effective_where) = if let Some(expr) = &select.selection { - let extraction = - crate::planner::subquery::extract_subqueries(expr, catalog, functions, temporal)?; + let extraction = crate::planner::subquery::extract_subqueries( + expr, &scope, catalog, functions, temporal, + )?; (extraction.joins, extraction.remaining_where) } else { (Vec::new(), None) @@ -194,13 +161,13 @@ pub(super) fn plan_select( // Check for search-triggering functions in WHERE. The resolved // SELECT target list is threaded through so the search plan // self-describes its output columns. - let where_projection = convert_projection(&select.projection)?; + let where_projection = convert_projection(&select.projection, &scope)?; if let Some(plan) = try_extract_where_search(expr, table, functions, &where_projection)? { - return Ok(plan); + return Ok(PlannedSelect { plan, scope }); } cached_projection = Some(where_projection); - convert_where_to_filters(expr)? + convert_where_to_filters(expr, &scope)? } None => Vec::new(), }; @@ -230,7 +197,7 @@ pub(super) fn plan_select( base_input = Box::new(SqlPlan::Join { left: base_input, right: Box::new(sq.inner_plan.clone()), - on: vec![(sq.outer_column.clone(), sq.inner_column.clone())], + on: sq.on.clone(), join_type: sq.join_type, condition: None, limit: None, @@ -248,7 +215,7 @@ pub(super) fn plan_select( plan = SqlPlan::Join { left: Box::new(plan), right: Box::new(sq.inner_plan), - on: vec![(sq.outer_column, sq.inner_column)], + on: sq.on, join_type: sq.join_type, condition: None, limit: None, @@ -256,18 +223,19 @@ pub(super) fn plan_select( filters: Vec::new(), }; } - return Ok(plan); + return Ok(PlannedSelect { plan, scope }); } // 7. Convert projection (reuse the WHERE-search conversion if we already // did it in step 5, to avoid converting the same projection twice). let projection = match cached_projection { Some(p) => p, - None => convert_projection(&select.projection)?, + None => convert_projection(&select.projection, &scope)?, }; // 8. Convert window functions (SELECT with OVER). - let window_functions = crate::planner::window::extract_window_functions(select, functions)?; + let window_functions = + crate::planner::window::extract_window_functions(select, functions, &scope)?; // 9. Build base scan plan. let scan_projection = if subquery_joins.is_empty() { @@ -289,7 +257,10 @@ pub(super) fn plan_select( // itself downstream — the same reason `scan_projection` is empty here. let (sort_keys, limit, offset) = if subquery_joins.is_empty() { let (limit, offset) = tail.limit_offset()?; - (tail.sort_keys()?, limit, offset) + // ORDER BY resolves against the output names the SELECT list + // introduces as well as the input columns. + let order_scope = scope.with_output_names(super::select_output_aliases(&select.projection)); + (tail.sort_keys(&order_scope)?, limit, offset) } else { (Vec::new(), None, 0) }; @@ -342,7 +313,7 @@ pub(super) fn plan_select( plan = SqlPlan::Join { left: Box::new(plan), right: Box::new(sq.inner_plan), - on: vec![(sq.outer_column, sq.inner_column)], + on: sq.on, join_type: sq.join_type, condition: None, limit: None, @@ -359,7 +330,7 @@ pub(super) fn plan_select( *join_projection = projection; } - Ok(plan) + Ok(PlannedSelect { plan, scope }) } /// Check if a filter expression contains a column-vs-column comparison @@ -388,12 +359,6 @@ fn has_column_comparison(expr: &SqlExpr) -> bool { } } -/// Extract the alias from the first table in a `TableWithJoins`. -fn extract_table_alias_from_twj(twj: &sqlparser::ast::TableWithJoins) -> Result> { - crate::parser::normalize::table_name_from_factor(&twj.relation) - .map(|relation| relation.map(|(name, alias)| alias.unwrap_or(name))) -} - /// Check if a SELECT has aggregation (GROUP BY or aggregate functions in projection). fn has_aggregation(select: &Select, functions: &FunctionRegistry) -> bool { let group_by_non_empty = match &select.group_by { diff --git a/nodedb-sql/src/planner/select/where_search.rs b/nodedb-sql/src/planner/select/where_search.rs index 9b9f498e9..e1be4403e 100644 --- a/nodedb-sql/src/planner/select/where_search.rs +++ b/nodedb-sql/src/planner/select/where_search.rs @@ -160,9 +160,15 @@ fn dispatch_trigger( } } -fn extra_filter_to_filters(extra: Option<&ast::Expr>) -> Result> { +fn extra_filter_to_filters( + extra: Option<&ast::Expr>, + table: &crate::resolver::columns::ResolvedTable, +) -> Result> { match extra { - Some(e) => convert_where_to_filters(e), + Some(e) => { + let scope = crate::resolver::columns::TableScope::single(table.clone())?; + convert_where_to_filters(e, &scope) + } None => Ok(Vec::new()), } } @@ -208,7 +214,7 @@ fn plan_text_from_where( collection: table.name.clone(), query: fts_query, top_k: 1000, - filters: extra_filter_to_filters(extra_filter)?, + filters: extra_filter_to_filters(extra_filter, table)?, score_alias: None, projection: projection.to_vec(), })) @@ -244,7 +250,7 @@ fn plan_vector_from_where( top_k: DEFAULT_TOP_K, ef_search, metric: metric_from_func_name(name), - filters: extra_filter_to_filters(extra_filter)?, + filters: extra_filter_to_filters(extra_filter, table)?, array_prefilter: None, ann_options, // Vector-primary skip-payload-fetch and payload-filter peeling are @@ -340,7 +346,7 @@ fn plan_spatial_from_where( predicate, query_geometry: geometry, distance_meters: distance, - attribute_filters: extra_filter_to_filters(extra_filter)?, + attribute_filters: extra_filter_to_filters(extra_filter, table)?, limit: 1000, projection: projection.to_vec(), })) diff --git a/nodedb-sql/src/planner/sort.rs b/nodedb-sql/src/planner/sort.rs index 37101f63a..93c020ba3 100644 --- a/nodedb-sql/src/planner/sort.rs +++ b/nodedb-sql/src/planner/sort.rs @@ -7,16 +7,22 @@ //! sort key extraction utilities. use crate::error::Result; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::SortKey; /// Convert sqlparser OrderByExpr list to SortKey list. -pub fn convert_sort_keys(exprs: &[sqlparser::ast::OrderByExpr]) -> Result> { +pub fn convert_sort_keys( + exprs: &[sqlparser::ast::OrderByExpr], + scope: &TableScope, +) -> Result> { + let scope = ColumnScope::Relations(scope); exprs .iter() .map(|o| { Ok(SortKey { - expr: convert_expr(&o.expr)?, + expr: convert_expr(&o.expr, &scope)?, ascending: o.options.asc.unwrap_or(true), nulls_first: o .options diff --git a/nodedb-sql/src/planner/window/extract.rs b/nodedb-sql/src/planner/window/extract.rs index 4b3b20dc7..21db3f862 100644 --- a/nodedb-sql/src/planner/window/extract.rs +++ b/nodedb-sql/src/planner/window/extract.rs @@ -17,6 +17,8 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::functions::registry::{FunctionCategory, FunctionRegistry}; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; use crate::resolver::expr::convert_expr; use crate::types::{SortKey, SqlExpr, WindowSpec}; use nodedb_query::{FrameBound, WindowFrame}; @@ -28,6 +30,7 @@ use super::named::{collect_named_windows, flatten_window_spec, resolve_named_def pub fn extract_window_functions( select: &ast::Select, functions: &FunctionRegistry, + scope: &TableScope, ) -> Result> { let named = collect_named_windows(&select.named_window)?; let mut specs = Vec::new(); @@ -40,7 +43,7 @@ pub fn extract_window_functions( if let ast::Expr::Function(func) = expr && func.over.is_some() { - specs.push(convert_window_spec(func, &alias, functions, &named)?); + specs.push(convert_window_spec(func, &alias, functions, &named, scope)?); } } Ok(specs) @@ -51,6 +54,7 @@ fn convert_window_spec( alias: &str, functions: &FunctionRegistry, named: &HashMap, + scope: &TableScope, ) -> Result { if func.name.0.len() > 1 { let qualified: String = func @@ -99,7 +103,7 @@ fn convert_window_spec( } } - let args = convert_window_args(func, &name)?; + let args = convert_window_args(func, &name, scope)?; validate_constant_args(&name, &args)?; // Resolve the OVER target into a flattened partition/order/frame. @@ -121,14 +125,14 @@ fn convert_window_spec( let pb = flat .partition_by .iter() - .map(convert_expr) + .map(|e| convert_expr(e, &ColumnScope::Relations(scope))) .collect::>>()?; let ob = flat .order_by .iter() .map(|o| { Ok(SortKey { - expr: convert_expr(&o.expr)?, + expr: convert_expr(&o.expr, &ColumnScope::Relations(scope))?, ascending: o.options.asc.unwrap_or(true), nulls_first: o .options @@ -185,7 +189,11 @@ fn convert_window_spec( /// An argument the converter cannot represent is an error, never a dropped /// argument: discarding one silently turns `SUM(price * qty) OVER (...)` into /// a windowed column of NULLs that still reports success. -fn convert_window_args(func: &ast::Function, name: &str) -> Result> { +fn convert_window_args( + func: &ast::Function, + name: &str, + scope: &TableScope, +) -> Result> { let ast::FunctionArguments::List(list) = &func.args else { return Ok(Vec::new()); }; @@ -193,7 +201,9 @@ fn convert_window_args(func: &ast::Function, name: &str) -> Result> let mut args = Vec::with_capacity(list.args.len()); for arg in &list.args { match arg { - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)) => args.push(convert_expr(e)?), + ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)) => { + args.push(convert_expr(e, &ColumnScope::Relations(scope))?) + } // `COUNT(*) OVER (...)` — a wildcard carries no value to evaluate. // The evaluator counts frame rows when no argument is present. ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard) => {} @@ -242,6 +252,7 @@ mod tests { use super::*; use crate::functions::registry::FunctionRegistry; use crate::parser::statement::parse_sql; + use crate::resolver::columns::test_support::open_scope; fn select_of(sql: &str) -> Box { match parse_sql(sql).unwrap().into_iter().next().unwrap() { @@ -261,7 +272,7 @@ mod tests { FROM ticks WINDOW w AS (PARTITION BY bucket ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", ); - let specs = extract_window_functions(&select, ®).unwrap(); + let specs = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap(); assert_eq!(specs.len(), 3); for s in &specs { assert_eq!( @@ -284,7 +295,7 @@ mod tests { fn undefined_named_window_is_rejected() { let reg = FunctionRegistry::new(); let select = select_of("SELECT row_number() OVER missing AS r FROM t"); - let err = extract_window_functions(&select, ®).unwrap_err(); + let err = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap_err(); assert!( format!("{err}").contains("missing"), "error must name the missing window: {err}" @@ -297,7 +308,7 @@ mod tests { let select = select_of( "SELECT sum(x) OVER w2 AS s FROM t WINDOW w1 AS (PARTITION BY a), w2 AS (w1 ORDER BY ts)", ); - let specs = extract_window_functions(&select, ®).unwrap(); + let specs = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap(); assert_eq!(specs.len(), 1); assert_eq!( specs[0].partition_by.len(), @@ -311,7 +322,7 @@ mod tests { fn circular_named_window_is_rejected() { let reg = FunctionRegistry::new(); let select = select_of("SELECT sum(x) OVER w1 AS s FROM t WINDOW w1 AS (w2), w2 AS (w1)"); - let err = extract_window_functions(&select, ®).unwrap_err(); + let err = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap_err(); assert!( format!("{err}").to_lowercase().contains("circular"), "got: {err}" @@ -331,7 +342,7 @@ mod tests { WINDOW w AS (PARTITION BY time_bucket('1m', ts), symbol), w_ord AS (w ORDER BY ts ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)", ); - let specs = extract_window_functions(&select, ®).unwrap(); + let specs = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap(); assert_eq!(specs.len(), 5); for s in &specs { assert_eq!( @@ -365,7 +376,7 @@ mod tests { let select = select_of( "SELECT sum(x) OVER (w ORDER BY ts) AS s FROM t WINDOW w AS (PARTITION BY a)", ); - let specs = extract_window_functions(&select, ®).unwrap(); + let specs = extract_window_functions(&select, ®, &open_scope("ticks")).unwrap(); assert_eq!(specs[0].partition_by.len(), 1); assert_eq!(specs[0].order_by.len(), 1); } diff --git a/nodedb-sql/src/resolver/expr/functions.rs b/nodedb-sql/src/resolver/expr/functions.rs index 33ff14794..f7bdf1980 100644 --- a/nodedb-sql/src/resolver/expr/functions.rs +++ b/nodedb-sql/src/resolver/expr/functions.rs @@ -7,6 +7,7 @@ use sqlparser::ast; use crate::error::{Result, SqlError}; use crate::functions::registry::FunctionRegistry; use crate::parser::normalize::{SCHEMA_QUALIFIED_MSG, normalize_ident}; +use crate::resolver::ColumnScope; use crate::types::*; use super::convert::convert_expr_depth; @@ -19,7 +20,11 @@ use super::convert::convert_expr_depth; /// `planner::const_fold::DEFAULT_REGISTRY`. static FUNCTION_REGISTRY: LazyLock = LazyLock::new(FunctionRegistry::new); -pub(super) fn convert_function_depth(func: &ast::Function, depth: &mut usize) -> Result { +pub(super) fn convert_function_depth( + func: &ast::Function, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result { // Intercept PG FTS surface functions and lower them to pg_* internal names // before the generic path runs. if func.name.0.len() == 1 { @@ -27,7 +32,7 @@ pub(super) fn convert_function_depth(func: &ast::Function, depth: &mut usize) -> ast::ObjectNamePart::Identifier(ident) => ident.value.to_ascii_lowercase(), _ => String::new(), }; - if let Some(expr) = intercept_fts_function(&raw_name, func, depth)? { + if let Some(expr) = intercept_fts_function(&raw_name, func, depth, scope)? { return Ok(expr); } } @@ -62,7 +67,7 @@ pub(super) fn convert_function_depth(func: &ast::Function, depth: &mut usize) -> // Fold to a literal array at parse time so `= ANY(current_schemas(...))` works // without threading session context into the data-plane evaluator. if (name == "current_schemas" || name == "current_schema") - && let Some(expr) = intercept_catalog_function(&name, func, depth)? + && let Some(expr) = intercept_catalog_function(&name, func, depth, scope)? { return Ok(expr); } @@ -81,31 +86,7 @@ pub(super) fn convert_function_depth(func: &ast::Function, depth: &mut usize) -> return Err(SqlError::UndefinedFunction { name }); } - let args = match &func.args { - ast::FunctionArguments::None => Vec::new(), - ast::FunctionArguments::Subquery(_) => { - return Err(SqlError::Unsupported { - detail: "subquery in function args".into(), - }); - } - ast::FunctionArguments::List(arg_list) => arg_list - .args - .iter() - .filter_map(|a| match a { - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(e)) => { - Some(convert_expr_depth(e, depth)) - } - ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard) => { - Some(Ok(SqlExpr::Wildcard)) - } - ast::FunctionArg::Named { - arg: ast::FunctionArgExpr::Expr(e), - .. - } => Some(convert_expr_depth(e, depth)), - _ => None, - }) - .collect::>>()?, - }; + let args = collect_function_args(func, depth, scope)?; let distinct = match &func.args { ast::FunctionArguments::List(arg_list) => { @@ -133,10 +114,24 @@ fn intercept_fts_function( name: &str, func: &ast::Function, depth: &mut usize, + scope: &ColumnScope<'_>, ) -> Result> { use crate::functions::fts_ops::pg_fts_funcs; - let args = collect_function_args(func, depth)?; + if !matches!( + name, + "to_tsvector" + | "to_tsquery" + | "plainto_tsquery" + | "phraseto_tsquery" + | "websearch_to_tsquery" + | "ts_rank" + | "ts_rank_cd" + | "ts_headline" + ) { + return Ok(None); + } + let args = collect_function_args(func, depth, scope)?; match name { "to_tsvector" => Ok(Some(SqlExpr::Function { name: "pg_to_tsvector".into(), @@ -175,8 +170,9 @@ fn intercept_catalog_function( name: &str, func: &ast::Function, depth: &mut usize, + scope: &ColumnScope<'_>, ) -> Result> { - let args = collect_function_args(func, depth)?; + let args = collect_function_args(func, depth, scope)?; match name { "current_schemas" => { // Determine whether to include implicit schemas (pg_catalog). @@ -198,7 +194,11 @@ fn intercept_catalog_function( } /// Collect function call arguments, converting each `Expr` to `SqlExpr`. -fn collect_function_args(func: &ast::Function, depth: &mut usize) -> Result> { +fn collect_function_args( + func: &ast::Function, + depth: &mut usize, + scope: &ColumnScope<'_>, +) -> Result> { match &func.args { ast::FunctionArguments::None => Ok(Vec::new()), ast::FunctionArguments::Subquery(_) => Err(SqlError::Unsupported { @@ -209,7 +209,7 @@ fn collect_function_args(func: &ast::Function, depth: &mut usize) -> Result { - Some(convert_expr_depth(e, depth)) + Some(convert_expr_depth(e, depth, scope)) } ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Wildcard) => { Some(Ok(SqlExpr::Wildcard)) @@ -217,7 +217,7 @@ fn collect_function_args(func: &ast::Function, depth: &mut usize) -> Result Some(convert_expr_depth(e, depth)), + } => Some(convert_expr_depth(e, depth, scope)), _ => None, }) .collect::>>(), @@ -259,7 +259,7 @@ mod tests { fn unknown_function_name_is_rejected() { let func = function_ast("SELECT totally_bogus_fn(1, 2)"); let mut depth = 0; - let err = convert_function_depth(&func, &mut depth).unwrap_err(); + let err = convert_function_depth(&func, &mut depth, &ColumnScope::Unchecked).unwrap_err(); match err { SqlError::UndefinedFunction { name } => assert_eq!(name, "totally_bogus_fn"), other => panic!("expected SqlError::UndefinedFunction, got {other:?}"), @@ -270,7 +270,7 @@ mod tests { fn known_scalar_function_name_resolves() { let func = function_ast("SELECT upper('x')"); let mut depth = 0; - let expr = convert_function_depth(&func, &mut depth).unwrap(); + let expr = convert_function_depth(&func, &mut depth, &ColumnScope::Unchecked).unwrap(); match expr { SqlExpr::Function { name, .. } => assert_eq!(name, "upper"), other => panic!("expected SqlExpr::Function, got {other:?}"), @@ -289,11 +289,11 @@ mod tests { let lower = function_ast("SELECT upper('x')"); let upper_quoted = function_ast(r#"SELECT "UPPER"('x')"#); assert!( - convert_function_depth(&lower, &mut depth).is_ok(), + convert_function_depth(&lower, &mut depth, &ColumnScope::Unchecked).is_ok(), "lowercase 'upper' must resolve" ); assert!( - convert_function_depth(&upper_quoted, &mut depth).is_ok(), + convert_function_depth(&upper_quoted, &mut depth, &ColumnScope::Unchecked).is_ok(), "quoted 'UPPER' must still resolve via case-insensitive registry lookup" ); } diff --git a/nodedb-sql/tests/sql_suite/cases/limit_offset_bounds.rs b/nodedb-sql/tests/sql_suite/cases/limit_offset_bounds.rs index 3d8c3bbac..a6a8ff99e 100644 --- a/nodedb-sql/tests/sql_suite/cases/limit_offset_bounds.rs +++ b/nodedb-sql/tests/sql_suite/cases/limit_offset_bounds.rs @@ -24,6 +24,20 @@ use nodedb_sql::types::{CollectionInfo, EngineType, SqlPlan}; use nodedb_sql::{SqlCatalog, SqlCatalogError, plan_sql}; use nodedb_types::DatabaseId; +/// A declared TEXT column, for the stub catalog below. +fn text_column(name: &str, is_primary_key: bool) -> nodedb_sql::types::ColumnInfo { + nodedb_sql::types::ColumnInfo { + name: name.into(), + data_type: nodedb_sql::types::SqlDataType::String, + nullable: !is_primary_key, + is_primary_key, + default: None, + raw_type: None, + int_width: None, + float_width: None, + } +} + struct Catalog; impl SqlCatalog for Catalog { @@ -36,7 +50,7 @@ impl SqlCatalog for Catalog { "articles" | "authors" => Some(CollectionInfo { name: name.into(), engine: EngineType::DocumentStrict, - columns: Vec::new(), + columns: vec![text_column("id", true), text_column("name", false)], primary_key: Some("id".into()), has_auto_tier: false, indexes: Vec::new(), @@ -44,6 +58,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), }), _ => None, }; diff --git a/nodedb-sql/tests/sql_suite/cases/on_conflict_update_range_check.rs b/nodedb-sql/tests/sql_suite/cases/on_conflict_update_range_check.rs index 742e63398..6ba0a0add 100644 --- a/nodedb-sql/tests/sql_suite/cases/on_conflict_update_range_check.rs +++ b/nodedb-sql/tests/sql_suite/cases/on_conflict_update_range_check.rs @@ -76,6 +76,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), }), _ => None, }; diff --git a/nodedb-sql/tests/sql_suite/cases/point_get_operand_order.rs b/nodedb-sql/tests/sql_suite/cases/point_get_operand_order.rs index b58e7ec29..d42e68aa8 100644 --- a/nodedb-sql/tests/sql_suite/cases/point_get_operand_order.rs +++ b/nodedb-sql/tests/sql_suite/cases/point_get_operand_order.rs @@ -28,7 +28,16 @@ impl SqlCatalog for Catalog { "articles" => Some(CollectionInfo { name: "articles".into(), engine: EngineType::DocumentStrict, - columns: Vec::new(), + columns: vec![nodedb_sql::types::ColumnInfo { + name: "id".into(), + data_type: nodedb_sql::types::SqlDataType::String, + nullable: false, + is_primary_key: true, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }], primary_key: Some("id".into()), has_auto_tier: false, indexes: Vec::new(), @@ -36,6 +45,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), }), _ => None, }; diff --git a/nodedb-sql/tests/sql_suite/cases/positional_insert_column_binding.rs b/nodedb-sql/tests/sql_suite/cases/positional_insert_column_binding.rs index bb6e22b10..30568f9a1 100644 --- a/nodedb-sql/tests/sql_suite/cases/positional_insert_column_binding.rs +++ b/nodedb-sql/tests/sql_suite/cases/positional_insert_column_binding.rs @@ -61,6 +61,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentStrict), }), // Schemaless collection: no declared column order exists to // bind positionally to. The pre-existing `col{i}` fallback is @@ -76,6 +77,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), // KV collection: key/value are separated by the KV insert path // matching column names against the "key"/"ttl" sentinels, not @@ -92,6 +94,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::KeyValue), }), _ => None, }; diff --git a/nodedb-sql/tests/sql_suite/cases/schema_qualified_rejection.rs b/nodedb-sql/tests/sql_suite/cases/schema_qualified_rejection.rs index af2214271..9b4ee28bb 100644 --- a/nodedb-sql/tests/sql_suite/cases/schema_qualified_rejection.rs +++ b/nodedb-sql/tests/sql_suite/cases/schema_qualified_rejection.rs @@ -30,6 +30,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), "orders" => Some(CollectionInfo { name: "orders".into(), @@ -42,6 +43,7 @@ impl SqlCatalog for Catalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::DocumentSchemaless), }), _ => None, }; diff --git a/nodedb-types/src/error/code.rs b/nodedb-types/src/error/code.rs index 44637fd6e..e689c0263 100644 --- a/nodedb-types/src/error/code.rs +++ b/nodedb-types/src/error/code.rs @@ -59,6 +59,10 @@ impl ErrorCode { pub const DIVISION_BY_ZERO: Self = Self(1204); /// A LIMIT/OFFSET/FETCH bound resolved outside `[0, usize::MAX]`. pub const INVALID_LIMIT_VALUE: Self = Self(1205); + /// A column reference names no column of any relation in scope. + pub const UNDEFINED_COLUMN: Self = Self(1206); + /// A bare column name resolves against more than one relation in scope. + pub const AMBIGUOUS_COLUMN: Self = Self(1207); // Engine ops (1300–1399) pub const ARRAY: Self = Self(1300); diff --git a/nodedb-types/src/error/code_table.rs b/nodedb-types/src/error/code_table.rs index 31dd1e7d3..6c4171681 100644 --- a/nodedb-types/src/error/code_table.rs +++ b/nodedb-types/src/error/code_table.rs @@ -87,6 +87,8 @@ error_code_table! { FAN_OUT_EXCEEDED => FanOutExceeded { shards_touched: 0, limit: 0 }, SQL_NOT_ENABLED => SqlNotEnabled, UNDEFINED_FUNCTION => UndefinedFunction { name: String::new() }, + UNDEFINED_COLUMN => UndefinedColumn { column: String::new() }, + AMBIGUOUS_COLUMN => AmbiguousColumn { column: String::new() }, DIVISION_BY_ZERO => DivisionByZero, INVALID_LIMIT_VALUE => InvalidLimitValue { clause: "remote".into(), value: message.to_owned() }, diff --git a/nodedb-types/src/error/ctors/read_query_auth.rs b/nodedb-types/src/error/ctors/read_query_auth.rs index c345c7149..446d5fd97 100644 --- a/nodedb-types/src/error/ctors/read_query_auth.rs +++ b/nodedb-types/src/error/ctors/read_query_auth.rs @@ -134,6 +134,32 @@ impl NodeDbError { } } + /// A column reference names no column of any relation in scope. Distinct + /// from `plan_error` so clients match on the code rather than parsing the + /// message. + pub fn undefined_column(column: impl Into) -> Self { + let column = column.into(); + Self { + code: ErrorCode::UNDEFINED_COLUMN, + message: format!("column \"{column}\" does not exist"), + details: ErrorDetails::UndefinedColumn { column }, + cause: None, + } + } + + /// A bare column name resolves against more than one relation in scope. + /// Distinct from `plan_error` so clients match on the code rather than + /// parsing the message. + pub fn ambiguous_column(column: impl Into) -> Self { + let column = column.into(); + Self { + code: ErrorCode::AMBIGUOUS_COLUMN, + message: format!("column reference \"{column}\" is ambiguous"), + details: ErrorDetails::AmbiguousColumn { column }, + cause: None, + } + } + /// Expression evaluation divided or took a modulus by zero. Distinct /// from `plan_error` so clients can match on the specific code /// (SQLSTATE `22012`, `division_by_zero`) rather than parsing the diff --git a/nodedb-types/src/error/details.rs b/nodedb-types/src/error/details.rs index 8b2729f1d..395544c01 100644 --- a/nodedb-types/src/error/details.rs +++ b/nodedb-types/src/error/details.rs @@ -101,6 +101,12 @@ pub enum ErrorDetails { /// A function call names no registered scalar/aggregate/window function. #[serde(rename = "undefined_function")] UndefinedFunction { name: String }, + /// A column reference names no column of any relation in scope. + #[serde(rename = "undefined_column")] + UndefinedColumn { column: String }, + /// A bare column name resolves against more than one relation in scope. + #[serde(rename = "ambiguous_column")] + AmbiguousColumn { column: String }, /// Expression evaluation divided or took a modulus by zero. #[serde(rename = "division_by_zero")] DivisionByZero, diff --git a/nodedb-types/src/error/msgpack/constants.rs b/nodedb-types/src/error/msgpack/constants.rs index 856895d45..2de2601e2 100644 --- a/nodedb-types/src/error/msgpack/constants.rs +++ b/nodedb-types/src/error/msgpack/constants.rs @@ -82,6 +82,8 @@ // | 76 | NotFound | // | 77 | CannotDropDefaultDatabase | // | 78 | InvalidLimitValue | +// | 79 | UndefinedColumn | +// | 80 | AmbiguousColumn | pub(super) const TAG_CONSTRAINT_VIOLATION: u16 = 1; pub(super) const TAG_WRITE_CONFLICT: u16 = 2; @@ -161,3 +163,5 @@ pub(super) const TAG_OBJECT_NOT_READY: u16 = 75; pub(super) const TAG_NOT_FOUND: u16 = 76; pub(super) const TAG_CANNOT_DROP_DEFAULT_DATABASE: u16 = 77; pub(super) const TAG_INVALID_LIMIT_VALUE: u16 = 78; +pub(super) const TAG_UNDEFINED_COLUMN: u16 = 79; +pub(super) const TAG_AMBIGUOUS_COLUMN: u16 = 80; diff --git a/nodedb-types/src/error/msgpack/decode/from_messagepack.rs b/nodedb-types/src/error/msgpack/decode/from_messagepack.rs index 80c6c842a..eb6763287 100644 --- a/nodedb-types/src/error/msgpack/decode/from_messagepack.rs +++ b/nodedb-types/src/error/msgpack/decode/from_messagepack.rs @@ -131,6 +131,14 @@ impl<'a> FromMessagePack<'a> for ErrorDetails { let (name,) = read1_str(reader, field_count)?; Ok(ErrorDetails::UndefinedFunction { name }) } + TAG_UNDEFINED_COLUMN => { + let (column,) = read1_str(reader, field_count)?; + Ok(ErrorDetails::UndefinedColumn { column }) + } + TAG_AMBIGUOUS_COLUMN => { + let (column,) = read1_str(reader, field_count)?; + Ok(ErrorDetails::AmbiguousColumn { column }) + } TAG_DIVISION_BY_ZERO => { skip_fields(reader, field_count)?; Ok(ErrorDetails::DivisionByZero) diff --git a/nodedb-types/src/error/msgpack/encode.rs b/nodedb-types/src/error/msgpack/encode.rs index 5a392d125..bbaef9712 100644 --- a/nodedb-types/src/error/msgpack/encode.rs +++ b/nodedb-types/src/error/msgpack/encode.rs @@ -159,6 +159,12 @@ impl ToMessagePack for ErrorDetails { ErrorDetails::UndefinedFunction { name } => { write1(writer, TAG_UNDEFINED_FUNCTION, name) } + ErrorDetails::UndefinedColumn { column } => { + write1(writer, TAG_UNDEFINED_COLUMN, column) + } + ErrorDetails::AmbiguousColumn { column } => { + write1(writer, TAG_AMBIGUOUS_COLUMN, column) + } ErrorDetails::DivisionByZero => write_unit(writer, TAG_DIVISION_BY_ZERO), ErrorDetails::InvalidLimitValue { clause, value } => { write2(writer, TAG_INVALID_LIMIT_VALUE, clause, value) diff --git a/nodedb-types/src/error/sqlstate.rs b/nodedb-types/src/error/sqlstate.rs index ac192ca7b..ffea17029 100644 --- a/nodedb-types/src/error/sqlstate.rs +++ b/nodedb-types/src/error/sqlstate.rs @@ -140,6 +140,10 @@ pub const UNDEFINED_OBJECT: &str = "42704"; /// relation in scope) pub const UNDEFINED_COLUMN: &str = "42703"; +/// `42702` — `ambiguous_column`: a bare column name that resolves against +/// more than one relation in scope. +pub const AMBIGUOUS_COLUMN: &str = "42702"; + /// `42846` — `cannot_coerce` pub const CANNOT_COERCE: &str = "42846"; @@ -348,6 +352,7 @@ mod tests { INSUFFICIENT_PRIVILEGE, SYNTAX_ERROR, UNDEFINED_COLUMN, + AMBIGUOUS_COLUMN, UNDEFINED_OBJECT, CANNOT_COERCE, UNDEFINED_TABLE, @@ -403,6 +408,7 @@ mod tests { fn spot_check_well_known_codes() { assert_eq!(UNIQUE_VIOLATION, "23505"); assert_eq!(UNDEFINED_COLUMN, "42703"); + assert_eq!(AMBIGUOUS_COLUMN, "42702"); assert_eq!(UNDEFINED_TABLE, "42P01"); assert_eq!(INSUFFICIENT_PRIVILEGE, "42501"); assert_eq!(QUERY_CANCELED, "57014"); diff --git a/nodedb-types/src/error/types.rs b/nodedb-types/src/error/types.rs index ccbf8a42b..2b877912d 100644 --- a/nodedb-types/src/error/types.rs +++ b/nodedb-types/src/error/types.rs @@ -99,6 +99,8 @@ impl NodeDbError { | ErrorDetails::Config | ErrorDetails::SqlNotEnabled | ErrorDetails::UndefinedFunction { .. } + | ErrorDetails::UndefinedColumn { .. } + | ErrorDetails::AmbiguousColumn { .. } | ErrorDetails::DivisionByZero | ErrorDetails::InvalidLimitValue { .. } | ErrorDetails::BackupTenantMismatch { .. } diff --git a/nodedb/src/bridge/envelope/error_code.rs b/nodedb/src/bridge/envelope/error_code.rs index b3fd4bc95..3c10d7422 100644 --- a/nodedb/src/bridge/envelope/error_code.rs +++ b/nodedb/src/bridge/envelope/error_code.rs @@ -202,6 +202,11 @@ impl From for ErrorCode { Self::TxnOverlayMemoryExceeded { limit } } crate::Error::DivisionByZero => Self::DivisionByZero, + crate::Error::UndefinedColumn { column } => Self::UndefinedColumn { column }, + // Same condition an undefined column reports at plan time, raised + // here by the strict encoder for a transport the planner never + // sees (native client, `COPY FROM`, CRDT delta merge). + crate::Error::UnknownStrictField { column, .. } => Self::UndefinedColumn { column }, // Already a Data-Plane verdict: hand back the same code rather // than re-wrapping it as `Internal` and losing its SQLSTATE. crate::Error::DataPlane(code) => code, diff --git a/nodedb/src/control/planner/context/query/planning.rs b/nodedb/src/control/planner/context/query/planning.rs index 021da8b93..8a845dcd7 100644 --- a/nodedb/src/control/planner/context/query/planning.rs +++ b/nodedb/src/control/planner/context/query/planning.rs @@ -48,6 +48,15 @@ fn map_plan_error(error: nodedb_sql::SqlError, tenant_id: crate::types::TenantId nodedb_sql::SqlError::InvalidLimitValue { clause, value } => { crate::Error::InvalidLimitValue { clause, value } } + nodedb_sql::SqlError::UnknownColumn { column, .. } => { + crate::Error::UndefinedColumn { column } + } + nodedb_sql::SqlError::AmbiguousColumn { column } => { + crate::Error::AmbiguousColumn { column } + } + // A target/expression count mismatch is a syntax error in PostgreSQL, + // so it renders 42601 through `BadRequest`. + nodedb_sql::SqlError::Arity { detail } => crate::Error::BadRequest { detail }, other => crate::Error::PlanError { detail: other.to_string(), }, diff --git a/nodedb/src/control/server/native/sqlstate_code.rs b/nodedb/src/control/server/native/sqlstate_code.rs index 1bc5105db..7c6c9d721 100644 --- a/nodedb/src/control/server/native/sqlstate_code.rs +++ b/nodedb/src/control/server/native/sqlstate_code.rs @@ -66,6 +66,8 @@ pub(crate) fn ndb_code_for_sqlstate(sqlstate_str: &str) -> u16 { sqlstate::INVALID_CATALOG_NAME => ErrorCode::DATABASE_NOT_FOUND, sqlstate::INSUFFICIENT_PRIVILEGE => ErrorCode::AUTHORIZATION_DENIED, sqlstate::UNDEFINED_FUNCTION => ErrorCode::UNDEFINED_FUNCTION, + sqlstate::UNDEFINED_COLUMN => ErrorCode::UNDEFINED_COLUMN, + sqlstate::AMBIGUOUS_COLUMN => ErrorCode::AMBIGUOUS_COLUMN, // Both a malformed request and a plan that cannot be built render as // `42601`, so this cannot say which. It does not have to: the two // differ in which side wrote the bad statement, not in how a client diff --git a/nodedb/src/control/server/pgwire/types/error_map.rs b/nodedb/src/control/server/pgwire/types/error_map.rs index 2d12e6c6a..d143ef89d 100644 --- a/nodedb/src/control/server/pgwire/types/error_map.rs +++ b/nodedb/src/control/server/pgwire/types/error_map.rs @@ -50,6 +50,19 @@ pub fn error_to_sqlstate(err: &crate::Error) -> (&'static str, &'static str, Str sqlstate::UNDEFINED_FUNCTION, format!("function {name}(...) does not exist"), ), + crate::Error::UndefinedColumn { column } => ( + "ERROR", + sqlstate::UNDEFINED_COLUMN, + format!("column \"{column}\" does not exist"), + ), + crate::Error::AmbiguousColumn { column } => ( + "ERROR", + sqlstate::AMBIGUOUS_COLUMN, + format!("column reference \"{column}\" is ambiguous"), + ), + crate::Error::UnknownStrictField { .. } => { + ("ERROR", sqlstate::UNDEFINED_COLUMN, err.to_string()) + } crate::Error::DivisionByZero => ("ERROR", sqlstate::DIVISION_BY_ZERO, err.to_string()), crate::Error::InvalidLimitValue { .. } => { ("ERROR", sqlstate::INVALID_LIMIT_VALUE, err.to_string()) @@ -184,6 +197,10 @@ pub(crate) fn numeric_code_to_sqlstate(code: nodedb_types::error::ErrorCode) -> Ec::BAD_REQUEST | Ec::PLAN_ERROR => sqlstate::SYNTAX_ERROR, // Mirrors the `UndefinedFunction` arm. Ec::UNDEFINED_FUNCTION => sqlstate::UNDEFINED_FUNCTION, + // Mirrors the `UndefinedColumn` arm. + Ec::UNDEFINED_COLUMN => sqlstate::UNDEFINED_COLUMN, + // Mirrors the `AmbiguousColumn` arm. + Ec::AMBIGUOUS_COLUMN => sqlstate::AMBIGUOUS_COLUMN, // Mirrors the `DivisionByZero` arm. Ec::DIVISION_BY_ZERO => sqlstate::DIVISION_BY_ZERO, // Mirrors the `InvalidLimitValue` arm. diff --git a/nodedb/src/control/server/shared/ddl/result.rs b/nodedb/src/control/server/shared/ddl/result.rs index d4088434a..bc902d31c 100644 --- a/nodedb/src/control/server/shared/ddl/result.rs +++ b/nodedb/src/control/server/shared/ddl/result.rs @@ -160,6 +160,8 @@ pub fn code_for_sqlstate(sqlstate_str: &str) -> ErrorCode { sqlstate::INVALID_CATALOG_NAME => ErrorCode::DATABASE_NOT_FOUND, sqlstate::INSUFFICIENT_PRIVILEGE => ErrorCode::AUTHORIZATION_DENIED, sqlstate::UNDEFINED_FUNCTION => ErrorCode::UNDEFINED_FUNCTION, + sqlstate::UNDEFINED_COLUMN => ErrorCode::UNDEFINED_COLUMN, + sqlstate::AMBIGUOUS_COLUMN => ErrorCode::AMBIGUOUS_COLUMN, // A malformed request and a plan that cannot be built both render as // `42601`; both are non-retriable client errors, so one code covers // both without losing anything a client acts on. diff --git a/nodedb/src/control/server/shared/ddl/sql_parse.rs b/nodedb/src/control/server/shared/ddl/sql_parse.rs index defc2151c..76ff88775 100644 --- a/nodedb/src/control/server/shared/ddl/sql_parse.rs +++ b/nodedb/src/control/server/shared/ddl/sql_parse.rs @@ -137,7 +137,10 @@ fn try_eval_scalar_function(s: &str) -> Option { | sqlparser::ast::SelectItem::ExprWithAlias { expr: e, .. } => e, _ => return None, }; - let sql_expr = nodedb_sql::resolver::expr::convert_expr(&ast_expr).ok()?; + // A DDL constant expression has no FROM clause, so no identifier is + // checkable against a relation here. + let scope = nodedb_sql::resolver::ColumnScope::Unchecked; + let sql_expr = nodedb_sql::resolver::expr::convert_expr(&ast_expr, &scope).ok()?; let folded = nodedb_sql::planner::const_fold::fold_constant_default(&sql_expr).ok()??; Some(sql_value_to_ndb_value(folded)) } diff --git a/nodedb/src/error/types.rs b/nodedb/src/error/types.rs index 2916c8279..ae23847b2 100644 --- a/nodedb/src/error/types.rs +++ b/nodedb/src/error/types.rs @@ -282,6 +282,25 @@ pub enum Error { #[error("function {name}(...) does not exist")] UndefinedFunction { name: String }, + /// A column reference resolved against no relation, output alias, or + /// synthetic column in scope. Propagated from `SqlError::UnknownColumn`; + /// the pgwire layer renders it as SQLSTATE `42703` (undefined_column). + #[error("column \"{column}\" does not exist")] + UndefinedColumn { column: String }, + + /// A bare column name resolved against more than one relation in scope. + /// Propagated from `SqlError::AmbiguousColumn`; the pgwire layer renders + /// it as SQLSTATE `42702` (ambiguous_column). + #[error("column reference \"{column}\" is ambiguous")] + AmbiguousColumn { column: String }, + + /// A write body carried a field the collection's strict schema does not + /// declare. The same condition the planner reports as an unknown column, + /// detected at encode time for the transports that bypass the planner: + /// the native client, `COPY FROM`, and CRDT delta merge. + #[error("column \"{column}\" of collection \"{collection}\" does not exist")] + UnknownStrictField { collection: String, column: String }, + /// Expression evaluation divided or took a modulus by zero. Rendered as /// SQLSTATE `22012` (division_by_zero) at the pgwire layer. #[error("division by zero")] diff --git a/nodedb/src/error_classify.rs b/nodedb/src/error_classify.rs index d0a0ef5e6..3026f7da3 100644 --- a/nodedb/src/error_classify.rs +++ b/nodedb/src/error_classify.rs @@ -148,6 +148,9 @@ pub(crate) fn classify(e: &Error) -> NodeDbError { } Error::PlanError { detail } => NodeDbError::plan_error(detail), Error::UndefinedFunction { name } => NodeDbError::undefined_function(name.clone()), + Error::UndefinedColumn { column } => NodeDbError::undefined_column(column.clone()), + Error::AmbiguousColumn { column } => NodeDbError::ambiguous_column(column.clone()), + Error::UnknownStrictField { column, .. } => NodeDbError::undefined_column(column.clone()), Error::DivisionByZero => NodeDbError::division_by_zero(), Error::InvalidLimitValue { clause, value } => { NodeDbError::invalid_limit_value(*clause, value.clone()) diff --git a/nodedb/src/error_from_data_plane.rs b/nodedb/src/error_from_data_plane.rs index 02262a194..a1cc96b19 100644 --- a/nodedb/src/error_from_data_plane.rs +++ b/nodedb/src/error_from_data_plane.rs @@ -110,9 +110,7 @@ pub(crate) fn data_plane_code_to_public(code: ErrorCode) -> NodeDbError { "WITH RECURSIVE CTE '{cte_name}' exceeded max recursion depth {max_depth}; \ add a stricter termination condition or raise max_recursion_depth" )), - ErrorCode::UndefinedColumn { column } => { - NodeDbError::bad_request(format!("column \"{column}\" does not exist")) - } + ErrorCode::UndefinedColumn { column } => NodeDbError::undefined_column(column), ErrorCode::Unsupported { detail } => NodeDbError::bad_request(detail), ErrorCode::DivisionByZero => NodeDbError::division_by_zero(), ErrorCode::TxnOverlayMemoryExceeded { limit } => NodeDbError::bad_request(format!( diff --git a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs index 18328ad84..3b1cab4c5 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_group_by_alias.rs @@ -29,7 +29,16 @@ impl SqlCatalog for TimeseriesCatalog { "dns_bench" => Some(CollectionInfo { name: "dns_bench".into(), engine: EngineType::Timeseries, - columns: Vec::new(), + columns: vec![nodedb_sql::types::ColumnInfo { + name: "timestamp".into(), + data_type: nodedb_sql::types::SqlDataType::Timestamp, + nullable: false, + is_primary_key: false, + default: None, + raw_type: None, + int_width: None, + float_width: None, + }], primary_key: None, has_auto_tier: false, indexes: Vec::new(), @@ -37,6 +46,7 @@ impl SqlCatalog for TimeseriesCatalog { primary: nodedb_types::PrimaryEngine::Document, vector_primary: None, partition_strategy: nodedb_types::PartitionStrategy::CollectionHomed, + open_schema: CollectionInfo::open_schema_for(EngineType::Timeseries), }), _ => None, }; diff --git a/nodedb/tests/wire/cases/mod.rs b/nodedb/tests/wire/cases/mod.rs index 421df98d6..0f7da3ea2 100644 --- a/nodedb/tests/wire/cases/mod.rs +++ b/nodedb/tests/wire/cases/mod.rs @@ -242,6 +242,9 @@ mod sql_transactions_upsert_overlay; mod sql_transactions_vector_overlay; mod sql_trigger_fuel; mod sql_typeguard_defaults; +mod sql_undefined_column; +mod sql_undefined_column_dml; +mod sql_undefined_column_subquery; mod sql_undefined_function; mod sql_update_expressions; mod sql_update_from; diff --git a/nodedb/tests/wire/cases/sql_undefined_column.rs b/nodedb/tests/wire/cases/sql_undefined_column.rs new file mode 100644 index 000000000..b8d75e2a2 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_undefined_column.rs @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! An identifier naming no column of a closed-schema collection raises +//! SQLSTATE `42703` at plan time, in every read clause. +//! Closed engines: `document_strict`, `kv`, `columnar`, `timeseries`, +//! `spatial`. `document_strict` carries the full clause matrix. Each other +//! closed engine gets one representative check. +//! `document_schemaless` stays open (`docs/documents.md:43`). It accepts +//! undeclared fields on write, so unknown identifiers resolve to NULL. + +use crate::harness::TestServer; + +async fn seed_strict(server: &TestServer, name: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (1, 10)")) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (2, 20)")) + .await + .unwrap(); +} + +#[tokio::test] +async fn projection_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_proj").await; + srv.expect_error("SELECT nonexistent_col FROM uc_proj", "42703") + .await; +} + +#[tokio::test] +async fn where_equality_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_where_eq").await; + srv.expect_error( + "SELECT count(*) FROM uc_where_eq WHERE nonexistent_col = 1", + "42703", + ) + .await; +} + +/// `IS NULL` on an unknown column must not fold to a silent match-all. +#[tokio::test] +async fn where_is_null_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_where_null").await; + srv.expect_error( + "SELECT count(*) FROM uc_where_null WHERE nonexistent_col IS NULL", + "42703", + ) + .await; +} + +#[tokio::test] +async fn order_by_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_order").await; + srv.expect_error("SELECT b FROM uc_order ORDER BY nonexistent_col", "42703") + .await; +} + +#[tokio::test] +async fn group_by_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_group").await; + srv.expect_error( + "SELECT count(*) FROM uc_group GROUP BY nonexistent_col", + "42703", + ) + .await; +} + +#[tokio::test] +async fn having_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_having").await; + srv.expect_error( + "SELECT count(*) FROM uc_having GROUP BY a HAVING nonexistent_col > 1", + "42703", + ) + .await; +} + +#[tokio::test] +async fn qualified_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_qual").await; + srv.expect_error("SELECT t.nonexistent_col FROM uc_qual AS t", "42703") + .await; +} + +#[tokio::test] +async fn join_on_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_join_l").await; + seed_strict(&srv, "uc_join_r").await; + srv.expect_error( + "SELECT l.a FROM uc_join_l l JOIN uc_join_r r ON l.nonexistent_col = r.a", + "42703", + ) + .await; +} + +#[tokio::test] +async fn window_partition_by_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_win_part").await; + srv.expect_error( + "SELECT count(*) OVER (PARTITION BY nonexistent_col) FROM uc_win_part", + "42703", + ) + .await; +} + +#[tokio::test] +async fn window_order_by_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_win_order").await; + srv.expect_error( + "SELECT count(*) OVER (ORDER BY nonexistent_col) FROM uc_win_order", + "42703", + ) + .await; +} + +#[tokio::test] +async fn in_subquery_unknown_source_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_in_outer").await; + seed_strict(&srv, "uc_in_source").await; + srv.expect_error( + "SELECT a FROM uc_in_outer WHERE a IN (SELECT nonexistent_col FROM uc_in_source)", + "42703", + ) + .await; +} + +/// Plan-time proof: pairs an empty collection with `LIMIT 0` so no row ever +/// reaches the evaluator. An error here can only come from planning. +#[tokio::test] +async fn unknown_column_errors_with_zero_rows_scanned() { + let srv = TestServer::start().await; + srv.exec( + "CREATE COLLECTION uc_zero_rows (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.expect_error("SELECT nonexistent_col FROM uc_zero_rows LIMIT 0", "42703") + .await; +} + +#[tokio::test] +async fn kv_engine_projection_unknown_column_errors() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION uc_kv_proj (k TEXT PRIMARY KEY, v TEXT) WITH (engine = 'kv')") + .await + .unwrap(); + srv.exec("INSERT INTO uc_kv_proj (k, v) VALUES ('k1', 'v1')") + .await + .unwrap(); + srv.expect_error("SELECT nonexistent_col FROM uc_kv_proj", "42703") + .await; +} + +#[tokio::test] +async fn kv_engine_where_unknown_column_errors() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION uc_kv_where (k TEXT PRIMARY KEY, v TEXT) WITH (engine = 'kv')") + .await + .unwrap(); + srv.exec("INSERT INTO uc_kv_where (k, v) VALUES ('k1', 'v1')") + .await + .unwrap(); + srv.expect_error( + "SELECT k FROM uc_kv_where WHERE nonexistent_col = 'x'", + "42703", + ) + .await; +} + +#[tokio::test] +async fn columnar_engine_projection_unknown_column_errors() { + let srv = TestServer::start().await; + srv.exec( + "CREATE COLLECTION uc_columnar \ + COLUMNS (id TEXT, region TEXT, revenue FLOAT) \ + WITH (engine='columnar')", + ) + .await + .unwrap(); + srv.exec("INSERT INTO uc_columnar (id, region, revenue) VALUES ('r1', 'us', 100.0)") + .await + .unwrap(); + srv.expect_error("SELECT nonexistent_col FROM uc_columnar", "42703") + .await; +} + +#[tokio::test] +async fn timeseries_engine_projection_unknown_column_errors() { + let srv = TestServer::start().await; + srv.exec( + "CREATE COLLECTION uc_timeseries (ts TIMESTAMP TIME_KEY, value FLOAT) \ + WITH (engine='timeseries')", + ) + .await + .unwrap(); + srv.exec("INSERT INTO uc_timeseries (ts, value) VALUES ('2020-01-01 00:00:00', 1.0)") + .await + .unwrap(); + srv.expect_error("SELECT nonexistent_col FROM uc_timeseries", "42703") + .await; +} + +#[tokio::test] +async fn spatial_engine_projection_unknown_column_errors() { + let srv = TestServer::start().await; + srv.exec( + "CREATE COLLECTION uc_spatial \ + COLUMNS (id TEXT, location GEOMETRY, name TEXT) \ + WITH (engine='spatial')", + ) + .await + .unwrap(); + srv.exec( + "INSERT INTO uc_spatial (id, location, name) \ + VALUES ('p1', ST_Point(-122.4, 37.8), 'SF')", + ) + .await + .unwrap(); + srv.expect_error("SELECT nonexistent_col FROM uc_spatial", "42703") + .await; +} + +/// Positive control: a declared column keeps selecting correctly. +#[tokio::test] +async fn declared_column_still_selects() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_declared").await; + let rows = srv + .query_rows("SELECT a, b FROM uc_declared ORDER BY a") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1", "10"], vec!["2", "20"]]); +} + +/// Positive control: `SELECT *` still returns every row. +#[tokio::test] +async fn select_star_still_returns_rows() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_star").await; + let rows = srv.query_rows("SELECT * FROM uc_star").await.unwrap(); + assert_eq!(rows.len(), 2); +} + +/// Positive control: ORDER BY on a SELECT output alias must not be mistaken +/// for an unknown column. +#[tokio::test] +async fn order_by_output_alias_still_works() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_order_alias").await; + let rows = srv + .query_rows("SELECT b AS bee FROM uc_order_alias ORDER BY bee") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["10"], vec!["20"]]); +} + +/// Positive control: GROUP BY on a SELECT output alias must not be mistaken +/// for an unknown column. +#[tokio::test] +async fn group_by_output_alias_still_works() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_group_alias").await; + let rows = srv + .query_rows("SELECT b AS bee, count(*) FROM uc_group_alias GROUP BY bee ORDER BY bee") + .await + .unwrap(); + assert_eq!(rows.len(), 2); +} + +/// Positive control: HAVING on a SELECT output alias must not be mistaken +/// for an unknown column. +#[tokio::test] +async fn having_output_alias_still_works() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_having_alias").await; + let rows = srv + .query_rows( + "SELECT b AS bee, count(*) AS cnt FROM uc_having_alias \ + GROUP BY bee HAVING cnt > 0 ORDER BY bee", + ) + .await + .unwrap(); + assert_eq!(rows.len(), 2); +} + +/// The schemaless engine accepts undeclared fields on write +/// (`docs/documents.md:43`), so a read must accept them too: an unknown +/// identifier resolves to NULL instead of erroring. This is the deliberate +/// open-schema boundary the closed-schema gate must not cross. +#[tokio::test] +async fn schemaless_unknown_column_resolves_to_null() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION uc_schemaless (id INT PRIMARY KEY, x INT)") + .await + .unwrap(); + srv.exec("INSERT INTO uc_schemaless (id, x) VALUES (1, 10)") + .await + .unwrap(); + srv.exec("INSERT INTO uc_schemaless (id, x) VALUES (2, 20)") + .await + .unwrap(); + + let rows = srv + .query_rows("SELECT nonexistent_col FROM uc_schemaless") + .await + .expect("an unknown identifier on a schemaless collection must not error"); + assert_eq!(rows.len(), 2); + for row in &rows { + assert!(row[0].is_empty(), "expected NULL, got {row:?}"); + } + + let count = srv + .query_text("SELECT count(*) FROM uc_schemaless WHERE nonexistent_col IS NULL") + .await + .unwrap(); + assert_eq!(count, vec!["2".to_string()]); +} + +/// An undefined table still reports `42P01`, distinct from the `42703` +/// undefined-column path this file otherwise covers. +#[tokio::test] +async fn undefined_table_still_errors_42p01() { + let srv = TestServer::start().await; + srv.expect_error("SELECT * FROM uc_this_collection_does_not_exist", "42P01") + .await; +} + +/// An aggregate call's argument resolves in the same scope as any other +/// expression. An unknown column inside `SUM(...)` raises `42703`. +#[tokio::test] +async fn aggregate_argument_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_agg_arg").await; + srv.expect_error("SELECT SUM(nonexistent_col) FROM uc_agg_arg", "42703") + .await; +} + +/// The same check inside a grouped aggregate. +#[tokio::test] +async fn grouped_aggregate_argument_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_agg_grouped").await; + srv.expect_error( + "SELECT a, AVG(nonexistent_col) FROM uc_agg_grouped GROUP BY a", + "42703", + ) + .await; +} + +/// `COUNT(DISTINCT ...)` takes the same argument path. +#[tokio::test] +async fn count_distinct_unknown_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_agg_distinct").await; + srv.expect_error( + "SELECT COUNT(DISTINCT nonexistent_col) FROM uc_agg_distinct", + "42703", + ) + .await; +} + +/// Positive control: a declared column in an aggregate still aggregates. +#[tokio::test] +async fn aggregate_argument_declared_column_still_works() { + let srv = TestServer::start().await; + seed_strict(&srv, "uc_agg_ok").await; + let rows = srv + .query_text("SELECT SUM(b) FROM uc_agg_ok") + .await + .expect("SUM over a declared column must succeed"); + assert_eq!(rows.len(), 1); +} diff --git a/nodedb/tests/wire/cases/sql_undefined_column_dml.rs b/nodedb/tests/wire/cases/sql_undefined_column_dml.rs new file mode 100644 index 000000000..073626fd6 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_undefined_column_dml.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! A write predicate or assignment target naming no column of a +//! closed-schema collection raises SQLSTATE `42703` at plan time. +//! Each test asserts the error and that stored rows are unchanged. +//! A silently-skipped write and a silently-wiped table are the two +//! outcomes guarded against. + +use crate::harness::TestServer; + +async fn seed_strict(server: &TestServer, name: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (1, 10)")) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (2, 20)")) + .await + .unwrap(); +} + +async fn rows_of(server: &TestServer, name: &str) -> Vec> { + server + .query_rows(&format!("SELECT a, b FROM {name} ORDER BY a")) + .await + .unwrap() +} + +#[tokio::test] +async fn update_where_unknown_column_errors_and_leaves_rows_unchanged() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_upd_where").await; + + srv.expect_error( + "UPDATE ucd_upd_where SET b = 99 WHERE nonexistent_col = 1", + "42703", + ) + .await; + + let rows = rows_of(&srv, "ucd_upd_where").await; + assert_eq!(rows, vec![vec!["1", "10"], vec!["2", "20"]]); +} + +#[tokio::test] +async fn update_set_unknown_target_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_upd_target").await; + + srv.expect_error( + "UPDATE ucd_upd_target SET nonexistent_col = 1 WHERE a = 1", + "42703", + ) + .await; + + let rows = rows_of(&srv, "ucd_upd_target").await; + assert_eq!(rows, vec![vec!["1", "10"], vec!["2", "20"]]); +} + +#[tokio::test] +async fn update_set_unknown_rhs_errors_and_leaves_column_unchanged() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_upd_rhs").await; + + srv.expect_error( + "UPDATE ucd_upd_rhs SET b = nonexistent_col WHERE a = 1", + "42703", + ) + .await; + + let rows = rows_of(&srv, "ucd_upd_rhs").await; + assert_eq!(rows[0], vec!["1", "10"]); + assert_ne!(rows[0][1], "", "b must keep its value"); +} + +#[tokio::test] +async fn delete_where_unknown_column_errors_and_row_count_unchanged() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_del_where").await; + + srv.expect_error( + "DELETE FROM ucd_del_where WHERE nonexistent_col = 1", + "42703", + ) + .await; + + let rows = rows_of(&srv, "ucd_del_where").await; + assert_eq!(rows.len(), 2); +} + +/// The catastrophic full-wipe shape: an unknown column folding to NULL +/// turns `IS NULL` into an unqualified `DELETE`. Every original row +/// must survive. +#[tokio::test] +async fn delete_where_is_null_on_unknown_column_errors_and_wipes_nothing() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_del_isnull").await; + + srv.expect_error( + "DELETE FROM ucd_del_isnull WHERE nonexistent_col IS NULL", + "42703", + ) + .await; + + let rows = rows_of(&srv, "ucd_del_isnull").await; + assert_eq!(rows, vec![vec!["1", "10"], vec!["2", "20"]]); +} + +#[tokio::test] +async fn kv_update_where_unknown_column_errors_and_value_unchanged() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION ucd_kv_upd (k TEXT PRIMARY KEY, v TEXT) WITH (engine = 'kv')") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_kv_upd (k, v) VALUES ('k1', 'orig')") + .await + .unwrap(); + + srv.expect_error( + "UPDATE ucd_kv_upd SET v = 'changed' WHERE nonexistent_col = 'x'", + "42703", + ) + .await; + + let rows = srv.query_rows("SELECT k, v FROM ucd_kv_upd").await.unwrap(); + assert_eq!(rows, vec![vec!["k1", "orig"]]); +} + +#[tokio::test] +async fn kv_delete_where_unknown_column_errors_and_row_survives() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION ucd_kv_del (k TEXT PRIMARY KEY, v TEXT) WITH (engine = 'kv')") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_kv_del (k, v) VALUES ('k1', 'orig')") + .await + .unwrap(); + + srv.expect_error( + "DELETE FROM ucd_kv_del WHERE nonexistent_col = 'x'", + "42703", + ) + .await; + + let rows = srv.query_rows("SELECT k FROM ucd_kv_del").await.unwrap(); + assert_eq!(rows, vec![vec!["k1"]]); +} + +/// `INSERT ... SELECT` plans its source through the ordinary SELECT path, so an +/// unknown column in the source `WHERE` must raise `42703`. The target stays +/// empty: an unknown column folding to NULL copies zero rows and reports success. +#[tokio::test] +async fn insert_select_unknown_source_column_errors_and_target_stays_empty() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_insel_src").await; + srv.exec( + "CREATE COLLECTION ucd_insel_dst (a INT4 PRIMARY KEY) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.expect_error( + "INSERT INTO ucd_insel_dst SELECT * FROM ucd_insel_src WHERE nonexistent_col = 1", + "42703", + ) + .await; + + let rows = rows_of_single(&srv, "ucd_insel_dst").await; + assert!(rows.is_empty(), "target must stay empty, got {rows:?}"); +} + +async fn rows_of_single(server: &TestServer, name: &str) -> Vec> { + server + .query_rows(&format!("SELECT a FROM {name}")) + .await + .unwrap() +} + +/// An explicit target column list does not open the source projection: an +/// unknown column named in the `SELECT` list must still raise `42703`, and +/// the target stays empty. +#[tokio::test] +async fn insert_select_explicit_projection_unknown_column_errors_and_target_stays_empty() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_insel_proj_src").await; + srv.exec( + "CREATE COLLECTION ucd_insel_proj_dst (a INT4 PRIMARY KEY) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.expect_error( + "INSERT INTO ucd_insel_proj_dst (a) SELECT nonexistent_col FROM ucd_insel_proj_src", + "42703", + ) + .await; + + let rows = rows_of_single(&srv, "ucd_insel_proj_dst").await; + assert!(rows.is_empty(), "target must stay empty, got {rows:?}"); +} + +/// An explicit target column list does not open the source `WHERE` clause +/// either: an unknown column there must raise `42703`, and the target +/// stays empty. +#[tokio::test] +async fn insert_select_explicit_projection_unknown_where_column_errors_and_target_stays_empty() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_insel_projwhere_src").await; + srv.exec( + "CREATE COLLECTION ucd_insel_projwhere_dst (a INT4 PRIMARY KEY) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.expect_error( + "INSERT INTO ucd_insel_projwhere_dst (a) SELECT a FROM ucd_insel_projwhere_src WHERE nonexistent_col = 1", + "42703", + ) + .await; + + let rows = rows_of_single(&srv, "ucd_insel_projwhere_dst").await; + assert!(rows.is_empty(), "target must stay empty, got {rows:?}"); +} + +/// Positive control: an explicit target column list over a real source +/// column succeeds and copies every source row. +#[tokio::test] +async fn insert_select_explicit_projection_known_column_copies_all_rows() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_insel_ok_src").await; + srv.exec( + "CREATE COLLECTION ucd_insel_ok_dst (a INT4 PRIMARY KEY) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.exec("INSERT INTO ucd_insel_ok_dst (a) SELECT a FROM ucd_insel_ok_src") + .await + .unwrap(); + + let rows = srv + .query_rows("SELECT a FROM ucd_insel_ok_dst ORDER BY a") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"], vec!["2"]]); +} + +/// Positive control: a `WHERE` clause on a real source column copies +/// exactly the matching row. +#[tokio::test] +async fn insert_select_explicit_projection_known_where_copies_matching_row() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_insel_okwhere_src").await; + srv.exec( + "CREATE COLLECTION ucd_insel_okwhere_dst (a INT4 PRIMARY KEY) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + + srv.exec( + "INSERT INTO ucd_insel_okwhere_dst (a) SELECT a FROM ucd_insel_okwhere_src WHERE a = 1", + ) + .await + .unwrap(); + + let rows = srv + .query_rows("SELECT a FROM ucd_insel_okwhere_dst") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"]]); +} + +async fn create_merge_target(server: &TestServer) { + server + .exec( + "CREATE COLLECTION ucd_merge_target (\ + id TEXT PRIMARY KEY, \ + name TEXT, \ + score INT) WITH (engine='document_strict')", + ) + .await + .unwrap(); +} + +async fn create_merge_source(server: &TestServer) { + server + .exec( + "CREATE COLLECTION ucd_merge_source (\ + id TEXT PRIMARY KEY, \ + name TEXT, \ + score INT) WITH (engine='document_strict')", + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn merge_on_condition_unknown_column_errors() { + let srv = TestServer::start().await; + create_merge_target(&srv).await; + create_merge_source(&srv).await; + srv.exec("INSERT INTO ucd_merge_target (id, name, score) VALUES ('a', 'alpha', 10)") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_merge_source (id, name, score) VALUES ('a', 'ALPHA_UPD', 99)") + .await + .unwrap(); + + srv.expect_error( + "MERGE INTO ucd_merge_target t \ + USING ucd_merge_source s ON t.nonexistent_col = s.id \ + WHEN MATCHED THEN UPDATE SET name = s.name", + "42703", + ) + .await; +} + +#[tokio::test] +async fn merge_when_matched_update_set_unknown_column_errors() { + let srv = TestServer::start().await; + create_merge_target(&srv).await; + create_merge_source(&srv).await; + srv.exec("INSERT INTO ucd_merge_target (id, name, score) VALUES ('a', 'alpha', 10)") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_merge_source (id, name, score) VALUES ('a', 'ALPHA_UPD', 99)") + .await + .unwrap(); + + srv.expect_error( + "MERGE INTO ucd_merge_target t \ + USING ucd_merge_source s ON t.id = s.id \ + WHEN MATCHED THEN UPDATE SET nonexistent_col = s.name", + "42703", + ) + .await; +} + +/// Positive control: UPDATE on a real column applies on `document_strict`. +#[tokio::test] +async fn update_on_declared_column_still_applies() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_upd_ok").await; + + srv.exec("UPDATE ucd_upd_ok SET b = 99 WHERE a = 1") + .await + .unwrap(); + + let rows = rows_of(&srv, "ucd_upd_ok").await; + assert_eq!(rows, vec![vec!["1", "99"], vec!["2", "20"]]); +} + +/// Positive control: DELETE on a real column removes exactly the matching +/// row. +#[tokio::test] +async fn delete_on_declared_column_still_removes_matching_row() { + let srv = TestServer::start().await; + seed_strict(&srv, "ucd_del_ok").await; + + srv.exec("DELETE FROM ucd_del_ok WHERE a = 1") + .await + .unwrap(); + + let rows = rows_of(&srv, "ucd_del_ok").await; + assert_eq!(rows, vec![vec!["2", "20"]]); +} + +/// The schemaless engine treats undeclared fields as NULL by design, so an +/// `UPDATE ... WHERE nonexistent_col IS NULL` succeeds and matches every +/// row instead of erroring. This is the deliberate open-schema boundary +/// the closed-schema write gate must not cross. +#[tokio::test] +async fn schemaless_update_where_unknown_column_is_null_succeeds() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION ucd_schemaless (id INT PRIMARY KEY, x INT)") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_schemaless (id, x) VALUES (1, 1)") + .await + .unwrap(); + srv.exec("INSERT INTO ucd_schemaless (id, x) VALUES (2, 1)") + .await + .unwrap(); + + srv.exec("UPDATE ucd_schemaless SET x = 99 WHERE nonexistent_col IS NULL") + .await + .expect("an unknown identifier folding to NULL must not error on schemaless"); + + let rows = srv + .query_rows("SELECT id, x FROM ucd_schemaless ORDER BY id") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1", "99"], vec!["2", "99"]]); +} From 97852e955f933de46d4443c31fc4b10cacb1f59e Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:36 +0800 Subject: [PATCH 5/7] feat(sql): support EXISTS subqueries and INSERT ... SELECT projections Two statement shapes the planner refused, both reachable from the column gate's own coverage. EXISTS and NOT EXISTS previously errored 42601 in every form. They now plan as semi and anti joins. The inner scope chains to the outer, so a correlated reference resolves and an unknown column on either side raises. A correlation splits into join keys by relation membership rather than operand position, and SubqueryJoin carries several keys instead of silently keeping the first. Shapes without an equality key are refused by name: correlated inequality, correlation under OR, and set operations. INSERT ... SELECT accepted only SELECT *. It now binds an arbitrary source projection to the target column list, carrying the binding as a column map on the plan, the physical op, and WAL replication, and shaping each copied row before the target surrogate is assigned. An arity mismatch raises. --- .../src/physical_plan/document/op.rs | 4 + nodedb-sql/src/planner/catalog_fold.rs | 2 + .../planner/dml_helpers/insert_select_bind.rs | 122 ++++++++ nodedb-sql/src/planner/dml_helpers/mod.rs | 3 + nodedb-sql/src/planner/subquery/exists.rs | 252 ++++++++++++++++ nodedb-sql/src/types/plan/variants.rs | 6 + .../src/visitor/plan_visitor/dispatch.rs | 3 +- .../src/visitor/plan_visitor/trait_def.rs | 1 + nodedb/src/control/insert_select/copy_rows.rs | 50 +++- .../control/insert_select/expand_staged.rs | 5 + .../src/control/insert_select/orchestrator.rs | 65 ++-- .../planner/sql_plan_convert/aggregate/mod.rs | 2 +- .../sql_plan_convert/aggregate/plan.rs | 2 + .../sql_plan_convert/aggregate/projection.rs | 32 +- .../control/planner/sql_plan_convert/expr.rs | 2 + .../planner/sql_plan_convert/set_ops.rs | 23 +- .../sql_plan_convert/visitor/arms_set_ops.rs | 9 +- .../native/dispatch/plan_builder/document.rs | 3 + .../shared/authorization/requirements.rs | 1 + nodedb/src/control/server/shared/returning.rs | 1 + .../predicate/txn_buffering/classify.rs | 3 + .../wal_replication/decode/document.rs | 2 + .../wal_replication/decode/entry_document.rs | 2 + .../wal_replication/encode/document.rs | 2 + .../wal_replication/encode/entry_document.rs | 2 + .../wal_replication/types/replicated_write.rs | 3 + .../cases/sql_undefined_column_subquery.rs | 282 ++++++++++++++++++ 27 files changed, 843 insertions(+), 41 deletions(-) create mode 100644 nodedb-sql/src/planner/dml_helpers/insert_select_bind.rs create mode 100644 nodedb-sql/src/planner/subquery/exists.rs create mode 100644 nodedb/tests/wire/cases/sql_undefined_column_subquery.rs diff --git a/nodedb-physical/src/physical_plan/document/op.rs b/nodedb-physical/src/physical_plan/document/op.rs index 09c3ab826..6dadee6c7 100644 --- a/nodedb-physical/src/physical_plan/document/op.rs +++ b/nodedb-physical/src/physical_plan/document/op.rs @@ -337,6 +337,10 @@ pub enum DocumentOp { source_collection: QualifiedCollection, source_filters: Vec, source_limit: usize, + /// zerompk-encoded `Vec`: one entry per target column, + /// its `alias` the target column name and its `expr` the source-row + /// expression. Empty means copy each source row unchanged. + column_map: Vec, }, /// Upsert: insert or merge. When `on_conflict_updates` is non-empty, diff --git a/nodedb-sql/src/planner/catalog_fold.rs b/nodedb-sql/src/planner/catalog_fold.rs index c2fee67c0..d29db868b 100644 --- a/nodedb-sql/src/planner/catalog_fold.rs +++ b/nodedb-sql/src/planner/catalog_fold.rs @@ -244,10 +244,12 @@ fn walk_plan( target, source, limit, + column_map, } => SqlPlan::InsertSelect { target, source: Box::new(walk_plan(*source, catalog, database_id, tenant_id)), limit, + column_map, }, SqlPlan::Aggregate { diff --git a/nodedb-sql/src/planner/dml_helpers/insert_select_bind.rs b/nodedb-sql/src/planner/dml_helpers/insert_select_bind.rs new file mode 100644 index 000000000..43bdc052c --- /dev/null +++ b/nodedb-sql/src/planner/dml_helpers/insert_select_bind.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Target-column binding for `INSERT ... SELECT`. + +use sqlparser::ast; + +use crate::error::{Result, SqlError}; +use crate::parser::normalize::normalize_ident; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +/// Bind a source `SELECT` list to the target column list. +/// +/// `target_columns` is the explicit `INSERT INTO t (a, b)` list, empty when +/// the statement gives none. An empty target list with a `SELECT *` source +/// copies the row unchanged and needs no per-column expression. +pub(crate) fn bind_insert_select_columns( + catalog: &dyn SqlCatalog, + target_columns: &[String], + select: &ast::Select, + target: &CollectionInfo, +) -> Result> { + let source_is_star = select.projection.iter().all(|item| { + matches!( + item, + ast::SelectItem::Wildcard(_) | ast::SelectItem::QualifiedWildcard(..) + ) + }); + + if source_is_star { + if target_columns.is_empty() { + return Ok(Vec::new()); + } + return Err(SqlError::Unsupported { + detail: "INSERT ... SELECT * cannot bind to an explicit target column list; name the \ + source columns explicitly" + .into(), + }); + } + + let names = target_names(target_columns, select)?; + + if !target.open_schema { + for name in &names { + if !target.columns.iter().any(|c| &c.name == name) { + return Err(SqlError::UnknownColumn { + table: target.name.clone(), + column: name.clone(), + }); + } + } + } + + // The source scope gates every column the SELECT list names: one the + // source does not carry raises `UnknownColumn` here, at plan time. + let source_scope = TableScope::resolve_from(catalog, &select.from)?; + let scope = ColumnScope::Relations(&source_scope); + + let mut bound = Vec::with_capacity(names.len()); + for (name, item) in names.into_iter().zip(select.projection.iter()) { + bound.push((name, convert_expr(projection_expr(item)?, &scope)?)); + } + Ok(bound) +} + +/// The target column each projection item writes, in projection order. +fn target_names(target_columns: &[String], select: &ast::Select) -> Result> { + if !target_columns.is_empty() { + if target_columns.len() != select.projection.len() { + return Err(SqlError::Arity { + detail: format!( + "INSERT has {} target columns but the SELECT list has {} expressions", + target_columns.len(), + select.projection.len() + ), + }); + } + return Ok(target_columns.to_vec()); + } + select.projection.iter().map(output_name).collect() +} + +/// The output name of a projection item: its alias, else the column it names. +fn output_name(item: &ast::SelectItem) -> Result { + match item { + ast::SelectItem::ExprWithAlias { alias, .. } => Ok(normalize_ident(alias)), + ast::SelectItem::UnnamedExpr(expr) => match expr { + ast::Expr::Identifier(ident) => Ok(normalize_ident(ident)), + ast::Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + Ok(normalize_ident(&parts[1])) + } + _ => Err(unnamed_projection_error()), + }, + ast::SelectItem::ExprWithAliases { .. } + | ast::SelectItem::Wildcard(_) + | ast::SelectItem::QualifiedWildcard(..) => Err(unnamed_projection_error()), + } +} + +fn unnamed_projection_error() -> SqlError { + SqlError::Unsupported { + detail: "INSERT ... SELECT needs a target column for every projected expression; add an \ + alias or an explicit target column list" + .into(), + } +} + +/// The expression a projection item evaluates per source row. +fn projection_expr(item: &ast::SelectItem) -> Result<&ast::Expr> { + match item { + ast::SelectItem::UnnamedExpr(expr) | ast::SelectItem::ExprWithAlias { expr, .. } => { + Ok(expr) + } + ast::SelectItem::ExprWithAliases { .. } + | ast::SelectItem::Wildcard(_) + | ast::SelectItem::QualifiedWildcard(..) => Err(SqlError::Unsupported { + detail: "INSERT ... SELECT cannot mix a wildcard with named projections".into(), + }), + } +} diff --git a/nodedb-sql/src/planner/dml_helpers/mod.rs b/nodedb-sql/src/planner/dml_helpers/mod.rs index 2864c4a58..40fdeb125 100644 --- a/nodedb-sql/src/planner/dml_helpers/mod.rs +++ b/nodedb-sql/src/planner/dml_helpers/mod.rs @@ -7,9 +7,11 @@ //! - [`ast_extract`] — table-name / primary-key point-lookup extraction //! - [`vector_primary_insert`] — vector-primary collection insert plans //! - [`kv_insert`] — KV engine insert plans +//! - [`insert_select_bind`] — `INSERT ... SELECT` target-column binding mod ast_extract; mod insert_columns; +mod insert_select_bind; mod kv_insert; mod range_check; mod value_convert; @@ -18,6 +20,7 @@ mod vector_primary_insert; pub use ast_extract::extract_point_keys; pub(super) use ast_extract::extract_table_name_from_table_with_joins; pub(super) use insert_columns::resolve_insert_columns; +pub(super) use insert_select_bind::bind_insert_select_columns; pub(super) use kv_insert::build_kv_insert_plan; pub(super) use range_check::{ check_declared_float_ranges_in_assignments, check_declared_int_ranges_in_assignments, diff --git a/nodedb-sql/src/planner/subquery/exists.rs b/nodedb-sql/src/planner/subquery/exists.rs new file mode 100644 index 000000000..971a4c205 --- /dev/null +++ b/nodedb-sql/src/planner/subquery/exists.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `EXISTS` / `NOT EXISTS` lowered to a semi / anti join. +//! +//! The inner query becomes the build side. Each AND-ed equality that links an +//! outer relation to an inner one becomes a probe key; every other conjunct +//! stays a local predicate on the inner scan. + +use std::ops::ControlFlow; + +use sqlparser::ast::{self, SetExpr}; + +use crate::error::{Result, SqlError}; +use crate::functions::registry::FunctionRegistry; +use crate::parser::normalize::normalize_ident; +use crate::planner::ast_helpers::{flatten_and_expr, rebuild_and_expr}; +use crate::planner::select::{convert_projection, plan_query}; +use crate::resolver::ColumnScope; +use crate::resolver::columns::TableScope; +use crate::resolver::expr::convert_expr; +use crate::types::*; + +use super::extract::SubqueryJoin; + +const SET_OPERATION: &str = "EXISTS over a set operation (UNION / INTERSECT / EXCEPT) \ + is not supported; wrap the set operation in a derived table and select from that"; + +const OUTER_ONLY_EQUALITY: &str = "an equality naming only outer relations inside EXISTS \ + is not supported; move it to the enclosing WHERE clause"; + +const CORRELATION_UNDER_OR: &str = "a correlated reference under OR inside EXISTS \ + is not supported; each correlation must be an AND-ed equality"; + +/// Plan `EXISTS (SELECT ... )` as a semi join, `NOT EXISTS` as an anti join. +/// +/// `outer` is the enclosing query's scope. The inner FROM clause resolves +/// nested inside it, so a qualifier naming no inner relation reaches the outer +/// query instead of erroring as an unknown relation. +pub(super) fn plan_exists_subquery( + subquery: &ast::Query, + negated: bool, + outer: &TableScope, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result { + let SetExpr::Select(select) = &*subquery.body else { + return Err(SqlError::Unsupported { + detail: SET_OPERATION.into(), + }); + }; + + let local = TableScope::resolve_from(catalog, &select.from)?; + let nested = local.clone().nested_in(outer.clone()); + + // EXISTS discards the projected values. The conversion still runs so a + // column the inner SELECT list names but no relation declares is rejected. + convert_projection(&select.projection, &nested)?; + + let mut on = Vec::new(); + let mut local_conjuncts = Vec::new(); + if let Some(where_expr) = &select.selection { + let mut conjuncts = Vec::new(); + flatten_and_expr(where_expr, &mut conjuncts); + for conjunct in conjuncts { + // Converting against the nested scope gates both inner and outer + // column names before the conjunct is classified. + convert_expr(&conjunct, &ColumnScope::Relations(&nested))?; + match correlation_pair(&conjunct, &local, outer)? { + Some(pair) => on.push(pair), + None => local_conjuncts.push(conjunct), + } + } + } + + let inner_plan = plan_local_query( + subquery, + select, + local_conjuncts, + catalog, + functions, + temporal, + )?; + + Ok(SubqueryJoin { + on, + inner_plan, + join_type: if negated { + JoinType::Anti + } else { + JoinType::Semi + }, + }) +} + +/// Plan the inner query carrying only its local predicates. +/// +/// The projection widens to `*`: EXISTS ignores the values, but the scan must +/// still carry the correlation key columns for the join probe. +fn plan_local_query( + subquery: &ast::Query, + select: &ast::Select, + local_conjuncts: Vec, + catalog: &dyn SqlCatalog, + functions: &FunctionRegistry, + temporal: crate::TemporalScope, +) -> Result { + let mut inner_select = select.clone(); + inner_select.projection = vec![ast::SelectItem::Wildcard( + ast::WildcardAdditionalOptions::default(), + )]; + inner_select.selection = if local_conjuncts.is_empty() { + None + } else { + Some(rebuild_and_expr(local_conjuncts)) + }; + + let mut inner_query = subquery.clone(); + *inner_query.body = SetExpr::Select(Box::new(inner_select)); + plan_query(&inner_query, catalog, functions, temporal) +} + +/// Which query a column reference belongs to. +enum Side { + /// A column of a relation in the subquery's own FROM clause. + Inner(String), + /// A column of a relation in the enclosing query. + Outer(String), + /// Anything else: a literal, a function call, a computed expression. + Other, +} + +/// Classify one conjunct of the inner WHERE clause. +/// +/// Returns the `(outer column, inner column)` probe key for a correlation, or +/// `None` for a predicate local to the inner query. A conjunct that reads an +/// outer relation in any other shape raises a typed error naming that shape. +fn correlation_pair( + expr: &ast::Expr, + inner: &TableScope, + outer: &TableScope, +) -> Result> { + let bare = unnest(expr); + if let ast::Expr::BinaryOp { left, op, right } = bare { + let l = classify(left, inner, outer); + let r = classify(right, inner, outer); + match (op, &l, &r) { + (ast::BinaryOperator::Eq, Side::Inner(ic), Side::Outer(oc)) + | (ast::BinaryOperator::Eq, Side::Outer(oc), Side::Inner(ic)) => { + return Ok(Some((oc.clone(), ic.clone()))); + } + (ast::BinaryOperator::Eq, Side::Outer(_), Side::Outer(_)) => { + return Err(SqlError::Unsupported { + detail: OUTER_ONLY_EQUALITY.into(), + }); + } + (ast::BinaryOperator::Or, _, _) + if references_outer(left, inner, outer) + || references_outer(right, inner, outer) => + { + return Err(SqlError::Unsupported { + detail: CORRELATION_UNDER_OR.into(), + }); + } + (op, Side::Inner(_), Side::Outer(_)) | (op, Side::Outer(_), Side::Inner(_)) + if is_inequality(op) => + { + return Err(SqlError::Unsupported { + detail: format!( + "a correlated inequality ('{op}') inside EXISTS is not supported; \ + a semi join probes on equality keys only" + ), + }); + } + _ => {} + } + } + + if references_outer(bare, inner, outer) { + return Err(SqlError::Unsupported { + detail: format!( + "the correlated predicate '{expr}' inside EXISTS is not supported; \ + a correlation must be an AND-ed equality between one outer column \ + and one inner column" + ), + }); + } + Ok(None) +} + +/// Decide which query a column reference belongs to by relation membership. +/// +/// A qualifier names the relation directly. A bare name goes to whichever +/// scope resolves it, inner first, matching the resolution order the +/// expression converter already applied. +fn classify(expr: &ast::Expr, inner: &TableScope, outer: &TableScope) -> Side { + match expr { + ast::Expr::CompoundIdentifier(parts) if parts.len() == 2 => { + let qualifier = normalize_ident(&parts[0]); + let column = normalize_ident(&parts[1]); + if inner.table_by_ref(&qualifier).is_some() { + Side::Inner(column) + } else if outer.table_by_ref(&qualifier).is_some() { + Side::Outer(column) + } else { + Side::Other + } + } + ast::Expr::Identifier(ident) => { + let column = normalize_ident(ident); + if inner.check_name(None, &column).is_ok() { + Side::Inner(column) + } else if outer.check_name(None, &column).is_ok() { + Side::Outer(column) + } else { + Side::Other + } + } + ast::Expr::Nested(nested) => classify(nested, inner, outer), + _ => Side::Other, + } +} + +/// Whether any column reference anywhere in `expr` names an outer relation. +fn references_outer(expr: &ast::Expr, inner: &TableScope, outer: &TableScope) -> bool { + let walk = ast::visit_expressions(expr, |node| { + if matches!(classify(node, inner, outer), Side::Outer(_)) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }); + walk.is_break() +} + +fn unnest(expr: &ast::Expr) -> &ast::Expr { + match expr { + ast::Expr::Nested(inner) => unnest(inner), + other => other, + } +} + +fn is_inequality(op: &ast::BinaryOperator) -> bool { + matches!( + op, + ast::BinaryOperator::Gt + | ast::BinaryOperator::GtEq + | ast::BinaryOperator::Lt + | ast::BinaryOperator::LtEq + | ast::BinaryOperator::NotEq + ) +} diff --git a/nodedb-sql/src/types/plan/variants.rs b/nodedb-sql/src/types/plan/variants.rs index f17e154ef..5e539e7b8 100644 --- a/nodedb-sql/src/types/plan/variants.rs +++ b/nodedb-sql/src/types/plan/variants.rs @@ -160,6 +160,12 @@ pub enum SqlPlan { target: String, source: Box, limit: usize, + /// `(target_column, source_expression)`, in target-column order. + /// + /// Empty means passthrough: `INSERT INTO t SELECT * FROM s` copies + /// each source row unchanged. Non-empty means every target column is + /// materialized from the paired expression over the source row. + column_map: Vec<(String, SqlExpr)>, }, Update { collection: String, diff --git a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs index 20be52ad7..1357c1b1d 100644 --- a/nodedb-sql/src/visitor/plan_visitor/dispatch.rs +++ b/nodedb-sql/src/visitor/plan_visitor/dispatch.rs @@ -135,7 +135,8 @@ pub fn dispatch(visitor: &mut V, plan: &SqlPlan) -> Result visitor.insert_select(target, source, *limit), + column_map, + } => visitor.insert_select(target, source, *limit, column_map), SqlPlan::Update { collection, engine, diff --git a/nodedb-sql/src/visitor/plan_visitor/trait_def.rs b/nodedb-sql/src/visitor/plan_visitor/trait_def.rs index 88427f310..6a4ce0b0b 100644 --- a/nodedb-sql/src/visitor/plan_visitor/trait_def.rs +++ b/nodedb-sql/src/visitor/plan_visitor/trait_def.rs @@ -91,6 +91,7 @@ pub trait PlanVisitor { target: &str, source: &SqlPlan, limit: usize, + column_map: &[(String, SqlExpr)], ) -> Result; /// Handle [`SqlPlan::Update`]. diff --git a/nodedb/src/control/insert_select/copy_rows.rs b/nodedb/src/control/insert_select/copy_rows.rs index e45d8f34c..be1376484 100644 --- a/nodedb/src/control/insert_select/copy_rows.rs +++ b/nodedb/src/control/insert_select/copy_rows.rs @@ -12,6 +12,7 @@ use nodedb_types::{DatabaseId, Surrogate, TenantId}; +use crate::bridge::expr_eval::ComputedColumn; use crate::bridge::scan_filter::ScanFilter; use crate::control::state::SharedState; use crate::control::target_identity::{ @@ -25,6 +26,9 @@ pub(crate) struct CopySpec { pub target_pk: TargetPk, /// Residual source `WHERE` predicate (deserialized `Vec`). pub filters: Vec, + /// One entry per target column, its `alias` the target column name and its + /// `expr` the source-row expression. Empty copies each row unchanged. + pub column_map: Vec, } /// Resolve the target PK and the residual source `WHERE` filter for one @@ -38,6 +42,7 @@ pub(crate) fn resolve_copy_spec( database_id: DatabaseId, target_collection: &str, source_filters: &[u8], + column_map: &[u8], ) -> crate::Result { let catalog = state.credentials.catalog(); @@ -62,7 +67,20 @@ pub(crate) fn resolve_copy_spec( })? }; - Ok(CopySpec { target_pk, filters }) + let column_map: Vec = if column_map.is_empty() { + Vec::new() + } else { + zerompk::from_msgpack(column_map).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("insert-select column map: {e}"), + })? + }; + + Ok(CopySpec { + target_pk, + filters, + column_map, + }) } /// Filter and assign fresh surrogates for one scanned source page. @@ -91,6 +109,13 @@ pub(crate) fn assign_page_rows( { continue; } + // Shape the source row into the target's column set. An empty map is + // passthrough — the source body is already the target body. + let value = if spec.column_map.is_empty() { + value + } else { + shape_row(&value, &spec.column_map)? + }; let surrogate = assign_target_surrogate( state, database_id, @@ -104,3 +129,26 @@ pub(crate) fn assign_page_rows( } Ok(out) } + +/// Evaluate each `(target_column, expression)` pair against the source row and +/// re-encode as the target body. A source column absent from the row yields +/// SQL NULL, matching every other projection path. +fn shape_row(source: &[u8], column_map: &[ComputedColumn]) -> crate::Result> { + let row = + nodedb_types::value_from_msgpack(source).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("insert-select source row: {e}"), + })?; + let mut shaped = std::collections::HashMap::with_capacity(column_map.len()); + for column in column_map { + // A division/modulo-by-zero in a bound expression fails the statement + // instead of silently writing NULL into the target. + shaped.insert(column.alias.clone(), column.expr.eval(&row)?); + } + nodedb_types::value_to_msgpack(&nodedb_types::Value::Object(shaped)).map_err(|e| { + crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("insert-select target row: {e}"), + } + }) +} diff --git a/nodedb/src/control/insert_select/expand_staged.rs b/nodedb/src/control/insert_select/expand_staged.rs index 251518b4d..bd2e1f1c7 100644 --- a/nodedb/src/control/insert_select/expand_staged.rs +++ b/nodedb/src/control/insert_select/expand_staged.rs @@ -42,6 +42,7 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( source_collection, source_filters, source_limit, + column_map, }) = &task.plan else { // Callers only pass an `InsertSelect` task; a mismatch is a bug. @@ -61,6 +62,7 @@ pub(crate) async fn resolve_and_emit_insert_select_ops( source_collection: source_collection.as_str(), source_filters, source_limit: *source_limit, + column_map, txn_id: task.txn_id, }, ) @@ -121,6 +123,7 @@ struct MaterializeCopy<'a> { source_collection: &'a str, source_filters: &'a [u8], source_limit: usize, + column_map: &'a [u8], txn_id: Option, } @@ -140,6 +143,7 @@ async fn materialize_copy( source_collection, source_filters, source_limit, + column_map, txn_id, } = args; let spec = resolve_copy_spec( @@ -148,6 +152,7 @@ async fn materialize_copy( database_id, target_collection, source_filters, + column_map, )?; let mut cursor: Vec = Vec::new(); diff --git a/nodedb/src/control/insert_select/orchestrator.rs b/nodedb/src/control/insert_select/orchestrator.rs index febb1d7bd..2377e1ce6 100644 --- a/nodedb/src/control/insert_select/orchestrator.rs +++ b/nodedb/src/control/insert_select/orchestrator.rs @@ -34,6 +34,7 @@ pub async fn run_authorized_insert_select( source_collection, source_filters, source_limit, + column_map, }) = task.plan else { return Err(crate::Error::BadRequest { @@ -44,20 +45,35 @@ pub async fn run_authorized_insert_select( state, task.tenant_id, task.database_id, - target_collection.as_str(), - source_collection.as_str(), - &source_filters, - source_limit, + &CopyRequest { + target_collection: target_collection.as_str(), + source_collection: source_collection.as_str(), + source_filters: &source_filters, + source_limit, + column_map: &column_map, + }, ) .await } +/// The plan-derived operands of one `INSERT ... SELECT` copy. +pub(crate) struct CopyRequest<'a> { + pub target_collection: &'a str, + pub source_collection: &'a str, + /// Serialized `Vec` residual `WHERE` predicate. + pub source_filters: &'a [u8], + /// Bounds how many source rows are copied. + pub source_limit: usize, + /// Serialized `Vec` shaping each copied row. Empty for a + /// plain copy. + pub column_map: &'a [u8], +} + /// Drive an `INSERT ... SELECT` from `source_collection` into `target_collection`. /// -/// `target_collection` / `source_collection` are the (db-qualified) collection -/// names as they appear in the `DocumentOp::InsertSelect` plan. `source_filters` -/// is the serialized `Vec` residual `WHERE` predicate; `source_limit` -/// bounds how many source rows are copied. +/// The copy operands travel in `CopyRequest`: `target_collection` / +/// `source_collection` are the (db-qualified) collection names as they +/// appear in the `DocumentOp::InsertSelect` plan. /// /// Returns a `{"inserted": N}` response mirroring the shape the autocommit /// dispatch loops shape as an `INSERT` command tag. @@ -65,21 +81,19 @@ pub(crate) async fn run_insert_select( state: &SharedState, tenant_id: TenantId, database_id: DatabaseId, - target_collection: &str, - source_collection: &str, - source_filters: &[u8], - source_limit: usize, + req: &CopyRequest<'_>, ) -> crate::Result { let spec = resolve_copy_spec( state, tenant_id, database_id, - target_collection, - source_filters, + req.target_collection, + req.source_filters, + req.column_map, )?; let mut cursor: Vec = Vec::new(); - let mut remaining = source_limit; + let mut remaining = req.source_limit; let mut total_inserted: usize = 0; let mut max_lsn = Lsn::ZERO; @@ -89,7 +103,7 @@ pub(crate) async fn run_insert_select( state, tenant_id, database_id, - source_collection, + req.source_collection, &cursor, None, None, @@ -102,7 +116,7 @@ pub(crate) async fn run_insert_select( state, tenant_id, database_id, - target_collection, + req.target_collection, &spec, entries, &mut remaining, @@ -127,7 +141,7 @@ pub(crate) async fn run_insert_select( crate::control::planner::materialized_sum::resolve_sum_targets_for_bodies( state, &page_bodies, - target_collection, + req.target_collection, tenant_id, database_id, crate::types::TraceId::ZERO, @@ -136,7 +150,7 @@ pub(crate) async fn run_insert_select( let plan = PhysicalPlan::Document(DocumentOp::BatchInsert { collection: nodedb_types::QualifiedCollection::from_stored( - target_collection.to_string(), + req.target_collection.to_string(), ), documents, surrogates, @@ -151,8 +165,15 @@ pub(crate) async fn run_insert_select( // deferred to a sibling task. deferred_sum_targets: Vec::new(), }); - let resp = dispatch_local(state, tenant_id, database_id, target_collection, plan, None) - .await?; + let resp = dispatch_local( + state, + tenant_id, + database_id, + req.target_collection, + plan, + None, + ) + .await?; if resp.status != Status::Ok { // Atomic page failure (e.g. constraint violation): the page's // rows did not land. Surface the DP error verbatim. @@ -166,7 +187,7 @@ pub(crate) async fn run_insert_select( &state.wal, tenant_id, database_id, - target_collection, + req.target_collection, &resp, )?; total_inserted += decode_inserted(&resp.payload).unwrap_or(page_len); diff --git a/nodedb/src/control/planner/sql_plan_convert/aggregate/mod.rs b/nodedb/src/control/planner/sql_plan_convert/aggregate/mod.rs index 0e96b8fa8..ac20801bd 100644 --- a/nodedb/src/control/planner/sql_plan_convert/aggregate/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/aggregate/mod.rs @@ -18,7 +18,7 @@ pub(in crate::control::planner::sql_plan_convert) use plan::{ }; pub(in crate::control::planner::sql_plan_convert) use projection::{ extract_computed_columns, extract_join_projection_specs, extract_projection_names, - serialize_join_computed_projection, serialize_window_functions, + serialize_column_map, serialize_join_computed_projection, serialize_window_functions, }; pub(in crate::control::planner::sql_plan_convert) use spec::{ agg_expr_to_pair, extract_collection_name, extract_scan_alias, inline_join_side, diff --git a/nodedb/src/control/planner/sql_plan_convert/aggregate/plan.rs b/nodedb/src/control/planner/sql_plan_convert/aggregate/plan.rs index 270dddfe0..2f9c4c3bc 100644 --- a/nodedb/src/control/planner/sql_plan_convert/aggregate/plan.rs +++ b/nodedb/src/control/planner/sql_plan_convert/aggregate/plan.rs @@ -115,6 +115,8 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_aggregate( // Populated by `rls_injection` after conversion, per side. left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), post_set_op: PostSetOp::None, txn_id: None, diff --git a/nodedb/src/control/planner/sql_plan_convert/aggregate/projection.rs b/nodedb/src/control/planner/sql_plan_convert/aggregate/projection.rs index 830697fcd..2591a3d9f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/aggregate/projection.rs +++ b/nodedb/src/control/planner/sql_plan_convert/aggregate/projection.rs @@ -89,9 +89,39 @@ pub(in crate::control::planner::sql_plan_convert) fn serialize_join_computed_pro .ok_or_else(|| crate::Error::BadRequest { detail: "wildcard join projection reached computed-expression lowering".into(), })?; + encode_computed_columns(computed, "join computed projection") +} + +/// Encode a `(target_column, source_expression)` binding as the same +/// `Vec` payload [`serialize_join_computed_projection`] +/// produces. Every pair is kept: a bare column binding still names the target +/// column it writes. Column references stay unqualified — the rows this map +/// shapes come from one collection, so their fields carry bare names. +pub(in crate::control::planner::sql_plan_convert) fn serialize_column_map( + column_map: &[(String, SqlExpr)], +) -> crate::Result> { + if column_map.is_empty() { + return Ok(Vec::new()); + } + let computed: Vec = column_map + .iter() + .map(|(name, expr)| crate::bridge::expr_eval::ComputedColumn { + alias: name.clone(), + expr: sql_expr_to_bridge_expr(expr), + }) + .collect(); + encode_computed_columns(computed, "insert-select column map") +} + +/// Encode a computed-column list as its MessagePack payload. `context` names +/// the caller, so an encode error says which payload failed. +fn encode_computed_columns( + computed: Vec, + context: &str, +) -> crate::Result> { zerompk::to_msgpack_vec(&computed).map_err(|e| crate::Error::Serialization { format: "msgpack".into(), - detail: format!("join computed projection: {e}"), + detail: format!("{context}: {e}"), }) } diff --git a/nodedb/src/control/planner/sql_plan_convert/expr.rs b/nodedb/src/control/planner/sql_plan_convert/expr.rs index 28ff9f101..80119f4ff 100644 --- a/nodedb/src/control/planner/sql_plan_convert/expr.rs +++ b/nodedb/src/control/planner/sql_plan_convert/expr.rs @@ -494,10 +494,12 @@ pub(super) fn inline_cte(plan: &SqlPlan, cte_name: &str, cte_plan: &SqlPlan) -> target, source, limit, + column_map, } => SqlPlan::InsertSelect { target: target.clone(), source: Box::new(inline_cte(source, cte_name, cte_plan)), limit: *limit, + column_map: column_map.clone(), }, // A post-processor produced by an earlier CTE definition: recurse into diff --git a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs index 6d3fcf97d..e671f0b30 100644 --- a/nodedb/src/control/planner/sql_plan_convert/set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/set_ops.rs @@ -2,7 +2,7 @@ //! Set operations and miscellaneous plan conversions (UNION, INTERSECT, EXCEPT, CTE, etc.). -use nodedb_sql::types::{Projection, SortKey, SqlPlan, SqlValue}; +use nodedb_sql::types::{Projection, SortKey, SqlExpr, SqlPlan, SqlValue}; use crate::bridge::envelope::PhysicalPlan; use crate::types::{TenantId, VShardId}; @@ -146,6 +146,7 @@ pub(super) fn convert_except( pub(super) fn convert_insert_select( target: &str, source: &SqlPlan, + column_map: &[(String, SqlExpr)], tenant_id: TenantId, ctx: &ConvertContext, ) -> crate::Result> { @@ -155,7 +156,6 @@ pub(super) fn convert_insert_select( let SqlPlan::Scan { collection, filters, - projection, sort_keys, limit, offset, @@ -169,18 +169,9 @@ pub(super) fn convert_insert_select( }); }; - let projection_is_passthrough = projection.is_empty() - || projection.iter().all(|p| { - matches!(p, Projection::Star) - || matches!(p, Projection::QualifiedStar(name) if name == collection) - }); - - if !projection_is_passthrough - || !sort_keys.is_empty() - || *offset != 0 - || *distinct - || !window_functions.is_empty() - { + // Ordering, offset, distinct, and window functions each need an ordered + // materialization the page-at-a-time copy does not provide. + if !sort_keys.is_empty() || *offset != 0 || *distinct || !window_functions.is_empty() { return Err(crate::Error::PlanError { detail: "INSERT ... SELECT currently supports only SELECT * with optional WHERE/LIMIT" .into(), @@ -188,6 +179,7 @@ pub(super) fn convert_insert_select( } let filter_bytes = super::filter::serialize_filters(filters)?; + let column_map_bytes = super::aggregate::serialize_column_map(column_map)?; let vshard = VShardId::from_collection_in_database(ctx.database_id, target); let qualified_source = nodedb_types::QualifiedCollection::new(ctx.database_id, collection); @@ -200,6 +192,7 @@ pub(super) fn convert_insert_select( source_collection: qualified_source, source_filters: filter_bytes, source_limit: limit.unwrap_or(10_000), + column_map: column_map_bytes, }), post_set_op: PostSetOp::None, txn_id: None, @@ -384,6 +377,7 @@ mod tests { let tasks = convert_insert_select( "batch_copy", &source, + &[], TenantId::new(1), &ConvertContext { purpose: crate::control::planner::sql_plan_convert::PlanningPurpose::Execute, @@ -442,6 +436,7 @@ mod tests { let tasks = convert_insert_select( "batch_copy", &source, + &[], TenantId::new(1), &ConvertContext { purpose: crate::control::planner::sql_plan_convert::PlanningPurpose::Execute, diff --git a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_set_ops.rs b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_set_ops.rs index 746c1b4eb..ac980de8d 100644 --- a/nodedb/src/control/planner/sql_plan_convert/visitor/arms_set_ops.rs +++ b/nodedb/src/control/planner/sql_plan_convert/visitor/arms_set_ops.rs @@ -61,8 +61,15 @@ macro_rules! impl_set_ops_arms_for_convert_visitor { target: &str, source: &nodedb_sql::types::SqlPlan, _limit: usize, + column_map: &[(String, nodedb_sql::types::SqlExpr)], ) -> crate::Result> { - super::super::set_ops::convert_insert_select(target, source, self.tenant_id, self.ctx) + super::super::set_ops::convert_insert_select( + target, + source, + column_map, + self.tenant_id, + self.ctx, + ) } fn cte( diff --git a/nodedb/src/control/server/native/dispatch/plan_builder/document.rs b/nodedb/src/control/server/native/dispatch/plan_builder/document.rs index e7fc4ab54..aec9862c1 100644 --- a/nodedb/src/control/server/native/dispatch/plan_builder/document.rs +++ b/nodedb/src/control/server/native/dispatch/plan_builder/document.rs @@ -443,6 +443,9 @@ pub(crate) fn build_insert_select( source_collection: QualifiedCollection::new(ctx.database_id(), &source), source_filters: filters, source_limit: limit, + // The native text-field form names no projection, so every source row + // copies unchanged. + column_map: Vec::new(), })) } diff --git a/nodedb/src/control/server/shared/authorization/requirements.rs b/nodedb/src/control/server/shared/authorization/requirements.rs index e538788d5..dc7b1e6c6 100644 --- a/nodedb/src/control/server/shared/authorization/requirements.rs +++ b/nodedb/src/control/server/shared/authorization/requirements.rs @@ -55,6 +55,7 @@ mod tests { source_collection: QualifiedCollection::new(DatabaseId::DEFAULT, "source"), source_filters: Vec::new(), source_limit: 0, + column_map: Vec::new(), }); assert_eq!( diff --git a/nodedb/src/control/server/shared/returning.rs b/nodedb/src/control/server/shared/returning.rs index e3973ec41..a8b2cfd6f 100644 --- a/nodedb/src/control/server/shared/returning.rs +++ b/nodedb/src/control/server/shared/returning.rs @@ -375,6 +375,7 @@ mod tests { source_collection: QualifiedCollection::new(DatabaseId::DEFAULT, "src"), source_filters: Vec::new(), source_limit: 0, + column_map: Vec::new(), }); let detail = refuse_unprojectable_insert_returning(&plan) .expect_err("an INSERT ... SELECT cannot carry the clause") diff --git a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs index 4a8790294..ac518e9f7 100644 --- a/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs +++ b/nodedb/src/control/server/shared/write_admission/predicate/txn_buffering/classify.rs @@ -596,6 +596,7 @@ mod tests { source_collection: QualifiedCollection::new(DatabaseId::DEFAULT, "s"), source_filters: Vec::new(), source_limit: 0, + column_map: Vec::new(), }), PhysicalPlan::Document(DocumentOp::Upsert { collection: QualifiedCollection::new(DatabaseId::DEFAULT, "c"), @@ -1563,6 +1564,8 @@ mod tests { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), PhysicalPlan::Query(QueryOp::ShuffleJoinConsume { build_path: String::new(), diff --git a/nodedb/src/control/wal_replication/decode/document.rs b/nodedb/src/control/wal_replication/decode/document.rs index d7d2c1f04..37665003b 100644 --- a/nodedb/src/control/wal_replication/decode/document.rs +++ b/nodedb/src/control/wal_replication/decode/document.rs @@ -361,6 +361,7 @@ pub(super) fn insert_select( source_collection: &str, source_filters: &[u8], source_limit: usize, + column_map: &[u8], ) -> PhysicalPlan { PhysicalPlan::Document(DocumentOp::InsertSelect { target_collection: nodedb_types::QualifiedCollection::from_stored( @@ -371,6 +372,7 @@ pub(super) fn insert_select( ), source_filters: source_filters.to_vec(), source_limit, + column_map: column_map.to_vec(), }) } diff --git a/nodedb/src/control/wal_replication/decode/entry_document.rs b/nodedb/src/control/wal_replication/decode/entry_document.rs index b843e1142..6f83e478d 100644 --- a/nodedb/src/control/wal_replication/decode/entry_document.rs +++ b/nodedb/src/control/wal_replication/decode/entry_document.rs @@ -194,11 +194,13 @@ pub(super) fn decode_arm(ctx: &DecodeCtx, write: &ReplicatedWrite) -> crate::Res source_collection, source_filters, source_limit, + column_map, } => Ok(document::insert_select( target_collection, source_collection, source_filters, *source_limit, + column_map, )), ReplicatedWrite::ApplyBalanceDelta { collection, diff --git a/nodedb/src/control/wal_replication/encode/document.rs b/nodedb/src/control/wal_replication/encode/document.rs index 102dd88cd..150420431 100644 --- a/nodedb/src/control/wal_replication/encode/document.rs +++ b/nodedb/src/control/wal_replication/encode/document.rs @@ -249,12 +249,14 @@ pub(super) fn insert_select( source_collection: &str, source_filters: &[u8], source_limit: usize, + column_map: &[u8], ) -> ReplicatedWrite { ReplicatedWrite::InsertSelect { target_collection: target_collection.to_owned(), source_collection: source_collection.to_owned(), source_filters: source_filters.to_vec(), source_limit, + column_map: column_map.to_vec(), } } diff --git a/nodedb/src/control/wal_replication/encode/entry_document.rs b/nodedb/src/control/wal_replication/encode/entry_document.rs index a966e6046..553103b8a 100644 --- a/nodedb/src/control/wal_replication/encode/entry_document.rs +++ b/nodedb/src/control/wal_replication/encode/entry_document.rs @@ -175,11 +175,13 @@ pub(super) fn document_write(op: &DocumentOp) -> Option { source_collection, source_filters, source_limit, + column_map, } => document::insert_select( target_collection.as_str(), source_collection.as_str(), source_filters, *source_limit, + column_map, ), DocumentOp::BatchInsert { diff --git a/nodedb/src/control/wal_replication/types/replicated_write.rs b/nodedb/src/control/wal_replication/types/replicated_write.rs index 598205341..8cc2c3a92 100644 --- a/nodedb/src/control/wal_replication/types/replicated_write.rs +++ b/nodedb/src/control/wal_replication/types/replicated_write.rs @@ -578,6 +578,9 @@ pub enum ReplicatedWrite { source_collection: String, source_filters: Vec, source_limit: usize, + /// See `DocumentOp::InsertSelect::column_map`. Each replica shapes the + /// rows it copies with the same bindings the leader used. + column_map: Vec, }, CrdtImportCollection { tenant_id: u64, diff --git a/nodedb/tests/wire/cases/sql_undefined_column_subquery.rs b/nodedb/tests/wire/cases/sql_undefined_column_subquery.rs new file mode 100644 index 000000000..e63cecda0 --- /dev/null +++ b/nodedb/tests/wire/cases/sql_undefined_column_subquery.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Unknown column references inside subquery, EXISTS, and LATERAL scopes +//! raise SQLSTATE `42703`, and the derived-alias column set is inferred +//! rather than treated as open. + +use crate::harness::TestServer; + +async fn seed_strict(server: &TestServer, name: &str) { + server + .exec(&format!( + "CREATE COLLECTION {name} (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')" + )) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (1, 10)")) + .await + .unwrap(); + server + .exec(&format!("INSERT INTO {name} (a, b) VALUES (2, 20)")) + .await + .unwrap(); +} + +#[tokio::test] +async fn uncorrelated_exists_unknown_inner_where_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_exists_unc_o").await; + seed_strict(&srv, "sqs_exists_unc_i").await; + srv.expect_error( + "SELECT a FROM sqs_exists_unc_o \ + WHERE EXISTS (SELECT 1 FROM sqs_exists_unc_i WHERE nonexistent_col = 1)", + "42703", + ) + .await; +} + +#[tokio::test] +async fn correlated_exists_unknown_outer_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_exists_outer_o").await; + seed_strict(&srv, "sqs_exists_outer_i").await; + srv.expect_error( + "SELECT a FROM sqs_exists_outer_o AS o \ + WHERE EXISTS (SELECT 1 FROM sqs_exists_outer_i AS i WHERE i.a = o.nonexistent_col)", + "42703", + ) + .await; +} + +#[tokio::test] +async fn correlated_exists_unknown_inner_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_exists_inner_o").await; + seed_strict(&srv, "sqs_exists_inner_i").await; + srv.expect_error( + "SELECT a FROM sqs_exists_inner_o AS o \ + WHERE EXISTS (SELECT 1 FROM sqs_exists_inner_i AS i WHERE i.nonexistent_col = o.a)", + "42703", + ) + .await; +} + +#[tokio::test] +async fn not_exists_unknown_inner_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_notexists_o").await; + seed_strict(&srv, "sqs_notexists_i").await; + srv.expect_error( + "SELECT a FROM sqs_notexists_o \ + WHERE NOT EXISTS (SELECT 1 FROM sqs_notexists_i WHERE nonexistent_col = 1)", + "42703", + ) + .await; +} + +#[tokio::test] +async fn in_subquery_unknown_inner_where_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_in_o").await; + seed_strict(&srv, "sqs_in_i").await; + srv.expect_error( + "SELECT a FROM sqs_in_o WHERE a IN (SELECT a FROM sqs_in_i WHERE nonexistent_col = 1)", + "42703", + ) + .await; +} + +/// The LATERAL derived alias `x` projects only `a`. Selecting an +/// unqualified name outside that set must not resolve as open. +#[tokio::test] +async fn lateral_alias_unknown_projected_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_lat_unk_o").await; + seed_strict(&srv, "sqs_lat_unk_i").await; + srv.expect_error( + "SELECT x.nonexistent_col FROM sqs_lat_unk_o AS o, \ + LATERAL (SELECT i.a FROM sqs_lat_unk_i AS i WHERE i.a = o.a) x", + "42703", + ) + .await; +} + +#[tokio::test] +async fn lateral_correlated_predicate_unknown_outer_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_lat_outerunk_o").await; + seed_strict(&srv, "sqs_lat_outerunk_i").await; + srv.expect_error( + "SELECT o.a FROM sqs_lat_outerunk_o AS o, \ + LATERAL (SELECT i.a FROM sqs_lat_outerunk_i AS i WHERE i.a = o.nonexistent_col) x", + "42703", + ) + .await; +} + +/// The non-LATERAL derived alias `t` projects only `a`. Selecting an +/// unqualified name outside that set must not resolve as open. +#[tokio::test] +async fn derived_subquery_alias_unknown_projected_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_derived_unk_src").await; + srv.expect_error( + "SELECT t.nonexistent_col FROM (SELECT a FROM sqs_derived_unk_src) AS t", + "42703", + ) + .await; +} + +/// Positive control: an uncorrelated EXISTS on a real column still guards +/// the outer scan — this must not be a false-positive `42703`. +#[tokio::test] +async fn uncorrelated_exists_known_column_filters_correctly() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_exists_pos_o").await; + seed_strict(&srv, "sqs_exists_pos_i").await; + let rows = srv + .query_rows("SELECT a FROM sqs_exists_pos_o WHERE EXISTS (SELECT 1 FROM sqs_exists_pos_i WHERE b = 20) ORDER BY a") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"], vec!["2"]]); +} + +/// Positive control: a correlated EXISTS on real columns on both sides +/// returns exactly the outer rows that have a matching inner row. +#[tokio::test] +async fn correlated_exists_known_columns_returns_matching_rows() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_exists_corr_o").await; + srv.exec( + "CREATE COLLECTION sqs_exists_corr_i (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + srv.exec("INSERT INTO sqs_exists_corr_i (a, b) VALUES (1, 100)") + .await + .unwrap(); + + let rows = srv + .query_rows( + "SELECT o.a FROM sqs_exists_corr_o AS o \ + WHERE EXISTS (SELECT 1 FROM sqs_exists_corr_i AS i WHERE i.a = o.a) \ + ORDER BY o.a", + ) + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"]]); +} + +/// Positive control: NOT EXISTS on the same setup returns the complementary +/// row set. +#[tokio::test] +async fn not_exists_known_columns_returns_complementary_rows() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_notexists_pos_o").await; + srv.exec( + "CREATE COLLECTION sqs_notexists_pos_i (a INT4 PRIMARY KEY, b INT8) WITH (engine = 'document_strict')", + ) + .await + .unwrap(); + srv.exec("INSERT INTO sqs_notexists_pos_i (a, b) VALUES (1, 100)") + .await + .unwrap(); + + let rows = srv + .query_rows( + "SELECT o.a FROM sqs_notexists_pos_o AS o \ + WHERE NOT EXISTS (SELECT 1 FROM sqs_notexists_pos_i AS i WHERE i.a = o.a) \ + ORDER BY o.a", + ) + .await + .unwrap(); + assert_eq!(rows, vec![vec!["2"]]); +} + +/// Positive control: a LATERAL alias projecting a real column still +/// selects and returns the expected value. +#[tokio::test] +async fn lateral_alias_known_projected_column_returns_values() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_lat_pos_o").await; + seed_strict(&srv, "sqs_lat_pos_i").await; + let rows = srv + .query_rows( + "SELECT x.a FROM sqs_lat_pos_o AS o, \ + LATERAL (SELECT i.a FROM sqs_lat_pos_i AS i WHERE i.a = o.a) x \ + ORDER BY x.a", + ) + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"], vec!["2"]]); +} + +/// Positive control: a non-LATERAL derived alias projecting a real column +/// still selects and returns the expected values. +#[tokio::test] +async fn derived_subquery_alias_known_projected_column_returns_values() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_derived_pos_src").await; + let rows = srv + .query_rows("SELECT t.a FROM (SELECT a FROM sqs_derived_pos_src) AS t ORDER BY t.a") + .await + .unwrap(); + assert_eq!(rows, vec![vec!["1"], vec!["2"]]); +} + +/// The schemaless engine accepts undeclared fields on write +/// (`docs/documents.md:43`). A derived alias that projects `*` over an +/// open-schema source carries that openness through: an unqualified name +/// outside the declared fields still resolves to NULL, never errors. +#[tokio::test] +async fn schemaless_derived_subquery_unknown_column_resolves_to_null() { + let srv = TestServer::start().await; + srv.exec("CREATE COLLECTION sqs_schemaless_src (id INT PRIMARY KEY, x INT)") + .await + .unwrap(); + srv.exec("INSERT INTO sqs_schemaless_src (id, x) VALUES (1, 10)") + .await + .unwrap(); + srv.exec("INSERT INTO sqs_schemaless_src (id, x) VALUES (2, 20)") + .await + .unwrap(); + + let rows = srv + .query_rows("SELECT t.nonexistent_col FROM (SELECT * FROM sqs_schemaless_src) AS t") + .await + .expect("an unknown identifier over a schemaless source must not error"); + assert_eq!(rows.len(), 2); + for row in &rows { + assert!(row[0].is_empty(), "expected NULL, got {row:?}"); + } +} + +/// The `IN (SELECT ...)` rewrite consumes the whole predicate, so the outer +/// operand never reaches the expression converter. An unknown outer column +/// must still raise `42703`. +#[tokio::test] +async fn in_subquery_unknown_outer_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_in_outer_o").await; + seed_strict(&srv, "sqs_in_outer_i").await; + srv.expect_error( + "SELECT a FROM sqs_in_outer_o WHERE nonexistent_col IN (SELECT a FROM sqs_in_outer_i)", + "42703", + ) + .await; +} + +/// The same rewrite with a qualified outer operand. +#[tokio::test] +async fn in_subquery_unknown_qualified_outer_column_errors() { + let srv = TestServer::start().await; + seed_strict(&srv, "sqs_in_qual_o").await; + seed_strict(&srv, "sqs_in_qual_i").await; + srv.expect_error( + "SELECT o.a FROM sqs_in_qual_o AS o \ + WHERE o.nonexistent_col IN (SELECT a FROM sqs_in_qual_i)", + "42703", + ) + .await; +} From d95529a6a95d21d863ac0fde365c16b34494ae03 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:45 +0800 Subject: [PATCH 6/7] fix(join): apply per-side scan predicates before the join An equi-correlated LATERAL lowers to a join, and its inner relation's local WHERE was dropped: the residual predicate sat on a Scan that convert_join never lowered, so the subquery returned every row. A LATERAL alias also named its output columns after the inner table's alias, so selecting one by the alias yielded an empty value. HashJoin carries left_scan_filters and right_scan_filters, applied to rows scanned locally before the join, at the same point as the row-level security filters and for the same reason: an excluded row must neither match a partner nor produce a null-extended outer row. The bitmap-prefiltered scan arm dropped the row-level security filters entirely, since its scan plan had no predicate slot. It now applies them alongside the new per-side predicates. --- nodedb-physical/src/physical_plan/query.rs | 9 + nodedb-physical/src/physical_plan/wire.rs | 2 + nodedb-sql/src/planner/lateral/plan.rs | 251 ++++++------------ .../control/planner/redaction_refusal/plan.rs | 2 + .../rls_injection/permission_tree/query.rs | 2 + .../control/planner/rls_injection/query.rs | 2 + .../planner/sql_plan_convert/filter.rs | 6 +- .../sql_plan_convert/filter_scan_side.rs | 52 ++++ .../control/planner/sql_plan_convert/mod.rs | 1 + .../planner/sql_plan_convert/scan/join.rs | 20 ++ .../src/control/server/exchange/full_scan.rs | 51 +++- .../exchange/resolve/exchange/dispatch.rs | 4 + .../resolve/exchange/hash_join_arm.rs | 8 +- .../server/exchange/resolve/materialize.rs | 4 + .../server/exchange/resolve/shuffle.rs | 14 +- .../server/response_shape/redaction/query.rs | 2 + nodedb/src/data/executor/dispatch/query.rs | 4 + .../executor/handlers/join/grace_drive.rs | 8 + .../executor/handlers/join/hash_handlers.rs | 143 ++++++++-- .../src/data/executor/handlers/join/params.rs | 5 + .../data/executor/handlers/join/row_source.rs | 17 +- nodedb/src/data/executor/scan_normalize.rs | 20 +- .../cases/cross_engine_bitmap_currency.rs | 2 + .../test_cross_type_join/cross_semi_joins.rs | 8 + .../test_cross_type_join/inline_hash_join.rs | 4 + .../test_cross_type_join/join_budget.rs | 16 ++ .../test_cross_type_join/multi_core_joins.rs | 6 + .../test_cross_type_join/single_core_joins.rs | 10 + nodedb/tests/wire/cases/sql_lateral.rs | 31 +++ 29 files changed, 481 insertions(+), 223 deletions(-) create mode 100644 nodedb/src/control/planner/sql_plan_convert/filter_scan_side.rs diff --git a/nodedb-physical/src/physical_plan/query.rs b/nodedb-physical/src/physical_plan/query.rs index f7dd9db69..96f087d3c 100644 --- a/nodedb-physical/src/physical_plan/query.rs +++ b/nodedb-physical/src/physical_plan/query.rs @@ -261,6 +261,15 @@ pub enum QueryOp { /// Row-level-security filters for rows scanned from /// `right_collection` locally. Same semantics as `left_rls_filters`. right_rls_filters: Vec, + /// Predicates from the left side's own `WHERE`, applied to rows + /// scanned from `left_collection` locally. Empty when `left_input` is + /// `Some`, because the child plan then carries its own predicates. + /// Applied per side before the join, for the same reason as + /// `left_rls_filters`. + left_scan_filters: Vec, + /// Predicates from the right side's own `WHERE`. Same semantics as + /// `left_scan_filters`. + right_scan_filters: Vec, }, /// Cross-node shuffle-join CONSUMER (E4b): run the node-local grace-hash diff --git a/nodedb-physical/src/physical_plan/wire.rs b/nodedb-physical/src/physical_plan/wire.rs index 6e202b2ca..e3c37969a 100644 --- a/nodedb-physical/src/physical_plan/wire.rs +++ b/nodedb-physical/src/physical_plan/wire.rs @@ -130,6 +130,8 @@ mod tests { right_input: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), left_bitmap: None, right_bitmap: None, }); diff --git a/nodedb-sql/src/planner/lateral/plan.rs b/nodedb-sql/src/planner/lateral/plan.rs index c17374152..19fcc1a31 100644 --- a/nodedb-sql/src/planner/lateral/plan.rs +++ b/nodedb-sql/src/planner/lateral/plan.rs @@ -6,9 +6,13 @@ use sqlparser::ast; use super::correlation::analyse_lateral_where; +use super::subquery::{ + extract_inner_alias, extract_inner_collection, inner_non_correlated_filters, limit_from_query, + reject_lateral_offset, +}; use crate::error::{Result, SqlError}; -use crate::parser::normalize::normalize_ident; -use crate::reserved::check_ast_identifier; +use crate::resolver::ColumnScope; +use crate::resolver::columns::{ResolvedTable, TableScope}; use crate::resolver::expr::convert_expr; use crate::temporal::TemporalScope; use crate::types::*; @@ -31,6 +35,9 @@ pub struct LateralJoinArgs<'a> { pub left_join: bool, /// SELECT list projection to apply after the lateral. pub outer_projection: Vec, + /// The enclosing query's column namespace. The inner body nests inside it + /// so a correlated reference to an outer relation resolves. + pub outer_scope: &'a TableScope, pub catalog: &'a dyn SqlCatalog, pub temporal: TemporalScope, } @@ -47,6 +54,7 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { lateral_alias, left_join, outer_projection, + outer_scope, catalog, temporal, } = args; @@ -63,6 +71,16 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { let analysis = analyse_lateral_where(subquery, &outer_alias_str); + // The outer operand of a correlation predicate is stripped from the inner + // WHERE before conversion, so it is checked here or nowhere. + let outer_qualifier = outer_alias.as_deref(); + for key in &analysis.equi_keys { + outer_scope.check_name(outer_qualifier, &key.outer_col)?; + } + for (_, outer_col) in &analysis.non_equi { + outer_scope.check_name(outer_qualifier, outer_col)?; + } + // Determine if this is the equi-correlated + TopK shape: // - At least one equi-key correlation. // - A LIMIT k on the subquery. @@ -83,6 +101,7 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { lateral_alias, left_join, outer_projection, + outer_scope, catalog, }) } else if has_equi && analysis.non_equi.is_empty() { @@ -93,7 +112,14 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { // WHERE; the outer alias never leaks into inner name resolution. The join // executor scans the inner collection by name, so the join key columns // are available on the merged rows. - let inner_plan = build_inner_scan(select, analysis.remaining, catalog, temporal)?; + let inner_plan = build_inner_scan(InnerScanArgs { + select, + residual_where: analysis.remaining, + lateral_alias, + catalog, + temporal, + outer_scope, + })?; let equi_on: Vec<(String, String)> = analysis .equi_keys .into_iter() @@ -126,7 +152,14 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { // not duplicated here. The subquery is not routed through `plan_query` // because its WHERE references the outer alias, which is not resolvable // in the inner FROM scope. - let inner_plan = build_inner_scan(select, analysis.remaining, catalog, temporal)?; + let inner_plan = build_inner_scan(InnerScanArgs { + select, + residual_where: analysis.remaining, + lateral_alias, + catalog, + temporal, + outer_scope, + })?; let correlation_predicates: Vec<(String, String)> = analysis .equi_keys .iter() @@ -145,18 +178,31 @@ pub fn plan_lateral_join(args: LateralJoinArgs<'_>) -> Result { } } +/// Parameters for [`build_inner_scan`]. +struct InnerScanArgs<'a> { + select: &'a sqlparser::ast::Select, + residual_where: Option, + lateral_alias: &'a str, + catalog: &'a dyn SqlCatalog, + temporal: TemporalScope, + outer_scope: &'a TableScope, +} + /// Build the inner `SqlPlan::Scan` for a LATERAL join. /// /// The scan carries the residual WHERE as filters. Correlated non-equi /// predicates survive as column-vs-column comparisons and lower to /// runtime-bound `*Column` filters downstream; the executor binds the outer /// operand per outer row. -fn build_inner_scan( - select: &sqlparser::ast::Select, - residual_where: Option, - catalog: &dyn SqlCatalog, - temporal: TemporalScope, -) -> Result { +fn build_inner_scan(args: InnerScanArgs<'_>) -> Result { + let InnerScanArgs { + select, + residual_where, + lateral_alias, + catalog, + temporal, + outer_scope, + } = args; let inner_collection = extract_inner_collection(select)?; let inner_alias = extract_inner_alias(select)?; let inner_info = catalog @@ -164,13 +210,26 @@ fn build_inner_scan( .ok_or_else(|| SqlError::UnknownTable { name: inner_collection.clone(), })?; + // A correlated residual predicate names the outer relation, so the inner + // scope nests inside it. + let inner_scope = TableScope::single(ResolvedTable { + name: inner_collection.clone(), + alias: inner_alias, + info: inner_info.clone(), + })? + .nested_in(outer_scope.clone()); let filters = match &residual_where { - Some(expr) => crate::planner::select::convert_where_to_filters(expr)?, + Some(expr) => crate::planner::select::convert_where_to_filters(expr, &inner_scope)?, None => Vec::new(), }; + // The lateral subquery's output relation is named by the LATERAL alias, and + // the inner table alias is private to the subquery. Downstream the scan + // alias qualifies the inner columns on a merged join row, so it carries the + // LATERAL alias and `x.a` resolves the way `LateralLoop` and `LateralTopK` + // already name their inner columns. Ok(SqlPlan::Scan { collection: inner_collection, - alias: inner_alias, + alias: Some(lateral_alias.to_string()), engine: inner_info.engine, filters, projection: Vec::new(), @@ -183,20 +242,6 @@ fn build_inner_scan( }) } -/// Extract the alias of the single-table inner SELECT, if present. -fn extract_inner_alias(select: &sqlparser::ast::Select) -> Result> { - let Some(from) = select.from.first() else { - return Ok(None); - }; - match &from.relation { - ast::TableFactor::Table { alias, .. } => alias - .as_ref() - .map(|alias| check_ast_identifier(&alias.name)) - .transpose(), - _ => Ok(None), - } -} - /// Parameters for [`plan_lateral_top_k`]. struct LateralTopKPlanArgs<'a> { outer_plan: SqlPlan, @@ -208,6 +253,7 @@ struct LateralTopKPlanArgs<'a> { lateral_alias: &'a str, left_join: bool, outer_projection: Vec, + outer_scope: &'a TableScope, catalog: &'a dyn SqlCatalog, } @@ -223,6 +269,7 @@ fn plan_lateral_top_k(args: LateralTopKPlanArgs<'_>) -> Result { lateral_alias, left_join, outer_projection, + outer_scope, catalog, } = args; // Build a bare inner Scan without correlation filters (those are injected @@ -236,7 +283,10 @@ fn plan_lateral_top_k(args: LateralTopKPlanArgs<'_>) -> Result { // The Top-K plan does not retain the inner alias, but it must still reject // malformed aliases before expressions referencing them are lowered. let _inner_alias = extract_inner_alias(select)?; - let inner_filters = inner_non_correlated_filters(select, outer_alias.as_deref().unwrap_or(""))?; + let inner_scope = + TableScope::resolve_from(catalog, &select.from)?.nested_in(outer_scope.clone()); + let inner_filters = + inner_non_correlated_filters(select, outer_alias.as_deref().unwrap_or(""), &inner_scope)?; // Extract ORDER BY from the inner subquery. // For LATERAL inner scans we only need simple column-expression sort keys; @@ -246,9 +296,9 @@ fn plan_lateral_top_k(args: LateralTopKPlanArgs<'_>) -> Result { match &order_by.kind { ast::OrderByKind::Expressions(exprs) => exprs .iter() - .filter_map(|o| { - convert_expr(&o.expr).ok().map(|expr| SortKey { - expr, + .map(|o| { + Ok(SortKey { + expr: convert_expr(&o.expr, &ColumnScope::Relations(&inner_scope))?, ascending: o.options.asc.unwrap_or(true), nulls_first: o .options @@ -256,7 +306,7 @@ fn plan_lateral_top_k(args: LateralTopKPlanArgs<'_>) -> Result { .unwrap_or(!o.options.asc.unwrap_or(true)), }) }) - .collect(), + .collect::>>()?, ast::OrderByKind::All(_) => Vec::new(), } } else { @@ -281,142 +331,3 @@ fn plan_lateral_top_k(args: LateralTopKPlanArgs<'_>) -> Result { left_join, }) } - -/// Extract the collection name from a single-table inner SELECT. -fn extract_inner_collection(select: &sqlparser::ast::Select) -> Result { - let from = select.from.first().ok_or_else(|| SqlError::Unsupported { - detail: "LATERAL subquery must have a FROM clause".into(), - })?; - crate::parser::normalize::table_name_from_factor(&from.relation)? - .map(|(name, _)| name) - .ok_or_else(|| SqlError::Unsupported { - detail: "LATERAL LateralTopK subquery must reference a plain table".into(), - }) -} - -/// Extract filters from the inner SELECT that do NOT reference the outer alias. -fn inner_non_correlated_filters( - select: &sqlparser::ast::Select, - outer_alias: &str, -) -> Result> { - let Some(where_expr) = &select.selection else { - return Ok(Vec::new()); - }; - let remaining = strip_outer_refs(where_expr, outer_alias); - match remaining { - Some(expr) => crate::planner::select::convert_where_to_filters(&expr), - None => Ok(Vec::new()), - } -} - -/// Remove all predicates referencing `outer_alias` from a WHERE expression. -fn strip_outer_refs(expr: &ast::Expr, outer_alias: &str) -> Option { - match expr { - ast::Expr::BinaryOp { - left, - op: ast::BinaryOperator::And, - right, - } => { - let l = strip_outer_refs(left, outer_alias); - let r = strip_outer_refs(right, outer_alias); - match (l, r) { - (None, None) => None, - (Some(e), None) | (None, Some(e)) => Some(e), - (Some(l), Some(r)) => Some(ast::Expr::BinaryOp { - left: Box::new(l), - op: ast::BinaryOperator::And, - right: Box::new(r), - }), - } - } - ast::Expr::BinaryOp { left, right, .. } => { - if refs_outer(left, outer_alias) || refs_outer(right, outer_alias) { - None - } else { - Some(expr.clone()) - } - } - ast::Expr::Nested(inner) => strip_outer_refs(inner, outer_alias), - _ => Some(expr.clone()), - } -} - -fn refs_outer(expr: &ast::Expr, outer_alias: &str) -> bool { - match expr { - ast::Expr::CompoundIdentifier(parts) if parts.len() == 2 => { - normalize_ident(&parts[0]).eq_ignore_ascii_case(outer_alias) - } - ast::Expr::BinaryOp { left, right, .. } => { - refs_outer(left, outer_alias) || refs_outer(right, outer_alias) - } - _ => false, - } -} - -/// Extract the LIMIT value from a query, or fail on a bound that does not -/// resolve to `[0, usize::MAX]`. `LIMIT NULL` / `LIMIT ALL` and an absent -/// clause all mean no bound, so both map to `None`. -fn limit_from_query(query: &ast::Query) -> Result> { - match &query.limit_clause { - Some(ast::LimitClause::LimitOffset { - limit: Some(limit), .. - }) - | Some(ast::LimitClause::OffsetCommaLimit { limit, .. }) => { - Ok(crate::coerce::checked_row_bound("LIMIT", limit)?.limit()) - } - Some(ast::LimitClause::LimitOffset { limit: None, .. }) | None => Ok(None), - } -} - -/// Reject an inner OFFSET on a LATERAL subquery. -/// -/// `SqlPlan::LateralTopK` carries no offset field and `SqlPlan::LateralLoop` -/// carries neither limit nor offset. A per-outer-row OFFSET needs a new plan -/// field plus Data Plane execution that skips rows per outer row, so this -/// rejects rather than silently drops the clause. `OFFSET 0` and `OFFSET -/// NULL` skip nothing and plan cleanly; a resolved offset above zero fails -/// with `SqlError::Unsupported`. An offset literal outside `[0, usize::MAX]` -/// fails first, inside `checked_row_bound`, with `SqlError::InvalidLimitValue`. -fn reject_lateral_offset(query: &ast::Query) -> Result<()> { - let offset_expr = match &query.limit_clause { - Some(ast::LimitClause::LimitOffset { - offset: Some(offset), - .. - }) => Some(&offset.value), - Some(ast::LimitClause::OffsetCommaLimit { offset, .. }) => Some(offset), - Some(ast::LimitClause::LimitOffset { offset: None, .. }) | None => None, - }; - let Some(expr) = offset_expr else { - return Ok(()); - }; - if crate::coerce::checked_row_bound("OFFSET", expr)?.offset() > 0 { - return Err(SqlError::Unsupported { - detail: "OFFSET inside a LATERAL subquery is not supported".into(), - }); - } - Ok(()) -} - -/// Extract and validate a LATERAL alias from a `TableFactor::Derived`. -pub fn lateral_alias_from_factor(factor: &ast::TableFactor) -> Result> { - match factor { - ast::TableFactor::Derived { alias, .. } => alias - .as_ref() - .map(|alias| check_ast_identifier(&alias.name)) - .transpose(), - _ => Ok(None), - } -} - -/// True when a `TableFactor` is a LATERAL derived subquery. -pub fn is_lateral_derived(factor: &ast::TableFactor) -> bool { - matches!(factor, ast::TableFactor::Derived { lateral: true, .. }) -} - -/// Extract the subquery from a `TableFactor::Derived`. -pub fn subquery_from_factor(factor: &ast::TableFactor) -> Option<&ast::Query> { - match factor { - ast::TableFactor::Derived { subquery, .. } => Some(subquery), - _ => None, - } -} diff --git a/nodedb/src/control/planner/redaction_refusal/plan.rs b/nodedb/src/control/planner/redaction_refusal/plan.rs index fd6930ea3..dd71f71eb 100644 --- a/nodedb/src/control/planner/redaction_refusal/plan.rs +++ b/nodedb/src/control/planner/redaction_refusal/plan.rs @@ -499,6 +499,8 @@ mod tests { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }) } diff --git a/nodedb/src/control/planner/rls_injection/permission_tree/query.rs b/nodedb/src/control/planner/rls_injection/permission_tree/query.rs index 154fa409d..3667a3723 100644 --- a/nodedb/src/control/planner/rls_injection/permission_tree/query.rs +++ b/nodedb/src/control/planner/rls_injection/permission_tree/query.rs @@ -245,6 +245,8 @@ mod tests { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }); assert_refused(apply(&mut plan, &cache), "users"); } diff --git a/nodedb/src/control/planner/rls_injection/query.rs b/nodedb/src/control/planner/rls_injection/query.rs index d76dd9bad..2c0bd05dd 100644 --- a/nodedb/src/control/planner/rls_injection/query.rs +++ b/nodedb/src/control/planner/rls_injection/query.rs @@ -263,6 +263,8 @@ mod tests { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }); assert_refused(inject(&mut plan, &store), "users"); } diff --git a/nodedb/src/control/planner/sql_plan_convert/filter.rs b/nodedb/src/control/planner/sql_plan_convert/filter.rs index 69fdca339..6cef153d8 100644 --- a/nodedb/src/control/planner/sql_plan_convert/filter.rs +++ b/nodedb/src/control/planner/sql_plan_convert/filter.rs @@ -45,7 +45,7 @@ pub(super) fn serialize_join_post_filters(filters: &[Filter]) -> crate::Result, ) -> crate::Result> { if filters.is_empty() { @@ -80,7 +80,9 @@ fn filter_to_join_scan_filters(expr: &FilterExpr) -> Vec Vec { +pub(super) fn filter_to_scan_filters( + expr: &FilterExpr, +) -> Vec { use nodedb_query::scan_filter::{FilterOp, ScanFilter}; match expr { diff --git a/nodedb/src/control/planner/sql_plan_convert/filter_scan_side.rs b/nodedb/src/control/planner/sql_plan_convert/filter_scan_side.rs new file mode 100644 index 000000000..dade0cec2 --- /dev/null +++ b/nodedb/src/control/planner/sql_plan_convert/filter_scan_side.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Serialization of one join side's own `WHERE` predicates. +//! +//! A side scanned by name is read as a bare collection scan, so its predicates +//! must address that collection's own rows — unlike the post-join filters in +//! `filter`, which address alias-prefixed merged rows. + +use nodedb_sql::types::{Filter, FilterExpr}; + +use super::filter::{encode_scan_filters, expr_filter, filter_to_scan_filters}; + +/// Serialize one join side's own `WHERE` predicates for a scan of that side's +/// collection. +/// +/// Column qualifiers are dropped: the predicate belongs to the side being +/// scanned, so every column names a field of that side's own rows. A qualifier +/// kept here becomes a literal `"e.score"` field lookup no stored row has, and +/// the predicate matches nothing. +pub(crate) fn serialize_scan_side_filters(filters: &[Filter]) -> crate::Result> { + if filters.is_empty() { + return Ok(Vec::new()); + } + let scan_filters = filters + .iter() + .flat_map(|filter| side_filter_to_scan_filters(&filter.expr)) + .collect::>(); + encode_scan_filters(&scan_filters) +} + +fn side_filter_to_scan_filters(expr: &FilterExpr) -> Vec { + use nodedb_query::scan_filter::{FilterOp, ScanFilter}; + + match expr { + FilterExpr::And(filters) => filters + .iter() + .flat_map(|filter| side_filter_to_scan_filters(&filter.expr)) + .collect(), + FilterExpr::Or(filters) => vec![ScanFilter { + field: String::new(), + op: FilterOp::Or, + value: nodedb_types::Value::Null, + clauses: filters + .iter() + .map(|filter| side_filter_to_scan_filters(&filter.expr)) + .collect(), + expr: None, + }], + FilterExpr::Expr(sql_expr) => vec![expr_filter(sql_expr)], + other => filter_to_scan_filters(other), + } +} diff --git a/nodedb/src/control/planner/sql_plan_convert/mod.rs b/nodedb/src/control/planner/sql_plan_convert/mod.rs index ba24816d6..bdfb309b6 100644 --- a/nodedb/src/control/planner/sql_plan_convert/mod.rs +++ b/nodedb/src/control/planner/sql_plan_convert/mod.rs @@ -8,6 +8,7 @@ pub mod convert; pub mod dml; pub mod expr; pub mod filter; +pub mod filter_scan_side; pub mod group_key_name; pub mod lateral; pub mod output_schema; diff --git a/nodedb/src/control/planner/sql_plan_convert/scan/join.rs b/nodedb/src/control/planner/sql_plan_convert/scan/join.rs index ed8fbbf18..f201e513f 100644 --- a/nodedb/src/control/planner/sql_plan_convert/scan/join.rs +++ b/nodedb/src/control/planner/sql_plan_convert/scan/join.rs @@ -15,6 +15,7 @@ use super::super::aggregate::{ }; use super::super::convert::convert_one; use super::super::filter::{expr_filter_qualified, serialize_join_post_filters}; +use super::super::filter_scan_side::serialize_scan_side_filters; use super::super::scan_params::JoinPlanParams; use super::super::value::sql_value_to_string; use nodedb_physical::physical_task::{PhysicalTask, PostSetOp}; @@ -49,6 +50,19 @@ fn shuffle_supports_join_tail( && post_filters.is_empty() } +/// Serialize a join side's own `WHERE` predicates when that side is scanned by +/// name. A side lowered to a child plan carries its predicates inside that +/// plan, so its slot stays empty. +fn side_scan_filters(plan: &SqlPlan, has_input: bool) -> crate::Result> { + if has_input { + return Ok(Vec::new()); + } + match plan { + SqlPlan::Scan { filters, .. } => serialize_scan_side_filters(filters), + _ => Ok(Vec::new()), + } +} + /// Build a `PhysicalPlan` bitmap-producer sub-plan from a `BitmapHint`. /// /// Returns `None` for hint shapes that cannot be represented as an @@ -129,6 +143,9 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_join( }; let right_input = super::super::aggregate::inline_join_side(right, tenant_id, ctx)?; + let mut left_scan_filters = side_scan_filters(left, left_input.is_some())?; + let mut right_scan_filters = side_scan_filters(right, right_input.is_some())?; + // RIGHT JOIN → swap sides and convert to LEFT JOIN. let mut on_keys = on.to_vec(); let mut left_input = left_input; @@ -138,6 +155,7 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_join( std::mem::swap(&mut left_raw, &mut right_raw); std::mem::swap(&mut left_alias, &mut right_alias); std::mem::swap(&mut left_input, &mut right_input); + std::mem::swap(&mut left_scan_filters, &mut right_scan_filters); on_keys = on_keys.into_iter().map(|(l, r)| (r, l)).collect(); "left".to_string() } else { @@ -225,6 +243,8 @@ pub(in crate::control::planner::sql_plan_convert) fn convert_join( // when that side is scanned locally (`*_input` is `None`). left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters, + right_scan_filters, }); let plan = if shuffle_eligible { diff --git a/nodedb/src/control/server/exchange/full_scan.rs b/nodedb/src/control/server/exchange/full_scan.rs index 27bce927d..58fb851d2 100644 --- a/nodedb/src/control/server/exchange/full_scan.rs +++ b/nodedb/src/control/server/exchange/full_scan.rs @@ -27,6 +27,7 @@ use nodedb_physical::physical_plan::{ColumnarOp, DocumentOp, KvOp, PhysicalPlan, TimeseriesOp}; use nodedb_types::{CollectionType, ColumnarProfile, DocumentMode, SystemTimeScope}; +use crate::bridge::scan_filter::decode_scan_filters; use crate::control::state::SharedState; use crate::types::{DatabaseId, TenantId}; @@ -39,23 +40,27 @@ use crate::types::{DatabaseId, TenantId}; /// overflowing. const COMPLETE_SCAN: usize = usize::MAX; -/// A collection to scan, paired with the row-level-security filters that apply -/// to it for the requesting identity. The pairing is the point: no caller can -/// scan one join side's rows under the other side's policy, whichever side the +/// A collection to scan, paired with the predicates that apply to it: the +/// row-level-security filters for the requesting identity and the side's own +/// `WHERE` predicates. The pairing is the point: no caller can scan one join +/// side's rows under the other side's policy or predicate, whichever side the /// planner drives from. pub struct ScanSide<'a> { collection: &'a str, rls_filters: &'a [u8], + scan_filters: &'a [u8], } impl<'a> ScanSide<'a> { /// One side of a join, under the compiled read filters the RLS pass - /// injected into that side's slot. Empty means no policy restricts this - /// identity on the collection. - pub fn join_side(collection: &'a str, rls_filters: &'a [u8]) -> Self { + /// injected into that side's slot plus that side's own `WHERE` predicates. + /// Empty `rls_filters` means no policy restricts this identity on the + /// collection; empty `scan_filters` means the side has no local predicate. + pub fn join_side(collection: &'a str, rls_filters: &'a [u8], scan_filters: &'a [u8]) -> Self { Self { collection, rls_filters, + scan_filters, } } @@ -66,6 +71,7 @@ impl<'a> ScanSide<'a> { Self { collection, rls_filters: &[], + scan_filters: &[], } } @@ -75,6 +81,26 @@ impl<'a> ScanSide<'a> { } } +/// AND the two predicate sets a scanned join side carries into one +/// MessagePack `Vec`. +/// +/// A set that fails to decode is an error, never an empty set: dropping either +/// returns rows the caller excluded. +fn combine_side_filters(rls_filters: &[u8], scan_filters: &[u8]) -> crate::Result> { + if scan_filters.is_empty() { + return Ok(rls_filters.to_vec()); + } + if rls_filters.is_empty() { + return Ok(scan_filters.to_vec()); + } + let mut combined = decode_scan_filters(rls_filters, "join side filter")?; + combined.extend(decode_scan_filters(scan_filters, "join side filter")?); + zerompk::to_msgpack_vec(&combined).map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!("join side filter serialization: {e}"), + }) +} + /// Build a full-collection scan plan for `side`, or `Ok(None)` when the /// catalog has no record for it on this node. /// @@ -89,7 +115,10 @@ pub fn full_scan_plan_for_collection( side: ScanSide<'_>, ) -> crate::Result> { let collection = side.collection; - let rls_filters = side.rls_filters; + // Both predicate sets travel in the slot the RLS filters already use: that + // slot is evaluated per row before any join match, which is what a local + // `WHERE` predicate needs too. + let side_filters = combine_side_filters(side.rls_filters, side.scan_filters)?; let catalog = state.credentials.catalog(); let stored = match catalog.get_collection(database_id, tenant_id.as_u64(), collection)? { Some(s) => s, @@ -103,7 +132,7 @@ pub fn full_scan_plan_for_collection( collection: nodedb_types::QualifiedCollection::from_stored(collection.to_string()), limit: COMPLETE_SCAN, offset: 0, - filters: rls_filters.to_vec(), + filters: side_filters, sort_keys: Vec::new(), distinct: false, projection: Vec::new(), @@ -118,7 +147,7 @@ pub fn full_scan_plan_for_collection( collection: nodedb_types::QualifiedCollection::from_stored(collection.to_string()), cursor: Vec::new(), count: COMPLETE_SCAN, - filters: rls_filters.to_vec(), + filters: side_filters, sort_keys: Vec::new(), match_pattern: None, surrogate_ceiling: None, @@ -131,7 +160,7 @@ pub fn full_scan_plan_for_collection( limit: COMPLETE_SCAN, filters: Vec::new(), sort_keys: Vec::new(), - rls_filters: rls_filters.to_vec(), + rls_filters: side_filters, system_time: SystemTimeScope::Current, valid_at_ms: None, prefilter: None, @@ -152,7 +181,7 @@ pub fn full_scan_plan_for_collection( aggregates: Vec::new(), gap_fill: String::new(), computed_columns: Vec::new(), - rls_filters: rls_filters.to_vec(), + rls_filters: side_filters, system_time: SystemTimeScope::Current, valid_at_ms: None, }) diff --git a/nodedb/src/control/server/exchange/resolve/exchange/dispatch.rs b/nodedb/src/control/server/exchange/resolve/exchange/dispatch.rs index f83237f6c..c312c65c4 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/dispatch.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/dispatch.rs @@ -114,6 +114,8 @@ pub(super) async fn resolve_exchange( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }) => { hash_join_arm::resolve_hash_join( state, @@ -139,6 +141,8 @@ pub(super) async fn resolve_exchange( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }, ) .await diff --git a/nodedb/src/control/server/exchange/resolve/exchange/hash_join_arm.rs b/nodedb/src/control/server/exchange/resolve/exchange/hash_join_arm.rs index 765f4cf11..e773ae057 100644 --- a/nodedb/src/control/server/exchange/resolve/exchange/hash_join_arm.rs +++ b/nodedb/src/control/server/exchange/resolve/exchange/hash_join_arm.rs @@ -37,6 +37,8 @@ pub(super) struct HashJoinFields { pub right_bitmap: Option>, pub left_rls_filters: Vec, pub right_rls_filters: Vec, + pub left_scan_filters: Vec, + pub right_scan_filters: Vec, } /// Resolve a `QueryOp::HashJoin` node: resolve `Broadcast` children embedded @@ -73,6 +75,8 @@ pub(super) async fn resolve_hash_join( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, } = fields; let left_input = resolve_join_input( @@ -122,7 +126,7 @@ pub(super) async fn resolve_hash_join( // The side's own collection and its own injected policy, // taken as one value: a planner that swaps build and probe // swaps both together, never one without the other. - ScanSide::join_side(&right_collection, &right_rls_filters), + ScanSide::join_side(&right_collection, &right_rls_filters, &right_scan_filters), trace_id, txn_id, captures, @@ -151,6 +155,8 @@ pub(super) async fn resolve_hash_join( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }, )))) } diff --git a/nodedb/src/control/server/exchange/resolve/materialize.rs b/nodedb/src/control/server/exchange/resolve/materialize.rs index fc53d4588..1d2a2c417 100644 --- a/nodedb/src/control/server/exchange/resolve/materialize.rs +++ b/nodedb/src/control/server/exchange/resolve/materialize.rs @@ -113,6 +113,8 @@ pub(super) async fn materialize_providers( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }) => { let left_input = match left_input { Some(p) => Some(Box::new( @@ -158,6 +160,8 @@ pub(super) async fn materialize_providers( right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, })) } diff --git a/nodedb/src/control/server/exchange/resolve/shuffle.rs b/nodedb/src/control/server/exchange/resolve/shuffle.rs index 6c2c90637..9a56c69a8 100644 --- a/nodedb/src/control/server/exchange/resolve/shuffle.rs +++ b/nodedb/src/control/server/exchange/resolve/shuffle.rs @@ -80,6 +80,8 @@ pub async fn resolve_shuffle_join( right_input, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, .. }) = child else { @@ -201,13 +203,21 @@ pub async fn resolve_shuffle_join( state, database_id, tenant_id, - ScanSide::join_side(right_collection.as_str(), &right_rls_filters), + ScanSide::join_side( + right_collection.as_str(), + &right_rls_filters, + &right_scan_filters, + ), )?; let probe_scan = require_scan_plan( state, database_id, tenant_id, - ScanSide::join_side(left_collection.as_str(), &left_rls_filters), + ScanSide::join_side( + left_collection.as_str(), + &left_rls_filters, + &left_scan_filters, + ), )?; let build_plan_bytes = plan_wire::encode(&build_scan).map_err(|e| crate::Error::Internal { detail: format!("shuffle join: encode build scan: {e}"), diff --git a/nodedb/src/control/server/response_shape/redaction/query.rs b/nodedb/src/control/server/response_shape/redaction/query.rs index 6d604689e..0b54ecd80 100644 --- a/nodedb/src/control/server/response_shape/redaction/query.rs +++ b/nodedb/src/control/server/response_shape/redaction/query.rs @@ -362,6 +362,8 @@ mod tests { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }) } diff --git a/nodedb/src/data/executor/dispatch/query.rs b/nodedb/src/data/executor/dispatch/query.rs index 2610fd817..180a112d0 100644 --- a/nodedb/src/data/executor/dispatch/query.rs +++ b/nodedb/src/data/executor/dispatch/query.rs @@ -107,6 +107,8 @@ impl CoreLoop { right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, .. } => self.execute_hash_join(HashJoinParams { join: JoinParams { @@ -130,6 +132,8 @@ impl CoreLoop { right_bitmap: right_bitmap.as_deref(), left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }), QueryOp::ShuffleJoinConsume { diff --git a/nodedb/src/data/executor/handlers/join/grace_drive.rs b/nodedb/src/data/executor/handlers/join/grace_drive.rs index 0297fca37..537481f4b 100644 --- a/nodedb/src/data/executor/handlers/join/grace_drive.rs +++ b/nodedb/src/data/executor/handlers/join/grace_drive.rs @@ -102,6 +102,10 @@ pub(super) struct LocalJoinSides<'a> { pub(super) left_rls_filters: &'a [u8], /// Row-level-security filters for the build (right) side. pub(super) right_rls_filters: &'a [u8], + /// The probe (left) side's own `WHERE` predicates. + pub(super) left_scan_filters: &'a [u8], + /// The build (right) side's own `WHERE` predicates. + pub(super) right_scan_filters: &'a [u8], } /// Per-side streaming accumulation state. Starts `Buffering`; transitions to @@ -149,6 +153,8 @@ impl CoreLoop { right_alias, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, } = sides; let probe_collection = left_alias.unwrap_or(left_collection); let index_collection = right_alias.unwrap_or(right_collection); @@ -177,12 +183,14 @@ impl CoreLoop { tenant_id: tid, collection: right_collection.to_string(), rls_filters: right_rls_filters.to_vec(), + scan_filters: right_scan_filters.to_vec(), }, probe: RowSource::LocalScan { database_id: did, tenant_id: tid, collection: left_collection.to_string(), rls_filters: left_rls_filters.to_vec(), + scan_filters: left_scan_filters.to_vec(), }, }; diff --git a/nodedb/src/data/executor/handlers/join/hash_handlers.rs b/nodedb/src/data/executor/handlers/join/hash_handlers.rs index 6de081cbb..388174c2b 100644 --- a/nodedb/src/data/executor/handlers/join/hash_handlers.rs +++ b/nodedb/src/data/executor/handlers/join/hash_handlers.rs @@ -9,6 +9,50 @@ use nodedb_query::msgpack_scan; use super::hash::{HashIndex, ProbeParams, probe_hash_index}; use super::params::HashJoinParams; +/// One locally-scanned join side: the collection to read and the two predicate +/// sets its rows must pass before the join sees them. +pub(super) struct JoinSideScan<'a> { + pub(super) database_id: u64, + pub(super) tenant_id: u64, + pub(super) collection: &'a str, + pub(super) limit: usize, + /// Row-level-security filters the planner injected for this side. + pub(super) rls_filters: &'a [u8], + /// This side's own `WHERE` predicates. + pub(super) scan_filters: &'a [u8], +} + +impl CoreLoop { + /// Scan one join side locally, keeping only the rows that pass BOTH the + /// side's policy filters and its own `WHERE` predicates. + /// + /// Both apply per side before the join: an excluded row must neither match + /// a partner nor produce a null-extended outer row, and a post-join filter + /// can do neither. + fn scan_join_side(&self, side: JoinSideScan<'_>) -> crate::Result)>> { + let docs = self.scan_collection_with_rls( + side.database_id, + side.tenant_id, + side.collection, + side.limit, + side.rls_filters, + )?; + self.retain_rows_matching(docs, side.scan_filters, "join side predicate") + } + + /// Apply a join side's policy filters and its own `WHERE` predicates to + /// rows a sub-plan already produced. + fn retain_join_side_rows( + &self, + docs: Vec<(String, Vec)>, + rls_filters: &[u8], + scan_filters: &[u8], + ) -> crate::Result)>> { + let kept = self.retain_rows_matching(docs, rls_filters, "RLS filter (join side)")?; + self.retain_rows_matching(kept, scan_filters, "join side predicate") + } +} + impl CoreLoop { pub(in crate::data::executor) fn execute_hash_join( &mut self, @@ -27,6 +71,8 @@ impl CoreLoop { right_bitmap, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, } = p; debug!( @@ -120,6 +166,8 @@ impl CoreLoop { right_alias, left_rls_filters, right_rls_filters, + left_scan_filters, + right_scan_filters, }, budget, ) @@ -193,12 +241,33 @@ impl CoreLoop { // Forward a failing sub-plan response (e.g. ResourcesExhausted // from the bitmap scan) instead of swallowing it to an empty // Vec, which would silently return a zero-row join. - match crate::data::executor::response_codec::decode_response_to_docs(&resp) { - Some(d) => d, - None => return resp, + // The prefiltered scan carries no predicate slot of its own, + // so both of this side's filter sets apply to its rows here. + let rows = + match crate::data::executor::response_codec::decode_response_to_docs(&resp) { + Some(d) => d, + None => return resp, + }; + match self.retain_join_side_rows(rows, left_rls_filters, left_scan_filters) { + Ok(d) => d, + Err(e) => { + return self.response_error( + join.task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } } } - None => match self.scan_collection_with_rls(join.task.request.database_id.as_u64(), tid, left_collection, scan_limit, left_rls_filters) { + None => match self.scan_join_side(JoinSideScan { + database_id: join.task.request.database_id.as_u64(), + tenant_id: tid, + collection: left_collection, + limit: scan_limit, + rls_filters: left_rls_filters, + scan_filters: left_scan_filters, + }) { Ok(d) => d, Err(e) => { return self.response_error( @@ -213,13 +282,14 @@ impl CoreLoop { let keys = join.on.iter().map(|(l, _)| l.clone()).collect(); (docs, keys) } else { - let docs = match self.scan_collection_with_rls( - join.task.request.database_id.as_u64(), - tid, - left_collection, - scan_limit, - left_rls_filters, - ) { + let docs = match self.scan_join_side(JoinSideScan { + database_id: join.task.request.database_id.as_u64(), + tenant_id: tid, + collection: left_collection, + limit: scan_limit, + rls_filters: left_rls_filters, + scan_filters: left_scan_filters, + }) { Ok(d) => d, Err(e) => { return self.response_error( @@ -263,18 +333,34 @@ impl CoreLoop { // Forward a failing sub-plan response (e.g. ResourcesExhausted // from the bitmap scan) instead of swallowing it to an empty // Vec, which would silently return a zero-row join. - match crate::data::executor::response_codec::decode_response_to_docs(&resp) { - Some(d) => d, - None => return resp, + // The prefiltered scan carries no predicate slot of its own, + // so both of this side's filter sets apply to its rows here. + let rows = + match crate::data::executor::response_codec::decode_response_to_docs(&resp) + { + Some(d) => d, + None => return resp, + }; + match self.retain_join_side_rows(rows, right_rls_filters, right_scan_filters) { + Ok(d) => d, + Err(e) => { + return self.response_error( + join.task, + ErrorCode::Internal { + detail: e.to_string(), + }, + ); + } } } - None => match self.scan_collection_with_rls( - join.task.request.database_id.as_u64(), - tid, - right_collection, - scan_limit, - right_rls_filters, - ) { + None => match self.scan_join_side(JoinSideScan { + database_id: join.task.request.database_id.as_u64(), + tenant_id: tid, + collection: right_collection, + limit: scan_limit, + rls_filters: right_rls_filters, + scan_filters: right_scan_filters, + }) { Ok(d) => d, Err(e) => { return self.response_error( @@ -287,13 +373,14 @@ impl CoreLoop { }, } } else { - match self.scan_collection_with_rls( - join.task.request.database_id.as_u64(), - tid, - right_collection, - scan_limit, - right_rls_filters, - ) { + match self.scan_join_side(JoinSideScan { + database_id: join.task.request.database_id.as_u64(), + tenant_id: tid, + collection: right_collection, + limit: scan_limit, + rls_filters: right_rls_filters, + scan_filters: right_scan_filters, + }) { Ok(d) => d, Err(e) => { return self.response_error( diff --git a/nodedb/src/data/executor/handlers/join/params.rs b/nodedb/src/data/executor/handlers/join/params.rs index 85aeb03dd..c5620115e 100644 --- a/nodedb/src/data/executor/handlers/join/params.rs +++ b/nodedb/src/data/executor/handlers/join/params.rs @@ -54,6 +54,11 @@ pub(crate) struct HashJoinParams<'a> { pub left_rls_filters: &'a [u8], /// Row-level-security filters for the right side. Same semantics. pub right_rls_filters: &'a [u8], + /// The left side's own `WHERE` predicates when it is scanned locally. + /// Applied with `left_rls_filters` before the join: both must pass. + pub left_scan_filters: &'a [u8], + /// The right side's own `WHERE` predicates. Same semantics. + pub right_scan_filters: &'a [u8], } /// Nested-loop join: O(N×M) fallback for non-equi, theta, and cross joins. diff --git a/nodedb/src/data/executor/handlers/join/row_source.rs b/nodedb/src/data/executor/handlers/join/row_source.rs index e9b356ce4..982bf069d 100644 --- a/nodedb/src/data/executor/handlers/join/row_source.rs +++ b/nodedb/src/data/executor/handlers/join/row_source.rs @@ -24,6 +24,7 @@ use std::path::PathBuf; use super::grace_repartition::FrameStreamReader; +use crate::bridge::scan_filter::decode_scan_filters; use crate::data::executor::core_loop::CoreLoop; /// One side of a join consumed through a uniform interface. @@ -53,6 +54,11 @@ pub(super) enum RowSource { /// through — a filter applied anywhere else would be one strategy's /// filter, not the join's. rls_filters: Vec, + /// This side's own `WHERE` predicates, as the MessagePack + /// `Vec` the planner serialized. Empty = no local + /// predicate. Applied at the same seam as `rls_filters`, and a row must + /// pass both. + scan_filters: Vec, }, /// Stream rows from a LOCAL staged shuffle file written by a cross-node /// exchange. The file is a sequence of `[u32 LE len][row-bytes]` frames, @@ -88,17 +94,16 @@ impl RowSource { tenant_id, collection, rls_filters, + scan_filters, } => { - if rls_filters.is_empty() { + if rls_filters.is_empty() && scan_filters.is_empty() { return core.scan_collection_for_each(*database_id, *tenant_id, collection, f); } - // Deserialize once, outside the per-row closure. A filter that + // Deserialize once, outside the per-row closure. A set that // fails to decode is an error, never an empty filter set: // dropping it would stream the unfiltered side into the join. - let filters: Vec = - zerompk::from_msgpack(rls_filters).map_err(|e| crate::Error::PlanError { - detail: format!("RLS filter deserialization failed (join side): {e}"), - })?; + let mut filters = decode_scan_filters(rls_filters, "RLS filter (join side)")?; + filters.extend(decode_scan_filters(scan_filters, "join side predicate")?); core.scan_collection_for_each(*database_id, *tenant_id, collection, |id, bytes| { if crate::bridge::scan_filter::ScanFilter::all_match_binary(&filters, bytes)? { f(id, bytes)?; diff --git a/nodedb/src/data/executor/scan_normalize.rs b/nodedb/src/data/executor/scan_normalize.rs index b25c068b8..1ee4d66c6 100644 --- a/nodedb/src/data/executor/scan_normalize.rs +++ b/nodedb/src/data/executor/scan_normalize.rs @@ -37,13 +37,27 @@ impl CoreLoop { rls_filters: &[u8], ) -> crate::Result)>> { let docs = self.scan_collection(did, tid, collection, limit)?; - if rls_filters.is_empty() { + self.retain_rows_matching(docs, rls_filters, "RLS filter (join side)") + } + + /// Keep the rows matching a MessagePack `Vec`. + /// + /// Empty `filter_bytes` keeps every row. A set that fails to deserialize is + /// an error, never an empty set: `context` names which set failed so the + /// caller's error says whether a policy or a query predicate was dropped. + pub(in crate::data::executor) fn retain_rows_matching( + &self, + docs: Vec<(String, Vec)>, + filter_bytes: &[u8], + context: &str, + ) -> crate::Result)>> { + if filter_bytes.is_empty() { return Ok(docs); } let filters: Vec = - zerompk::from_msgpack(rls_filters).map_err(|e| crate::Error::PlanError { - detail: format!("RLS filter deserialization failed (join side): {e}"), + zerompk::from_msgpack(filter_bytes).map_err(|e| crate::Error::PlanError { + detail: format!("{context} deserialization failed: {e}"), })?; let mut kept = Vec::with_capacity(docs.len()); diff --git a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs index 0f3b91493..7be2d6d08 100644 --- a/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs +++ b/nodedb/tests/inproc/cases/cross_engine_bitmap_currency.rs @@ -511,6 +511,8 @@ fn document_scan_bitmap_filters_columnar_aggregate() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); // The join result has prefixed keys: "catalog.id", "metrics.id", etc. diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/cross_semi_joins.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/cross_semi_joins.rs index 67e1a5204..9ae7c2668 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/cross_semi_joins.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/cross_semi_joins.rs @@ -110,6 +110,8 @@ fn cross_join_uses_inline_right_scalar_aggregate_for_post_filter() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -227,6 +229,8 @@ fn cross_join_uses_unaliased_scalar_aggregate_key_for_post_filter() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -394,11 +398,15 @@ fn semi_join_uses_nested_scalar_subquery_result_as_inline_right() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }))), left_bitmap: None, right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/inline_hash_join.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/inline_hash_join.rs index a0789ea97..8ce8e1420 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/inline_hash_join.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/inline_hash_join.rs @@ -114,6 +114,8 @@ fn inline_hash_join_honors_qualified_left_keys() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -188,6 +190,8 @@ fn inline_hash_join_honors_qualified_left_keys() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/join_budget.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/join_budget.rs index 0ea0d3472..e6370374f 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/join_budget.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/join_budget.rs @@ -127,6 +127,8 @@ fn hash_join_completeness_past_50k_cap() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -327,6 +329,8 @@ fn hash_join_left_side_over_budget_streams_and_completes() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -397,6 +401,8 @@ fn hash_join_right_side_over_budget_spills_and_completes() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -466,6 +472,8 @@ fn hash_join_build_side_spill_returns_all_matches_across_partitions() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -545,6 +553,8 @@ fn hash_join_probe_side_spill_returns_all_matches() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -848,6 +858,8 @@ fn no_limit_join_within_budget_returns_all_rows_past_10k() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -912,6 +924,8 @@ fn explicit_limit_join_caps_at_k_regardless_of_budget() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -972,6 +986,8 @@ fn join_budget_zero_is_unlimited() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); } diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/multi_core_joins.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/multi_core_joins.rs index 683a25976..4c99430cf 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/multi_core_joins.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/multi_core_joins.rs @@ -139,6 +139,8 @@ fn multi_core_broadcast_inner_join() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -284,6 +286,8 @@ fn multi_core_broadcast_left_join() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -469,6 +473,8 @@ fn multi_core_broadcast_merge_simulation() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }) }; diff --git a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/single_core_joins.rs b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/single_core_joins.rs index f4b712086..247bcf397 100644 --- a/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/single_core_joins.rs +++ b/nodedb/tests/inproc/cases/executor_tests/test_cross_type_join/single_core_joins.rs @@ -99,6 +99,8 @@ fn single_core_cross_type_hash_join() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -209,6 +211,8 @@ fn single_core_left_join_with_nulls() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -311,6 +315,8 @@ fn single_core_self_join_respects_aliases_in_filter_and_projection() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -387,6 +393,8 @@ fn single_core_self_join_star_keeps_both_sides() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); @@ -507,6 +515,8 @@ fn schemaless_self_join_matches_on_canonicalized_object_fields() { right_bitmap: None, left_rls_filters: Vec::new(), right_rls_filters: Vec::new(), + left_scan_filters: Vec::new(), + right_scan_filters: Vec::new(), }), ); diff --git a/nodedb/tests/wire/cases/sql_lateral.rs b/nodedb/tests/wire/cases/sql_lateral.rs index 972c0ef9d..3908ec79a 100644 --- a/nodedb/tests/wire/cases/sql_lateral.rs +++ b/nodedb/tests/wire/cases/sql_lateral.rs @@ -333,3 +333,34 @@ async fn lateral_loop_outer_row_cap_returns_error() { "small LateralLoop should succeed with 1 outer row, got {rows:?}" ); } + +/// The equi-correlated LATERAL branch lowers to a join. Its inner `WHERE` +/// carries the correlation predicate plus local predicates on the inner +/// relation. Both must survive: dropping the local ones returns every inner +/// row, silently widening the result. +/// +/// `u1` has scores 10, 30, 20 and `u2` has 50, 40, so `score > 25` matches one +/// `u1` row and two `u2` rows. Without the local predicate the count is 5. +#[tokio::test] +async fn lateral_equi_correlated_keeps_local_inner_predicate() { + let server = TestServer::start().await; + setup_users_events(&server).await; + + let rows = server + .query_text( + "SELECT u.id \ + FROM lat_users u, \ + LATERAL (\ + SELECT e.id FROM lat_events e \ + WHERE e.user_id = u.id AND e.score > 25\ + ) x", + ) + .await + .expect("equi-correlated LATERAL with a local predicate must plan"); + + assert_eq!( + rows.len(), + 3, + "expected one u1 row and two u2 rows, got {rows:?}" + ); +} From e674b2863adb77564fb41f91faccf345020e7696 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 7 Sep 2026 21:28:55 +0800 Subject: [PATCH 7/7] fix(executor): raise row errors instead of skipping the row The UPDATE paths logged a warning and skipped any row they could not decode, re-encode, or evaluate, then reported success with a smaller affected count. A row the caller asked to update was silently left untouched. One site was worse: an undecodable row left the match set before the affected count was computed. Each of those now raises a typed error naming the collection and the document. A row genuinely outside the update set, such as a target with no joined source, still skips. An undecodable stored row also files a diagnostics report at the detection site, grouped by collection so a scan over many bad rows produces one report rather than a storm. The strict encoder's unknown-field error is typed and carries the collection, so it reports 42703 through every caller instead of being flattened into a string and re-wrapped as an internal error. The planner gate covers SQL; this path still serves the native client, COPY FROM, CRDT delta merge, and schemaless-to-strict conversion. Deduplicate the scan-filter decode onto one helper beside the ScanFilter type, and share the computed-column encode tail. --- nodedb/src/bridge/mod.rs | 12 +- nodedb/src/bridge/scan_filter.rs | 21 ++++ .../enforcement/materialized_sum/apply.rs | 2 +- .../handlers/bulk_dml/update_project.rs | 78 ++++++------- nodedb/src/data/executor/handlers/convert.rs | 3 +- .../executor/handlers/document/read/decode.rs | 4 +- .../handlers/point/apply_put/stored_body.rs | 13 ++- .../handlers/point/update/post_image.rs | 13 ++- .../handlers/transaction/resolve/document.rs | 2 +- .../handlers/transaction/resolve/entry.rs | 8 +- .../handlers/transaction/stage_write/body.rs | 24 ++-- .../transaction/stage_write/stage_upsert.rs | 12 +- .../handlers/update_from_join_collect.rs | 103 ++++++++++-------- .../src/data/executor/strict_format/coerce.rs | 22 ++-- .../src/data/executor/strict_format/decode.rs | 9 ++ .../src/data/executor/strict_format/encode.rs | 36 ++++-- nodedb/src/data/executor/strict_format/mod.rs | 4 +- nodedb/src/diag/context/mod.rs | 3 +- nodedb/src/diag/context/write_path.rs | 44 ++++++++ nodedb/src/diag/mod.rs | 4 +- nodedb/src/diag/recording/mod.rs | 2 +- nodedb/src/diag/recording/recovery.rs | 18 +++ 22 files changed, 292 insertions(+), 145 deletions(-) create mode 100644 nodedb/src/bridge/scan_filter.rs diff --git a/nodedb/src/bridge/mod.rs b/nodedb/src/bridge/mod.rs index 2d449d216..0d18936ac 100644 --- a/nodedb/src/bridge/mod.rs +++ b/nodedb/src/bridge/mod.rs @@ -5,16 +5,14 @@ pub mod dispatch; pub mod envelope; pub mod quiesce; -// Re-export shared query engine from nodedb-query crate. -// Origin's internal code continues to use `crate::bridge::expr_eval`, -// `crate::bridge::json_ops`, `crate::bridge::scan_filter`, and -// `crate::bridge::window_func` — they now resolve to nodedb-query. +// Shared query engine re-exports. Origin's internal code names +// `crate::bridge::expr_eval`, `crate::bridge::json_ops`, +// `crate::bridge::scan_filter`, and `crate::bridge::window_func`; the types +// behind them come from nodedb-query. pub mod expr_eval { pub use nodedb_query::expr::{BinaryOp, CastType, ComputedColumn, SqlExpr}; } -pub mod scan_filter { - pub use nodedb_query::scan_filter::*; -} +pub mod scan_filter; pub mod window_func { pub use nodedb_query::window::*; } diff --git a/nodedb/src/bridge/scan_filter.rs b/nodedb/src/bridge/scan_filter.rs new file mode 100644 index 000000000..cfaf2152e --- /dev/null +++ b/nodedb/src/bridge/scan_filter.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: BUSL-1.1 + +//! Scan-filter predicates, shared by the Control Plane and the Data Plane. +//! +//! The types come from `nodedb-query`, which Lite shares. Decoding needs the +//! Origin error type, so it lives here. + +pub use nodedb_query::scan_filter::*; + +/// Decode one MessagePack `Vec` set. +/// +/// Empty bytes carry no predicate. `context` names the set, so a decode error +/// says which one failed. +pub fn decode_scan_filters(bytes: &[u8], context: &str) -> crate::Result> { + if bytes.is_empty() { + return Ok(Vec::new()); + } + zerompk::from_msgpack(bytes).map_err(|e| crate::Error::PlanError { + detail: format!("{context} deserialization failed: {e}"), + }) +} diff --git a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs index 057442cae..e1d44f415 100644 --- a/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs +++ b/nodedb/src/data/executor/enforcement/materialized_sum/apply.rs @@ -445,7 +445,7 @@ mod tests { row.insert("id".to_string(), Value::String(ACCOUNT.into())); row.insert("owner".to_string(), Value::String("alice".into())); row.insert("balance".to_string(), Value::String("100".into())); - let tuple = strict_format::value_to_binary_tuple(&Value::Object(row), &schema) + let tuple = strict_format::value_to_binary_tuple(&Value::Object(row), &schema, TARGET) .expect("encode seed tuple"); let target_key = surrogate_to_doc_id(TARGET_SURROGATE); core.sparse diff --git a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs index 35d79b51a..66d5c9e93 100644 --- a/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs +++ b/nodedb/src/data/executor/handlers/bulk_dml/update_project.rs @@ -43,9 +43,9 @@ pub(in crate::data::executor) struct ProjectUpdateRows<'a> { } impl CoreLoop { - /// Compute the post-image of every matched row. A row that is gone or - /// cannot decode/re-encode is skipped, same as the apply loop. A failure - /// that means the statement itself is wrong is an error, not a skip. + /// Compute the post-image of every matched row. A row deleted between the + /// match and this pass is skipped — it is no longer in the update set. A + /// row the engine cannot decode, re-encode, or evaluate is an error. pub(in crate::data::executor) fn project_bulk_update_rows( &self, p: ProjectUpdateRows<'_>, @@ -70,18 +70,18 @@ impl CoreLoop { continue; }; - // Decode current value — format depends on storage mode. + // Decode current value — format depends on storage mode. A row the + // statement matched but cannot decode fails the statement rather + // than under-reporting the affected count. let mut doc = match strict_schema { - Some(schema) => { - match crate::data::executor::strict_format::binary_tuple_to_json( - ¤t_bytes, - schema, - ) { - Some(v) => v, - None => continue, - } - } - // Fails the statement rather than silently under-reporting affected. + Some(schema) => crate::data::executor::strict_format::binary_tuple_to_json( + ¤t_bytes, + schema, + ) + .ok_or_else(|| { + crate::diag::strict_row_undecodable(collection, doc_id, "bulk_update_project"); + crate::data::executor::strict_format::undecodable_strict_row(collection, doc_id) + })?, None => doc_format::decode_document(¤t_bytes)?, }; @@ -93,14 +93,16 @@ impl CoreLoop { if let Some(obj) = doc.as_object_mut() { for (field, update_val) in updates { let val: serde_json::Value = match update_val { - UpdateValue::Literal(bytes) => match nodedb_types::json_from_msgpack(bytes) - { - Ok(v) => v, - Err(_) => continue, - }, + UpdateValue::Literal(bytes) => nodedb_types::json_from_msgpack(bytes) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!( + "literal assigned to \"{field}\" for document \"{doc_id}\" \ + of collection \"{collection}\" does not decode: {e}" + ), + })?, + // Division or modulo by zero fails the statement. UpdateValue::Expr(expr) => { - // Unlike the literal-decode skip above, a division/ - // modulo-by-zero here fails the whole statement. let result: nodedb_types::Value = expr.eval(&eval_doc)?; result.into() } @@ -109,43 +111,31 @@ impl CoreLoop { } } - // Recompute generated columns if any dependency changed. + // Recompute generated columns if any dependency changed. A column + // the engine cannot recompute fails the statement. if let Some(config) = self.doc_configs.get(&config_key) && !config.enforcement.generated_columns.is_empty() && super::super::generated::needs_recomputation( updates, &config.enforcement.generated_columns, ) - && let Err(e) = super::super::generated::evaluate_generated_columns( + { + super::super::generated::evaluate_generated_columns( &mut doc, &config.enforcement.generated_columns, ) - { - tracing::warn!( - %doc_id, - error = ?e, - "generated column recomputation failed, skipping document" - ); - continue; + .map_err(crate::Error::DataPlane)?; } - // Re-encode — format depends on storage mode. + // Re-encode — format depends on storage mode. An encode error + // carries its own typed cause, such as a field the strict schema + // does not declare. let updated_bytes = match strict_schema { Some(schema) => { let ndb_val: nodedb_types::Value = doc.clone().into(); - match crate::data::executor::strict_format::value_to_binary_tuple( - &ndb_val, schema, - ) { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!( - %doc_id, - error = %e, - "strict re-encode failed, skipping document" - ); - continue; - } - } + crate::data::executor::strict_format::value_to_binary_tuple( + &ndb_val, schema, collection, + )? } None => doc_format::encode_to_msgpack(&doc), }; diff --git a/nodedb/src/data/executor/handlers/convert.rs b/nodedb/src/data/executor/handlers/convert.rs index de3368e19..318598716 100644 --- a/nodedb/src/data/executor/handlers/convert.rs +++ b/nodedb/src/data/executor/handlers/convert.rs @@ -138,7 +138,8 @@ impl CoreLoop { let mut errors = 0u64; for (doc_id, doc_bytes) in &docs { - match super::super::strict_format::bytes_to_binary_tuple(doc_bytes, &schema) { + match super::super::strict_format::bytes_to_binary_tuple(doc_bytes, &schema, collection) + { Ok(tuple_bytes) => { if let Err(e) = self.sparse diff --git a/nodedb/src/data/executor/handlers/document/read/decode.rs b/nodedb/src/data/executor/handlers/document/read/decode.rs index 45c87a477..412855af0 100644 --- a/nodedb/src/data/executor/handlers/document/read/decode.rs +++ b/nodedb/src/data/executor/handlers/document/read/decode.rs @@ -104,7 +104,7 @@ mod tests { map.insert("name".into(), Value::String("Ada".into())); map.insert("age".into(), Value::Integer(42)); - let tuple = strict_format::value_to_binary_tuple(&Value::Object(map), &schema) + let tuple = strict_format::value_to_binary_tuple(&Value::Object(map), &schema, "docs") .expect("encode strict tuple"); let decoded = decode_scanned_document(&tuple, SparseBodyFormatRef::Strict(&schema)) @@ -230,7 +230,7 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert("id".into(), Value::String("u1".into())); map.insert("name".into(), Value::String("Ada".into())); - let tuple = strict_format::value_to_binary_tuple(&Value::Object(map), &schema) + let tuple = strict_format::value_to_binary_tuple(&Value::Object(map), &schema, "docs") .expect("encode strict tuple"); let image = crate::data::executor::scan_normalize::sparse_body_to_msgpack( diff --git a/nodedb/src/data/executor/handlers/point/apply_put/stored_body.rs b/nodedb/src/data/executor/handlers/point/apply_put/stored_body.rs index 8809a961f..63c8a0e11 100644 --- a/nodedb/src/data/executor/handlers/point/apply_put/stored_body.rs +++ b/nodedb/src/data/executor/handlers/point/apply_put/stored_body.rs @@ -113,6 +113,7 @@ impl CoreLoop { }; let value = value_with_rowid.unwrap_or(value); + let collection = &config_key.2; let stored = if bitemporal && schema.bitemporal { strict_format::bytes_to_binary_tuple_bitemporal( &value, @@ -120,13 +121,17 @@ impl CoreLoop { sys_from_ms, valid_from_ms, valid_until_ms, + collection, ) } else { - strict_format::bytes_to_binary_tuple(&value, schema) + strict_format::bytes_to_binary_tuple(&value, schema, collection) } - .map_err(|e| crate::Error::Serialization { - format: "binary_tuple".into(), - detail: e.to_string(), + .map_err(|e| match e { + crate::Error::UnknownStrictField { .. } => e, + other => crate::Error::Serialization { + format: "binary_tuple".into(), + detail: other.to_string(), + }, })?; Ok(StoredBody { value, stored }) diff --git a/nodedb/src/data/executor/handlers/point/update/post_image.rs b/nodedb/src/data/executor/handlers/point/update/post_image.rs index c1c67be72..0f64fc9bd 100644 --- a/nodedb/src/data/executor/handlers/point/update/post_image.rs +++ b/nodedb/src/data/executor/handlers/point/update/post_image.rs @@ -84,6 +84,7 @@ impl CoreLoop { }); }; let ndb_val: nodedb_types::Value = doc.clone().into(); + let collection = &config_key.2; let result = if bitemporal && schema.bitemporal { strict_format::value_to_binary_tuple_bitemporal( &ndb_val, @@ -91,12 +92,18 @@ impl CoreLoop { sys_from_ms, i64::MIN, i64::MAX, + collection, ) } else { - strict_format::value_to_binary_tuple(&ndb_val, schema) + strict_format::value_to_binary_tuple(&ndb_val, schema, collection) }; - result.map_err(|e| ErrorCode::Internal { - detail: format!("strict re-encode: {e}"), + result.map_err(|e| match e { + crate::Error::UnknownStrictField { column, .. } => { + ErrorCode::UndefinedColumn { column } + } + other => ErrorCode::Internal { + detail: format!("strict re-encode: {other}"), + }, }) } diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/document.rs b/nodedb/src/data/executor/handlers/transaction/resolve/document.rs index 27809ac05..e10a59fc0 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/document.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/document.rs @@ -180,7 +180,7 @@ mod tests { let mut obj = std::collections::HashMap::new(); obj.insert("_rowid".to_string(), Value::Integer(rowid)); obj.insert("body".to_string(), Value::String(body.to_string())); - strict_format::value_to_binary_tuple(&Value::Object(obj), &strict_schema()) + strict_format::value_to_binary_tuple(&Value::Object(obj), &strict_schema(), "docs") .expect("encode binary tuple") } diff --git a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs index 62fb01954..98ac4f28e 100644 --- a/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs +++ b/nodedb/src/data/executor/handlers/transaction/resolve/entry.rs @@ -1320,8 +1320,12 @@ mod tests { "body".to_string(), nodedb_types::Value::String(body.to_string()), ); - strict_format::value_to_binary_tuple(&nodedb_types::Value::Object(obj), &strict_schema()) - .expect("encode binary tuple") + strict_format::value_to_binary_tuple( + &nodedb_types::Value::Object(obj), + &strict_schema(), + "docs", + ) + .expect("encode binary tuple") } /// A schemaless document body in canonical storage encoding, not the raw diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/body.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/body.rs index c7bf23e21..8e48ddba4 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/body.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/body.rs @@ -97,13 +97,17 @@ impl CoreLoop { sys_from_ms, i64::MIN, i64::MAX, + collection, ) } else { - strict_format::bytes_to_binary_tuple(&encoded_input, schema) + strict_format::bytes_to_binary_tuple(&encoded_input, schema, collection) } - .map_err(|e| crate::Error::Serialization { - format: "binary_tuple".into(), - detail: e.to_string(), + .map_err(|e| match e { + crate::Error::UnknownStrictField { .. } => e, + other => crate::Error::Serialization { + format: "binary_tuple".into(), + detail: other.to_string(), + }, })?; Ok(stored) } else { @@ -204,13 +208,17 @@ impl CoreLoop { sys_from_ms, i64::MIN, i64::MAX, + collection, ) } else { - strict_format::value_to_binary_tuple(&ndb_val, schema) + strict_format::value_to_binary_tuple(&ndb_val, schema, collection) } - .map_err(|e| crate::Error::Serialization { - format: "binary_tuple".into(), - detail: e.to_string(), + .map_err(|e| match e { + crate::Error::UnknownStrictField { .. } => e, + other => crate::Error::Serialization { + format: "binary_tuple".into(), + detail: other.to_string(), + }, })?; Ok(bytes) } diff --git a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_upsert.rs b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_upsert.rs index dd851ea73..7e7e4936f 100644 --- a/nodedb/src/data/executor/handlers/transaction/stage_write/stage_upsert.rs +++ b/nodedb/src/data/executor/handlers/transaction/stage_write/stage_upsert.rs @@ -170,13 +170,17 @@ impl CoreLoop { sys_from_ms, i64::MIN, i64::MAX, + ctx.collection, ) } else { - strict_format::value_to_binary_tuple(&merged, schema) + strict_format::value_to_binary_tuple(&merged, schema, ctx.collection) }; - result.map_err(|e| crate::Error::Serialization { - format: "binary_tuple".into(), - detail: e.to_string(), + result.map_err(|e| match e { + crate::Error::UnknownStrictField { .. } => e, + other => crate::Error::Serialization { + format: "binary_tuple".into(), + detail: other.to_string(), + }, }) } else { nodedb_types::value_to_msgpack(&merged).map_err(|e| crate::Error::Serialization { diff --git a/nodedb/src/data/executor/handlers/update_from_join_collect.rs b/nodedb/src/data/executor/handlers/update_from_join_collect.rs index 31b585f84..a84280ea7 100644 --- a/nodedb/src/data/executor/handlers/update_from_join_collect.rs +++ b/nodedb/src/data/executor/handlers/update_from_join_collect.rs @@ -100,15 +100,23 @@ impl CoreLoop { let mut rows: Vec = Vec::new(); for (doc_id, current_bytes) in target_rows { + // A row the statement matched but cannot decode fails the + // statement. Skipping it leaves the row untouched under a smaller + // affected count that reports success. let mut target_doc = if let Some(schema) = strict_schema { - match super::super::strict_format::binary_tuple_to_json(¤t_bytes, schema) { - Some(v) => v, - None => continue, - } + super::super::strict_format::binary_tuple_to_json(¤t_bytes, schema) + .ok_or_else(|| { + crate::diag::strict_row_undecodable( + target_collection, + &doc_id, + "update_from_join_collect", + ); + super::super::strict_format::undecodable_strict_row( + target_collection, + &doc_id, + ) + })? } else { - // A target row skipped here is one the UPDATE silently leaves - // untouched while reporting a smaller affected count as the - // truth. doc_format::decode_document(¤t_bytes)? }; @@ -140,16 +148,15 @@ impl CoreLoop { if let Some(target_obj) = target_doc.as_object_mut() { for (field, update_val) in updates { let val: serde_json::Value = match update_val { - UpdateValue::Literal(bytes) => match nodedb_types::json_from_msgpack(bytes) - { - Ok(v) => v, - Err(_) => continue, - }, - // Division/modulo by zero fails the statement, same - // as the literal decode-failure arm above would if - // it propagated instead of skipping (kept as-is; - // only the newly-fallible expr path is threaded - // here). + UpdateValue::Literal(bytes) => nodedb_types::json_from_msgpack(bytes) + .map_err(|e| crate::Error::Serialization { + format: "msgpack".into(), + detail: format!( + "literal assigned to \"{field}\" for document \"{doc_id}\" \ + of collection \"{target_collection}\" does not decode: {e}" + ), + })?, + // Division or modulo by zero fails the statement. UpdateValue::Expr(expr) => { expr.eval(&merged_ndb).map_err(crate::Error::from)?.into() } @@ -158,40 +165,32 @@ impl CoreLoop { } } - // Recompute generated columns if any dependency changed. + // Recompute generated columns if any dependency changed. A column + // the engine cannot recompute fails the statement. if let Some(config) = self.doc_configs.get(config_key) && !config.enforcement.generated_columns.is_empty() && super::generated::needs_recomputation( updates, &config.enforcement.generated_columns, ) - && let Err(e) = super::generated::evaluate_generated_columns( + { + super::generated::evaluate_generated_columns( &mut target_doc, &config.enforcement.generated_columns, ) - { - tracing::warn!( - %doc_id, - error = ?e, - "generated column recomputation failed during UpdateFromJoin, skipping" - ); - continue; + .map_err(crate::Error::DataPlane)?; } // Re-encode the post-image (strict Binary Tuple or MessagePack). + // An encode error carries its own typed cause, such as a field the + // strict schema does not declare. let updated_bytes = if let Some(schema) = strict_schema { let ndb_val: nodedb_types::Value = target_doc.clone().into(); - match super::super::strict_format::value_to_binary_tuple(&ndb_val, schema) { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!( - %doc_id, - error = %e, - "strict re-encode failed during UpdateFromJoin, skipping" - ); - continue; - } - } + super::super::strict_format::value_to_binary_tuple( + &ndb_val, + schema, + target_collection, + )? } else { doc_format::encode_to_msgpack(&target_doc) }; @@ -256,18 +255,32 @@ impl CoreLoop { for entry in range.flatten() { let key = entry.0.value(); let value_bytes = entry.1.value(); + let Some(doc_id) = key.strip_prefix(&prefix) else { + continue; + }; + // A stored row that does not decode fails the statement. + // Treating it as a non-match drops it from the update set + // while the statement reports success. let matches = if let Some(schema) = strict_schema { - match super::super::strict_format::binary_tuple_to_json(value_bytes, schema) { - Some(doc) => { - let msgpack = doc_format::encode_to_msgpack(&doc); - ScanFilter::all_match_binary(target_filters, &msgpack)? - } - None => false, - } + let doc = + super::super::strict_format::binary_tuple_to_json(value_bytes, schema) + .ok_or_else(|| { + crate::diag::strict_row_undecodable( + target_collection, + doc_id, + "update_from_join_scan", + ); + super::super::strict_format::undecodable_strict_row( + target_collection, + doc_id, + ) + })?; + let msgpack = doc_format::encode_to_msgpack(&doc); + ScanFilter::all_match_binary(target_filters, &msgpack)? } else { ScanFilter::all_match_binary(target_filters, value_bytes)? }; - if matches && let Some(doc_id) = key.strip_prefix(&prefix) { + if matches { rows.push((doc_id.to_string(), value_bytes.to_vec())); } } diff --git a/nodedb/src/data/executor/strict_format/coerce.rs b/nodedb/src/data/executor/strict_format/coerce.rs index 006b663b3..d2ec8eec1 100644 --- a/nodedb/src/data/executor/strict_format/coerce.rs +++ b/nodedb/src/data/executor/strict_format/coerce.rs @@ -318,7 +318,8 @@ mod tests { map.insert("age".into(), Value::Integer(30)); let tuple_bytes = - super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema).unwrap(); + super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema, "docs") + .unwrap(); let decoded = super::super::decode::binary_tuple_to_json(&tuple_bytes, &schema).unwrap(); assert_eq!(decoded["id"], "u1"); assert_eq!(decoded["name"], "Alice"); @@ -333,7 +334,8 @@ mod tests { map.insert("name".into(), Value::String("Bob".into())); let tuple_bytes = - super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema).unwrap(); + super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema, "docs") + .unwrap(); let decoded = super::super::decode::binary_tuple_to_json(&tuple_bytes, &schema).unwrap(); assert_eq!(decoded["id"], "u2"); assert!(decoded["age"].is_null()); @@ -345,7 +347,8 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert("id".into(), Value::String("u3".into())); - let result = super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema); + let result = + super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema, "docs"); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("NOT NULL")); } @@ -367,6 +370,7 @@ mod tests { 1_700_000_000_000, 0, i64::MAX, + "docs", ) .unwrap(); @@ -391,6 +395,7 @@ mod tests { 0, 0, 0, + "docs", ); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("not bitemporal")); @@ -404,13 +409,10 @@ mod tests { map.insert("name".into(), Value::String("Eve".into())); map.insert("extra".into(), Value::String("boom".into())); - let result = super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema); + let result = + super::super::encode::value_to_binary_tuple(&Value::Object(map), &schema, "docs"); assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("unknown field 'extra'") - ); + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("\"extra\"") && msg.contains("does not exist")); } } diff --git a/nodedb/src/data/executor/strict_format/decode.rs b/nodedb/src/data/executor/strict_format/decode.rs index 6d1ebf29b..600eb00fc 100644 --- a/nodedb/src/data/executor/strict_format/decode.rs +++ b/nodedb/src/data/executor/strict_format/decode.rs @@ -71,6 +71,15 @@ pub fn binary_tuple_to_msgpack(tuple_bytes: &[u8], schema: &StrictSchema) -> Opt nodedb_types::value_to_msgpack(&val).ok() } +/// The error for a stored Binary Tuple that does not decode against the +/// collection's strict schema. Names the row so an operator can find it. +pub fn undecodable_strict_row(collection: &str, doc_id: &str) -> crate::Error { + crate::Error::Serialization { + format: "binary_tuple".into(), + detail: format!("document \"{doc_id}\" of collection \"{collection}\" does not decode"), + } +} + /// Decode a Binary Tuple to a JSON object using the schema (for pgwire output). pub fn binary_tuple_to_json( tuple_bytes: &[u8], diff --git a/nodedb/src/data/executor/strict_format/encode.rs b/nodedb/src/data/executor/strict_format/encode.rs index 5ea37c078..efb064b1d 100644 --- a/nodedb/src/data/executor/strict_format/encode.rs +++ b/nodedb/src/data/executor/strict_format/encode.rs @@ -11,17 +11,29 @@ use super::coerce::coerce_value; /// /// Accepts zerompk bytes (from planner) and decodes them internally. /// Missing nullable columns become NULL; missing non-nullable columns error. -pub fn bytes_to_binary_tuple(bytes: &[u8], schema: &StrictSchema) -> crate::Result> { +/// `collection` names the target in an unknown-field error and carries no +/// other meaning. +pub fn bytes_to_binary_tuple( + bytes: &[u8], + schema: &StrictSchema, + collection: &str, +) -> crate::Result> { let value = nodedb_types::value_from_msgpack(bytes).map_err(|e| crate::Error::Serialization { format: "msgpack".to_string(), detail: format!("zerompk decode: {e}"), })?; - value_to_binary_tuple(&value, schema) + value_to_binary_tuple(&value, schema, collection) } /// Encode a `nodedb_types::Value` as a Binary Tuple according to the schema. -pub fn value_to_binary_tuple(value: &Value, schema: &StrictSchema) -> crate::Result> { +/// `collection` names the target in an unknown-field error and carries no +/// other meaning. +pub fn value_to_binary_tuple( + value: &Value, + schema: &StrictSchema, + collection: &str, +) -> crate::Result> { let map = match value { Value::Object(m) => m, _ => { @@ -34,8 +46,9 @@ pub fn value_to_binary_tuple(value: &Value, schema: &StrictSchema) -> crate::Res let schema_columns: std::collections::HashSet<&str> = schema.columns.iter().map(|c| c.name.as_str()).collect(); if let Some(unknown) = map.keys().find(|k| !schema_columns.contains(k.as_str())) { - return Err(crate::Error::BadRequest { - detail: format!("unknown field '{unknown}' not present in strict schema"), + return Err(crate::Error::UnknownStrictField { + collection: collection.to_string(), + column: unknown.clone(), }); } @@ -67,12 +80,15 @@ pub fn value_to_binary_tuple(value: &Value, schema: &StrictSchema) -> crate::Res /// Bitemporal variant: decode msgpack to `Value`, then encode as a Binary /// Tuple with reserved slots 0/1/2 populated from the supplied timestamps. +/// `collection` names the target in an unknown-field error and carries no +/// other meaning. pub fn bytes_to_binary_tuple_bitemporal( bytes: &[u8], schema: &StrictSchema, system_from_ms: i64, valid_from_ms: i64, valid_until_ms: i64, + collection: &str, ) -> crate::Result> { let value = nodedb_types::value_from_msgpack(bytes).map_err(|e| crate::Error::Serialization { @@ -85,17 +101,20 @@ pub fn bytes_to_binary_tuple_bitemporal( system_from_ms, valid_from_ms, valid_until_ms, + collection, ) } /// Bitemporal variant: encode a user-supplied `Value::Object` together -/// with the three reserved bitemporal timestamps. +/// with the three reserved bitemporal timestamps. `collection` names the +/// target in an unknown-field error and carries no other meaning. pub fn value_to_binary_tuple_bitemporal( value: &Value, schema: &StrictSchema, system_from_ms: i64, valid_from_ms: i64, valid_until_ms: i64, + collection: &str, ) -> crate::Result> { if !schema.bitemporal { return Err(crate::Error::BadRequest { @@ -133,8 +152,9 @@ pub fn value_to_binary_tuple_bitemporal( !user_names.contains(k.as_str()) && !nodedb_types::columnar::BITEMPORAL_RESERVED_COLUMNS.contains(&k.as_str()) }) { - return Err(crate::Error::BadRequest { - detail: format!("unknown field '{unknown}' not present in strict schema"), + return Err(crate::Error::UnknownStrictField { + collection: collection.to_string(), + column: unknown.clone(), }); } diff --git a/nodedb/src/data/executor/strict_format/mod.rs b/nodedb/src/data/executor/strict_format/mod.rs index ad7fc5ca9..f2dc6e30a 100644 --- a/nodedb/src/data/executor/strict_format/mod.rs +++ b/nodedb/src/data/executor/strict_format/mod.rs @@ -9,7 +9,9 @@ mod coerce; mod decode; mod encode; -pub(crate) use decode::{binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_value}; +pub(crate) use decode::{ + binary_tuple_to_json, binary_tuple_to_msgpack, binary_tuple_to_value, undecodable_strict_row, +}; pub(super) use encode::{ bytes_to_binary_tuple, bytes_to_binary_tuple_bitemporal, value_to_binary_tuple, value_to_binary_tuple_bitemporal, diff --git a/nodedb/src/diag/context/mod.rs b/nodedb/src/diag/context/mod.rs index 2a4dd2f8b..276849977 100644 --- a/nodedb/src/diag/context/mod.rs +++ b/nodedb/src/diag/context/mod.rs @@ -34,5 +34,6 @@ pub(in crate::diag) use recovery::{ReplayRecordUnapplied, WalArchivalFailedTrunc pub(in crate::diag) use retention::RetentionAutowireOrphaned; pub(in crate::diag) use vector::VectorIndexNotApplied; pub(in crate::diag) use write_path::{ - BatchInsertWithoutSurrogates, FtsIndexUpdateFailed, WriteAckedWithoutDurability, + BatchInsertWithoutSurrogates, FtsIndexUpdateFailed, StrictRowUndecodable, + WriteAckedWithoutDurability, }; diff --git a/nodedb/src/diag/context/write_path.rs b/nodedb/src/diag/context/write_path.rs index e526e1e0d..15edcc6d5 100644 --- a/nodedb/src/diag/context/write_path.rs +++ b/nodedb/src/diag/context/write_path.rs @@ -127,3 +127,47 @@ impl DomainContext for BatchInsertWithoutSurrogates<'_> { }) } } + +/// A stored Binary Tuple that does not decode against its collection's +/// strict schema. The bytes on disk are wrong, so the statement that read +/// them is refused rather than applied over a partial row set. +pub(in crate::diag) struct StrictRowUndecodable<'a> { + /// Collection whose stored row did not decode. + pub collection: &'a str, + /// Storage key of the row that did not decode. + pub doc_id: &'a str, + /// Detection site inside the UPDATE path. + pub site: &'static str, +} + +impl DomainContext for StrictRowUndecodable<'_> { + fn domain_kind(&self) -> &'static str { + "nodedb.strict_row_undecodable" + } + + fn grouping_key(&self) -> String { + // Collection names the root cause: one schema, one stored form. The + // row id is the occurrence, so a scan over many bad rows files one + // report with a rising count. + format!("collection={}", self.collection) + } + + fn to_json(&self) -> Value { + json!({ + "collection": self.collection, + "doc_id": self.doc_id, + "site": self.site, + "why_fatal": "the row is stored state that no longer matches the schema this \ + build decodes it with, so every statement reading it is refused \ + from here on. The bytes are already on disk, so the damage \ + outlives the statement and outlives the process — a restart \ + re-reads the same row and fails the same way", + "operator_action": "read the named row of the named collection directly: a \ + single bad row points at a truncated or partially written \ + body, while every row failing points at a schema whose \ + stored column layout no longer matches the catalog. \ + Restore the collection from a snapshot or rewrite the \ + named row", + }) + } +} diff --git a/nodedb/src/diag/mod.rs b/nodedb/src/diag/mod.rs index d3000c80f..ce7dc1332 100644 --- a/nodedb/src/diag/mod.rs +++ b/nodedb/src/diag/mod.rs @@ -16,6 +16,6 @@ pub use recording::{ ilp_invalid_utf8_drop, ilp_line_read_drop, metadata_apply_wedged, quota_row_invalid, quota_row_undecodable, quota_row_write_failed, quota_scope_purge_incomplete, quota_scope_replay_aborted, replay_record_unapplied, retention_autowire_orphaned, - scope_quota_not_installed, synonym_group_not_applied, vector_index_not_applied, - wal_archival_failed_truncation_held, write_acked_without_durability, + scope_quota_not_installed, strict_row_undecodable, synonym_group_not_applied, + vector_index_not_applied, wal_archival_failed_truncation_held, write_acked_without_durability, }; diff --git a/nodedb/src/diag/recording/mod.rs b/nodedb/src/diag/recording/mod.rs index cc30c5bf2..5f91b032b 100644 --- a/nodedb/src/diag/recording/mod.rs +++ b/nodedb/src/diag/recording/mod.rs @@ -33,7 +33,7 @@ pub use quota::{ }; pub use recovery::{ batch_insert_without_surrogates, fts_index_update_failed, replay_record_unapplied, - wal_archival_failed_truncation_held, write_acked_without_durability, + strict_row_undecodable, wal_archival_failed_truncation_held, write_acked_without_durability, }; pub use retention::retention_autowire_orphaned; pub use shared::entry_kind; diff --git a/nodedb/src/diag/recording/recovery.rs b/nodedb/src/diag/recording/recovery.rs index 20aad6d14..99e1558c9 100644 --- a/nodedb/src/diag/recording/recovery.rs +++ b/nodedb/src/diag/recording/recovery.rs @@ -118,3 +118,21 @@ pub fn wal_archival_failed_truncation_held( None => capture.emit(), }; } + +/// Report a stored Binary Tuple that does not decode against its +/// collection's strict schema. Called from each UPDATE site that detects +/// the row, alongside the error it returns; `site` names that site. +pub fn strict_row_undecodable(collection: &str, doc_id: &str, site: &'static str) { + let ctx = context::StrictRowUndecodable { + collection, + doc_id, + site, + }; + let _ = Capture::new( + EventKind::Corruption, + "stored strict row did not decode, so the statement reading it is refused", + ) + .domain(&ctx) + .with_backtrace() + .emit(); +}