fix(fts): sum all boolean must scores

docs/src/quickstart/full-text-search.md

@@ -242,6 +242,9 @@ query_result = ds.to_table(full_text_query=(q1 & q2))
 
 To combine `OR` queries via operators, use the pattern `q1 | q2`.
 
+Every query combined with `AND` becomes a scoring `MUST` clause: all clauses must match,
+and every matching clause contributes to the final `_score`.
+
 #### Exclude terms: `NOT`
 
 Queries that exclude specific keywords are explicitly written using `BooleanQuery`/`Occur`

java/src/main/java/org/lance/ipc/FullTextQuery.java

@@ -45,8 +45,11 @@ public enum Operator {
   }
 
   public enum Occur {
+    /** The clause may match and contributes its score when it does. */
     SHOULD,
+    /** The clause must match and contributes its score. */
     MUST,
+    /** The clause must not match and never contributes to the score. */
     MUST_NOT
   }
 

python/python/lance/query.py

@@ -270,7 +270,9 @@ def __init__(self, queries: list[tuple[Occur, FullTextQuery]]):
         Parameters
         ----------
         queries : list[tuple(Occur, FullTextQuery)]
-            The list of queries with their occurrence requirements.
+            The list of queries with their occurrence requirements. Every MUST
+            clause must match and contributes its score; matching SHOULD scores
+            are also added, while MUST_NOT clauses only exclude documents.
         """
         self._inner = PyFullTextQuery.boolean_query(
             [(occur.value, query.inner) for occur, query in queries]

rust/lance-index/src/scalar/inverted/compound.rs

@@ -1182,8 +1182,7 @@ impl ComposableScorer for DisjunctionScorer<'_> {
     }
 }
 
-/// Intersection scorer preserving the existing Boolean MUST score contract:
-/// all children filter membership, while the first MUST child supplies score.
+/// Intersection scorer that requires and scores every Boolean MUST child.
 pub(super) struct RequiredConjunctionScorer<'a> {
     children: Vec<BoxScorer<'a>>,
     current: Option<u64>,
@@ -1289,7 +1288,11 @@ impl ComposableScorer for RequiredConjunctionScorer<'_> {
                 "FTS conjunction score requested for an unconfirmed document",
             ));
         }
-        self.children[0].score()
+        let mut score = 0.0_f32;
+        for child in &mut self.children {
+            score += child.score()?;
+        }
+        checked_score(score, "FTS conjunction")
     }
 
     fn advance_shallow(&mut self, target: u64) -> Result<u64> {
@@ -1302,13 +1305,25 @@ impl ComposableScorer for RequiredConjunctionScorer<'_> {
     }
 
     fn score_bounds(&mut self, up_to: u64) -> Result<ScoreBounds> {
-        self.children[0].score_bounds(up_to)
+        let mut bounds = ScoreBounds::ZERO;
+        for child in &mut self.children {
+            bounds = bounds.add(child.score_bounds(up_to)?);
+        }
+        Ok(bounds)
     }
 
     fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> {
-        // Only the first MUST child contributes score. The remaining children
-        // are exact membership filters and must never be score-pruned.
-        self.children[0].set_min_competitive_score(min_score)
+        if min_score.is_nan() {
+            return Err(Error::invalid_input(
+                "minimum competitive FTS score cannot be NaN",
+            ));
+        }
+        // Propagating the full conjunction floor to one child is unsafe because
+        // individually sub-threshold MUST scores may sum to a competitive hit.
+        if self.children.len() == 1 {
+            self.children[0].set_min_competitive_score(min_score)?;
+        }
+        Ok(())
     }
 
     fn matches(&mut self) -> Result<bool> {
@@ -1323,7 +1338,9 @@ impl ComposableScorer for RequiredConjunctionScorer<'_> {
     }
 
     fn scores_non_negative(&self) -> bool {
-        self.children[0].scores_non_negative()
+        self.children
+            .iter()
+            .all(|child| child.scores_non_negative())
     }
 }
 
@@ -2455,7 +2472,30 @@ mod tests {
     }
 
     #[test]
-    fn boolean_and_dismax_preserve_exact_scores_and_order() {
+    fn required_conjunction_uses_all_must_scores_for_competitive_bounds() {
+        let left = Box::new(
+            MaterializedScorer::try_new(rows(&[(1, 3.0), (3, 1.0)]))
+                .unwrap()
+                .with_block_size(1),
+        );
+        let right = Box::new(
+            MaterializedScorer::try_new(rows(&[(1, 30.0), (3, 10.0)]))
+                .unwrap()
+                .with_block_size(1),
+        );
+        let mut scorer = RequiredConjunctionScorer::try_new(vec![left, right]).unwrap();
+        let competitive_score = Arc::new(CompetitiveScore::default());
+        competitive_score.raise(10.0);
+
+        let results = TopKCollector::with_competitive_score(10, competitive_score)
+            .collect(&mut scorer)
+            .unwrap();
+
+        assert_eq!(results, rows(&[(1, 33.0), (3, 11.0)]));
+    }
+
+    #[test]
+    fn boolean_sums_all_matching_clause_scores() {
         let must = vec![
             materialized(&[(1, 3.0), (2, 2.0), (3, 1.0)]),
             materialized(&[(1, 30.0), (3, 10.0)]),
@@ -2471,7 +2511,7 @@ mod tests {
             results,
             vec![ScoredRow {
                 row_id: 3,
-                score: 7.0
+                score: 17.0
             }]
         );
 

rust/lance-index/src/scalar/inverted/query.rs

@@ -592,8 +592,11 @@ impl FtsQueryNode for MultiMatchQuery {
 }
 
 pub enum Occur {
+    /// The clause may match and contributes its score when it does.
     Should,
+    /// The clause must match and contributes its score.
     Must,
+    /// The clause must not match and never contributes to the score.
     MustNot,
 }
 
@@ -624,8 +627,11 @@ impl From<Occur> for &'static str {
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct BooleanQuery {
+    /// Optional scoring clauses, at least one of which must match when there are no MUST clauses.
     pub should: Vec<FtsQuery>,
+    /// Required scoring clauses whose scores are summed.
     pub must: Vec<FtsQuery>,
+    /// Prohibited non-scoring clauses.
     pub must_not: Vec<FtsQuery>,
 }
 

rust/lance/src/dataset/mem_wal/index/fts.rs

@@ -154,7 +154,7 @@ pub enum FtsQueryExpr {
     },
     /// Boolean combination of queries.
     Boolean {
-        /// All MUST clauses must match for a document to be included.
+        /// All MUST clauses must match and contribute to the score.
         must: Vec<Self>,
         /// At least one SHOULD clause should match (adds to score).
         should: Vec<Self>,
@@ -4720,14 +4720,35 @@ mod tests {
         let batch = create_boolean_test_batch(&schema);
         index.insert(&batch, 0).unwrap();
 
+        let rust = FtsQueryExpr::match_query("rust").with_boost(2.0);
+        let programming = FtsQueryExpr::match_query("programming").with_boost(3.0);
+        let rust_score = index
+            .search_query(&rust)
+            .into_iter()
+            .find(|entry| entry.row_position == 0)
+            .unwrap()
+            .score;
+        let programming_score = index
+            .search_query(&programming)
+            .into_iter()
+            .find(|entry| entry.row_position == 0)
+            .unwrap()
+            .score;
         let query = FtsQueryExpr::boolean()
-            .must(FtsQueryExpr::match_query("rust"))
-            .must(FtsQueryExpr::match_query("programming"))
+            .must(rust.clone())
+            .must(programming.clone())
             .build();
 
         let entries = index.search_query(&query);
         assert_eq!(entries.len(), 1);
         assert_eq!(entries[0].row_position, 0);
+        let expected_score = rust_score + programming_score;
+        assert!((entries[0].score - expected_score).abs() < 1e-6);
+
+        let reversed = FtsQueryExpr::boolean().must(programming).must(rust).build();
+        let reversed_entries = index.search_query(&reversed);
+        assert_eq!(reversed_entries.len(), 1);
+        assert!((reversed_entries[0].score - expected_score).abs() < 1e-6);
     }
 
     #[test]

rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs

@@ -76,7 +76,7 @@ pub enum FtsQueryType {
     },
     /// Boolean query with MUST/SHOULD/MUST_NOT.
     Boolean {
-        /// Terms that must match.
+        /// Terms that must match and contribute to the score.
         must: Vec<String>,
         /// Terms that should match (adds to score).
         should: Vec<String>,

rust/lance/src/dataset/tests/dataset_index.rs

@@ -961,6 +961,171 @@ async fn assert_compound_fts_top_k(dataset: &Dataset, query: FtsQuery, limit: us
     assert_eq!(limited, exhaustive[..limit]);
 }
 
+fn expected_must_score_sum(left: Vec<(u64, f32)>, right: Vec<(u64, f32)>) -> Vec<(u64, f32)> {
+    let right = right.into_iter().collect::<HashMap<_, _>>();
+    let mut expected = left
+        .into_iter()
+        .filter_map(|(row_id, left_score)| {
+            right
+                .get(&row_id)
+                .map(|right_score| (row_id, left_score + right_score))
+        })
+        .collect::<Vec<_>>();
+    expected.sort_unstable_by(|(left_row_id, left_score), (right_row_id, right_score)| {
+        right_score
+            .total_cmp(left_score)
+            .then_with(|| left_row_id.cmp(right_row_id))
+    });
+    expected
+}
+
+#[tokio::test]
+async fn test_boolean_must_scores_sum_across_execution_paths() {
+    let batch = arrow_array::record_batch!(
+        (
+            "title",
+            Utf8,
+            [
+                "alpha beta delta",
+                "alpha alpha beta delta delta",
+                "alpha delta",
+                "beta delta",
+                "alpha beta beta delta delta delta"
+            ]
+        ),
+        (
+            "body",
+            Utf8,
+            ["gamma", "gamma gamma", "gamma", "gamma", "other"]
+        )
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let mut dataset = Dataset::write(
+        RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema),
+        "memory://",
+        None,
+    )
+    .await
+    .unwrap();
+    create_fragmented_fts_index(&mut dataset, "title", false).await;
+    create_fragmented_fts_index(&mut dataset, "body", false).await;
+    const LIMIT: usize = 2;
+
+    let match_query = |term: &str, column: &str, boost: f32| -> FtsQuery {
+        MatchQuery::new(term.to_owned())
+            .with_column(Some(column.to_owned()))
+            .with_boost(boost)
+            .into()
+    };
+
+    let same_column_left = match_query("alpha", "title", 2.0);
+    let same_column_right = match_query("beta", "title", 3.0);
+    let expected = expected_must_score_sum(
+        compound_fts_results(&dataset, same_column_left.clone(), None).await,
+        compound_fts_results(&dataset, same_column_right.clone(), None).await,
+    );
+    assert!(expected.len() > LIMIT);
+    let same_column_query: FtsQuery = BooleanQuery::new([
+        (Occur::Must, same_column_left.clone()),
+        (Occur::Must, same_column_right.clone()),
+    ])
+    .into();
+    let actual =
+        compound_fts_results(&dataset, same_column_query.clone(), Some(LIMIT as i64)).await;
+    assert_eq!(actual, expected[..LIMIT]);
+    let reversed_same_column_query: FtsQuery = BooleanQuery::new([
+        (Occur::Must, same_column_right),
+        (Occur::Must, same_column_left),
+    ])
+    .into();
+    assert_eq!(
+        compound_fts_results(&dataset, reversed_same_column_query, Some(LIMIT as i64)).await,
+        expected[..LIMIT]
+    );
+
+    let nested_left = match_query("alpha", "title", 2.0);
+    let nested_middle = match_query("beta", "title", 3.0);
+    let nested_right = match_query("delta", "title", 5.0);
+    let expected = expected_must_score_sum(
+        expected_must_score_sum(
+            compound_fts_results(&dataset, nested_left.clone(), None).await,
+            compound_fts_results(&dataset, nested_middle.clone(), None).await,
+        ),
+        compound_fts_results(&dataset, nested_right.clone(), None).await,
+    );
+    assert!(expected.len() > LIMIT);
+    let nested_pair: FtsQuery =
+        BooleanQuery::new([(Occur::Must, nested_left), (Occur::Must, nested_middle)]).into();
+    let nested_query: FtsQuery =
+        BooleanQuery::new([(Occur::Must, nested_pair), (Occur::Must, nested_right)]).into();
+    assert_eq!(
+        compound_fts_results(&dataset, nested_query, Some(LIMIT as i64)).await,
+        expected[..LIMIT]
+    );
+    let reversed_nested_pair: FtsQuery = BooleanQuery::new([
+        (Occur::Must, match_query("beta", "title", 3.0)),
+        (Occur::Must, match_query("alpha", "title", 2.0)),
+    ])
+    .into();
+    let reversed_nested_query: FtsQuery = BooleanQuery::new([
+        (Occur::Must, match_query("delta", "title", 5.0)),
+        (Occur::Must, reversed_nested_pair),
+    ])
+    .into();
+    assert_eq!(
+        compound_fts_results(&dataset, reversed_nested_query, Some(LIMIT as i64)).await,
+        expected[..LIMIT]
+    );
+
+    let mut scanner = dataset.scan();
+    scanner
+        .full_text_search(FullTextSearchQuery::new_query(same_column_query))
+        .unwrap();
+    scanner.limit(Some(LIMIT as i64), None).unwrap();
+    let plan = scanner.explain_plan(false).await.unwrap();
+    assert!(
+        plan.contains("CompoundFtsScorer"),
+        "same-column MUST should exercise the composable scorer:\n{plan}"
+    );
+
+    let cross_column_left = match_query("alpha", "title", 2.0);
+    let cross_column_right = match_query("gamma", "body", 3.0);
+    let expected = expected_must_score_sum(
+        compound_fts_results(&dataset, cross_column_left.clone(), None).await,
+        compound_fts_results(&dataset, cross_column_right.clone(), None).await,
+    );
+    assert!(expected.len() > LIMIT);
+    let cross_column_query: FtsQuery = BooleanQuery::new([
+        (Occur::Must, cross_column_left.clone()),
+        (Occur::Must, cross_column_right.clone()),
+    ])
+    .into();
+    let actual =
+        compound_fts_results(&dataset, cross_column_query.clone(), Some(LIMIT as i64)).await;
+    assert_eq!(actual, expected[..LIMIT]);
+    let reversed_cross_column_query: FtsQuery = BooleanQuery::new([
+        (Occur::Must, cross_column_right),
+        (Occur::Must, cross_column_left),
+    ])
+    .into();
+    assert_eq!(
+        compound_fts_results(&dataset, reversed_cross_column_query, Some(LIMIT as i64)).await,
+        expected[..LIMIT]
+    );
+
+    let mut scanner = dataset.scan();
+    scanner
+        .full_text_search(FullTextSearchQuery::new_query(cross_column_query))
+        .unwrap();
+    scanner.limit(Some(LIMIT as i64), None).unwrap();
+    let plan = scanner.explain_plan(false).await.unwrap();
+    assert!(
+        plan.contains("HashJoinExec"),
+        "cross-column MUST should exercise the exact fallback:\n{plan}"
+    );
+}
+
 #[tokio::test]
 async fn test_nested_multimatch_limit_propagation() {
     let batch = arrow_array::record_batch!(