fix(KeywordSearch): keep punctuation handling consistent in case-sensitive mode

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala

@@ -19,16 +19,17 @@
 package org.apache.texera.amber.operator.keywordSearch
 
 import org.apache.lucene.analysis.{Analyzer, TokenStream}
-import org.apache.lucene.analysis.core.WhitespaceTokenizer
+import org.apache.lucene.analysis.standard.StandardTokenizer
 import org.apache.lucene.analysis.CharArraySet
 import org.apache.lucene.analysis.StopFilter
 import org.apache.lucene.analysis.Analyzer.TokenStreamComponents
 
-// Achieves case sensitivity by skipping the lowercasing and normalization
-// pipeline used in StandardAnalyzer.
+// Mirrors StandardAnalyzer but omits the LowerCaseFilter, so tokens keep their
+// case while still splitting on Unicode word boundaries. A bare WhitespaceTokenizer
+// would instead glue punctuation to tokens (e.g. "perfect." would not match "perfect").
 class CaseSensitiveAnalyzer extends Analyzer {
   override protected def createComponents(fieldName: String): TokenStreamComponents = {
-    val tokenizer = new WhitespaceTokenizer()
+    val tokenizer = new StandardTokenizer()
     val stream: TokenStream = new StopFilter(tokenizer, CharArraySet.EMPTY_SET)
     new TokenStreamComponents(tokenizer, stream)
   }

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzerSpec.scala

@@ -76,11 +76,11 @@ class CaseSensitiveAnalyzerSpec extends AnyFlatSpec {
   }
 
   // ---------------------------------------------------------------------------
-  // Whitespace tokenization — the underlying tokenizer is
-  // WhitespaceTokenizer; pin its splitting behavior.
+  // Word-boundary tokenization — the underlying tokenizer is
+  // StandardTokenizer (Unicode UAX#29); pin its splitting behavior.
   // ---------------------------------------------------------------------------
 
-  "CaseSensitiveAnalyzer (whitespace tokenizer)" should
+  "CaseSensitiveAnalyzer (standard tokenizer)" should
     "split on a single space" in {
     assert(tokensOf("body", "a b c") == List("a", "b", "c"))
   }
@@ -94,16 +94,18 @@ class CaseSensitiveAnalyzerSpec extends AnyFlatSpec {
   }
 
   // ---------------------------------------------------------------------------
-  // Punctuation — WhitespaceTokenizer keeps punctuation attached to
-  // adjacent characters (it only splits on whitespace).
+  // Punctuation — StandardTokenizer splits on Unicode word boundaries, so
+  // punctuation is a boundary and is stripped rather than kept attached.
+  // This is the whole point of the fix: case-sensitive matching must not
+  // change punctuation handling relative to the default StandardAnalyzer.
   // ---------------------------------------------------------------------------
 
   it should
-    "leave punctuation attached to tokens (WhitespaceTokenizer only splits on whitespace)" in {
-    // `"abc,def"` has no whitespace inside, so it stays one token.
-    assert(tokensOf("body", "abc,def") == List("abc,def"))
-    // Sentence-final punctuation also stays attached.
-    assert(tokensOf("body", "Hello, world!") == List("Hello,", "world!"))
+    "split on punctuation and strip it (StandardTokenizer word boundaries)" in {
+    // `,` is a word boundary, so `"abc,def"` splits into two tokens.
+    assert(tokensOf("body", "abc,def") == List("abc", "def"))
+    // Sentence punctuation is a boundary and is dropped, not kept attached.
+    assert(tokensOf("body", "Hello, world!") == List("Hello", "world"))
   }
 
   // ---------------------------------------------------------------------------

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExecSpec.scala

@@ -50,7 +50,8 @@ class KeywordSearchOpExecSpec extends AnyFlatSpec with BeforeAndAfter {
     createTuple("Twitter"),
     createTuple("안녕하세요"),
     createTuple("你好"),
-    createTuple("_!@,-")
+    createTuple("_!@,-"),
+    createTuple("everything was absolutely perfect.")
   )
 
   it should "find exact match with single number" in {
@@ -238,4 +239,19 @@ class KeywordSearchOpExecSpec extends AnyFlatSpec with BeforeAndAfter {
     opExec.close()
     opDesc.isCaseSensitive = false
   }
+
+  it should "match a word adjacent to punctuation when case-sensitive" in {
+    // Regression test: case sensitivity must change only case handling, not
+    // punctuation -- "perfect" should still match "...perfect." (word-boundary split).
+    opDesc.attribute = "text"
+    opDesc.keyword = "perfect"
+    opDesc.isCaseSensitive = true
+    val opExec = new KeywordSearchOpExec(objectMapper.writeValueAsString(opDesc))
+    opExec.open()
+    val results = testData.filter(t => opExec.processTuple(t, inputPort).hasNext)
+    assert(results.length == 1)
+    assert(results.head.getField[String]("text") == "everything was absolutely perfect.")
+    opExec.close()
+    opDesc.isCaseSensitive = false
+  }
 }