fix: match sequence patterns against whole tokens

Signed-off-by: Joseph Kato <joseph@jdkato.io>

Author: jdkato

internal/check/sequence.go

@@ -35,8 +35,16 @@ type NLPToken struct {
 	// words".
 	Target bool
 
+	// re finds candidate positions by scanning the whole sentence, so it must
+	// not be anchored.
 	re *rx.Regexp
 
+	// tokenRe tests one token, so it must be. A `pattern` names the word a
+	// position accepts: without anchoring, `self` also accepts the single
+	// token `self-worth`, and a rule for `your self` fires on `your
+	// self-worth`.
+	tokenRe *rx.Regexp
+
 	// wordRe narrows a universal tag to the words it can apply to.
 	//
 	// Kept apart from `re` because that one doubles as the anchor and is run
@@ -123,6 +131,17 @@ func NewSequence(cfg *core.Config, generic baseCheck, path string) (Sequence, er
 				return rule, core.NewE201FromPosition(errc.Error(), path, 1)
 			}
 			rule.Tokens[i].re = re
+
+			anchored := fmt.Sprintf(tokenTemplate, token.Pattern)
+			if rule.Ignorecase {
+				anchored = ignoreCase + anchored
+			}
+
+			tre, terr := rx.Compile(anchored)
+			if terr != nil {
+				return rule, core.NewE201FromPosition(terr.Error(), path, 1)
+			}
+			rule.Tokens[i].tokenRe = tre
 		}
 	}
 
@@ -175,7 +194,8 @@ func tokensMatch(token NLPToken, word tag.Token) bool {
 	}
 
 	failedTag = failedTag == token.Negate
-	failedTok := token.re != nil && token.re.MatchStringStd(word.Text) == token.Negate
+	failedTok := token.tokenRe != nil &&
+		token.tokenRe.MatchStringStd(word.Text) == token.Negate
 
 	// A universal tag that Penn cannot express also restricts which words
 	// qualify -- `upos: AUX` is "a verb, and one of these words".

internal/check/sequence_test.go

@@ -0,0 +1,59 @@
+package check
+
+import (
+	"testing"
+
+	"github.com/errata-ai/vale/v3/internal/core"
+	"github.com/errata-ai/vale/v3/internal/nlp"
+)
+
+// A sequence position names the word it accepts, so it has to match the whole
+// token.
+//
+// The regex that finds candidate positions is deliberately unanchored -- it is
+// run against the whole sentence -- and reusing it to test a token made
+// `pattern: self` accept the single token `self-worth`, so a rule for
+// `your self` fired on `your self-worth`.
+func TestSequenceMatchesWholeTokens(t *testing.T) {
+	rule, err := NewSequence(testConfig(), baseCheck{
+		"extends": "sequence",
+		"name":    "Test.Yourself",
+		"level":   "error",
+		"message": "Did you mean 'yourself'?",
+		"tokens": []interface{}{
+			map[string]interface{}{"pattern": "your"},
+			map[string]interface{}{"pattern": "self"},
+		},
+	}, "Test.Yourself")
+	if err != nil {
+		t.Fatalf("building rule: %v", err)
+	}
+
+	cases := []struct {
+		name string
+		text string
+		want int
+	}{
+		{"exact words", "Ask your self what matters.", 1},
+		{"hyphenated compound", "Question your self-worth sometimes.", 0},
+		{"longer word", "Consider your selfishness here.", 0},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			f := &core.File{NLP: nlp.Info{}}
+
+			alerts, rerr := rule.Run(nlp.NewBlock(c.text, c.text, "text"), f, testConfig())
+			if rerr != nil {
+				t.Fatalf("running rule: %v", rerr)
+			}
+			if len(alerts) != c.want {
+				t.Errorf("%q produced %d alerts, want %d", c.text, len(alerts), c.want)
+			}
+		})
+	}
+}
+
+func testConfig() *core.Config {
+	return &core.Config{WordTemplate: wordTemplate}
+}