feat: match universal POS tags in sequence

Signed-off-by: Joseph Kato <joseph@jdkato.io>

Author: jdkato

internal/check/sequence.go

@@ -18,6 +18,15 @@ type NLPToken struct {
 	Tag     string
 	Skip    int
 
+	// UPOS matches a universal part-of-speech tag -- NOUN, VERB, ADJ and so
+	// on -- rather than a Penn Treebank one.
+	//
+	// It compiles down to the equivalent Penn tags, plus a word constraint for
+	// the few categories Penn cannot express (see upos.go). Rules written
+	// against universal tags are portable; rules written against Penn tags are
+	// more precise.
+	UPOS string
+
 	// Target narrows the alert to this token alone.
 	//
 	// Without it a match spans every token in the sequence. Marking one lets a
@@ -26,7 +35,15 @@ type NLPToken struct {
 	// words".
 	Target bool
 
-	re       *rx.Regexp
+	re *rx.Regexp
+
+	// wordRe narrows a universal tag to the words it can apply to.
+	//
+	// Kept apart from `re` because that one doubles as the anchor and is run
+	// against the whole sentence; this is only ever tested against a single
+	// token's text.
+	wordRe *rx.Regexp
+
 	Negate   bool
 	optional bool
 	start    bool
@@ -61,6 +78,33 @@ func NewSequence(cfg *core.Config, generic baseCheck, path string) (Sequence, er
 	}
 
 	for i, token := range rule.Tokens {
+		if token.UPOS != "" {
+			if token.Tag != "" {
+				return rule, core.NewE201FromPosition(
+					"a token cannot set both `tag` and `upos`", path, 1)
+			}
+
+			pattern, uerr := uposTagPattern(token.UPOS)
+			if uerr != nil {
+				return rule, core.NewE201FromPosition(uerr.Error(), path, 1)
+			}
+			rule.Tokens[i].Tag = pattern
+			token.Tag = pattern
+
+			// A category Penn cannot express on its own also constrains the
+			// word. Only applied when the rule did not ask for a pattern of
+			// its own, which is the more specific request.
+			if token.Pattern == "" {
+				if words := uposWordPattern(token.UPOS); words != "" {
+					wre, werr := rx.Compile(words)
+					if werr != nil {
+						return rule, core.NewE201FromPosition(werr.Error(), path, 1)
+					}
+					rule.Tokens[i].wordRe = wre
+				}
+			}
+		}
+
 		if !rule.needsTagging && token.Tag != "" {
 			rule.needsTagging = true
 		}
@@ -112,7 +156,7 @@ func makeTokens(s *Sequence, generic baseCheck) error {
 			s.Tokens = append(s.Tokens, tok)
 		}
 
-		if tok.Pattern != "" || tok.Tag != "" {
+		if tok.Pattern != "" || tok.Tag != "" || tok.UPOS != "" {
 			tok.optional = false
 			tok.end = true
 			s.Tokens = append(s.Tokens, tok)
@@ -133,6 +177,12 @@ func tokensMatch(token NLPToken, word tag.Token) bool {
 	failedTag = failedTag == token.Negate
 	failedTok := token.re != nil && token.re.MatchStringStd(word.Text) == token.Negate
 
+	// A universal tag that Penn cannot express also restricts which words
+	// qualify -- `upos: AUX` is "a verb, and one of these words".
+	if token.wordRe != nil && token.wordRe.MatchStringStd(word.Text) == token.Negate {
+		return false
+	}
+
 	if (token.Pattern == "" && failedTag) ||
 		(token.Tag == "" && failedTok) ||
 		(token.Tag != "" && token.Pattern != "") && (failedTag || failedTok) {
@@ -356,10 +406,14 @@ func (s Sequence) Run(blk nlp.Block, f *core.File, _ *core.Config) ([]core.Alert
 	positioned := f.NLP.Endpoint == ""
 
 	txt := blk.Text
-	for idx, tok := range s.Tokens {
-		if !tok.Negate && tok.Pattern != "" {
-			// We're looking for our "anchor" ...
-			for _, loc := range tok.re.FindAllStringIndex(txt, -1) {
+	idx, tok, ok := s.anchor()
+	if ok {
+		{
+			// Each candidate position for the anchor is one possible
+			// violation. A `pattern` anchor enumerates them by searching the
+			// text; a tag-only anchor has nothing to search for, so we let
+			// sequenceMatches walk the words and stop when it runs out.
+			for _, loc := range s.candidates(txt, tok, len(words)) {
 				// These are all possible violations in `txt`:
 				m := sequenceMatches(idx, s, tok, words, history)
 				history = append(history, m.index)
@@ -392,17 +446,48 @@ func (s Sequence) Run(blk nlp.Block, f *core.File, _ *core.Config) ([]core.Alert
 
 					alerts = append(alerts, a)
 					offset = []string{}
-				} else {
+				} else if loc != nil {
 					converted, err := re2Loc(txt, loc)
 					if err != nil {
 						return alerts, err
 					}
 					offset = append(offset, converted)
 				}
 			}
-			break
 		}
 	}
 
 	return alerts, nil
 }
+
+// anchor picks the token the search starts from.
+//
+// A `pattern` token is preferred because it can be located in the text
+// directly. Failing that any tagged token will do -- without this, a rule made
+// only of tags matched nothing at all, silently.
+func (s Sequence) anchor() (int, NLPToken, bool) {
+	for i, tok := range s.Tokens {
+		if !tok.Negate && tok.Pattern != "" {
+			return i, tok, true
+		}
+	}
+	for i, tok := range s.Tokens {
+		if !tok.Negate && tok.Tag != "" {
+			return i, tok, true
+		}
+	}
+	return 0, NLPToken{}, false
+}
+
+// candidates returns one entry per position the anchor might occupy.
+//
+// For a `pattern` anchor each entry is the match's location in the text, which
+// the caller reports as an offset when the surrounding sequence does not pan
+// out. A tag-only anchor has no such location, so the entries are nil and the
+// count simply bounds how many times to try.
+func (s Sequence) candidates(txt string, tok NLPToken, words int) [][]int {
+	if tok.re != nil {
+		return tok.re.FindAllStringIndex(txt, -1)
+	}
+	return make([][]int, words)
+}

internal/check/upos.go

@@ -0,0 +1,102 @@
+package check
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// Universal POS tags, expressed in terms of the Penn Treebank tags Vale's
+// tagger actually produces.
+//
+// Penn is the finer tagset, so most of this is a straight widening: NOUN is
+// "NN or NNS", ADJ is "JJ, JJR or JJS". Three categories are not expressible
+// that way, because the distinction Penn omits is syntactic rather than
+// morphological:
+//
+//   - Penn has no AUX at all. "have" is VBP whether it is the main verb of
+//     "I have a car" or the auxiliary of "I have eaten".
+//   - IN covers both ADP and SCONJ: "after dinner" and "after we ate".
+//   - TO is PART in "to eat" but ADP in "to the store".
+//
+// Those are approximated with a word list alongside the tag; see uposWords.
+// The approximation is stated in the docs rather than hidden, because a rule
+// author needs to know that `upos: AUX` means "an auxiliary-looking verb", not
+// "an auxiliary".
+var uposTags = map[string]string{
+	"ADJ":   `JJ|JJR|JJS`,
+	"ADP":   `IN|RP`,
+	"ADV":   `RB|RBR|RBS|WRB`,
+	"AUX":   `MD|VB|VBD|VBG|VBN|VBP|VBZ`,
+	"CCONJ": `CC`,
+	"DET":   `DT|PDT|WDT|WP\$`,
+	"INTJ":  `UH`,
+	"NOUN":  `NN|NNS`,
+	"NUM":   `CD`,
+	"PART":  `TO|RP|POS`,
+	"PRON":  `PRP|PRP\$|WP|EX`,
+	"PROPN": `NNP|NNPS`,
+	"PUNCT": `[.,:]|''|` + "``" + `|-LRB-|-RRB-|\(|\)`,
+	"SCONJ": `IN|WRB`,
+	"SYM":   `SYM|\$|#`,
+	"VERB":  `VB|VBD|VBG|VBN|VBP|VBZ`,
+	"X":     `FW|LS`,
+}
+
+// uposWords narrows the categories Penn cannot distinguish on its own.
+//
+// A token must carry both the tag and one of these words to match. Without
+// this, `upos: AUX` would match every verb in the document.
+var uposWords = map[string][]string{
+	"AUX": {
+		"am", "is", "are", "was", "were", "be", "been", "being",
+		"have", "has", "had", "having",
+		"do", "does", "did", "doing",
+		"will", "would", "shall", "should", "can", "could",
+		"may", "might", "must", "ought",
+		"'s", "'re", "'m", "'ve", "'ll", "'d",
+	},
+	"SCONJ": {
+		"after", "although", "as", "because", "before", "even", "if",
+		"since", "so", "than", "that", "though", "unless", "until",
+		"when", "whenever", "where", "whereas", "wherever", "whether",
+		"while", "why", "how",
+	},
+}
+
+// uposTagPattern returns the Penn-tag regex for a universal tag.
+func uposTagPattern(name string) (string, error) {
+	pattern, ok := uposTags[strings.ToUpper(name)]
+	if !ok {
+		return "", fmt.Errorf("unknown universal POS tag %q; expected one of %s",
+			name, strings.Join(uposNames(), ", "))
+	}
+	return "^(?:" + pattern + ")$", nil
+}
+
+// uposWordPattern returns the word-level constraint for a universal tag, or
+// "" when the tag needs none.
+func uposWordPattern(name string) string {
+	words, ok := uposWords[strings.ToUpper(name)]
+	if !ok {
+		return ""
+	}
+
+	escaped := make([]string, 0, len(words))
+	for _, w := range words {
+		escaped = append(escaped, regexp.QuoteMeta(w))
+	}
+
+	return "^(?:" + strings.Join(escaped, "|") + ")$"
+}
+
+// uposNames lists the supported tags, sorted, for error messages.
+func uposNames() []string {
+	names := make([]string, 0, len(uposTags))
+	for name := range uposTags {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return names
+}

testdata/features/checks.feature

@@ -178,6 +178,7 @@ Feature: Checks
             test.md:25:1:LanguageTool.Metadata:Use data and metadata as plural nouns.
             test.md:29:1:LanguageTool.Metadata:Use data and metadata as plural nouns.
             test.md:31:17:LanguageTool.ARE_USING:Use 'by using' instead of 'using' when it follows a noun for clarity and grammatical correctness.
+            test.txt:1:16:LanguageTool.AMBIG:Avoid ambiguous pronouns
             test.txt:3:4:LanguageTool.WOULD_BE_JJ_VB:The infinitive 'write' after 'be' requries 'to'. Did you mean 'be great *to* write'?
             test.txt:9:88:LanguageTool.WOULD_BE_JJ_VB:The infinitive 'come' after 'be' requries 'to'. Did you mean 'be available *to* come'?
             test.txt:11:7:LanguageTool.AMBIG:Avoid ambiguous pronouns