Skip to content

Thread the regex grammar (--regextype) through the parser to the matchers (#85) #313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
helly25 merged 1 commit into from
Jul 10, 2026
Merged

Thread the regex grammar (--regextype) through the parser to the matchers (#85) #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions xff/parser/ast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <string>
#include <vector>

#include "xff/regex/regex.h"
#include "xff/registry/descriptor.h"

namespace xff::regex {
Expand Down Expand Up @@ -85,6 +86,10 @@ struct Command {
std::vector<std::string> globals;
std::vector<std::string> roots;
ExprPtr expression;
// The regex grammar for every matcher in this command, resolved once from --regextype at parse
// time (default RE2). The pattern predicates (-regex/-rxc/-grep + the -capture extraction regex)
// compile with it; ApplyCaseMode's recompile reuses it.
regex::Grammar grammar = regex::Grammar::kRe2;
};

} // namespace xff::parser
Expand Down
56 changes: 39 additions & 17 deletions xff/parser/parser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ bool IsXorTier(std::string_view t) {
// pattern does not compile (then no-match).
std::shared_ptr<const regex::Matcher> CompileNodeRegex(
const registry::Descriptor& descriptor,
const std::vector<std::string>& args) {
const std::vector<std::string>& args,
regex::Grammar grammar) {
std::string_view pattern;
if ((descriptor.name == "-regex" || descriptor.name == "-iregex" || descriptor.name == "-rxc"
|| descriptor.name == "-irxc" || descriptor.name == "-grep")
Expand All @@ -96,18 +97,18 @@ std::shared_ptr<const regex::Matcher> CompileNodeRegex(
} else {
return nullptr;
}
absl::StatusOr<regex::Matcher> matcher = regex::Matcher::Compile(pattern, descriptor.fold_case);
absl::StatusOr<regex::Matcher> matcher = regex::Matcher::Compile(pattern, descriptor.fold_case, grammar);
if (!matcher.ok()) {
return nullptr;
}
return std::make_shared<const regex::Matcher>(*std::move(matcher));
}

ExprPtr MakePredicate(const registry::Descriptor* descriptor, std::vector<std::string> args) {
ExprPtr MakePredicate(const registry::Descriptor* descriptor, std::vector<std::string> args, regex::Grammar grammar) {
auto expr = std::make_unique<Expr>();
expr->kind = Expr::Kind::kPredicate;
expr->descriptor = descriptor;
expr->matcher = CompileNodeRegex(*descriptor, args); // compile once, here; eval just reads it
expr->matcher = CompileNodeRegex(*descriptor, args, grammar); // compile once, here; eval just reads it
expr->args = std::move(args);
return expr;
}
Expand Down Expand Up @@ -137,7 +138,7 @@ ExprPtr MakeBinary(Expr::Kind kind, ExprPtr lhs, ExprPtr rhs) {
// prim := '(' list ')' | PREDICATE arg{arity}
class ExprParser {
public:
explicit ExprParser(const std::vector<std::string>& tokens) : tokens_(tokens) {}
ExprParser(const std::vector<std::string>& tokens, regex::Grammar grammar) : tokens_(tokens), grammar_(grammar) {}

absl::StatusOr<ExprPtr> Parse() {
ExprPtr expr = ParseComma();
Expand Down Expand Up @@ -277,7 +278,7 @@ class ExprParser {
for (std::string& cmd_token : command) {
args.push_back(std::move(cmd_token));
}
return MakePredicate(descriptor, std::move(args));
return MakePredicate(descriptor, std::move(args), grammar_);
}
// A Binding::kFormat primary (-grep) carries an attached =FORMAT output
// template on its own token; the whole payload after the first '=' is the
Expand All @@ -295,7 +296,7 @@ class ExprParser {
}
args.push_back(tokens_[pos_++]);
}
ExprPtr node = MakePredicate(descriptor, std::move(args));
ExprPtr node = MakePredicate(descriptor, std::move(args), grammar_);
if (node != nullptr) {
node->grep_template = std::make_shared<const fields::Template>(fields::Template::Compile(format));
}
Expand Down Expand Up @@ -332,7 +333,7 @@ class ExprParser {
}
args.push_back(tokens_[pos_++]);
}
ExprPtr node = MakePredicate(descriptor, std::move(args));
ExprPtr node = MakePredicate(descriptor, std::move(args), grammar_);
if (node != nullptr) {
node->diff_style = style;
}
Expand All @@ -345,7 +346,7 @@ class ExprParser {
descriptor != nullptr && descriptor->binding == registry::Binding::kHash) {
const std::string spec = token.substr(eq + 1);
++pos_; // consume the `<name>=SPEC` token
ExprPtr node = MakePredicate(descriptor, {});
ExprPtr node = MakePredicate(descriptor, {}, grammar_);
if (node != nullptr) {
node->hash_spec = spec;
}
Expand Down Expand Up @@ -393,7 +394,7 @@ class ExprParser {
return nullptr;
}
++pos_; // consume ';' or '+'
ExprPtr node = MakePredicate(descriptor, std::move(command));
ExprPtr node = MakePredicate(descriptor, std::move(command), grammar_);
if (node != nullptr) {
node->exec_batch = batch;
}
Expand All @@ -407,10 +408,11 @@ class ExprParser {
}
args.push_back(tokens_[pos_++]);
}
return MakePredicate(descriptor, std::move(args));
return MakePredicate(descriptor, std::move(args), grammar_);
}

const std::vector<std::string>& tokens_;
regex::Grammar grammar_; // the regex grammar for this command's matchers (from --regextype)
std::size_t pos_ = 0;
absl::Status status_ = absl::OkStatus();
};
Expand Down Expand Up @@ -505,6 +507,22 @@ const Expr* FirstXffPrintfField(const Expr* expr) {
return FirstXffPrintfField(expr->rhs.get());
}

// The regex grammar for the command's matchers, from `--regextype=` (last occurrence wins). Only
// PCRE2 selects a non-default grammar; RE2 / EXACT (and the -grep MATCH placeholder) stay RE2 here.
// This is lenient by design: an unknown or PCRE2-not-built-in value is left as RE2 and rejected by
// run.cc's ResolveGrepLiteral (the validating reader) before the walk, so it never reaches a matcher.
regex::Grammar GrammarFromGlobals(const std::vector<std::string>& globals) {
constexpr std::string_view kPrefix = "--regextype=";
regex::Grammar grammar = regex::Grammar::kRe2;
for (const std::string& global : globals) {
if (global.starts_with(kPrefix)) {
grammar =
std::string_view(global).substr(kPrefix.size()) == "PCRE2" ? regex::Grammar::kPcre2 : regex::Grammar::kRe2;
}
}
return grammar;
}

} // namespace

absl::StatusOr<Command> Parse(const std::vector<std::string>& args) {
Expand All @@ -524,6 +542,9 @@ absl::StatusOr<Command> Parse(const std::vector<std::string>& args) {
break;
}
}
// The regex grammar (from --regextype) is fixed for the whole command, so resolve it once here and
// compile every matcher with it (below, and on the ApplyCaseMode recompile).
cmd.grammar = GrammarFromGlobals(cmd.globals);

// Roots: operands until the expression begins.
for (; i < args.size(); ++i) {
Expand All @@ -536,7 +557,7 @@ absl::StatusOr<Command> Parse(const std::vector<std::string>& args) {
// Expression: the remaining tokens, parsed to a tree.
const std::vector<std::string> expr_tokens(args.begin() + static_cast<std::ptrdiff_t>(i), args.end());
if (!expr_tokens.empty()) {
ExprParser parser(expr_tokens);
ExprParser parser(expr_tokens, cmd.grammar);
MBO_ASSIGN_OR_RETURN(cmd.expression, parser.Parse());
}
return cmd;
Expand Down Expand Up @@ -611,7 +632,7 @@ bool ShouldFold(CaseMode mode, std::string_view pattern) {
// (-name/-path/-lname/-content) via Expr::case_fold, the pre-compiled regex ones
// (-regex/-rxc/-grep) by recompiling `matcher` case-insensitively. Leaves the -i variants
// and non-matcher nodes untouched; recurses over the whole tree.
void ApplyCaseModeToNode(Expr* expr, CaseMode mode) {
void ApplyCaseModeToNode(Expr* expr, CaseMode mode, regex::Grammar grammar) {
if (expr == nullptr) {
return;
}
Expand All @@ -623,7 +644,8 @@ void ApplyCaseModeToNode(Expr* expr, CaseMode mode) {
const bool regex = name == "-regex" || name == "-rxc" || name == "-grep";
if ((glob_or_content || regex) && ShouldFold(mode, pattern)) {
if (regex) {
if (absl::StatusOr<regex::Matcher> matcher = regex::Matcher::Compile(pattern, /*case_insensitive=*/true);
if (absl::StatusOr<regex::Matcher> matcher =
regex::Matcher::Compile(pattern, /*case_insensitive=*/true, grammar);
matcher.ok()) {
expr->matcher = std::make_shared<const regex::Matcher>(*std::move(matcher));
}
Expand All @@ -632,8 +654,8 @@ void ApplyCaseModeToNode(Expr* expr, CaseMode mode) {
}
}
}
ApplyCaseModeToNode(expr->lhs.get(), mode);
ApplyCaseModeToNode(expr->rhs.get(), mode);
ApplyCaseModeToNode(expr->lhs.get(), mode, grammar);
ApplyCaseModeToNode(expr->rhs.get(), mode, grammar);
}

} // namespace
Expand All @@ -642,7 +664,7 @@ void ApplyCaseMode(Command& command, CaseMode mode) {
if (mode == CaseMode::kSensitive) {
return; // nothing to fold; the -i variants already handle their own case
}
ApplyCaseModeToNode(command.expression.get(), mode);
ApplyCaseModeToNode(command.expression.get(), mode, command.grammar);
}

} // namespace xff::parser
21 changes: 21 additions & 0 deletions xff/parser/parser_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,5 +386,26 @@ TEST_F(ParserTest, EnforceStyleAcceptsXffOperatorsUnderXff) {
EXPECT_THAT(EnforceStyle(cmd, registry::Style::kXff), IsOk());
}

TEST_F(ParserTest, RegextypeSelectsTheMatcherGrammar) {
// The grammar is resolved once from --regextype and stored on the Command, so every matcher (and
// the ApplyCaseMode recompile) uses it. RE2 is the default; PCRE2 is the only non-default value.
ASSERT_OK_AND_ASSIGN(const Command def, Parse({".", "-regex", ".*"}));
EXPECT_THAT(def.grammar, regex::Grammar::kRe2); // no --regextype -> RE2

ASSERT_OK_AND_ASSIGN(const Command re2, Parse({"--regextype=RE2", ".", "-regex", ".*"}));
EXPECT_THAT(re2.grammar, regex::Grammar::kRe2);

ASSERT_OK_AND_ASSIGN(const Command pcre2, Parse({"--regextype=PCRE2", ".", "-regex", ".*"}));
EXPECT_THAT(pcre2.grammar, regex::Grammar::kPcre2);

// EXACT is a -grep literal selector, not a regex engine, so the grammar stays RE2.
ASSERT_OK_AND_ASSIGN(const Command exact, Parse({"--regextype=EXACT", ".", "-grep", "x"}));
EXPECT_THAT(exact.grammar, regex::Grammar::kRe2);

// Last occurrence wins (mirrors run.cc's ResolveGrepLiteral).
ASSERT_OK_AND_ASSIGN(const Command last, Parse({"--regextype=PCRE2", "--regextype=RE2", ".", "-regex", ".*"}));
EXPECT_THAT(last.grammar, regex::Grammar::kRe2);
}

} // namespace
} // namespace xff::parser
Loading