Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions xff/cli/globals.cc
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ constexpr std::array kGlobals = std::to_array<GlobalFlag>({
},
{
.name = "--regextype",
.display = "--regextype=RE2|PCRE2|EXACT",
.display = "--regextype=RE2|EXACT|PCRE2",
.group = "matching",
.header = "Matching",
.summary = "regex grammar: RE2 (default) or PCRE2 (a build extra), or EXACT literal for -grep",
.summary = "match engine: RE2 (default), EXACT (literal), or PCRE2 (a build extra)",
},
{
.name = "--exclude",
Expand Down
30 changes: 11 additions & 19 deletions xff/engine/evaluate.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1183,9 +1183,9 @@ bool EvalHash(const parser::Expr& expr, EvalContext& ctx) {

// xff -grep PATTERN: the line-output companion of -rxc. Prints each line of the
// file's content that matches, as `path:lineno:text` (grep's piped form). The
// pattern is an RE2 regex by default (pre-compiled by the parser) or a literal
// substring under --regextype=EXACT (ctx.grep_literal). Matching is per line, so a
// pattern with no '\n' selects individual lines the way grep does; non-regular,
// pattern is pre-compiled by the parser under the run's --regextype grammar (RE2 by
// default, the literal engine under EXACT, PCRE2 when built in). Matching is per line,
// so a pattern with no '\n' selects individual lines the way grep does; non-regular,
// unreadable, and binary files yield nothing (see ContentToSearch). Returns true
// iff at least one line was printed, so the action's truth reflects "found a match"
// for -o / -q.
Expand All @@ -1197,18 +1197,15 @@ bool EvalGrep(const parser::Expr& expr, EvalContext& ctx) {
if (!content.has_value()) {
return false;
}
// --regextype=EXACT matches the pattern as a literal substring; the default is the
// pre-compiled RE2 regex (a null matcher -- unparseable pattern -- matches nothing,
// mirroring -rxc). The same choice drives the line filter and, for -grep=FORMAT,
// the per-line {match}/{column} span.
const std::string_view needle = expr.args.front();
const MatcherRef matcher = ctx.grep_literal ? MatcherRef{} : AsRef(expr.matcher);
if (!ctx.grep_literal && !matcher.has_value()) {
// The pattern is pre-compiled by the parser into a matcher under the run's grammar (RE2 by
// default, the literal engine under --regextype=EXACT, PCRE2 when built in). A null matcher (an
// unparseable pattern) matches nothing, mirroring -rxc. The matcher drives the line filter and,
// for -grep=FORMAT, the per-line {match}/{column} span, so EXACT and RE2 share one code path.
const MatcherRef matcher = AsRef(expr.matcher);
if (!matcher.has_value()) {
return false;
}
const auto is_match = [&](std::string_view line) {
return ctx.grep_literal ? absl::StrContains(line, needle) : matcher->get().PartialMatch(line);
};
const auto is_match = [&](std::string_view line) { return matcher->get().PartialMatch(line); };
if (ctx.grep_count) {
// --count / -c (rg -c): one path:count per file with matches, in place of the
// lines (and any -grep=FORMAT); files with no match emit nothing. Context is ignored.
Expand Down Expand Up @@ -1245,12 +1242,7 @@ bool EvalGrep(const parser::Expr& expr, EvalContext& ctx) {
// match line; on a context line they stay empty.
std::string_view match_text;
std::optional<std::size_t> match_column;
if (line.is_match && ctx.grep_literal) {
if (const std::size_t pos = line.text.find(needle); pos != std::string_view::npos) {
match_text = line.text.substr(pos, needle.size());
match_column = pos + 1;
}
} else if (line.is_match) {
if (line.is_match) {
if (const std::optional<std::pair<std::size_t, std::size_t>> span = matcher->get().FindFirst(line.text)) {
match_text = line.text.substr(span->first, span->second);
match_column = span->first + 1;
Expand Down
4 changes: 0 additions & 4 deletions xff/engine/evaluate.h
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,6 @@ struct EvalContext {
// false in the find style and under --exact (byte-exact matching). The `i`
// variants (-iname/-ipath) fold regardless.
bool fold_name_case = false;
// --regextype=EXACT: -grep matches its pattern as a literal substring per line
// instead of the default RE2 regex. The driver resolves it once from --regextype
// (RE2 the default; MATCH/PCRE reserved for #85). Only -grep consults it today.
bool grep_literal = false;
// --count / -c: -grep prints one `path:count` per file (its matching-line count)
// instead of the lines, rg -c style; supersedes -grep=FORMAT. Only -grep reads it.
bool grep_count = false;
Expand Down
25 changes: 19 additions & 6 deletions xff/engine/evaluate_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ struct EvaluateTest : ::testing::Test {
file_emitted_.clear();
control_ = {};
std::vector<std::string> argv;
argv.reserve(expr.size() + 1);
argv.reserve(expr.size() + 2);
if (!regextype_.empty()) {
argv.push_back(absl::StrCat("--regextype=", regextype_)); // a leading global, before the root
}
argv.emplace_back(".");
argv.insert(argv.end(), expr.begin(), expr.end());
const auto command = parser::Parse(argv);
Expand All @@ -77,7 +80,6 @@ struct EvaluateTest : ::testing::Test {
.now = now_,
.tz = tz_,
.fold_name_case = fold_name_case_,
.grep_literal = grep_literal_,
.grep_count = grep_count_,
.control = control_,
.exec_fields = exec_fields_,
Expand Down Expand Up @@ -131,7 +133,7 @@ struct EvaluateTest : ::testing::Test {
Control control_; // set by Match from the most recent evaluation (-prune/-quit)
bool exec_fields_ = false; // when true, Match enables --exec-fields token substitution
bool fold_name_case_ = false; // when true, Match sets EvalContext::fold_name_case (FS-native fold)
bool grep_literal_ = false; // when true, Match sets EvalContext::grep_literal (-grep EXACT mode)
std::string regextype_; // when set (e.g. "EXACT"), Match prepends --regextype=<v> as a global
bool grep_count_ = false; // when true, Match sets EvalContext::grep_count (-grep --count mode)
std::vector<std::string> captures_; // -regex groups captured during the most recent (gated) Match
std::map<std::string, std::string> outputs_; // -capture results from the most recent Match
Expand Down Expand Up @@ -661,6 +663,17 @@ TEST_F(EvaluateTest, RxcMatchesRegexAnywhere) {
EXPECT_FALSE(Match({"-rxc", "^name="}, visit)); // 'name=' is not at the start of the content
}

TEST_F(EvaluateTest, RxcUnderExactMatchesLiterally) {
// --regextype=EXACT reaches -rxc too (not just -grep): the content predicate matches the argument
// as a literal substring, so metacharacters are plain text.
const std::string path = WriteContentFile("rx_exact.txt", "value = a[0-9]b here\n");
vfs::Metadata md;
const Visit visit = MakeVisit(path, "rx_exact.txt", vfs::FileType::kRegular, md);
regextype_ = "EXACT";
EXPECT_TRUE(Match({"-rxc", "a[0-9]b"}, visit)); // the literal bracket text is present
EXPECT_FALSE(Match({"-rxc", "a5b"}, visit)); // the regex interpretation is gone under EXACT
}

TEST_F(EvaluateTest, IrxcFoldsCase) {
const std::string path = WriteContentFile("irx.txt", "STATUS: OK");
vfs::Metadata md;
Expand Down Expand Up @@ -717,7 +730,7 @@ TEST_F(EvaluateTest, GrepExactModeMatchesLiterally) {
const std::string path = WriteContentFile("grep_exact.txt", "price 3.50\nprice 3X50\n");
vfs::Metadata md;
const Visit visit = MakeVisit(path, "grep_exact.txt", vfs::FileType::kRegular, md);
grep_literal_ = true;
regextype_ = "EXACT";
EXPECT_TRUE(Match({"-grep", "3.50"}, visit));
EXPECT_EQ(emitted_, absl::StrCat(path, ":1:price 3.50\n")); // only the literal 3.50, not 3X50
}
Expand All @@ -727,7 +740,7 @@ TEST_F(EvaluateTest, GrepExactModeAcceptsRegexMetacharactersAsLiterals) {
const std::string path = WriteContentFile("grep_lit.txt", "call foo(bar) now\n");
vfs::Metadata md;
const Visit visit = MakeVisit(path, "grep_lit.txt", vfs::FileType::kRegular, md);
grep_literal_ = true;
regextype_ = "EXACT";
EXPECT_TRUE(Match({"-grep", "foo(bar"}, visit));
EXPECT_EQ(emitted_, absl::StrCat(path, ":1:call foo(bar) now\n"));
}
Expand Down Expand Up @@ -763,7 +776,7 @@ TEST_F(EvaluateTest, GrepFormatMatchInExactModeUsesTheLiteralSpan) {
const std::string path = WriteContentFile("grep_o2.txt", "aXbXc\n");
vfs::Metadata md;
const Visit visit = MakeVisit(path, "grep_o2.txt", vfs::FileType::kRegular, md);
grep_literal_ = true;
regextype_ = "EXACT";
EXPECT_TRUE(Match({"-grep={column} {match}", "X"}, visit));
EXPECT_EQ(emitted_, "2 X\n"); // first literal X at column 2
}
Expand Down
44 changes: 20 additions & 24 deletions xff/engine/run.cc
Original file line number Diff line number Diff line change
Expand Up @@ -743,39 +743,37 @@ absl::Status ResolveBlockSize(const std::vector<std::string>& globals, std::uint
return absl::OkStatus();
}

// --regextype=RE2|PCRE2|EXACT: the regex grammar for the pattern predicates (RE2 default, or PCRE2),
// or EXACT (a literal substring, -grep only). This validates the value for the whole run (it is
// called unconditionally, so it also guards -regex/-rxc, not just -grep) and returns whether -grep
// matches literally. PCRE2 is a build-time extra: when it is not linked into this binary it is a
// usage error here -- never a silent RE2 fallback. MATCH is still reserved. An unknown value is a
// usage error. All are refused before the walk (exit 2). Last occurrence wins.
absl::StatusOr<bool> ResolveGrepLiteral(const std::vector<std::string>& globals) {
// --regextype=RE2|EXACT|PCRE2: validates the grammar selector for the whole run. The grammar itself
// is resolved by the parser (parser::GrammarFromGlobals) and pre-compiled into each matcher; this is
// the single validating reader, called unconditionally so it guards every pattern predicate
// (-regex/-rxc/-grep). RE2 (default) and EXACT (literal) are core engines, always available. PCRE2
// is a build-time extra: when its backend is not linked it is a usage error here, never a silent RE2
// fallback. MATCH is still reserved. An unknown value is a usage error. All are refused before the
// walk (exit 2). Last occurrence wins (the parser agrees).
absl::Status ValidateRegextype(const std::vector<std::string>& globals) {
constexpr std::string_view kPrefix = "--regextype=";
bool literal = false;
for (const std::string& global : globals) {
if (!global.starts_with(kPrefix)) {
continue;
}
const std::string_view value = std::string_view(global).substr(kPrefix.size());
if (value == "RE2") {
literal = false;
} else if (value == "PCRE2") {
if (value == "RE2" || value == "EXACT") {
continue; // core engines, always linked
}
if (value == "PCRE2") {
if (!regex::Pcre2Available()) {
return absl::InvalidArgumentError(
"--regextype=PCRE2 is not built into this binary (the PCRE2 backend is a build extra)");
}
literal = false;
} else if (value == "EXACT") {
literal = true;
} else if (value == "MATCH") {
return absl::InvalidArgumentError(
absl::StrCat("--regextype=", value, " is not supported yet (planned via #85); use RE2, PCRE2 or EXACT"));
absl::StrCat("--regextype=", value, " is reserved and not supported yet; use RE2, EXACT or PCRE2"));
} else {
return absl::InvalidArgumentError(
absl::StrCat("unknown --regextype '", value, "'; expected RE2, PCRE2 or EXACT"));
absl::StrCat("unknown --regextype '", value, "'; expected RE2, EXACT or PCRE2"));
}
}
return literal;
return absl::OkStatus();
}

// --time-format=NAME sets the default format for a time field rendered without an
Expand Down Expand Up @@ -1614,12 +1612,11 @@ int RunFind(
on_error("--block-size", size_status);
return 2; // do not traverse
}
// --regextype=RE2|PCRE2|EXACT: the regex grammar (RE2 default, or PCRE2) plus EXACT (-grep
// literal). Validated for the whole run; PCRE2 when not built into this binary, MATCH (reserved),
// and unknown values are usage errors.
const absl::StatusOr<bool> grep_literal = ResolveGrepLiteral(command.globals);
if (!grep_literal.ok()) {
on_error("--regextype", grep_literal.status());
// --regextype=RE2|EXACT|PCRE2: the grammar is resolved by the parser and pre-compiled into each
// matcher; here we only validate the selector for the whole run. PCRE2 when not built into this
// binary, MATCH (reserved), and unknown values are usage errors, refused before the walk.
if (const absl::Status regextype = ValidateRegextype(command.globals); !regextype.ok()) {
on_error("--regextype", regextype);
return 2; // do not traverse
}
// --count / -c: -grep emits a per-file matching-line count instead of the lines.
Expand Down Expand Up @@ -1965,7 +1962,6 @@ int RunFind(
.time_format = time_format,
.block_size = block_size,
.fold_name_case = fold_name_case,
.grep_literal = *grep_literal,
.grep_count = grep_count,
.grep_before = grep_before,
.grep_after = grep_after,
Expand Down
21 changes: 14 additions & 7 deletions xff/parser/parser.cc
Original file line number Diff line number Diff line change
Expand Up @@ -507,17 +507,24 @@ const Expr* FirstXffPrintfField(const Expr* expr) {
return FirstXffPrintfField(expr->rhs.get());
}

// The regex grammar for the command's matchers, from `--regextype=` (last occurrence wins). Only
// PCRE2 selects a non-default grammar; RE2 / EXACT (and the -grep MATCH placeholder) stay RE2 here.
// This is lenient by design: an unknown or PCRE2-not-built-in value is left as RE2 and rejected by
// run.cc's ResolveGrepLiteral (the validating reader) before the walk, so it never reaches a matcher.
// The regex grammar for the command's matchers, from `--regextype=` (last occurrence wins). EXACT
// selects the literal engine, PCRE2 the Perl engine; RE2 (and the reserved MATCH placeholder) stay
// RE2. This is lenient by design: an unknown or PCRE2-not-built-in value is left as RE2 and rejected
// by run.cc's ValidateRegextype (the validating reader) before the walk, so it never reaches a matcher.
regex::Grammar GrammarFromGlobals(const std::vector<std::string>& globals) {
constexpr std::string_view kPrefix = "--regextype=";
regex::Grammar grammar = regex::Grammar::kRe2;
for (const std::string& global : globals) {
if (global.starts_with(kPrefix)) {
grammar =
std::string_view(global).substr(kPrefix.size()) == "PCRE2" ? regex::Grammar::kPcre2 : regex::Grammar::kRe2;
if (!global.starts_with(kPrefix)) {
continue;
}
const std::string_view value = std::string_view(global).substr(kPrefix.size());
if (value == "EXACT") {
grammar = regex::Grammar::kExact;
} else if (value == "PCRE2") {
grammar = regex::Grammar::kPcre2;
} else {
grammar = regex::Grammar::kRe2; // RE2 / MATCH / unknown -> RE2 (run.cc validates the value)
}
}
return grammar;
Expand Down
4 changes: 2 additions & 2 deletions xff/parser/parser_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,9 @@ TEST_F(ParserTest, RegextypeSelectsTheMatcherGrammar) {
ASSERT_OK_AND_ASSIGN(const Command pcre2, Parse({"--regextype=PCRE2", ".", "-regex", ".*"}));
EXPECT_THAT(pcre2.grammar, regex::Grammar::kPcre2);

// EXACT is a -grep literal selector, not a regex engine, so the grammar stays RE2.
// EXACT selects the literal engine (a core grammar, applies to every pattern predicate).
ASSERT_OK_AND_ASSIGN(const Command exact, Parse({"--regextype=EXACT", ".", "-grep", "x"}));
EXPECT_THAT(exact.grammar, regex::Grammar::kRe2);
EXPECT_THAT(exact.grammar, regex::Grammar::kExact);

// Last occurrence wins (mirrors run.cc's ResolveGrepLiteral).
ASSERT_OK_AND_ASSIGN(const Command last, Parse({"--regextype=PCRE2", "--regextype=RE2", ".", "-regex", ".*"}));
Expand Down
74 changes: 73 additions & 1 deletion xff/regex/regex.cc
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@

#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/ascii.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "re2/re2.h"
#include "xff/regex/backend.h"
Expand Down Expand Up @@ -79,6 +81,73 @@ class Re2Backend final : public RegexBackend {
std::unique_ptr<RE2> re_;
};

// The kExact grammar: a literal string match, no metacharacters. A core engine (always linked, no
// dependency), so --regextype=EXACT is always available. FullMatch is equality, PartialMatch a
// substring test, FindFirst the first occurrence, Rewrite a literal find/replace (no
// backreferences); `case_insensitive` folds ASCII case on both sides. There is no pattern to
// compile, so Compile(kExact) never fails.
class ExactBackend final : public RegexBackend {
public:
ExactBackend(std::string pattern, bool case_insensitive)
: pattern_(std::move(pattern)),
case_insensitive_(case_insensitive),
needle_(case_insensitive_ ? absl::AsciiStrToLower(pattern_) : pattern_) {}

bool FullMatch(std::string_view text) const override {
return case_insensitive_ ? absl::EqualsIgnoreCase(text, pattern_) : text == pattern_;
}

bool PartialMatch(std::string_view text) const override { return FindFirst(text).has_value(); }

std::optional<std::pair<std::size_t, std::size_t>> FindFirst(std::string_view text) const override {
if (pattern_.empty()) {
return std::make_pair(std::size_t{0}, std::size_t{0}); // empty needle matches at the start
}
// ASCII case-folding preserves byte positions, so the offset found in the lowered copy maps back
// to `text` unchanged (the reported length is the pattern's).
const std::size_t pos = case_insensitive_ ? absl::AsciiStrToLower(text).find(needle_) : text.find(needle_);
if (pos == std::string::npos) {
return std::nullopt;
}
return std::make_pair(pos, pattern_.size());
}

std::optional<std::vector<std::string>> FullMatchCaptures(std::string_view text) const override {
if (!FullMatch(text)) {
return std::nullopt;
}
return std::vector<std::string>{std::string(text)}; // index 0 = the whole match; no groups
}

std::string Rewrite(std::string_view text, std::string_view replacement, bool global) const override {
if (pattern_.empty()) {
return std::string(text); // an empty needle rewrites nothing (avoids an infinite loop)
}
const std::string haystack = case_insensitive_ ? absl::AsciiStrToLower(text) : std::string(text);
std::string out;
std::size_t pos = 0;
while (true) {
const std::size_t hit = haystack.find(needle_, pos);
if (hit == std::string::npos) {
break;
}
out.append(text, pos, hit - pos); // copy from the original text (preserves case)
out.append(replacement);
pos = hit + needle_.size();
if (!global) {
break;
}
}
out.append(text.substr(pos));
return out;
}

private:
std::string pattern_;
bool case_insensitive_;
std::string needle_; // == pattern_, ASCII-lowered when case_insensitive_
};

// The process-wide PCRE2 backend factory, empty when no PCRE2 backend is linked. Set once at
// static-init by the real backend's Pcre2Registrar (full build only); a Meyers static so the
// registrar in another TU can safely write it during static initialization.
Expand Down Expand Up @@ -109,13 +178,16 @@ absl::StatusOr<Matcher> Matcher::Compile(std::string_view pattern, bool case_ins
}
return Matcher(std::make_unique<Re2Backend>(std::move(re)));
}
case Grammar::kExact:
// A literal match: no pattern to compile, so this never fails.
return Matcher(std::make_unique<ExactBackend>(std::string(pattern), case_insensitive));
case Grammar::kPcre2: {
// PCRE2 is a build-time extra: the real backend self-registers a factory (full build only).
// When none is registered (lean build) the grammar is not available -- a distinct Unimplemented
// state from an InvalidArgument bad pattern, and never a silent fallback to RE2.
const Pcre2Factory& factory = Pcre2FactorySlot();
if (!factory) {
return absl::UnimplementedError("the PCRE2 regex grammar (-regextype=pcre) is not built into this binary");
return absl::UnimplementedError("the PCRE2 regex grammar (--regextype=PCRE2) is not built into this binary");
}
absl::StatusOr<std::unique_ptr<const RegexBackend>> backend = factory(pattern, case_insensitive);
if (!backend.ok()) {
Expand Down
Loading
Loading