From 5757bff6940297e9b738faf45967c2f5eb908c36 Mon Sep 17 00:00:00 2001 From: helly25 <6420169+helly25@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:23:58 +0100 Subject: [PATCH] feat(#109): -verify matcher for hash verification Add the `-verify EXPECTED` predicate: TRUE when the file's computed digest equals EXPECTED, a {field} template rendered per entry - the manifest- verification companion of -hash. So: xff . -type f -verify {def.SUMS} # list files matching their manifest xff . -type f ! -verify {def.SUMS} # list drift / corruption `-verify=ALGO[/ENCODING]` shares the -hash spec grammar (sha256/hex default; sha1/sha512/md5, hex/base64); hex comparison folds case (sha256sum lowercases, SRI-style tools upper-case) while base64 stays case-sensitive. Style::kXff, so the find style rejects it; a bad spec is a pre-walk usage error. Wiring mirrors -cmp (arity-1 field-template target) + -hash (kHash binding); the parser's kHash =SPEC branch now consumes the descriptor's arity operands so `-verify=sha256 EXPECTED` parses. ValidateHashArgs covers -verify too. Tests: xff/cli/verify_test.sh (match/mismatch, {def.X} source, hex case-fold, ALGO/ENCODING selectors, `! -verify` drift, bad-spec + find-style errors). Regenerated XFF.md; CHANGELOG + TODO updated. --summary tallies deferred to a follow-up slice. --- CHANGELOG.md | 4 ++ TODO.md | 15 ++--- XFF.md | 2 + xff/cli/BUILD.bazel | 10 ++++ xff/cli/verify_test.sh | 116 +++++++++++++++++++++++++++++++++++++++ xff/engine/evaluate.cc | 53 +++++++++++++++++- xff/parser/parser.cc | 17 ++++-- xff/registry/registry.cc | 17 ++++++ 8 files changed, 221 insertions(+), 13 deletions(-) create mode 100755 xff/cli/verify_test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 276f485..65394e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ # 0.2.0 +- Added the `-verify EXPECTED` matcher (#109): true when the file's digest equals EXPECTED, a + `{field}` template rendered per entry (so `-verify {def.SUMS}` checks a sidecar value and + `! -verify …` selects drift). `-verify=ALGO[/ENCODING]` shares the `-hash` spec grammar; hex + comparison folds case. - Added ASN.1 `GeneralizedTime` time-format presets: `asn1` (aka `generalizedtime`, `YYYYMMDDHHMMSS` in local time, its optional zone added by `--time-zone-suffix=always`) and `asn1z` (UTC with a mandatory `Z`). diff --git a/TODO.md b/TODO.md index cd9f0a0..5cf5dfa 100644 --- a/TODO.md +++ b/TODO.md @@ -290,13 +290,14 @@ remains below is the design-forked / larger work. non-regular file (`content::FileLineCount` + `CountLines`, reusing the grep NUL-byte binary heuristic). **Remaining:** surfacing it as an aggregate (sum + a distribution across matches), which is the `lines` metric of the histograms work (#81), not a separate item. -- **Hash-verification workflow (#109) - [DISCUSS].** The hashing primitives shipped (#105: - `xff/hash` + the `{hash}` / `{hash:sha256}` field + the `-hash` action + hex/base64 via - `mbo::digest`). Still to design and build: read an expected hash into a variable, an - `-eval`-style matcher that compares a computed `{hash}` against it (verify a manifest, detect - drift), and `--summary` tallies of verified vs failed (plus `--summary=hash` grouping for - dedup). **Open:** where expected hashes come from (a sidecar manifest file, a `{def.X}` value, - or a per-entry target like `-cmp`) and the matcher's spelling / polarity. Not yet designed. +- **Hash-verification workflow (#109) - matcher SHIPPED, tallies remain.** The hashing primitives + (#105) and now the `-verify EXPECTED` matcher are in: `-verify` computes the file's digest and is + true when it equals EXPECTED, a `{field}` template rendered per entry (so `-verify {def.SUMS}` + checks a sidecar value and `! -verify …` selects drift); `-verify=ALGO[/ENCODING]` shares the + `-hash` spec grammar, and hex comparison folds case. **Still to build:** `--summary` tallies of + verified vs failed (and `--summary=hash` grouping for dedup) - the reduction-side aggregation over + the matcher's result. **Deferred producer:** a sidecar-manifest reader that populates `{def.X}` + from a `sha256sum`-style file, so `-verify` needs no bespoke manifest parser. - **Smart-case matching (`--smart-case`) - [DISCUSS].** The rg / fd convention: an all-lowercase pattern matches case-insensitively, a pattern with any uppercase matches case-sensitively. Already referenced as an rg-flavor default (the `xfd`-drop and flavor-table notes below) but diff --git a/XFF.md b/XFF.md index 7abe7a5..53ab418 100644 --- a/XFF.md +++ b/XFF.md @@ -304,6 +304,8 @@ A dangerous directive (the exec family -exec/-execdir/-ok/-capture, or -delete) - `-eofcrlf` - match a regular file whose content ends with CRLF, or is empty (xff) _(test, xff)_ TRUE for a regular, readable file whose content ends with CRLF (or is empty). The Windows / -text=windows final terminator, and the CRLF analogue of -eofnl: compose -text=windows -eofcrlf for a well-formed CRLF-terminated file, or -text=windows ! -eofcrlf for the missing final CRLF. Stricter than -eofnl (which any LF-ending file, including CRLF, satisfies). Reads the file (expensive). An xff extension --config=find rejects. - `-cmp ARG` - true when the file's content is byte-identical to TARGET (a field template) (xff) _(test, xff)_ +- `-verify ARG` - true when the digest equals EXPECTED (a field template); -verify=ALGO[/ENC] (xff) _(test, xff)_ + Computes the file's digest and is true when it equals EXPECTED - a {field} template evaluated per entry, so it can name a sidecar value like `{def.SUMS}` or a capture. `-verify=ALGO[/ENCODING]` picks the algorithm (sha256 default; also sha1/sha512/...) and encoding (hex default, or base64); the same grammar as -hash / {hash}. The hex comparison is case-insensitive. `! -verify` selects files whose digest differs (drift / corruption). Reads the whole file, so it is expensive. - `-type ARG` - match the file type (f, d, l, b, c, p, s) _(test, find)_ Matches the entry's type by letter: f=regular file, d=directory, l=symlink, b/c=block / char device, p=FIFO, s=socket. A GNU-style comma list is any-of, so `-type f,l` matches regular files or symlinks. Under the default -P a symlink is type l; -xtype tests its target's type instead. - `-xtype ARG` - match the file type of a symlink's target _(test, find)_ diff --git a/xff/cli/BUILD.bazel b/xff/cli/BUILD.bazel index b8a54bf..63ba51f 100644 --- a/xff/cli/BUILD.bazel +++ b/xff/cli/BUILD.bazel @@ -663,6 +663,16 @@ bashtest( data = [":xff"], ) +# End-to-end test of -verify (the manifest-verification companion of -hash): TRUE when the file's +# digest equals the EXPECTED field template; -verify=ALGO[/ENCODING]; hex case-insensitivity; +# `! -verify` selects drift; the bad-spec usage error; and the find-style rejection. +bashtest( + name = "verify_test", + size = "small", + srcs = ["verify_test.sh"], + data = [":xff"], +) + # End-to-end test of --histogram: ASCII vs Unicode bars (--unicode), count-per-bucket sorted by # height, suppression of the per-match listing, combining with --summary, and the bad-bucket error. bashtest( diff --git a/xff/cli/verify_test.sh b/xff/cli/verify_test.sh new file mode 100755 index 0000000..113cb3b --- /dev/null +++ b/xff/cli/verify_test.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) The helly25 authors (helly25.com) +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# End-to-end test of -verify (the manifest-verification companion of -hash): TRUE when the file's +# computed digest equals the EXPECTED field template (a literal or {def.NAME}), -verify=ALGO +# [/ENCODING], hex case-insensitivity, `! -verify` selecting drift, the bad-spec usage error, and +# the find-style rejection of this xff extension. Drives the real binary (reads files). + +set -euo pipefail + +# shellcheck disable=SC1090,SC1091,SC2154 +source "${helly25_bashtest}" + +_xff_bin() { + local bin="${TEST_SRCDIR}/${TEST_WORKSPACE}/xff/cli/xff" + if [[ ! -x "${bin}" ]]; then + bin="$(find "${TEST_SRCDIR}" -type f -name xff -path '*xff/cli/xff' 2>/dev/null | head -1)" + fi + echo "${bin}" +} + +# The sha256 / md5 / base64 of "abc" are stable, spec-pinned vectors (mbo::digest owns +# conformance); here they verify the CLI plumbing end to end. +_SHA256_ABC="ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +_MD5_ABC="900150983cd24fb0d6963f7d28e17f72" +_B64_ABC="ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" + +test::verify_matches_and_mismatches() { + local dir out + dir="${TEST_TMPDIR}/vfy" + mkdir -p "${dir}" + printf 'abc' >"${dir}/f.txt" + # A matching EXPECTED (the sha256 of "abc") makes -verify true, so the implicit print emits it. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify "${_SHA256_ABC}" 2>&1)" + expect_output_contains "${dir}/f.txt" "${out}" + # A non-matching EXPECTED makes -verify false, so nothing prints. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify "deadbeef" 2>&1)" + expect_output_not_contains "${dir}/f.txt" "${out}" +} + +test::verify_reads_expected_from_a_define() { + local dir out + dir="${TEST_TMPDIR}/vfydef" + mkdir -p "${dir}" + printf 'abc' >"${dir}/f.txt" + # EXPECTED is a field template, so a {def.NAME} value drives the comparison (a sidecar manifest). + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" --define="SUMS=${_SHA256_ABC}" \ + "${dir}" -name f.txt -verify '{def.SUMS}' 2>&1)" + expect_output_contains "${dir}/f.txt" "${out}" +} + +test::verify_hex_is_case_insensitive() { + local dir out upper + dir="${TEST_TMPDIR}/vfycase" + mkdir -p "${dir}" + printf 'abc' >"${dir}/f.txt" + upper="$(printf '%s' "${_SHA256_ABC}" | tr 'a-f' 'A-F')" + # An upper-cased hex expected still matches the lower-cased computed digest (hex folds case). + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify "${upper}" 2>&1)" + expect_output_contains "${dir}/f.txt" "${out}" +} + +test::verify_algo_and_encoding_selectors() { + local dir out + dir="${TEST_TMPDIR}/vfyalgo" + mkdir -p "${dir}" + printf 'abc' >"${dir}/f.txt" + # -verify=md5 checks against the md5 digest ... + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify=md5 "${_MD5_ABC}" 2>&1)" + expect_output_contains "${dir}/f.txt" "${out}" + # ... and -verify=sha256/base64 against the base64-encoded digest. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify=sha256/base64 "${_B64_ABC}" 2>&1)" + expect_output_contains "${dir}/f.txt" "${out}" +} + +test::not_verify_selects_drift() { + local dir out + dir="${TEST_TMPDIR}/vfydrift" + mkdir -p "${dir}" + printf 'abc' >"${dir}/good.txt" + printf 'xyz' >"${dir}/bad.txt" + # `! -verify EXPECTED` with the sha256 of "abc" selects the file whose content changed. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -type f ! -verify "${_SHA256_ABC}" 2>&1)" + expect_output_contains "${dir}/bad.txt" "${out}" + expect_output_not_contains "${dir}/good.txt" "${out}" +} + +test::verify_bad_spec_and_find_style_are_usage_errors() { + local dir out rc + dir="${TEST_TMPDIR}/vfyerr" + mkdir -p "${dir}" + printf 'abc' >"${dir}/f.txt" + # An unknown algorithm in -verify=ALGO is a usage error. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" "${dir}" -name f.txt -verify=crc32 x 2>&1)" && rc=0 || rc=$? + expect_eq "2" "${rc}" + expect_output_contains 'unknown algorithm or encoding' "${out}" + # -verify is an xff extension; the find style rejects it. + out="$(XFF_CONFIG="${TEST_TMPDIR}/none" "$(_xff_bin)" --config=find "${dir}" -verify x 2>&1)" && rc=0 || rc=$? + expect_eq "2" "${rc}" + expect_output_contains 'find style' "${out}" +} + +test_runner diff --git a/xff/engine/evaluate.cc b/xff/engine/evaluate.cc index 9573339..57502f6 100644 --- a/xff/engine/evaluate.cc +++ b/xff/engine/evaluate.cc @@ -1291,6 +1291,52 @@ bool EvalHash(const parser::Expr& expr, EvalContext& ctx) { return true; } +// xff -verify EXPECTED: TRUE when the entry's digest equals EXPECTED, the manifest-verification +// companion of -hash. EXPECTED is a field template rendered per entry (so `-verify {def.SUMS}` +// checks against a sidecar value, and `! -verify {def.SUMS}` lists drift). The algorithm / encoding +// come from `-verify=ALGO[/ENCODING]` (or the --hash-algorithm / --hash-encoding defaults), exactly +// like -hash; the hex comparison is case-insensitive since sha256sum and SRI differ only in case. +// An empty EXPECTED, an unreadable file, or a bad spec is FALSE (no match), so drift-selection is +// safe. Cost::kExpensive (reads the whole file). +bool EvalVerify(const parser::Expr& expr, EvalContext& ctx) { + if (expr.args.empty()) { + return false; + } + const std::string link = LinkTarget(ctx); // owns the {target} text for the render below + const std::string expected = fields::Template::Compile(expr.args.front()) + .Render( + fields::RenderContext{ + .path = ctx.visit.path, + .root = ctx.visit.root, + .link_target = link, + .metadata = ctx.visit.metadata, + .depth = ctx.visit.depth, + .tz = ctx.tz, + .time_format = ctx.time_format, + .zone_suffix = ctx.zone_suffix, + .hash_algorithm = ctx.hash_algorithm, + .hash_encoding = ctx.hash_encoding, + .captures = ctx.captures, + .defines = ctx.defines, + .outputs = ctx.outputs}); + if (expected.empty()) { + return false; // no expected hash resolved (e.g. an unset {def.X}) -> treat as a mismatch + } + const std::string_view default_algo = ctx.hash_algorithm.empty() ? "sha256" : ctx.hash_algorithm; + const hash::Encoding default_encoding = hash::ParseEncoding(ctx.hash_encoding).value_or(hash::Encoding::kHex); + const std::optional spec = hash::ParseSpec(expr.hash_spec, default_algo, default_encoding); + if (!spec.has_value()) { + return false; // defensively no-op; ValidateHashArgs rejects a bad spec before the walk + } + const std::optional digest = hash::HashFile(spec->algo, ctx.visit.path, spec->encoding); + if (!digest.has_value()) { + return false; // unreadable / non-regular file -> mismatch (selected by `! -verify`) + } + // Hex digests fold case (sha256sum lowercases, some SRI-style tools upper-case), but base64 is + // case-sensitive by definition (A-Z and a-z are distinct symbols), so only hex compares loosely. + return spec->encoding == hash::Encoding::kHex ? absl::EqualsIgnoreCase(*digest, expected) : *digest == expected; +} + // xff -grep PATTERN: the line-output companion of -rxc. Prints each line of the // file's content that matches, as `path:lineno:text` (grep's piped form). The // pattern is pre-compiled by the parser under the run's --regextype grammar (RE2 by @@ -2096,6 +2142,7 @@ constexpr auto kDispatch = mbo::container::MakeLimitedMap( DispatchPair{"-uid", {&EvalUid}}, DispatchPair{"-used", {&EvalUsed}}, DispatchPair{"-user", {&EvalUser}}, + DispatchPair{"-verify", {&EvalVerify}}, DispatchPair{"-wholename", {&EvalPath}}, DispatchPair{"-writable", {&EvalWritable}}, DispatchPair{"-xtype", {&EvalXtype}}); @@ -2244,12 +2291,14 @@ absl::Status ValidateSizeArgs(const parser::Expr& expr) { absl::Status ValidateHashArgs(const parser::Expr& expr) { if (expr.kind == parser::Expr::Kind::kPredicate) { - if (expr.descriptor != nullptr && expr.descriptor->name == "-hash" && !expr.hash_spec.empty()) { + // -hash and -verify share the =ALGO[/ENCODING] spec grammar (Binding::kHash), so both validate here. + if (expr.descriptor != nullptr && (expr.descriptor->name == "-hash" || expr.descriptor->name == "-verify") + && !expr.hash_spec.empty()) { // Only the spec's explicit parts matter here, so validate against a concrete default. if (!hash::ParseSpec(expr.hash_spec, "sha256", hash::Encoding::kHex).has_value()) { return absl::InvalidArgumentError( absl::StrCat( - "'-hash=", expr.hash_spec, + "'", expr.descriptor->name, "=", expr.hash_spec, "': unknown algorithm or encoding (ALGO[/ENCODING]; encoding is hex or base64)")); } } diff --git a/xff/parser/parser.cc b/xff/parser/parser.cc index d9d1bf1..c627cb6 100644 --- a/xff/parser/parser.cc +++ b/xff/parser/parser.cc @@ -374,14 +374,23 @@ class ExprParser { } return node; } - // A Binding::kHash primary (-hash) carries an attached =ALGO[/ENCODING] token and takes no - // operand. The spec is stored raw and validated before the walk (engine::ValidateHashArgs); - // a bare -hash (no '=') falls through to the default (--hash-algorithm / --hash-encoding). + // A Binding::kHash primary carries an attached =ALGO[/ENCODING] token, then its arity operands + // (none for -hash, the EXPECTED template for -verify). The spec is stored raw and validated + // before the walk (engine::ValidateHashArgs); a bare `` (no '=') falls through to the + // default (--hash-algorithm / --hash-encoding). if (const registry::Descriptor* const descriptor = registry::Lookup(base); descriptor != nullptr && descriptor->binding == registry::Binding::kHash) { const std::string spec = token.substr(eq + 1); ++pos_; // consume the `=SPEC` token - ExprPtr node = MakePredicate(descriptor, {}, grammar_); + std::vector args; + for (int i = 0; i < descriptor->arity; ++i) { + if (AtEnd()) { + Fail(absl::StrCat("predicate '", base, "' is missing an argument")); + return nullptr; + } + args.push_back(tokens_[pos_++]); + } + ExprPtr node = MakePredicate(descriptor, std::move(args), grammar_); if (node != nullptr) { node->hash_spec = spec; } diff --git a/xff/registry/registry.cc b/xff/registry/registry.cc index f951d43..7a3fee7 100644 --- a/xff/registry/registry.cc +++ b/xff/registry/registry.cc @@ -280,6 +280,23 @@ constexpr std::array kDescriptors = std::to_array({ .style = Style::kXff, .cost = Cost::kExpensive, }, + // xff -verify EXPECTED: true when the file's digest equals EXPECTED (a field template rendered + // per entry, so it can name a sidecar value like `{def.SUMS}` or a capture). `-verify=ALGO + // [/ENCODING]` picks the algorithm / encoding, like -hash. `! -verify` selects files that drift. + { + .name = "-verify", + .summary = "true when the digest equals EXPECTED (a field template); -verify=ALGO[/ENC] (xff)", + .details = "Computes the file's digest and is true when it equals EXPECTED - a {field} template evaluated per " + "entry, so it can name a sidecar value like `{def.SUMS}` or a capture. `-verify=ALGO[/ENCODING]` " + "picks the algorithm (sha256 default; also sha1/sha512/...) and encoding (hex default, or base64); " + "the same grammar as -hash / {hash}. The hex comparison is case-insensitive. `! -verify` selects " + "files whose digest differs (drift / corruption). Reads the whole file, so it is expensive.", + .kind = Kind::kTest, + .arity = 1, + .binding = Binding::kHash, + .style = Style::kXff, + .cost = Cost::kExpensive, + }, { .name = "-type", .summary = "match the file type (f, d, l, b, c, p, s)",