From e1d6ec0d0a9da2849565ec339eab9743c698776c Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Mon, 3 Aug 2026 23:52:48 -0300 Subject: [PATCH] Publish and resolve the deployed scrubber allowlist identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scrubber's link allowlist is baked in from config/assembler.yml at build time, so the deployed allowlist was not observable: backfill planning could only validate links against the local checkout, and allowlist skew surfaced only as silent link stripping on publication. The build workflow now computes the embedded allowlist's SHA-256 and build commit, and the release workflow attaches the identity document to the GitHub release only after a successful Lambda deploy — asset presence attests the deploy. A new `changelog scrubber-allowlist` command (and ScrubberAllowlistIdentityService for programmatic consumers) resolves the identity from the newest release carrying the asset, or a specific tag, and compares it against a local assembler.yml. Closes elastic/docs-eng-team#671 Co-Authored-By: Claude Fable 5 --- .../build-changelog-scrubber-lambda.yml | 28 +++ .github/workflows/release.yml | 11 +- docs/cli-schema.json | 94 +++++++++ docs/cli/changelog/cmd-scrubber-allowlist.md | 44 +++++ .../docs-lambda-changelog-scrubber/README.md | 7 + .../ScrubberAllowlistIdentity.cs | 115 +++++++++++ .../ScrubberAllowlistIdentityService.cs | 178 +++++++++++++++++ .../GitHub/GitHubReleaseService.cs | 117 +++++++++-- .../GitHub/IGitHubReleaseService.cs | 43 ++++ .../docs-builder/Commands/ChangelogCommand.cs | 50 +++++ .../ScrubberAllowlistIdentityServiceTests.cs | 183 ++++++++++++++++++ .../ScrubberAllowlistIdentityTests.cs | 110 +++++++++++ 12 files changed, 967 insertions(+), 13 deletions(-) create mode 100644 docs/cli/changelog/cmd-scrubber-allowlist.md create mode 100644 src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentity.cs create mode 100644 src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentityService.cs create mode 100644 tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityServiceTests.cs create mode 100644 tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityTests.cs diff --git a/.github/workflows/build-changelog-scrubber-lambda.yml b/.github/workflows/build-changelog-scrubber-lambda.yml index 2852b19d0a..af8e6bbaf1 100644 --- a/.github/workflows/build-changelog-scrubber-lambda.yml +++ b/.github/workflows/build-changelog-scrubber-lambda.yml @@ -44,3 +44,31 @@ jobs: retention-days: 1 if-no-files-found: error path: ${{ env.BINARY_PATH }} + # The scrubber's link allowlist is embedded from config/assembler.yml at this ref, so this + # hash identifies exactly which allowlist the deployed Lambda runs with. The release workflow + # attaches the document to the GitHub release after a successful deploy, making the deployed + # identity observable (docs-eng-team#671). + - name: Compute allowlist identity + env: + GIT_REF: ${{ inputs.ref || github.ref }} + # language=bash + run: | + set -euo pipefail + sha="sha256:$(sha256sum config/assembler.yml | awk '{print $1}')" + commit="$(git rev-parse HEAD)" + built_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + jq -n \ + --arg sha "$sha" \ + --arg commit "$commit" \ + --arg ref "$GIT_REF" \ + --arg built_at "$built_at" \ + '{schema_version: 1, artifact: "scrubber-allowlist-identity", allowlist_sha256: $sha, deployment_commit: $commit, git_ref: $ref, built_at: $built_at}' \ + > changelog-scrubber-allowlist.json + cat changelog-scrubber-allowlist.json + - name: Archive allowlist identity + uses: actions/upload-artifact@v7 + with: + name: changelog-scrubber-allowlist-identity + retention-days: 1 + if-no-files-found: error + path: changelog-scrubber-allowlist.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 615c909e25..575bdd9997 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,6 +208,11 @@ jobs: with: name: changelog-scrubber-lambda-binary + - name: Download allowlist identity + uses: actions/download-artifact@v8 + with: + name: changelog-scrubber-allowlist-identity + - name: Create zip run: | zip -j "${ZIP_FILE}" ./bootstrap @@ -223,12 +228,16 @@ jobs: --function-name elastic-docs-v3-changelog-scrubber \ --zip-file "fileb://${ZIP_FILE}" + # The allowlist identity is attached only after update-function-code succeeded, so the + # presence of this asset on a release attests that the release's allowlist was deployed. + # Consumers resolve the deployed identity from the newest release carrying the asset + # (`docs-builder changelog scrubber-allowlist`, docs-eng-team#671). - name: Attach Distribution to release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG_NAME: ${{ needs.release-drafter.outputs.tag_name }} REPO: ${{ github.repository }} - run: gh release upload --repo "$REPO" "$TAG_NAME" "${ZIP_FILE}" + run: gh release upload --repo "$REPO" "$TAG_NAME" "${ZIP_FILE}" changelog-scrubber-allowlist.json release: needs: diff --git a/docs/cli-schema.json b/docs/cli-schema.json index ef677c35be..4b362e7d9d 100644 --- a/docs/cli-schema.json +++ b/docs/cli-schema.json @@ -4299,6 +4299,100 @@ } ] }, + { + "path": [ + "changelog" + ], + "name": "scrubber-allowlist", + "summary": "Resolve the link allowlist identity of the deployed changelog scrubber.", + "notes": "The scrubber Lambda embeds its link allowlist from config/assembler.yml at build time, so the\ndeployed allowlist can differ from any local checkout. The release pipeline attaches a\nchangelog-scrubber-allowlist.json asset to the GitHub release after each successful scrubber\ndeploy; this command resolves the identity from that asset. Without a --tag, the newest release\ncarrying the asset wins \u2014 the most recent deploy that passed the gated pipeline. Exits non-zero when\nno identity can be resolved: backfill plans must pin this identity and cannot be approved without it.", + "usage": "docs-builder changelog scrubber-allowlist [options]", + "examples": [], + "parameters": [ + { + "role": "flag", + "name": "tag", + "type": "string", + "required": false, + "summary": "Release tag to resolve the identity from (e.g., \u0022v5.7.0\u0022). Defaults to the newest release carrying the identity asset." + }, + { + "role": "flag", + "name": "assembler", + "type": "string", + "required": false, + "summary": "Path to a local assembler.yml to compare against the deployed allowlist. Defaults to config/assembler.yml when it exists; a mismatch is reported as a warning, not an error.", + "validations": [ + { + "kind": "rejectSymbolicLinks" + }, + { + "kind": "existing" + }, + { + "kind": "fileExtensions", + "values": [ + "yml", + "yaml" + ] + } + ] + }, + { + "role": "flag", + "name": "owner", + "type": "string", + "required": false, + "summary": "GitHub owner of the repository whose releases carry the identity asset.", + "defaultValue": "elastic" + }, + { + "role": "flag", + "name": "repo", + "type": "string", + "required": false, + "summary": "GitHub repository whose releases carry the identity asset.", + "defaultValue": "docs-builder" + }, + { + "role": "flag", + "name": "log-level", + "shortName": "l", + "type": "enum", + "required": false, + "summary": "Minimum log level. Default: information", + "enumValues": [ + "trace", + "debug", + "information", + "warning", + "error", + "critical", + "none" + ] + }, + { + "role": "flag", + "name": "config-source", + "shortName": "c", + "type": "enum", + "required": false, + "summary": "Override the configuration source: local, remote", + "enumValues": [ + "local", + "remote", + "embedded" + ] + }, + { + "role": "flag", + "name": "skip-private-repositories", + "type": "boolean", + "required": false, + "summary": "Skip cloning private repositories" + } + ] + }, { "path": [ "changelog" diff --git a/docs/cli/changelog/cmd-scrubber-allowlist.md b/docs/cli/changelog/cmd-scrubber-allowlist.md new file mode 100644 index 0000000000..4d50a5707f --- /dev/null +++ b/docs/cli/changelog/cmd-scrubber-allowlist.md @@ -0,0 +1,44 @@ +## Description + +Resolve the link allowlist identity of the deployed changelog scrubber. + +The changelog scrubber Lambda embeds its link allowlist from `config/assembler.yml` at build time, so the allowlist the deployed scrubber actually runs with can differ from any local checkout. Links attributed to repositories that are not on the deployed allowlist are silently stripped on publication, which makes the deployed identity a required input for backfill planning and public verification. + +The release pipeline attaches a `changelog-scrubber-allowlist.json` asset to the GitHub release **after** the scrubber deploy succeeded, so the presence of the asset attests that the release's allowlist was deployed. This command resolves the identity from that asset: + +- Without `--tag`, the newest (non-draft) release carrying the asset wins — that is the most recent deploy that passed the gated pipeline. +- With `--tag`, the identity is read from that specific release and the command fails when the release does not carry the asset (it predates identity publication, or its scrubber deploy never completed). + +When a local `assembler.yml` is available (`--assembler`, or `config/assembler.yml` in the current directory), its hash is compared against the deployed identity. A mismatch is reported as a **warning**, not an error: it means link decisions must be validated against the deployed allowlist, not the local checkout. + +The command exits non-zero when no identity can be resolved. Backfill plans pin this identity, and a plan cannot be approved without it. + +## Identity document + +The resolved asset is a small JSON document: + +```json +{ + "schema_version": 1, + "artifact": "scrubber-allowlist-identity", + "allowlist_sha256": "sha256:<64 hex characters>", + "deployment_commit": "", + "git_ref": "v5.7.0", + "built_at": "2026-08-01T12:00:00Z" +} +``` + +`allowlist_sha256` is the SHA-256 of the raw `config/assembler.yml` bytes at the release tag — the same value `sha256sum config/assembler.yml` reports at that ref, and the same bytes the Lambda embeds as its allowlist source. + +## Examples + +```sh +# Resolve the identity of the most recent gated deploy +docs-builder changelog scrubber-allowlist + +# Resolve the identity a specific release deployed +docs-builder changelog scrubber-allowlist --tag v5.7.0 + +# Compare an explicit local assembler.yml against the deployed allowlist +docs-builder changelog scrubber-allowlist --assembler ./config/assembler.yml +``` diff --git a/src/infra/docs-lambda-changelog-scrubber/README.md b/src/infra/docs-lambda-changelog-scrubber/README.md index 2f993f56f2..1bff941781 100644 --- a/src/infra/docs-lambda-changelog-scrubber/README.md +++ b/src/infra/docs-lambda-changelog-scrubber/README.md @@ -8,6 +8,13 @@ The public repo allowlist is derived from `config/assembler.yml` (baked into the Lambda image as an embedded resource at build time). Changes to `assembler.yml` trigger a Lambda redeploy via CI. +The deployed allowlist's identity (SHA-256 of the embedded `assembler.yml`, plus the +build commit) is published as a `changelog-scrubber-allowlist.json` asset on the GitHub +release, attached only after a successful deploy. Resolve it with +`docs-builder changelog scrubber-allowlist` — backfill planning and verification pin +this identity so link decisions are checked against the deployed allowlist, not a +local checkout (docs-eng-team#671). + ## Build From a linux `x86_64` machine (or Docker): diff --git a/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentity.cs b/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentity.cs new file mode 100644 index 0000000000..68191dba60 --- /dev/null +++ b/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentity.cs @@ -0,0 +1,115 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace Elastic.Changelog.AllowlistIdentity; + +/// +/// Identifies exactly which link allowlist a changelog-scrubber Lambda deployment is running with. +/// The allowlist is embedded from config/assembler.yml at the release tag the Lambda was +/// built from, so the deployed identity is fully determined by that tag. The release pipeline +/// attaches this document as a release asset () only after the Lambda +/// deploy succeeded, which makes "the newest release carrying the asset" the identity of the +/// most recent gated deploy. +/// +public sealed partial record ScrubberAllowlistIdentity +{ + /// The name of the release asset this document is published as. + public const string AssetName = "changelog-scrubber-allowlist.json"; + + /// The artifact discriminator every identity document must carry. + public const string ArtifactKind = "scrubber-allowlist-identity"; + + /// The schema version this reader understands. + public const int CurrentSchemaVersion = 1; + + [GeneratedRegex("^sha256:[0-9a-f]{64}$")] + private static partial Regex Sha256Format(); + + [GeneratedRegex("^[0-9a-f]{40}$")] + private static partial Regex CommitFormat(); + + /// Version of this document's shape; readers reject versions they don't understand. + [JsonPropertyName("schema_version")] + public required int SchemaVersion { get; init; } + + /// What kind of document this is; always . + [JsonPropertyName("artifact")] + public required string Artifact { get; init; } + + /// Hash of the embedded config/assembler.yml bytes, as sha256: + 64 lower-case hex characters. + [JsonPropertyName("allowlist_sha256")] + public required string AllowlistSha256 { get; init; } + + /// The docs-builder commit the deployed scrubber was built from (full 40-character SHA). + [JsonPropertyName("deployment_commit")] + public required string DeploymentCommit { get; init; } + + /// The git ref (release tag) the scrubber build checked out. + [JsonPropertyName("git_ref")] + public string? GitRef { get; init; } + + /// When the scrubber binary embedding this allowlist was built, in UTC. + [JsonPropertyName("built_at")] + public DateTimeOffset? BuiltAt { get; init; } + + /// Adds a plain-English description of every problem in this identity to . + public void Validate(IList problems) + { + if (SchemaVersion != CurrentSchemaVersion) + problems.Add($"Unsupported allowlist identity schema version {SchemaVersion}; this reader understands version {CurrentSchemaVersion}."); + if (!string.Equals(Artifact, ArtifactKind, StringComparison.Ordinal)) + problems.Add($"Expected artifact '{ArtifactKind}' but found '{Artifact}'."); + if (string.IsNullOrWhiteSpace(AllowlistSha256) || !Sha256Format().IsMatch(AllowlistSha256)) + problems.Add($"The allowlist hash must look like sha256: plus 64 lower-case hex characters, but found '{AllowlistSha256}'."); + if (string.IsNullOrWhiteSpace(DeploymentCommit) || !CommitFormat().IsMatch(DeploymentCommit)) + problems.Add($"The deployment commit must be a full 40-character lower-case hex SHA, but found '{DeploymentCommit}'."); + } + + /// + /// Parses an identity document from JSON. Returns false with the reasons in + /// when the document is malformed or fails validation. + /// + public static bool TryParse(string json, [NotNullWhen(true)] out ScrubberAllowlistIdentity? identity, out IReadOnlyList problems) + { + var found = new List(); + identity = null; + try + { + identity = JsonSerializer.Deserialize(json, ScrubberAllowlistIdentityJsonContext.Default.ScrubberAllowlistIdentity); + } + catch (JsonException e) + { + found.Add($"The identity document is not valid JSON: {e.Message}"); + } + + if (identity is null && found.Count == 0) + found.Add("The identity document deserialized to null."); + + identity?.Validate(found); + if (found.Count > 0) + identity = null; + + problems = found; + return identity is not null; + } + + /// + /// Computes the identity hash of an allowlist source (the raw bytes of config/assembler.yml), + /// as sha256: + 64 lower-case hex characters — the same value sha256sum reports in CI. + /// + public static string ComputeSha256(Stream content) + { + var hash = SHA256.HashData(content); + return $"sha256:{Convert.ToHexStringLower(hash)}"; + } +} + +[JsonSerializable(typeof(ScrubberAllowlistIdentity))] +internal sealed partial class ScrubberAllowlistIdentityJsonContext : JsonSerializerContext; diff --git a/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentityService.cs b/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentityService.cs new file mode 100644 index 0000000000..8771c908d2 --- /dev/null +++ b/src/services/Elastic.Changelog/AllowlistIdentity/ScrubberAllowlistIdentityService.cs @@ -0,0 +1,178 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Elastic.Changelog.GitHub; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; + +namespace Elastic.Changelog.AllowlistIdentity; + +public record ResolveScrubberAllowlistArguments +{ + /// GitHub owner of the repository whose releases carry the identity asset. + public string Owner { get; init; } = "elastic"; + + /// GitHub repository whose releases carry the identity asset. + public string Repo { get; init; } = "docs-builder"; + + /// + /// Release tag to resolve the identity from. When null, the newest release carrying the + /// identity asset wins — that is the most recent deploy that passed the gated pipeline. + /// + public string? Tag { get; init; } + + /// Optional path to a local assembler.yml to compare against the deployed identity. + public string? AssemblerPath { get; init; } +} + +/// The deployed identity together with where it was found and how it compares to the local checkout. +public record ResolvedScrubberAllowlist +{ + /// The deployed allowlist identity document. + public required ScrubberAllowlistIdentity Identity { get; init; } + + /// The release tag the identity asset was found on. + public required string ReleaseTag { get; init; } + + /// Hash of the local assembler.yml, when a local path was given. Same format as the identity hash. + public string? LocalSha256 { get; init; } + + /// Whether the local allowlist matches the deployed one; null when no local path was given. + public bool? MatchesLocal => LocalSha256 is null ? null : string.Equals(LocalSha256, Identity.AllowlistSha256, StringComparison.Ordinal); +} + +/// +/// Resolves which link allowlist the deployed changelog scrubber is actually running with, from the +/// identity asset the release pipeline attaches after each successful scrubber deploy. Backfill +/// planning pins this identity in every plan and ledger so "which links survive publication" is +/// always answered against the deployed allowlist, never against the local checkout. +/// +public class ScrubberAllowlistIdentityService( + ILoggerFactory logFactory, + IGitHubReleaseService releaseService, + IFileSystem fileSystem +) : IService +{ + /// How many releases back to look for the identity asset when no tag is given. + private const int ReleaseLookback = 20; + + private readonly ILogger _logger = logFactory.CreateLogger(); + + /// + /// Resolves the deployed allowlist identity. Returns null after emitting errors when no + /// identity can be resolved — an unresolvable identity must block plan approval, not degrade + /// into a guess. + /// + public async Task ResolveDeployedAsync( + IDiagnosticsCollector collector, + ResolveScrubberAllowlistArguments args, + Cancel ctx = default) + { + var located = await LocateIdentityAssetAsync(collector, args, ctx); + if (located is null) + return null; + + var (release, asset) = located.Value; + var json = await releaseService.DownloadAssetTextAsync(asset, ctx); + if (json is null) + { + collector.EmitError(string.Empty, + $"Failed to download release asset '{asset.Name}' from {args.Owner}/{args.Repo}@{release.TagName}."); + return null; + } + + if (!ScrubberAllowlistIdentity.TryParse(json, out var identity, out var problems)) + { + foreach (var problem in problems) + collector.EmitError(string.Empty, $"Invalid allowlist identity on {args.Owner}/{args.Repo}@{release.TagName}: {problem}"); + return null; + } + + var localSha = ComputeLocalSha256(collector, args.AssemblerPath); + var resolved = new ResolvedScrubberAllowlist + { + Identity = identity, + ReleaseTag = release.TagName, + LocalSha256 = localSha + }; + + _logger.LogInformation("Deployed scrubber allowlist: {Sha256} (commit {Commit}, release {Tag})", + identity.AllowlistSha256, identity.DeploymentCommit, release.TagName); + + if (resolved.MatchesLocal == false) + { + collector.EmitWarning(string.Empty, + $"Local assembler.yml ({localSha}) differs from the deployed scrubber allowlist ({identity.AllowlistSha256}, release {release.TagName}). " + + "Links must be validated against the deployed allowlist, not the local checkout."); + } + else if (resolved.MatchesLocal == true) + { + _logger.LogInformation("Local assembler.yml matches the deployed allowlist"); + } + + return resolved; + } + + private async Task<(GitHubReleaseInfo Release, GitHubReleaseAsset Asset)?> LocateIdentityAssetAsync( + IDiagnosticsCollector collector, + ResolveScrubberAllowlistArguments args, + Cancel ctx) + { + if (!string.IsNullOrWhiteSpace(args.Tag)) + { + var release = await releaseService.FetchReleaseAsync(args.Owner, args.Repo, args.Tag, ctx); + if (release is null) + { + collector.EmitError(string.Empty, + $"Release '{args.Tag}' was not found on {args.Owner}/{args.Repo}. Ensure the tag exists and credentials are set."); + return null; + } + + var asset = FindIdentityAsset(release); + if (asset is null) + { + collector.EmitError(string.Empty, + $"Release {args.Owner}/{args.Repo}@{release.TagName} does not carry the '{ScrubberAllowlistIdentity.AssetName}' asset: " + + "either the release predates allowlist identity publication, or its scrubber deploy never completed."); + return null; + } + + return (release, asset); + } + + var releases = await releaseService.FetchReleasesAsync(args.Owner, args.Repo, ReleaseLookback, ctx); + foreach (var release in releases.Where(r => !r.Draft)) + { + var asset = FindIdentityAsset(release); + if (asset is not null) + return (release, asset); + _logger.LogDebug("Release {Tag} has no allowlist identity asset; looking further back", release.TagName); + } + + collector.EmitError(string.Empty, + $"No release among the latest {ReleaseLookback} on {args.Owner}/{args.Repo} carries the '{ScrubberAllowlistIdentity.AssetName}' asset, " + + "so the deployed scrubber allowlist identity cannot be resolved. A backfill plan cannot be approved without it."); + return null; + } + + private static GitHubReleaseAsset? FindIdentityAsset(GitHubReleaseInfo release) => + release.Assets.FirstOrDefault(a => string.Equals(a.Name, ScrubberAllowlistIdentity.AssetName, StringComparison.Ordinal)); + + private string? ComputeLocalSha256(IDiagnosticsCollector collector, string? assemblerPath) + { + if (string.IsNullOrWhiteSpace(assemblerPath)) + return null; + + if (!fileSystem.File.Exists(assemblerPath)) + { + collector.EmitWarning(string.Empty, $"Local assembler.yml not found at '{assemblerPath}'; skipping the local comparison."); + return null; + } + + using var stream = fileSystem.File.OpenRead(assemblerPath); + return ScrubberAllowlistIdentity.ComputeSha256(stream); + } +} diff --git a/src/services/Elastic.Changelog/GitHub/GitHubReleaseService.cs b/src/services/Elastic.Changelog/GitHub/GitHubReleaseService.cs index c28a0b7437..bd92317e40 100644 --- a/src/services/Elastic.Changelog/GitHub/GitHubReleaseService.cs +++ b/src/services/Elastic.Changelog/GitHub/GitHubReleaseService.cs @@ -69,14 +69,86 @@ static GitHubReleaseService() } } - private async Task FetchReleaseFromUrl(string url, CancellationToken ctx) + /// + public async Task> FetchReleasesAsync( + string owner, + string repo, + int count, + CancellationToken ctx = default) + { + try + { + var url = $"https://api.github.com/repos/{owner}/{repo}/releases?per_page={count}"; + using var request = CreateRequest(url); + _logger.LogDebug("Fetching releases from: {ApiUrl}", url); + + var response = await HttpClient.SendAsync(request, ctx); + if (!response.IsSuccessStatusCode) + { + _logger.LogDebug("Failed to fetch releases. Status: {StatusCode}, Reason: {ReasonPhrase}", + response.StatusCode, response.ReasonPhrase); + return []; + } + + var jsonContent = await response.Content.ReadAsStringAsync(ctx); + var releases = JsonSerializer.Deserialize(jsonContent, GitHubReleaseJsonContext.Default.GitHubReleaseResponseArray); + return releases == null ? [] : releases.Select(ToReleaseInfo).ToArray(); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "HTTP error fetching releases from GitHub"); + return []; + } + catch (TaskCanceledException) + { + _logger.LogWarning("Request timeout fetching releases from GitHub"); + return []; + } + } + + /// + public async Task DownloadAssetTextAsync(GitHubReleaseAsset asset, CancellationToken ctx = default) + { + try + { + using var request = CreateRequest(asset.BrowserDownloadUrl); + _logger.LogDebug("Downloading release asset: {AssetUrl}", asset.BrowserDownloadUrl); + + var response = await HttpClient.SendAsync(request, ctx); + if (!response.IsSuccessStatusCode) + { + _logger.LogDebug("Failed to download asset {AssetName}. Status: {StatusCode}, Reason: {ReasonPhrase}", + asset.Name, response.StatusCode, response.ReasonPhrase); + return null; + } + + return await response.Content.ReadAsStringAsync(ctx); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "HTTP error downloading release asset {AssetName}", asset.Name); + return null; + } + catch (TaskCanceledException) + { + _logger.LogWarning("Request timeout downloading release asset {AssetName}", asset.Name); + return null; + } + } + + private static HttpRequestMessage CreateRequest(string url) { // Add GitHub token if available (for rate limiting and private repos) var githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN"); - using var request = new HttpRequestMessage(HttpMethod.Get, url); + var request = new HttpRequestMessage(HttpMethod.Get, url); if (!string.IsNullOrEmpty(githubToken)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", githubToken); + return request; + } + private async Task FetchReleaseFromUrl(string url, CancellationToken ctx) + { + using var request = CreateRequest(url); _logger.LogDebug("Fetching release info from: {ApiUrl}", url); var response = await HttpClient.SendAsync(request, ctx); @@ -96,16 +168,33 @@ static GitHubReleaseService() return null; } - return new GitHubReleaseInfo - { - TagName = releaseData.TagName ?? string.Empty, - Name = releaseData.Name ?? string.Empty, - Body = releaseData.Body ?? string.Empty, - Prerelease = releaseData.Prerelease, - Draft = releaseData.Draft, - HtmlUrl = releaseData.HtmlUrl ?? string.Empty, - PublishedAt = releaseData.PublishedAt - }; + return ToReleaseInfo(releaseData); + } + + private static GitHubReleaseInfo ToReleaseInfo(GitHubReleaseResponse releaseData) => new() + { + TagName = releaseData.TagName ?? string.Empty, + Name = releaseData.Name ?? string.Empty, + Body = releaseData.Body ?? string.Empty, + Prerelease = releaseData.Prerelease, + Draft = releaseData.Draft, + HtmlUrl = releaseData.HtmlUrl ?? string.Empty, + PublishedAt = releaseData.PublishedAt, + Assets = releaseData.Assets is { Count: > 0 } + ? releaseData.Assets + .Where(a => a is { Name: not null, BrowserDownloadUrl: not null }) + .Select(a => new GitHubReleaseAsset { Name = a.Name!, BrowserDownloadUrl = a.BrowserDownloadUrl! }) + .ToArray() + : [] + }; + + private sealed class GitHubReleaseAssetResponse + { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("browser_download_url")] + public string? BrowserDownloadUrl { get; set; } } private sealed class GitHubReleaseResponse @@ -130,8 +219,12 @@ private sealed class GitHubReleaseResponse [JsonPropertyName("published_at")] public DateTimeOffset? PublishedAt { get; set; } + + [JsonPropertyName("assets")] + public List? Assets { get; set; } } [JsonSerializable(typeof(GitHubReleaseResponse))] + [JsonSerializable(typeof(GitHubReleaseResponse[]))] private sealed partial class GitHubReleaseJsonContext : JsonSerializerContext; } diff --git a/src/services/Elastic.Changelog/GitHub/IGitHubReleaseService.cs b/src/services/Elastic.Changelog/GitHub/IGitHubReleaseService.cs index 20f984ff22..fdb507ff6f 100644 --- a/src/services/Elastic.Changelog/GitHub/IGitHubReleaseService.cs +++ b/src/services/Elastic.Changelog/GitHub/IGitHubReleaseService.cs @@ -4,6 +4,22 @@ namespace Elastic.Changelog.GitHub; +/// +/// A single downloadable asset attached to a GitHub release +/// +public record GitHubReleaseAsset +{ + /// + /// The asset's file name (e.g., "changelog-scrubber-allowlist.json") + /// + public required string Name { get; init; } + + /// + /// Direct download URL for the asset's content + /// + public required string BrowserDownloadUrl { get; init; } +} + /// /// Information about a GitHub release /// @@ -43,6 +59,11 @@ public record GitHubReleaseInfo /// The date and time when this release was published on GitHub /// public DateTimeOffset? PublishedAt { get; init; } + + /// + /// The downloadable assets attached to this release + /// + public IReadOnlyList Assets { get; init; } = []; } /// @@ -63,4 +84,26 @@ public interface IGitHubReleaseService string repo, string? version, CancellationToken ctx = default); + + /// + /// Fetches the most recent releases from GitHub, newest first + /// + /// Repository owner + /// Repository name + /// Maximum number of releases to fetch + /// Cancellation token + /// The releases, or an empty list if the fetch fails + Task> FetchReleasesAsync( + string owner, + string repo, + int count, + CancellationToken ctx = default); + + /// + /// Downloads a release asset's content as text + /// + /// The asset to download + /// Cancellation token + /// The asset content, or null if the download fails + Task DownloadAssetTextAsync(GitHubReleaseAsset asset, CancellationToken ctx = default); } diff --git a/src/tooling/docs-builder/Commands/ChangelogCommand.cs b/src/tooling/docs-builder/Commands/ChangelogCommand.cs index 6a31995e2b..3084cb4f1f 100644 --- a/src/tooling/docs-builder/Commands/ChangelogCommand.cs +++ b/src/tooling/docs-builder/Commands/ChangelogCommand.cs @@ -11,6 +11,7 @@ using Actions.Core.Services; using Documentation.Builder.Arguments; using Elastic.Changelog; +using Elastic.Changelog.AllowlistIdentity; using Elastic.Changelog.Bundling; using Elastic.Changelog.Creation; using Elastic.Changelog.Evaluation; @@ -1675,6 +1676,55 @@ static async (s, c, state, ct) => await s.Upload(c, state, ct) return await serviceInvoker.InvokeAsync(ctx); } + /// Resolve the link allowlist identity of the deployed changelog scrubber. + /// + /// The scrubber Lambda embeds its link allowlist from config/assembler.yml at build time, so the + /// deployed allowlist can differ from any local checkout. The release pipeline attaches a + /// changelog-scrubber-allowlist.json asset to the GitHub release after each successful scrubber + /// deploy; this command resolves the identity from that asset. Without a --tag, the newest release + /// carrying the asset wins — the most recent deploy that passed the gated pipeline. Exits non-zero when + /// no identity can be resolved: backfill plans must pin this identity and cannot be approved without it. + /// + /// Release tag to resolve the identity from (e.g., "v5.7.0"). Defaults to the newest release carrying the identity asset. + /// Path to a local assembler.yml to compare against the deployed allowlist. Defaults to config/assembler.yml when it exists; a mismatch is reported as a warning, not an error. + /// GitHub owner of the repository whose releases carry the identity asset. + /// GitHub repository whose releases carry the identity asset. + /// Cancellation token + [NoOptionsInjection] + public async Task ScrubberAllowlist( + string? tag = null, + [Existing, ExpandUserProfile, RejectSymbolicLinks, FileExtensions(Extensions = "yml,yaml")] FileInfo? assembler = null, + string owner = "elastic", + string repo = "docs-builder", + CancellationToken ct = default + ) + { + var ctx = ct; + await using var serviceInvoker = new ServiceInvoker(collector); + + // Default the local comparison to config/assembler.yml relative to cwd when present. + var assemblerPath = assembler?.FullName; + if (assemblerPath is null) + { + var candidate = _fileSystem.Path.Join(Directory.GetCurrentDirectory(), "config", "assembler.yml"); + if (_fileSystem.File.Exists(candidate)) + assemblerPath = candidate; + } + + var service = new ScrubberAllowlistIdentityService(logFactory, new GitHubReleaseService(logFactory), _fileSystem); + var args = new ResolveScrubberAllowlistArguments + { + Owner = owner, + Repo = repo, + Tag = tag, + AssemblerPath = assemblerPath + }; + serviceInvoker.AddCommand(service, args, + static async (s, c, state, ct) => await s.ResolveDeployedAsync(c, state, ct) is not null + ); + return await serviceInvoker.InvokeAsync(ctx); + } + /// Resolves the authoring repo/owner/branch for uploads (CLI flags > bundle.{repo,owner} > git); owner falls back to the owner/ prefix of repo () before git, reducing the repo to a single path segment. private async Task<(string? Repo, string? Owner, string? Branch)> ResolveUploadRepoOwnerBranch(string? repoCli, string? ownerCli, string? branchCli, string? configPath, string? uploadDirectory, CancellationToken ctx) { diff --git a/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityServiceTests.cs b/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityServiceTests.cs new file mode 100644 index 0000000000..8372d422d2 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityServiceTests.cs @@ -0,0 +1,183 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions.TestingHelpers; +using AwesomeAssertions; +using Elastic.Changelog.AllowlistIdentity; +using Elastic.Changelog.GitHub; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Elastic.Changelog.Tests.AllowlistIdentity; + +[SuppressMessage("Usage", "CA1001:Types that own disposable fields should be disposable")] +public class ScrubberAllowlistIdentityServiceTests(ITestOutputHelper output) +{ + private const string ValidSha = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + private const string ValidCommit = "0123456789abcdef0123456789abcdef01234567"; + + private static readonly string ValidAssetJson = + $$""" + { + "schema_version": 1, + "artifact": "scrubber-allowlist-identity", + "allowlist_sha256": "{{ValidSha}}", + "deployment_commit": "{{ValidCommit}}", + "git_ref": "v1.2.3", + "built_at": "2026-08-01T12:00:00Z" + } + """; + + private readonly IGitHubReleaseService _releaseService = A.Fake(); + private readonly MockFileSystem _fileSystem = new(); + private readonly TestDiagnosticsCollector _collector = new(output); + + private ScrubberAllowlistIdentityService CreateService() => + new(NullLoggerFactory.Instance, _releaseService, _fileSystem); + + private static GitHubReleaseInfo Release(string tag, bool withAsset, bool draft = false) => new() + { + TagName = tag, + Draft = draft, + Assets = withAsset + ? [new GitHubReleaseAsset { Name = ScrubberAllowlistIdentity.AssetName, BrowserDownloadUrl = $"https://example/{tag}" }] + : [new GitHubReleaseAsset { Name = "docs-builder.zip", BrowserDownloadUrl = $"https://example/{tag}/zip" }] + }; + + private void AssetDownloadReturns(string? content) => + A.CallTo(() => _releaseService.DownloadAssetTextAsync( + A.That.Matches(a => a.Name == ScrubberAllowlistIdentity.AssetName), A._)) + .Returns(Task.FromResult(content)); + + [Fact] + public async Task ResolveDeployedAsync_LatestReleaseCarriesAsset_ResolvesIt() + { + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>([Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(ValidAssetJson); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, new ResolveScrubberAllowlistArguments(), TestContext.Current.CancellationToken); + + resolved.Should().NotBeNull(); + resolved.ReleaseTag.Should().Be("v2.0.0"); + resolved.Identity.AllowlistSha256.Should().Be(ValidSha); + resolved.MatchesLocal.Should().BeNull(); + } + + [Fact] + public async Task ResolveDeployedAsync_NewestReleaseMissingAsset_FallsBackToPreviousRelease() + { + // The newest release exists but its scrubber deploy never completed (no asset); the one + // before it is the most recent gated deploy and must win. + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>( + [Release("v2.1.0", withAsset: false), Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(ValidAssetJson); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, new ResolveScrubberAllowlistArguments(), TestContext.Current.CancellationToken); + + resolved.Should().NotBeNull(); + resolved.ReleaseTag.Should().Be("v2.0.0"); + } + + [Fact] + public async Task ResolveDeployedAsync_DraftReleasesAreSkipped() + { + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>( + [Release("v2.1.0", withAsset: true, draft: true), Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(ValidAssetJson); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, new ResolveScrubberAllowlistArguments(), TestContext.Current.CancellationToken); + + resolved.Should().NotBeNull(); + resolved.ReleaseTag.Should().Be("v2.0.0"); + } + + [Fact] + public async Task ResolveDeployedAsync_NoReleaseCarriesAsset_FailsWithError() + { + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>([Release("v2.1.0", withAsset: false)])); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, new ResolveScrubberAllowlistArguments(), TestContext.Current.CancellationToken); + + resolved.Should().BeNull(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("cannot be resolved")); + } + + [Fact] + public async Task ResolveDeployedAsync_ExplicitTagWithoutAsset_FailsWithError() + { + A.CallTo(() => _releaseService.FetchReleaseAsync("elastic", "docs-builder", "v1.0.0", A._)) + .Returns(Task.FromResult(Release("v1.0.0", withAsset: false))); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, + new ResolveScrubberAllowlistArguments { Tag = "v1.0.0" }, TestContext.Current.CancellationToken); + + resolved.Should().BeNull(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("predates")); + } + + [Fact] + public async Task ResolveDeployedAsync_ExplicitTagNotFound_FailsWithError() + { + A.CallTo(() => _releaseService.FetchReleaseAsync("elastic", "docs-builder", "v9.9.9", A._)) + .Returns(Task.FromResult(null)); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, + new ResolveScrubberAllowlistArguments { Tag = "v9.9.9" }, TestContext.Current.CancellationToken); + + resolved.Should().BeNull(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("was not found")); + } + + [Fact] + public async Task ResolveDeployedAsync_MalformedAsset_FailsWithError() + { + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>([Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(/*lang=json,strict*/ """{ "schema_version": 1, "artifact": "scrubber-allowlist-identity", "allowlist_sha256": "nope", "deployment_commit": "nope" }"""); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, new ResolveScrubberAllowlistArguments(), TestContext.Current.CancellationToken); + + resolved.Should().BeNull(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("Invalid allowlist identity")); + } + + [Fact] + public async Task ResolveDeployedAsync_LocalAssemblerMatches_ReportsMatch() + { + // sha256 of "hello\n" + const string helloSha = "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"; + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>([Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(ValidAssetJson.Replace(ValidSha, helloSha)); + _fileSystem.AddFile("/repo/config/assembler.yml", new MockFileData("hello\n")); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, + new ResolveScrubberAllowlistArguments { AssemblerPath = "/repo/config/assembler.yml" }, TestContext.Current.CancellationToken); + + resolved.Should().NotBeNull(); + resolved.LocalSha256.Should().Be(helloSha); + resolved.MatchesLocal.Should().BeTrue(); + } + + [Fact] + public async Task ResolveDeployedAsync_LocalAssemblerDiffers_WarnsButResolves() + { + A.CallTo(() => _releaseService.FetchReleasesAsync("elastic", "docs-builder", A._, A._)) + .Returns(Task.FromResult>([Release("v2.0.0", withAsset: true)])); + AssetDownloadReturns(ValidAssetJson); + _fileSystem.AddFile("/repo/config/assembler.yml", new MockFileData("different content\n")); + + var resolved = await CreateService().ResolveDeployedAsync(_collector, + new ResolveScrubberAllowlistArguments { AssemblerPath = "/repo/config/assembler.yml" }, TestContext.Current.CancellationToken); + + resolved.Should().NotBeNull(); + resolved.MatchesLocal.Should().BeFalse(); + _collector.Diagnostics.Should().Contain(d => d.Message.Contains("differs from the deployed scrubber allowlist")); + } +} diff --git a/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityTests.cs b/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityTests.cs new file mode 100644 index 0000000000..eba6902f6a --- /dev/null +++ b/tests/Elastic.Changelog.Tests/AllowlistIdentity/ScrubberAllowlistIdentityTests.cs @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Globalization; +using System.Text; +using AwesomeAssertions; +using Elastic.Changelog.AllowlistIdentity; + +namespace Elastic.Changelog.Tests.AllowlistIdentity; + +public class ScrubberAllowlistIdentityTests +{ + private const string ValidSha = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + private const string ValidCommit = "0123456789abcdef0123456789abcdef01234567"; + + private static string ValidJson( + int schemaVersion = ScrubberAllowlistIdentity.CurrentSchemaVersion, + string artifact = ScrubberAllowlistIdentity.ArtifactKind, + string sha = ValidSha, + string commit = ValidCommit) => + $$""" + { + "schema_version": {{schemaVersion}}, + "artifact": "{{artifact}}", + "allowlist_sha256": "{{sha}}", + "deployment_commit": "{{commit}}", + "git_ref": "v1.2.3", + "built_at": "2026-08-01T12:00:00Z" + } + """; + + [Fact] + public void TryParse_ValidDocument_ReturnsIdentity() + { + var result = ScrubberAllowlistIdentity.TryParse(ValidJson(), out var identity, out var problems); + + result.Should().BeTrue(); + problems.Should().BeEmpty(); + identity!.AllowlistSha256.Should().Be(ValidSha); + identity.DeploymentCommit.Should().Be(ValidCommit); + identity.GitRef.Should().Be("v1.2.3"); + identity.BuiltAt.Should().Be(DateTimeOffset.Parse("2026-08-01T12:00:00Z", CultureInfo.InvariantCulture)); + } + + [Fact] + public void TryParse_UnsupportedSchemaVersion_Fails() + { + var result = ScrubberAllowlistIdentity.TryParse(ValidJson(schemaVersion: 2), out var identity, out var problems); + + result.Should().BeFalse(); + identity.Should().BeNull(); + problems.Should().ContainSingle(p => p.Contains("schema version 2")); + } + + [Fact] + public void TryParse_WrongArtifactKind_Fails() + { + var result = ScrubberAllowlistIdentity.TryParse(ValidJson(artifact: "something-else"), out _, out var problems); + + result.Should().BeFalse(); + problems.Should().ContainSingle(p => p.Contains("something-else")); + } + + [Theory] + [InlineData("")] + [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")] + [InlineData("sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")] + [InlineData("sha256:abc")] + public void TryParse_MalformedSha256_Fails(string sha) + { + var result = ScrubberAllowlistIdentity.TryParse(ValidJson(sha: sha), out _, out var problems); + + result.Should().BeFalse(); + problems.Should().Contain(p => p.Contains("sha256:")); + } + + [Theory] + [InlineData("")] + [InlineData("abc123")] + [InlineData("0123456789ABCDEF0123456789ABCDEF01234567")] + public void TryParse_MalformedCommit_Fails(string commit) + { + var result = ScrubberAllowlistIdentity.TryParse(ValidJson(commit: commit), out _, out var problems); + + result.Should().BeFalse(); + problems.Should().Contain(p => p.Contains("40-character")); + } + + [Fact] + public void TryParse_InvalidJson_FailsWithoutThrowing() + { + var result = ScrubberAllowlistIdentity.TryParse("not json at all {", out var identity, out var problems); + + result.Should().BeFalse(); + identity.Should().BeNull(); + problems.Should().ContainSingle(p => p.Contains("not valid JSON")); + } + + [Fact] + public void ComputeSha256_KnownContent_MatchesSha256Sum() + { + // printf 'hello\n' | sha256sum + using var stream = new MemoryStream(Encoding.UTF8.GetBytes("hello\n")); + + var hash = ScrubberAllowlistIdentity.ComputeSha256(stream); + + hash.Should().Be("sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"); + } +}