diff --git a/src/services/Elastic.Changelog/Backfill/BackfillArtifactKind.cs b/src/services/Elastic.Changelog/Backfill/BackfillArtifactKind.cs new file mode 100644 index 000000000..8a7a5b9e6 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillArtifactKind.cs @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// +/// The kinds of document the backfill pipeline passes between its stages. +/// Each persisted file contains exactly one of these, named in its envelope +/// (e.g. "artifact": "semantic-model") so a reader knows what it is +/// looking at before parsing the payload. See the README in this folder for +/// a tour of who writes and reads each one. +/// +public enum BackfillArtifactKind +{ + /// The census: which products and release-note sources exist, where they came from, and what we decided about each. + Inventory, + + /// Manual corrections an operator feeds into planning, each with a reason attached. + Overrides, + + /// The release notes reduced to their meaning, with formatting stripped away. + SemanticModel, + + /// Exactly what we intend to create in S3, pinned to all of its inputs. + Plan, + + /// The evidence trail: why we believe each recovered fact about an entry or release. + Provenance, + + /// What actually happened when a plan was applied: every attempted step and its outcome. + Ledger +} + +/// +/// Converts between values and the names used in +/// files (inventory, overrides, semantic-model, plan, +/// provenance, ledger). Kept as explicit code, not reflection, so the +/// on-disk names can never drift by accident and the library stays AOT-friendly. +/// +public static class BackfillArtifactKinds +{ + /// The name written to files for , e.g. semantic-model. + public static string Name(BackfillArtifactKind kind) => kind switch + { + BackfillArtifactKind.Inventory => "inventory", + BackfillArtifactKind.Overrides => "overrides", + BackfillArtifactKind.SemanticModel => "semantic-model", + BackfillArtifactKind.Plan => "plan", + BackfillArtifactKind.Provenance => "provenance", + BackfillArtifactKind.Ledger => "ledger", + _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unknown backfill artifact kind") + }; + + /// + /// Looks up the kind for a name read from a file. Returns false for anything unknown. + /// Case-sensitive on purpose: these files are written by tools, so a case mismatch + /// means something is wrong and should surface rather than be papered over. + /// + public static bool TryParse(string? name, out BackfillArtifactKind kind) + { + switch (name) + { + case "inventory": + kind = BackfillArtifactKind.Inventory; + return true; + case "overrides": + kind = BackfillArtifactKind.Overrides; + return true; + case "semantic-model": + kind = BackfillArtifactKind.SemanticModel; + return true; + case "plan": + kind = BackfillArtifactKind.Plan; + return true; + case "provenance": + kind = BackfillArtifactKind.Provenance; + return true; + case "ledger": + kind = BackfillArtifactKind.Ledger; + return true; + default: + kind = default; + return false; + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/BackfillDocumentException.cs b/src/services/Elastic.Changelog/Backfill/BackfillDocumentException.cs new file mode 100644 index 000000000..2f7d948a5 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillDocumentException.cs @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// +/// Thrown when a backfill document cannot be safely read or written: the file is not +/// valid JSON, it is not the kind of document the caller asked for, it was written +/// with a schema version this code does not support, or a required field is missing +/// or invalid. The message always says what is wrong in plain terms and, where +/// possible, what to do about it — callers should surface it rather than swallow it, +/// because a half-understood document must never flow further down the pipeline. +/// +public sealed class BackfillDocumentException : Exception +{ + /// Creates the exception with a plain-English description of the problem. + public BackfillDocumentException(string message) : base(message) { } + + /// Creates the exception, keeping the lower-level parse error as the inner exception. + public BackfillDocumentException(string message, Exception innerException) : base(message, innerException) { } +} diff --git a/src/services/Elastic.Changelog/Backfill/BackfillDocuments.cs b/src/services/Elastic.Changelog/Backfill/BackfillDocuments.cs new file mode 100644 index 000000000..a2c03cc37 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillDocuments.cs @@ -0,0 +1,176 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace Elastic.Changelog.Backfill; + +/// +/// The one place to read, write, and hash backfill documents. Writing wraps the document +/// in its envelope (artifact name + schema version) and validates it first; reading checks +/// the envelope before parsing the payload and fails with a clear +/// on anything unexpected — a wrong document kind, +/// an unsupported schema version, invalid JSON, or a missing/invalid field. There is no +/// silent best-effort mode: a document either reads cleanly or not at all. +/// +public static class BackfillDocuments +{ + /// + /// Safety limit on document size (in characters). Backfill documents are review-sized + /// artifacts; anything this large is a bug or garbage input, and refusing it early + /// keeps a corrupt file from exhausting memory. + /// + public const int MaxDocumentCharacters = 64 * 1024 * 1024; + + /// + /// Validates , wraps it in its envelope, and returns + /// indented JSON ready to persist. The indentation is for human review only — it has + /// no effect on the document's hash, which is computed over the canonical form. + /// + public static string Serialize(T document) where T : class, IBackfillDocument + { + ArgumentNullException.ThrowIfNull(document); + ThrowIfInvalid(document); + + var envelope = new BackfillEnvelope + { + Artifact = BackfillArtifactKinds.Name(T.Kind), + SchemaVersion = BackfillSchemaVersions.Current(T.Kind), + Payload = document + }; + return JsonSerializer.Serialize(envelope, EnvelopeTypeInfo()); + } + + /// + /// Reads a document of type from . + /// Throws — never returns a half-parsed + /// document — when the text is not valid JSON, contains a different kind of document, + /// was written with a schema version this code does not support, or fails validation. + /// + public static T Deserialize(string json) where T : class, IBackfillDocument + { + ArgumentNullException.ThrowIfNull(json); + ThrowIfTooLarge(json); + + var expectedName = BackfillArtifactKinds.Name(T.Kind); + CheckEnvelopeHeader(json, expectedName, BackfillSchemaVersions.Current(T.Kind)); + + BackfillEnvelope? envelope; + try + { + envelope = JsonSerializer.Deserialize(json, EnvelopeTypeInfo()); + } + catch (JsonException e) + { + throw new BackfillDocumentException($"The '{expectedName}' document could not be parsed: {e.Message}", e); + } + + if (envelope?.Payload is not { } document) + throw new BackfillDocumentException($"The '{expectedName}' document has no payload."); + + ThrowIfInvalid(document); + return document; + } + + /// + /// The document's stable identity: SHA-256 over the canonical form of its envelope + /// (see ), as sha256: + 64 hex characters. The same + /// content always produces the same hash, regardless of formatting or the order + /// fields were assembled in — this is what makes plans content-addressed. + /// + public static string ComputeHash(T document) where T : class, IBackfillDocument => + BackfillHash.Compute(CanonicalJson.Canonicalize(Serialize(document))); + + /// + /// Hashes an already-serialized document exactly as + /// would. Useful for hashing a file as it sits on disk without knowing its type. + /// + public static string ComputeHash(string json) + { + ArgumentNullException.ThrowIfNull(json); + ThrowIfTooLarge(json); + return BackfillHash.Compute(CanonicalJson.Canonicalize(json)); + } + + private static void ThrowIfTooLarge(string json) + { + if (json.Length > MaxDocumentCharacters) + throw new BackfillDocumentException( + $"The document is {json.Length} characters long, over the {MaxDocumentCharacters} character safety limit for backfill documents."); + } + + private static void ThrowIfInvalid(T document) where T : class, IBackfillDocument + { + var problems = new List(); + document.Validate(problems); + if (problems.Count == 0) + return; + + var name = BackfillArtifactKinds.Name(T.Kind); + throw new BackfillDocumentException( + $"The '{name}' document is invalid:\n - {string.Join("\n - ", problems)}"); + } + + /// + /// Checks the two header fields before any payload parsing, so the errors for "wrong + /// file" and "wrong version" are specific instead of a generic parse failure. + /// + private static void CheckEnvelopeHeader(string json, string expectedName, int expectedVersion) + { + JsonDocument parsed; + try + { + parsed = JsonDocument.Parse(json); + } + catch (JsonException e) + { + throw new BackfillDocumentException($"The document is not valid JSON: {e.Message}", e); + } + + using (parsed) + { + var root = parsed.RootElement; + if (root.ValueKind != JsonValueKind.Object) + throw new BackfillDocumentException( + "This is not a backfill document: the top level must be a JSON object with 'artifact' and 'schema_version' fields."); + + CheckArtifactField(root, expectedName); + CheckSchemaVersionField(root, expectedName, expectedVersion); + } + } + + private static void CheckArtifactField(JsonElement root, string expectedName) + { + if (!root.TryGetProperty("artifact", out var artifact) || artifact.ValueKind != JsonValueKind.String) + throw new BackfillDocumentException( + "This is not a backfill document: the 'artifact' field naming the document kind is missing."); + + var name = artifact.GetString() ?? ""; + if (!BackfillArtifactKinds.TryParse(name, out _)) + throw new BackfillDocumentException( + $"Unknown document kind '{name}'. Expected one of: inventory, overrides, semantic-model, plan, provenance, ledger."); + + if (!string.Equals(name, expectedName, StringComparison.Ordinal)) + throw new BackfillDocumentException( + $"This file contains a '{name}' document, but a '{expectedName}' document was requested. Check that the right file is being read."); + } + + private static void CheckSchemaVersionField(JsonElement root, string expectedName, int expectedVersion) + { + if (!root.TryGetProperty("schema_version", out var version) || version.ValueKind != JsonValueKind.Number || !version.TryGetInt32(out var value)) + throw new BackfillDocumentException( + $"This '{expectedName}' document is missing the 'schema_version' field, so it cannot be read safely."); + + if (value != expectedVersion) + throw new BackfillDocumentException( + $"This '{expectedName}' document was written with schema version {value}, but this code only understands version {expectedVersion}. " + + "Regenerate the document with matching tooling, or upgrade to a version of this code that understands it."); + } + + private static JsonTypeInfo> EnvelopeTypeInfo() where T : IBackfillDocument => + BackfillJsonContext.Default.GetTypeInfo(typeof(BackfillEnvelope)) as JsonTypeInfo> + ?? throw new InvalidOperationException( + $"BackfillEnvelope<{typeof(T).Name}> is not registered on BackfillJsonContext; add a [JsonSerializable] attribute for it."); +} diff --git a/src/services/Elastic.Changelog/Backfill/BackfillEnvelope.cs b/src/services/Elastic.Changelog/Backfill/BackfillEnvelope.cs new file mode 100644 index 000000000..738ad18b2 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillEnvelope.cs @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// +/// Implemented by the root type of each backfill document family (inventory, overrides, +/// semantic model, plan, provenance, ledger). Ties the type to its artifact name so the +/// reading and writing helpers in can check "is this file +/// really the kind of document you asked for?" before parsing the payload. +/// +public interface IBackfillDocument +{ + /// Which of the six document families this type is the root of. + static abstract BackfillArtifactKind Kind { get; } + + /// + /// Adds a plain-English description of every problem found in this document to + /// . An empty list afterwards means the document is valid. + /// Run automatically on every read and write. + /// + void Validate(IList problems); +} + +/// +/// The small header wrapper around every persisted backfill document. It records what +/// kind of document the file contains () and which schema version +/// wrote it (), so a reader can fail fast — with a clear +/// error — on files it does not understand, instead of guessing at a payload shape. +/// +public sealed record BackfillEnvelope +{ + /// The document family name, e.g. inventory or semantic-model. See . + public required string Artifact { get; init; } + + /// The schema version of at the time of writing. Compared against on read. + public required int SchemaVersion { get; init; } + + /// The document itself. + public required T Payload { get; init; } +} diff --git a/src/services/Elastic.Changelog/Backfill/BackfillHash.cs b/src/services/Elastic.Changelog/Backfill/BackfillHash.cs new file mode 100644 index 000000000..c4fbbf263 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillHash.cs @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Security.Cryptography; +using System.Text; + +namespace Elastic.Changelog.Backfill; + +/// +/// Computes and checks the hashes that give backfill documents a stable identity. +/// A hash is SHA-256 over the UTF-8 bytes of a document's canonical form (see +/// ), written as sha256: followed by 64 lower-case +/// hex characters. Because the canonical form is stable, the same content always +/// produces the same hash — which is what lets a plan be pinned to the exact inputs +/// it was computed from. +/// +public static class BackfillHash +{ + /// Every hash value starts with this, so a reader can tell the algorithm at a glance. + public const string Prefix = "sha256:"; + + private const int HexLength = 64; + + /// + /// Hashes and returns e.g. + /// sha256:9f86d08…. The caller is responsible for passing canonical text; + /// to hash a document, prefer , + /// which canonicalizes first. + /// + public static string Compute(string canonicalText) + { + ArgumentNullException.ThrowIfNull(canonicalText); + var digest = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalText)); + return Prefix + Convert.ToHexStringLower(digest); + } + + /// True when is a well-formed hash: sha256: plus 64 lower-case hex characters. + public static bool IsWellFormed(string? value) + { + if (value is null || value.Length != Prefix.Length + HexLength) + return false; + if (!value.StartsWith(Prefix, StringComparison.Ordinal)) + return false; + + for (var i = Prefix.Length; i < value.Length; i++) + { + var c = value[i]; + if (c is (< '0' or > '9') and (< 'a' or > 'f')) + return false; + } + return true; + } +} diff --git a/src/services/Elastic.Changelog/Backfill/BackfillJsonContext.cs b/src/services/Elastic.Changelog/Backfill/BackfillJsonContext.cs new file mode 100644 index 000000000..44ee76daa --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillJsonContext.cs @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// +/// Source-generated JSON serialization for the backfill documents (required for AOT: +/// every serialized type must be registered here or serialization fails at runtime once +/// the binary is trimmed). Property names are snake_case, enums serialize as their +/// kebab-case names, and null properties are omitted — the same convention the rest of +/// the changelog pipeline uses on the wire. +/// +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + RespectNullableAnnotations = true +)] +[JsonSerializable(typeof(BackfillEnvelope))] +[JsonSerializable(typeof(BackfillEnvelope))] +[JsonSerializable(typeof(BackfillEnvelope))] +[JsonSerializable(typeof(BackfillEnvelope))] +[JsonSerializable(typeof(BackfillEnvelope))] +[JsonSerializable(typeof(BackfillEnvelope))] +public sealed partial class BackfillJsonContext : JsonSerializerContext; diff --git a/src/services/Elastic.Changelog/Backfill/BackfillSchemaVersions.cs b/src/services/Elastic.Changelog/Backfill/BackfillSchemaVersions.cs new file mode 100644 index 000000000..a97a5a5a1 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/BackfillSchemaVersions.cs @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// +/// The schema version this code reads and writes for each document family. +/// A document's envelope records the version it was written with; readers compare +/// it against these numbers and refuse anything different, so a shape change can +/// never be half-understood. Bump a family's number here when its shape changes +/// in a way old readers cannot safely ignore. +/// +public static class BackfillSchemaVersions +{ + /// Current version of the inventory document shape. + public const int Inventory = 1; + + /// Current version of the overrides document shape. + public const int Overrides = 1; + + /// Current version of the semantic-model document shape. + public const int SemanticModel = 1; + + /// Current version of the plan document shape. + public const int Plan = 1; + + /// Current version of the provenance document shape. + public const int Provenance = 1; + + /// Current version of the ledger document shape. + public const int Ledger = 1; + + /// The current version for the given document family. + public static int Current(BackfillArtifactKind kind) => kind switch + { + BackfillArtifactKind.Inventory => Inventory, + BackfillArtifactKind.Overrides => Overrides, + BackfillArtifactKind.SemanticModel => SemanticModel, + BackfillArtifactKind.Plan => Plan, + BackfillArtifactKind.Provenance => Provenance, + BackfillArtifactKind.Ledger => Ledger, + _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unknown backfill artifact kind") + }; +} diff --git a/src/services/Elastic.Changelog/Backfill/CanonicalJson.cs b/src/services/Elastic.Changelog/Backfill/CanonicalJson.cs new file mode 100644 index 000000000..8a8b64901 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/CanonicalJson.cs @@ -0,0 +1,123 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text; +using System.Text.Json; + +namespace Elastic.Changelog.Backfill; + +/// +/// Rewrites a JSON document into one agreed-upon form — the "canonical" form — so the +/// same content always produces exactly the same text, no matter how the file was +/// formatted or in what order a producer happened to add keys. Hashing that text (see +/// ) then gives every document a stable identity. +/// +/// +/// The rules, in plain terms: +/// +/// object keys are sorted by ordinal (byte-order) comparison; +/// all insignificant whitespace is removed; +/// Windows (\r\n) and old-Mac (\r) line endings inside strings become \n; +/// object properties whose value is null are dropped — absent and null mean the same thing; +/// array items keep their order (order is part of the meaning) and null items are kept; +/// numbers keep the exact text they were written with (all contract numbers are integers). +/// +/// Dictionaries in the contracts serialize as JSON objects, so the key sorting here is what +/// makes dictionary insertion order irrelevant to a document's hash. +/// +public static class CanonicalJson +{ + /// + /// Returns the canonical form of . Throws + /// when the text is not valid JSON or an + /// object contains the same key twice (a duplicate key would make the content ambiguous). + /// + public static string Canonicalize(string json) + { + ArgumentNullException.ThrowIfNull(json); + + JsonDocument document; + try + { + document = JsonDocument.Parse(json); + } + catch (JsonException e) + { + throw new BackfillDocumentException($"Cannot canonicalize: the text is not valid JSON. {e.Message}", e); + } + + using (document) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + WriteCanonical(writer, document.RootElement); + return Encoding.UTF8.GetString(stream.ToArray()); + } + } + + private static void WriteCanonical(Utf8JsonWriter writer, JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + WriteCanonicalObject(writer, element); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + WriteCanonical(writer, item); + writer.WriteEndArray(); + break; + case JsonValueKind.String: + writer.WriteStringValue(NormalizeLineEndings(element.GetString()!)); + break; + case JsonValueKind.Number: + // Written verbatim: re-parsing a number could change its text (e.g. trailing zeros). + writer.WriteRawValue(element.GetRawText()); + break; + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + default: + throw new BackfillDocumentException($"Cannot canonicalize: unexpected JSON value kind '{element.ValueKind}'."); + } + } + + private static void WriteCanonicalObject(Utf8JsonWriter writer, JsonElement element) + { + var properties = element.EnumerateObject() + .Where(p => p.Value.ValueKind != JsonValueKind.Null) + .OrderBy(p => p.Name, StringComparer.Ordinal) + .ToList(); + + writer.WriteStartObject(); + string? previousName = null; + foreach (var property in properties) + { + if (string.Equals(previousName, property.Name, StringComparison.Ordinal)) + throw new BackfillDocumentException($"Cannot canonicalize: the key '{property.Name}' appears more than once in the same object, which makes the content ambiguous."); + + previousName = property.Name; + writer.WritePropertyName(NormalizeLineEndings(property.Name)); + WriteCanonical(writer, property.Value); + } + writer.WriteEndObject(); + } + + private static string NormalizeLineEndings(string value) + { + if (!value.Contains('\r', StringComparison.Ordinal)) + return value; + + return value + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace("\r", "\n", StringComparison.Ordinal); + } +} diff --git a/src/services/Elastic.Changelog/Backfill/EntryIdentity.cs b/src/services/Elastic.Changelog/Backfill/EntryIdentity.cs new file mode 100644 index 000000000..03f7728c3 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/EntryIdentity.cs @@ -0,0 +1,140 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace Elastic.Changelog.Backfill; + +/// The ways a changelog entry can be identified. See . +public enum EntryIdentityKind +{ + /// Identified by the pull request that made the change (the strongest common identity). + [JsonStringEnumMemberName("pull-request")] + PullRequest, + + /// Identified by an issue, used when the source explicitly treats the issue as the entry's primary reference. + [JsonStringEnumMemberName("issue")] + Issue, + + /// + /// Identified by a made-up but stable file name plus a checksum of the entry's content. + /// Used when no PR or issue is available; also what bundle amend files match on when + /// retracting a previously published entry. + /// + [JsonStringEnumMemberName("synthetic-file")] + SyntheticFile +} + +/// +/// The stable identity of one changelog entry — the answer to "is this the same entry +/// we already have?". Used for de-duplication, for matching a highlights mention to its +/// entry, and for reconciling against entries already published in live bundles. +/// Exactly one of the identity fields is set, matching : a canonical +/// GitHub URL for and +/// , or a name-plus-checksum pair for +/// . +/// +public sealed record EntryIdentity +{ + /// Which way this entry is identified. + public required EntryIdentityKind Kind { get; init; } + + /// + /// The canonical GitHub URL, e.g. https://github.com/elastic/kibana/pull/12345. + /// Always the full URL — never a bare number, because a bare number is only meaningful + /// relative to some repository. Set for PR and issue identities, null for synthetic files. + /// + public string? Url { get; init; } + + /// The stable name and checksum. Set for synthetic-file identities, null otherwise. + public SyntheticFileIdentity? File { get; init; } + + /// Builds a pull-request identity with the canonical URL for the given repository and PR number. + public static EntryIdentity ForPullRequest(string owner, string repository, int number) => + new() { Kind = EntryIdentityKind.PullRequest, Url = $"https://github.com/{owner}/{repository}/pull/{number}" }; + + /// Builds an issue identity with the canonical URL for the given repository and issue number. + public static EntryIdentity ForIssue(string owner, string repository, int number) => + new() { Kind = EntryIdentityKind.Issue, Url = $"https://github.com/{owner}/{repository}/issues/{number}" }; + + /// Builds a synthetic-file identity from a stable name and a checksum of the entry's content. + public static EntryIdentity ForFile(string name, string checksum) => + new() { Kind = EntryIdentityKind.SyntheticFile, File = new SyntheticFileIdentity { Name = name, Checksum = checksum } }; + + /// Adds a plain-English description of every problem in this identity to . + public void Validate(IList problems) + { + switch (Kind) + { + case EntryIdentityKind.PullRequest: + if (File is not null) + problems.Add("A pull-request identity must not carry a file block."); + if (Url is null || !CanonicalGitHubUrls.IsPullRequestUrl(Url)) + problems.Add($"A pull-request identity needs a canonical URL like https://github.com/{{owner}}/{{repo}}/pull/{{number}}, but found '{Url}'."); + break; + case EntryIdentityKind.Issue: + if (File is not null) + problems.Add("An issue identity must not carry a file block."); + if (Url is null || !CanonicalGitHubUrls.IsIssueUrl(Url)) + problems.Add($"An issue identity needs a canonical URL like https://github.com/{{owner}}/{{repo}}/issues/{{number}}, but found '{Url}'."); + break; + case EntryIdentityKind.SyntheticFile: + if (Url is not null) + problems.Add("A synthetic-file identity must not carry a URL."); + if (File is null) + problems.Add("A synthetic-file identity needs a file block with a name and checksum."); + else + File.Validate(problems); + break; + default: + problems.Add($"Unknown identity kind '{Kind}'."); + break; + } + } +} + +/// +/// A made-up but stable file name plus a checksum of the entry's content. Entries created +/// by the backfill never exist as real files, but they still carry this block because +/// published bundles can only be corrected by amend files that match entries on exactly +/// these two fields. +/// +public sealed record SyntheticFileIdentity +{ + /// The stable file name, e.g. backfill-elasticsearch-9.0.0-0001.yaml. Derived from the entry's identity so reruns produce the same name. + public required string Name { get; init; } + + /// Checksum of the entry's canonical content, so a changed entry is never mistaken for the original. + public required string Checksum { get; init; } + + /// Adds a plain-English description of every problem in this file identity to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Name)) + problems.Add("A synthetic file identity needs a non-empty name."); + if (string.IsNullOrWhiteSpace(Checksum)) + problems.Add("A synthetic file identity needs a non-empty checksum."); + } +} + +/// +/// Recognizes canonical GitHub PR and issue URLs. The backfill always stores full URLs +/// (never bare numbers) because a bare number is ambiguous across repositories and the +/// scrubber's link allowlist matches on URLs. +/// +public static partial class CanonicalGitHubUrls +{ + [GeneratedRegex(@"^https://github\.com/[A-Za-z0-9-]+/[A-Za-z0-9._-]+/pull/[1-9][0-9]*$")] + private static partial Regex PullRequestUrl(); + + [GeneratedRegex(@"^https://github\.com/[A-Za-z0-9-]+/[A-Za-z0-9._-]+/issues/[1-9][0-9]*$")] + private static partial Regex IssueUrl(); + + /// True when is a canonical PR URL like https://github.com/elastic/kibana/pull/12345. + public static bool IsPullRequestUrl(string url) => PullRequestUrl().IsMatch(url); + + /// True when is a canonical issue URL like https://github.com/elastic/kibana/issues/12345. + public static bool IsIssueUrl(string url) => IssueUrl().IsMatch(url); +} diff --git a/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs b/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs new file mode 100644 index 000000000..c9b8a0c70 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/InventoryDocument.cs @@ -0,0 +1,229 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.ObjectModel; +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// +/// What the census concluded about one release-note source: does it have history worth +/// backfilling, and if not, why not. Only the first two classifications produce backfill +/// work; the rest stay visible in the census so nothing silently disappears. +/// +public enum SourceClassification +{ + /// Published release-note pages exist for this source, so there is history to backfill. + [JsonStringEnumMemberName("published-history-found")] + PublishedHistoryFound, + + /// Native changelog entries or bundles already exist for this source (the strongest input for backfilling). + [JsonStringEnumMemberName("native-artifacts-found")] + NativeArtifactsFound, + + /// The published page mixes hand-written history with live {changelog} output, so only the hand-written part needs backfilling. + [JsonStringEnumMemberName("hybrid-page")] + HybridPage, + + /// The product declares release notes but no published history could be found. + [JsonStringEnumMemberName("declared-no-history")] + DeclaredNoHistory, + + /// We could not work out where this product's release notes live; needs a human. + [JsonStringEnumMemberName("source-unresolved")] + SourceUnresolved, + + /// The source's history predates the product's migration cutoff, so it is out of scope. + [JsonStringEnumMemberName("outside-cutoff")] + OutsideCutoff, + + /// Live artifacts already fully cover this source; nothing to backfill. + [JsonStringEnumMemberName("already-live")] + AlreadyLive +} + +/// How far a repository has adopted the live changelog workflows. +public enum AdoptionState +{ + /// The repository does not use the changelog workflows at all yet. + [JsonStringEnumMemberName("not-adopted")] + NotAdopted, + + /// The repository uses some of the changelog workflows, so live and historical data may overlap mid-release. + [JsonStringEnumMemberName("partially-adopted")] + PartiallyAdopted, + + /// The repository fully uses the changelog workflows. + [JsonStringEnumMemberName("fully-adopted")] + FullyAdopted +} + +/// How a product names its releases, which decides how targets are parsed and compared. +public enum TargetScheme +{ + /// Semantic versions like 9.0.0. + [JsonStringEnumMemberName("semver")] + Semver, + + /// Calendar dates like 2025-11-04. + [JsonStringEnumMemberName("date")] + Date, + + /// Months like 2025-11, used by products that ship monthly. + [JsonStringEnumMemberName("monthly")] + Monthly +} + +/// Whether a cutoff boundary is expressed as a version or as a date. +public enum CutoffKind +{ + /// The boundary is a version, e.g. everything from stack 9.0.0 onwards. + [JsonStringEnumMemberName("version")] + Version, + + /// The boundary is a date, e.g. everything published after 2025-01-01. + [JsonStringEnumMemberName("date")] + Date +} + +/// +/// The line between "backfill this" and "leave it alone" for one source: releases on or +/// after the boundary are in scope, anything older is not. Products without stack +/// versioning get a date boundary instead of a version. +/// +public sealed record BackfillCutoff +{ + /// Whether is a version or a date. + public required CutoffKind Kind { get; init; } + + /// The boundary itself, e.g. 9.0.0 or 2025-01-01. + public required string Value { get; init; } + + /// Optional free-text explanation of why this boundary was chosen. + public string? Notes { get; init; } + + /// Adds a plain-English description of every problem in this cutoff to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Value)) + problems.Add("A cutoff needs a non-empty value (a version like 9.0.0 or a date like 2025-01-01)."); + } +} + +/// +/// A repository that entries from this source attribute their changes to, together with +/// whether the deployed scrubber's link allowlist knows it. Links to repositories that are +/// not on the allowlist get silently stripped on publication, so planning must check this +/// flag before upload instead of finding out afterwards. +/// +public sealed record AttributedRepository +{ + /// The repository entries attribute changes to. + public required GitRepository Repository { get; init; } + + /// True when the deployed scrubber's link allowlist includes this repository, so its links survive publication. + public required bool OnScrubberAllowlist { get; init; } + + /// Adds a plain-English description of every problem in this attribution to . + public void Validate(IList problems) => Repository.Validate(problems); +} + +/// +/// Everything the census recorded about one product/repository source: where its release +/// notes live, which products they feed, where the backfill boundary sits, and what we +/// concluded about it. One inventory document holds one of these per source. +/// +public sealed record InventorySource +{ + /// The repository the release-note content lives in. + public required GitRepository SourceRepository { get; init; } + + /// The git ref (branch, tag, or commit) the census read the content at. + public required string GitRef { get; init; } + + /// The docset the content belongs to, when the repository hosts more than one. + public string? Docset { get; init; } + + /// Paths inside the repository where the release-note content lives. + public IReadOnlyList Paths { get; init; } = []; + + /// The product IDs (as known to products.yml) this source produces release notes for. + public required IReadOnlyList ProductIds { get; init; } + + /// How this product names its releases (versions, dates, or months). + public required TargetScheme TargetScheme { get; init; } + + /// Where hand-written history ends and live workflow data begins. Null when the census could not determine one yet. + public BackfillCutoff? Cutoff { get; init; } + + /// Docset variable substitutions (like {{es}} → Elasticsearch) needed to expand the source text. + public IReadOnlyDictionary Substitutions { get; init; } = ReadOnlyDictionary.Empty; + + /// Known mappings from links as written in the source to the destinations they should resolve to. + public IReadOnlyDictionary LinkMappings { get; init; } = ReadOnlyDictionary.Empty; + + /// The repositories entries attribute changes to, each with its scrubber-allowlist status. + public IReadOnlyList AttributedRepositories { get; init; } = []; + + /// The repository to attribute content to when an entry does not say where it came from. Null when every entry is attributed. + public GitRepository? DefaultRepository { get; init; } + + /// The file-name pattern bundles for this source are expected to use, e.g. {repo}-{target}.yaml. + public string? BundleFilenameConvention { get; init; } + + /// How far this repository has adopted the live changelog workflows. + public required AdoptionState AdoptionState { get; init; } + + /// What the census concluded about this source. + public required SourceClassification Classification { get; init; } + + /// IDs of overrides (from the overrides document) that apply to this source. + public IReadOnlyList AppliedOverrideIds { get; init; } = []; + + /// Open questions about this source that a human still needs to answer, in plain text. + public IReadOnlyList UnresolvedItems { get; init; } = []; + + /// Adds a plain-English description of every problem in this source to . + public void Validate(IList problems) + { + SourceRepository.Validate(problems); + if (string.IsNullOrWhiteSpace(GitRef)) + problems.Add("An inventory source needs a non-empty git ref."); + if (ProductIds.Count == 0) + problems.Add("An inventory source needs at least one product ID."); + if (ProductIds.Any(string.IsNullOrWhiteSpace)) + problems.Add("An inventory source's product IDs must all be non-empty."); + Cutoff?.Validate(problems); + foreach (var attributed in AttributedRepositories) + attributed.Validate(problems); + DefaultRepository?.Validate(problems); + } +} + +/// +/// The census: which products and release-note sources exist, where they came from, and +/// what we decided about each. Produced by the inventory stage before any planning; +/// read by planning and by humans reviewing what a backfill run will cover. Sources that +/// produce no backfill work stay listed with their classification, so "we looked and +/// decided no" is always distinguishable from "we never looked". +/// +public sealed record InventoryDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.Inventory; + + /// One record per product/repository source the census examined. + public required IReadOnlyList Sources { get; init; } + + /// + public void Validate(IList problems) + { + for (var i = 0; i < Sources.Count; i++) + { + var before = problems.Count; + Sources[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"sources[{i}]"); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/LedgerDocument.cs b/src/services/Elastic.Changelog/Backfill/LedgerDocument.cs new file mode 100644 index 000000000..bb30d13c2 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/LedgerDocument.cs @@ -0,0 +1,208 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// How one attempted step of an apply run ended. +public enum LedgerActionOutcome +{ + /// The object was created. + [JsonStringEnumMemberName("created")] + Created, + + /// Nothing was written: the object already existed with the expected bytes. + [JsonStringEnumMemberName("skipped")] + Skipped, + + /// Nothing was written: the key already existed with different bytes, and normal runs never overwrite. + [JsonStringEnumMemberName("conflict")] + Conflict, + + /// The step failed; the detail says how. + [JsonStringEnumMemberName("failed")] + Failed +} + +/// How refreshing one registry manifest ended. +public enum RegistryRefreshOutcome +{ + /// The manifest was rewritten with the run's changes merged in. + [JsonStringEnumMemberName("updated")] + Updated, + + /// The manifest already matched; nothing was written. + [JsonStringEnumMemberName("unchanged")] + Unchanged, + + /// The manifest could not be updated. The run is incomplete until this is reconciled. + [JsonStringEnumMemberName("failed")] + Failed +} + +/// The overall answer from the post-apply verification. +public enum VerificationOutcome +{ + /// The published result matches the expected semantic model. + [JsonStringEnumMemberName("passed")] + Passed, + + /// The published result does not match; the details say where. + [JsonStringEnumMemberName("failed")] + Failed, + + /// Verification never ran, e.g. because the run was interrupted first. + [JsonStringEnumMemberName("not-run")] + NotRun +} + +/// One attempted step of an apply run and how it ended. +public sealed record LedgerAction +{ + /// What the plan wanted this step to do. + public required PlanActionKind PlannedKind { get; init; } + + /// The S3 key the step worked on. + public required string Key { get; init; } + + /// How the step ended. + public required LedgerActionOutcome Outcome { get; init; } + + /// Extra detail when the outcome needs explaining, e.g. the error for a failed step. + public string? Detail { get; init; } + + /// Adds a plain-English description of every problem in this action to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Key)) + problems.Add("A ledger action needs a non-empty key."); + if (Outcome == LedgerActionOutcome.Failed && string.IsNullOrWhiteSpace(Detail)) + problems.Add("A failed ledger action needs a detail saying what went wrong."); + } +} + +/// The state one registry manifest ended up in after the run. +public sealed record RegistryRefresh +{ + /// The manifest's S3 key, e.g. bundle/elasticsearch/registry.json. + public required string Key { get; init; } + + /// How refreshing this manifest ended. + public required RegistryRefreshOutcome Outcome { get; init; } + + /// Extra detail when the outcome needs explaining. + public string? Detail { get; init; } + + /// Adds a plain-English description of every problem in this refresh record to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Key)) + problems.Add("A registry refresh record needs a non-empty key."); + } +} + +/// What the post-apply verification concluded. +public sealed record VerificationResult +{ + /// The overall answer. + public required VerificationOutcome Outcome { get; init; } + + /// Plain-English findings, e.g. each semantic difference when verification failed. + public IReadOnlyList Details { get; init; } = []; + + /// Adds a plain-English description of every problem in this result to . + public void Validate(IList problems) + { + if (Outcome == VerificationOutcome.Failed && Details.Count == 0) + problems.Add("A failed verification needs at least one detail saying what did not match."); + } +} + +/// +/// What actually happened when a plan was applied: every attempted step and its outcome, +/// what was created (with content hashes), what was skipped, what conflicted, how the +/// registries ended up, and whether verification passed. Written by the apply stage; read +/// by reruns (to resume an interrupted run safely — created keys are never re-attempted) +/// and by anyone auditing a run. Timestamps are recorded in UTC. +/// +public sealed record LedgerDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.Ledger; + + /// Hash of the plan this run executed, proving exactly which approved plan the outcomes belong to. + public required string PlanHash { get; init; } + + /// The source repositories the run's inputs came from, pinned to exact commits. + public required IReadOnlyList InputRefs { get; init; } + + /// Hash of the uploaded content for every created S3 key, so an auditor can verify the objects byte-for-byte. + public required IReadOnlyDictionary CreatedObjectHashes { get; init; } + + /// Every attempted step and how it ended, in the order they ran. Created, skipped, conflicting, and failed keys are all here. + public required IReadOnlyList Actions { get; init; } + + /// The state each touched registry manifest ended up in. + public IReadOnlyList RegistryState { get; init; } = []; + + /// What the post-apply verification concluded. + public required VerificationResult Verification { get; init; } + + /// When the run started, in UTC. + public required DateTimeOffset StartedAt { get; init; } + + /// When the run finished, in UTC. Null when the run was interrupted before finishing. + public DateTimeOffset? FinishedAt { get; init; } + + /// + public void Validate(IList problems) + { + if (!BackfillHash.IsWellFormed(PlanHash)) + problems.Add($"The plan hash must look like sha256: plus 64 lower-case hex characters, but found '{PlanHash}'."); + + for (var i = 0; i < InputRefs.Count; i++) + { + var before = problems.Count; + InputRefs[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"input_refs[{i}]"); + } + + foreach (var (key, hash) in CreatedObjectHashes) + { + if (string.IsNullOrWhiteSpace(key)) + problems.Add("created_object_hashes: keys must be non-empty S3 keys."); + if (!BackfillHash.IsWellFormed(hash)) + problems.Add($"created_object_hashes['{key}']: the hash must look like sha256: plus 64 lower-case hex characters, but found '{hash}'."); + } + + for (var i = 0; i < Actions.Count; i++) + { + var before = problems.Count; + Actions[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"actions[{i}]"); + } + + for (var i = 0; i < RegistryState.Count; i++) + { + var before = problems.Count; + RegistryState[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"registry_state[{i}]"); + } + + var beforeVerification = problems.Count; + Verification.Validate(problems); + ValidationProblems.PrefixNew(problems, beforeVerification, "verification"); + + if (StartedAt.Offset != TimeSpan.Zero) + problems.Add($"The started-at timestamp must be in UTC, but found offset {StartedAt.Offset}."); + if (FinishedAt is { } finished) + { + if (finished.Offset != TimeSpan.Zero) + problems.Add($"The finished-at timestamp must be in UTC, but found offset {finished.Offset}."); + if (finished < StartedAt) + problems.Add("The run cannot finish before it started."); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/OverridesDocument.cs b/src/services/Elastic.Changelog/Backfill/OverridesDocument.cs new file mode 100644 index 000000000..c767c5728 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/OverridesDocument.cs @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// What an override does to the value it targets. +public enum OverrideOperation +{ + /// Replace the value (or add it when it was missing). + [JsonStringEnumMemberName("set")] + Set, + + /// Remove the value entirely. + [JsonStringEnumMemberName("remove")] + Remove +} + +/// +/// Which slice of the backfill an override applies to: always a product, optionally +/// narrowed to one contributing repository and/or one release target. +/// +public sealed record OverrideScope +{ + /// The product the override applies to. + public required string Product { get; init; } + + /// The contributing repository, when the override only applies to one. Null means all repositories. + public GitRepository? Repository { get; init; } + + /// The release target (e.g. 9.0.0), when the override only applies to one. Null means all targets. + public string? Target { get; init; } + + /// Adds a plain-English description of every problem in this scope to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("An override scope needs a non-empty product."); + Repository?.Validate(problems); + } +} + +/// +/// One manual correction: "in this scope, set (or remove) this field, and here is why". +/// Overrides live in their own document instead of being edits to generated output, so a +/// rerun of the pipeline reproduces the same corrected result instead of losing the fix. +/// +public sealed record BackfillOverride +{ + /// A stable, unique ID for this override, so inventories and plans can say which overrides they applied. + public required string Id { get; init; } + + /// Which slice of the backfill this override applies to. + public required OverrideScope Scope { get; init; } + + /// + /// Plain path to the field being corrected, e.g. entries[3].title or + /// release_date. Interpreted by the planning stage against the scoped document. + /// + public required string Path { get; init; } + + /// Whether the override sets a new value or removes the existing one. + public required OverrideOperation Operation { get; init; } + + /// The replacement value, as text. Required when is ; must be absent for removals. + public string? Value { get; init; } + + /// Why this correction is right — required, because a correction nobody can explain later is a liability. + public required string Reason { get; init; } + + /// Adds a plain-English description of every problem in this override to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Id)) + problems.Add("An override needs a non-empty id."); + Scope.Validate(problems); + if (string.IsNullOrWhiteSpace(Path)) + problems.Add("An override needs a non-empty path saying which field it corrects."); + if (string.IsNullOrWhiteSpace(Reason)) + problems.Add("An override needs a non-empty reason."); + if (Operation == OverrideOperation.Set && Value is null) + problems.Add("An override that sets a value needs the value to set."); + if (Operation == OverrideOperation.Remove && Value is not null) + problems.Add("An override that removes a value must not also carry a value."); + } +} + +/// +/// Manual corrections an operator feeds into planning, each with a reason attached. +/// Written by a human, read by the planning stage. Kept separate from all generated +/// documents so reruns stay reproducible: the pipeline output is always +/// "generated result + these corrections", never a hand-edited file. +/// +public sealed record OverridesDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.Overrides; + + /// The corrections, in no particular order. IDs must be unique. + public required IReadOnlyList Overrides { get; init; } + + /// + public void Validate(IList problems) + { + var seenIds = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < Overrides.Count; i++) + { + var before = problems.Count; + Overrides[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"overrides[{i}]"); + + var id = Overrides[i].Id; + if (!string.IsNullOrWhiteSpace(id) && !seenIds.Add(id)) + problems.Add($"overrides[{i}]: The id '{id}' is used by more than one override; ids must be unique."); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/PlanDocument.cs b/src/services/Elastic.Changelog/Backfill/PlanDocument.cs new file mode 100644 index 000000000..c88779985 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/PlanDocument.cs @@ -0,0 +1,255 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// The things a plan can decide to do (or flag) for one product/target. +public enum PlanActionKind +{ + /// Create a brand-new bundle for a target that has none from this contributor. + [JsonStringEnumMemberName("create-bundle")] + CreateBundle, + + /// Add missing entries to an existing bundle via an amend file, when the parent bundle is unambiguous. + [JsonStringEnumMemberName("create-amend")] + CreateAmend, + + /// Create an additional bundle for a target that already has one, when amending would risk picking the wrong parent. + [JsonStringEnumMemberName("create-supplemental-bundle")] + CreateSupplementalBundle, + + /// Do nothing: the key already exists with exactly the bytes we would have written. + [JsonStringEnumMemberName("skip-existing")] + SkipExisting, + + /// A human must decide; the plan explains why in the action's reason. + [JsonStringEnumMemberName("manual-review")] + ManualReview, + + /// The key already exists with different bytes. Never overwritten in a normal run. + [JsonStringEnumMemberName("conflict")] + Conflict +} + +/// Which slice of the backfill a plan covers: one product, optionally narrowed to one repository and a target range. +public sealed record PlanScope +{ + /// The product this plan covers. + public required string Product { get; init; } + + /// The contributing repository, when the plan covers only one. + public GitRepository? Repository { get; init; } + + /// Human-readable description of the covered targets, e.g. 9.0.0..9.3.0 or 2025-01 onwards. + public string? TargetRange { get; init; } + + /// Adds a plain-English description of every problem in this scope to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("A plan scope needs a non-empty product."); + Repository?.Validate(problems); + } +} + +/// +/// Identifies exactly which link allowlist the deployed scrubber Lambda is running with. +/// The allowlist is baked into the Lambda at deploy time, so the local checkout can be +/// ahead of or behind it; a plan pins the deployed identity so "which links will survive +/// publication" was answered against reality, not against the local files. At least one +/// of the two fields is set. +/// +public sealed record ScrubberAllowlist +{ + /// Hash of the deployed allowlist content, as sha256: + 64 hex characters. + public string? Sha256 { get; init; } + + /// The docs-builder commit the deployed scrubber was built from (full 40-character SHA). + public string? DeploymentCommit { get; init; } + + /// Adds a plain-English description of every problem in this allowlist identity to . + public void Validate(IList problems) + { + if (Sha256 is null && DeploymentCommit is null) + problems.Add("A scrubber allowlist identity needs its content hash, its deployment commit, or both — otherwise the plan cannot say which allowlist it was checked against."); + if (Sha256 is not null && !BackfillHash.IsWellFormed(Sha256)) + problems.Add($"The scrubber allowlist hash must look like sha256: plus 64 lower-case hex characters, but found '{Sha256}'."); + } +} + +/// +/// One object that existed in the private bucket when the plan was computed. The plan +/// records these so the apply stage can detect that the world moved under it: if a key +/// the plan saw (or didn't see) has changed, the plan is stale and must be recomputed. +/// +public sealed record RemoteObject +{ + /// The object's S3 key, e.g. bundle/elasticsearch/elasticsearch-9.0.0.yaml. + public required string Key { get; init; } + + /// The object's ETag as reported by S3, when available. + public string? ETag { get; init; } + + /// Hash of the object's content as sha256: + 64 hex characters, when it was downloaded and hashed. + public string? Sha256 { get; init; } + + /// Adds a plain-English description of every problem in this object record to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Key)) + problems.Add("A remote object needs a non-empty key."); + if (Sha256 is not null && !BackfillHash.IsWellFormed(Sha256)) + problems.Add($"A remote object's hash must look like sha256: plus 64 lower-case hex characters, but found '{Sha256}'."); + } +} + +/// +/// One decision in a plan: what to create (with the exact content, identified by hash), +/// what to skip, or what needs a human. Create actions name the S3 key they will write +/// and the hash of the bytes they will write there, so apply can verify it wrote exactly +/// what was approved. +/// +public sealed record PlanAction +{ + /// What this action does. + public required PlanActionKind Kind { get; init; } + + /// The product the action belongs to. + public required string Product { get; init; } + + /// The release target the action belongs to, e.g. 9.0.0. + public required string Target { get; init; } + + /// The S3 key involved. Required for create and skip/conflict actions; may be null for manual-review when no key was determined. + public string? Key { get; init; } + + /// Hash of the exact bytes a create action will upload, as sha256: + 64 hex characters. Required for create actions. + public string? ContentSha256 { get; init; } + + /// For : the S3 key of the bundle the amend attaches to. + public string? ParentKey { get; init; } + + /// Why the plan decided this — required for skips, manual reviews, and conflicts so a reviewer never has to guess. + public string? Reason { get; init; } + + /// Adds a plain-English description of every problem in this action to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("A plan action needs a non-empty product."); + if (string.IsNullOrWhiteSpace(Target)) + problems.Add("A plan action needs a non-empty target."); + + var creates = Kind is PlanActionKind.CreateBundle or PlanActionKind.CreateAmend or PlanActionKind.CreateSupplementalBundle; + if (creates) + { + if (string.IsNullOrWhiteSpace(Key)) + problems.Add($"A {DescribeKind()} action needs the S3 key it will create."); + if (!BackfillHash.IsWellFormed(ContentSha256)) + problems.Add($"A {DescribeKind()} action needs the hash of the content it will upload (sha256: plus 64 lower-case hex characters), but found '{ContentSha256}'."); + } + if (Kind == PlanActionKind.CreateAmend && string.IsNullOrWhiteSpace(ParentKey)) + problems.Add("A create-amend action needs the key of the parent bundle it amends."); + if (Kind is PlanActionKind.SkipExisting or PlanActionKind.ManualReview or PlanActionKind.Conflict && string.IsNullOrWhiteSpace(Reason)) + problems.Add($"A {DescribeKind()} action needs a reason so a reviewer never has to guess."); + if (Kind is PlanActionKind.SkipExisting or PlanActionKind.Conflict && string.IsNullOrWhiteSpace(Key)) + problems.Add($"A {DescribeKind()} action needs the existing S3 key it refers to."); + } + + private string DescribeKind() => Kind switch + { + PlanActionKind.CreateBundle => "create-bundle", + PlanActionKind.CreateAmend => "create-amend", + PlanActionKind.CreateSupplementalBundle => "create-supplemental-bundle", + PlanActionKind.SkipExisting => "skip-existing", + PlanActionKind.ManualReview => "manual-review", + PlanActionKind.Conflict => "conflict", + _ => Kind.ToString() + }; +} + +/// +/// Exactly what we intend to create in S3, pinned to all of its inputs. Written by the +/// planning stage; approved by a human; executed by the apply stage. A plan is +/// "content-addressed": its identity is the hash of its canonical content (see +/// ), so the same inputs always produce +/// a plan with the same identity, and the ledger can prove which plan a run executed. +/// No S3 write ever happens from an unapproved or stale plan. +/// +public sealed record PlanDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.Plan; + + /// Which slice of the backfill this plan covers. + public required PlanScope Scope { get; init; } + + /// The source repositories, pinned to the exact commits the plan was computed from. + public required IReadOnlyList SourceRefs { get; init; } + + /// Hash of the inventory document the plan consumed. + public required string InventoryHash { get; init; } + + /// Hash of the semantic-model document the plan consumed. + public required string SemanticModelHash { get; init; } + + /// Hash of the overrides document the plan consumed. Null when no overrides were in play. + public string? OverridesHash { get; init; } + + /// Hash of the enrichment snapshot (recovered GitHub metadata) the plan consumed. Null when enrichment was not used. + public string? EnrichmentSnapshotHash { get; init; } + + /// Which link allowlist the deployed scrubber was running with when the plan was computed. + public required ScrubberAllowlist ScrubberAllowlist { get; init; } + + /// The objects that existed in the plan's slice of the private bucket when it was computed. Empty when the slice was empty. + public required IReadOnlyList CurrentState { get; init; } + + /// The plan's decisions, one per product/target/key. + public required IReadOnlyList Actions { get; init; } + + /// + public void Validate(IList problems) + { + var before = problems.Count; + Scope.Validate(problems); + ValidationProblems.PrefixNew(problems, before, "scope"); + + for (var i = 0; i < SourceRefs.Count; i++) + { + before = problems.Count; + SourceRefs[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"source_refs[{i}]"); + } + + if (!BackfillHash.IsWellFormed(InventoryHash)) + problems.Add($"The inventory hash must look like sha256: plus 64 lower-case hex characters, but found '{InventoryHash}'."); + if (!BackfillHash.IsWellFormed(SemanticModelHash)) + problems.Add($"The semantic-model hash must look like sha256: plus 64 lower-case hex characters, but found '{SemanticModelHash}'."); + if (OverridesHash is not null && !BackfillHash.IsWellFormed(OverridesHash)) + problems.Add($"The overrides hash must look like sha256: plus 64 lower-case hex characters, but found '{OverridesHash}'."); + if (EnrichmentSnapshotHash is not null && !BackfillHash.IsWellFormed(EnrichmentSnapshotHash)) + problems.Add($"The enrichment snapshot hash must look like sha256: plus 64 lower-case hex characters, but found '{EnrichmentSnapshotHash}'."); + + before = problems.Count; + ScrubberAllowlist.Validate(problems); + ValidationProblems.PrefixNew(problems, before, "scrubber_allowlist"); + + for (var i = 0; i < CurrentState.Count; i++) + { + before = problems.Count; + CurrentState[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"current_state[{i}]"); + } + + for (var i = 0; i < Actions.Count; i++) + { + before = problems.Count; + Actions[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"actions[{i}]"); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/ProvenanceDocument.cs b/src/services/Elastic.Changelog/Backfill/ProvenanceDocument.cs new file mode 100644 index 000000000..f48d3067b --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/ProvenanceDocument.cs @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// +/// Where a recovered fact came from, ordered from strongest to weakest. When two sources +/// disagree, the stronger one wins; when only a default remains, the fact is marked as +/// such so quality metrics can count it. +/// +public enum EvidenceSource +{ + /// An existing native changelog entry or bundle already said so. + [JsonStringEnumMemberName("native-artifact")] + NativeArtifact, + + /// The release-note source itself said so — its structure, substitutions, or comments. + [JsonStringEnumMemberName("release-note-source")] + ReleaseNoteSource, + + /// The repository's changelog configuration mapped it, e.g. a label-to-type mapping in changelog.yml. + [JsonStringEnumMemberName("changelog-config")] + ChangelogConfig, + + /// GitHub metadata said so: labels, PR body, linked issues, milestone, or release data. + [JsonStringEnumMemberName("github-metadata")] + GithubMetadata, + + /// Nothing better was available; a deterministic default rule filled it in. + [JsonStringEnumMemberName("default-rule")] + DefaultRule +} + +/// How much we trust a recovered fact. +public enum EvidenceConfidence +{ + /// Direct, unambiguous evidence. + [JsonStringEnumMemberName("high")] + High, + + /// Good evidence with some interpretation involved. + [JsonStringEnumMemberName("medium")] + Medium, + + /// A guess a human may want to double-check, e.g. a default rule. + [JsonStringEnumMemberName("low")] + Low +} + +/// +/// The paper trail for one recovered fact: which field of which entry (or release) got +/// which value, based on what evidence, and how sure we are. A reviewer reading one of +/// these should be able to re-check the fact by hand. +/// +public sealed record ProvenanceRecord +{ + /// The product the fact belongs to. + public required string Product { get; init; } + + /// The release target the fact belongs to, e.g. 9.0.0. + public required string Target { get; init; } + + /// The entry the fact is about. Null when the fact is about the release itself (e.g. its release date). + public EntryIdentity? Entry { get; init; } + + /// The field that was filled in, named as in the semantic model, e.g. precise_type or release_date. + public required string Field { get; init; } + + /// The value the field was given, as text. + public required string Value { get; init; } + + /// Where the value came from, on the strongest-to-weakest ladder. + public required EvidenceSource Source { get; init; } + + /// How much we trust the value. + public required EvidenceConfidence Confidence { get; init; } + + /// Human-readable pointer to the evidence itself, e.g. a label name or a URL, so the fact can be re-checked by hand. + public string? Evidence { get; init; } + + /// Where in the source the evidence sits, when it came from a file. + public SourceLocation? Location { get; init; } + + /// Adds a plain-English description of every problem in this record to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("A provenance record needs a non-empty product."); + if (string.IsNullOrWhiteSpace(Target)) + problems.Add("A provenance record needs a non-empty target."); + if (string.IsNullOrWhiteSpace(Field)) + problems.Add("A provenance record needs a non-empty field name."); + if (string.IsNullOrWhiteSpace(Value)) + problems.Add("A provenance record needs a non-empty value."); + Entry?.Validate(problems); + Location?.Validate(problems); + } +} + +/// +/// The evidence trail: why we believe each recovered fact. Written by the parser and the +/// enrichment stage as they fill in fields; read by humans reviewing a scope before it is +/// approved. Not consumed by the apply stage — it exists so decisions can be audited, not +/// so they can be replayed. +/// +public sealed record ProvenanceDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.Provenance; + + /// One record per recovered fact. + public required IReadOnlyList Records { get; init; } + + /// + public void Validate(IList problems) + { + for (var i = 0; i < Records.Count; i++) + { + var before = problems.Count; + Records[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"records[{i}]"); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/README.md b/src/services/Elastic.Changelog/Backfill/README.md new file mode 100644 index 000000000..072894e89 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/README.md @@ -0,0 +1,55 @@ +# Backfill artifact contracts + +The changelog backfill pipeline (epic [elastic/docs-eng-team#656](https://github.com/elastic/docs-eng-team/issues/656)) +runs as a series of stages, and each stage hands its result to the next one as a JSON document. +The types in this folder define what those documents look like, so every stage — and every human +reviewing a run — reads and writes the same shapes. + +## The six document families + +| Family | What it is | Who writes it | Who reads it | +|---|---|---|---| +| **inventory** | The census: which products and release-note sources exist, where they came from, and what we decided about each. | The inventory stage | Planning; humans reviewing scope | +| **overrides** | Manual corrections an operator feeds into planning, each with a reason attached. | A human operator | Planning | +| **semantic-model** | The release notes reduced to their meaning, with formatting stripped away. | The parser | Planning; the fidelity gate | +| **plan** | Exactly what we intend to create in S3, pinned to all of its inputs. | Planning | A human approver; the apply stage | +| **provenance** | The evidence trail: why we believe each recovered fact (an entry's type, a release date, …). | The parser and enrichment | Humans reviewing a scope | +| **ledger** | What actually happened when a plan was applied: every attempted step and its outcome. | The apply stage | Reruns (to resume safely); auditors | + +## How they flow + +1. The inventory stage takes the census and writes an **inventory**; a human adds **overrides** where the census got something wrong. +2. The parser turns release-note sources into a **semantic-model** (plus **provenance** for every recovered fact). +3. Planning combines inventory + overrides + semantic model + the current S3 state into a **plan**. +4. A human approves the plan; the apply stage executes it and writes a **ledger** of what really happened. +5. If a run is interrupted, the next run reads the ledger and the (unchanged, content-addressed) plan and picks up where it left off. + +## Envelopes and versions + +Every persisted document is wrapped in a small envelope that records what kind of document the +file contains (`artifact`) and which schema version wrote it (`schema_version`). Readers check +both **before** parsing the payload and fail with a clear error on anything they don't +understand — there is no silent best-effort parsing. All six families are currently at +version 1; bump a family's version in `BackfillSchemaVersions` when its shape changes in a way +old readers cannot safely ignore. + +## Canonical form and hashing + +Documents are hashed so a plan can be pinned to the exact inputs it was computed from, and so +the same plan content always has the same identity ("content-addressed"). Because the same JSON +can be written many equivalent ways, hashing first rewrites the document into one agreed-upon +**canonical form**: + +- object keys sorted by ordinal (byte-order) comparison; +- no insignificant whitespace; +- `\r\n` and `\r` inside strings normalized to `\n`; +- properties whose value is null are omitted entirely (absent and null mean the same thing); +- timestamps are stored in UTC and dates as `yyyy-MM-dd`, so the text never depends on machine culture or timezone; +- array order is preserved — it is part of the meaning; dictionaries become sorted JSON objects, so insertion order never matters. + +The hash is SHA-256 over the UTF-8 bytes of the canonical JSON of the **whole envelope** +(so the schema version is covered too), written as `sha256:` + 64 lower-case hex characters. +The pretty-printed form on disk is irrelevant to the hash — reading a file back and re-hashing +it always gives the same answer. + +Use `BackfillDocuments` to read, write, and hash documents; it applies all of the above. diff --git a/src/services/Elastic.Changelog/Backfill/SemanticModelDocument.cs b/src/services/Elastic.Changelog/Backfill/SemanticModelDocument.cs new file mode 100644 index 000000000..5dc171b72 --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/SemanticModelDocument.cs @@ -0,0 +1,371 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json.Serialization; + +namespace Elastic.Changelog.Backfill; + +/// +/// The section of a release-notes page an entry belongs to. Published pages often merge +/// related types into one section (features with enhancements, fixes with security +/// fixes), so this is deliberately coarser than the exact changelog type — the family is +/// always known from the page structure, while is +/// only filled in when the evidence actually distinguishes it. +/// +public enum EntryCategoryFamily +{ + /// New features and improvements — pages often merge these into one section. + [JsonStringEnumMemberName("features-and-enhancements")] + FeaturesAndEnhancements, + + /// Bug fixes and security fixes — pages often merge these into one section. + [JsonStringEnumMemberName("fixes-and-security")] + FixesAndSecurity, + + /// Changes that break existing behavior and need user action. + [JsonStringEnumMemberName("breaking-changes")] + BreakingChanges, + + /// Functionality that still works but is on its way out. + [JsonStringEnumMemberName("deprecations")] + Deprecations, + + /// Problems known to exist in the release. + [JsonStringEnumMemberName("known-issues")] + KnownIssues, + + /// Major documentation changes or reorganizations. + [JsonStringEnumMemberName("docs")] + Docs, + + /// Functionality that stopped working or now behaves incorrectly. + [JsonStringEnumMemberName("regressions")] + Regressions, + + /// Anything that does not fit the other families. + [JsonStringEnumMemberName("other")] + Other +} + +/// +/// The exact changelog type of an entry, using the same vocabulary as live changelog +/// entries. Only recorded when the source or enrichment evidence actually distinguishes +/// it — a merged "Fixes" section, for example, gives every entry the +/// family but no precise type. +/// +public enum PreciseEntryType +{ + /// A new feature. + [JsonStringEnumMemberName("feature")] + Feature, + + /// An improvement to an existing feature. + [JsonStringEnumMemberName("enhancement")] + Enhancement, + + /// A fix or advisory for a security vulnerability. + [JsonStringEnumMemberName("security")] + Security, + + /// A bug fix. + [JsonStringEnumMemberName("bug-fix")] + BugFix, + + /// A change that breaks documented behavior. + [JsonStringEnumMemberName("breaking-change")] + BreakingChange, + + /// Functionality that is deprecated and will be removed later. + [JsonStringEnumMemberName("deprecation")] + Deprecation, + + /// A problem known to exist in the product. + [JsonStringEnumMemberName("known-issue")] + KnownIssue, + + /// A major documentation change. + [JsonStringEnumMemberName("docs")] + Docs, + + /// Functionality that stopped working or behaves incorrectly. + [JsonStringEnumMemberName("regression")] + Regression, + + /// Anything that does not fit the other types. + [JsonStringEnumMemberName("other")] + Other +} + +/// What part of a breaking change breaks, using the same vocabulary as live changelog entries. +public enum BreakingChangeSubtype +{ + /// Breaks an API. + [JsonStringEnumMemberName("api")] + Api, + + /// Breaks the way something works. + [JsonStringEnumMemberName("behavioral")] + Behavioral, + + /// Breaks configuration. + [JsonStringEnumMemberName("configuration")] + Configuration, + + /// Breaks a dependency, such as a third-party product. + [JsonStringEnumMemberName("dependency")] + Dependency, + + /// Breaks licensing or subscription behavior. + [JsonStringEnumMemberName("subscription")] + Subscription, + + /// Breaks a plugin. + [JsonStringEnumMemberName("plugin")] + Plugin, + + /// Breaks authentication, authorization, or permissions. + [JsonStringEnumMemberName("security")] + Security, + + /// A breaking change that does not fit the other subtypes. + [JsonStringEnumMemberName("other")] + Other +} + +/// How mature a feature is for a given product, using the same vocabulary as live changelog entries. +public enum ReleaseLifecycle +{ + /// Technical preview. + [JsonStringEnumMemberName("preview")] + Preview, + + /// Beta. + [JsonStringEnumMemberName("beta")] + Beta, + + /// Generally available. + [JsonStringEnumMemberName("ga")] + Ga, + + /// Experimental. + [JsonStringEnumMemberName("experimental")] + Experimental +} + +/// How serious a triage finding is. +public enum TriageSeverity +{ + /// The scope cannot be published until a human resolves this. + [JsonStringEnumMemberName("blocker")] + Blocker, + + /// Worth a look, but does not block publication (e.g. missing optional metadata). + [JsonStringEnumMemberName("warning")] + Warning +} + +/// A product touched by an entry, with the release it lands in and, when known, the feature's maturity. +public sealed record ReleaseProductReference +{ + /// The product ID, as known to products.yml. + public required string Product { get; init; } + + /// The release target for that product, e.g. 9.0.0 or 2025-11. + public required string Target { get; init; } + + /// The feature's maturity for this product, when the source or enrichment says so. + public ReleaseLifecycle? Lifecycle { get; init; } + + /// Adds a plain-English description of every problem in this reference to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("A product reference needs a non-empty product."); + if (string.IsNullOrWhiteSpace(Target)) + problems.Add("A product reference needs a non-empty target."); + } +} + +/// +/// One release-note entry reduced to its meaning: what changed, how it is categorized, +/// and what it links to. Everything about presentation — anchors, heading text, dropdown +/// versus list rendering, ordering — is deliberately absent, so a renderer redesign can +/// never make two equal entries look different to the pipeline. +/// +public sealed record ReleaseEntry +{ + /// The section family the entry belongs to. Always known, because it comes from the page structure. + public required EntryCategoryFamily CategoryFamily { get; init; } + + /// The exact changelog type, when the evidence distinguishes it. Null when only the family is known. + public PreciseEntryType? PreciseType { get; init; } + + /// What part of a breaking change breaks. Only meaningful for breaking changes; filled in by enrichment when known. + public BreakingChangeSubtype? Subtype { get; init; } + + /// The entry's one-line summary of what changed. + public required string Title { get; init; } + + /// Longer explanation of the change, when the source has one. Markdown structure, not rendered HTML. + public string? Description { get; init; } + + /// For breaking changes: what the change does to existing users. + public string? Impact { get; init; } + + /// For breaking changes: what users must do about it. + public string? Action { get; init; } + + /// + /// True when the release presents this entry as a highlight. A highlighted entry + /// appears in both the regular sections and the highlights output; it is never a + /// separate entry, because that would make it show up twice. + /// + public bool? Highlight { get; init; } + + /// The products this entry applies to, each with its release target. + public IReadOnlyList ProductReferences { get; init; } = []; + + /// Links the entry carries besides PRs and issues, as absolute URLs after substitutions are applied. + public IReadOnlyList Links { get; init; } = []; + + /// Pull requests behind the change, as full canonical URLs — never bare numbers. + public IReadOnlyList Prs { get; init; } = []; + + /// Issues related to the change, as full canonical URLs — never bare numbers. + public IReadOnlyList Issues { get; init; } = []; + + /// Product areas the entry touches, e.g. Search, when the source or enrichment says so. + public IReadOnlyList Areas { get; init; } = []; + + /// The entry's stable identity, once identity resolution has run. Null straight out of the parser. + public EntryIdentity? Identity { get; init; } + + /// Where in the source this entry was parsed from, so a human can check the original text. + public required SourceLocation SourceLocation { get; init; } + + /// Adds a plain-English description of every problem in this entry to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Title)) + problems.Add("An entry needs a non-empty title."); + foreach (var reference in ProductReferences) + reference.Validate(problems); + foreach (var pr in Prs) + { + if (!CanonicalGitHubUrls.IsPullRequestUrl(pr)) + problems.Add($"PR references must be full canonical URLs like https://github.com/{{owner}}/{{repo}}/pull/{{number}}, but found '{pr}'."); + } + foreach (var issue in Issues) + { + if (!CanonicalGitHubUrls.IsIssueUrl(issue)) + problems.Add($"Issue references must be full canonical URLs like https://github.com/{{owner}}/{{repo}}/issues/{{number}}, but found '{issue}'."); + } + Identity?.Validate(problems); + SourceLocation.Validate(problems); + } +} + +/// +/// One release of one product with everything we learned about it: when it shipped (if +/// known), its introductory text, and its entries. This is the ProductRelease +/// shape from the backfill epic. +/// +public sealed record ProductRelease +{ + /// The product ID, as known to products.yml. + public required string Product { get; init; } + + /// The release target, e.g. 9.0.0, 2025-11, or 2025-11-04, matching the product's target scheme. + public required string Target { get; init; } + + /// The day the release shipped, when the source or enrichment recovered it. + public DateOnly? ReleaseDate { get; init; } + + /// Introductory text for the release as a whole, when the source has one. + public string? Description { get; init; } + + /// The release's entries. Order carries no meaning; the fidelity gate compares releases as unordered collections. + public required IReadOnlyList Entries { get; init; } + + /// Adds a plain-English description of every problem in this release to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Product)) + problems.Add("A release needs a non-empty product."); + if (string.IsNullOrWhiteSpace(Target)) + problems.Add("A release needs a non-empty target."); + for (var i = 0; i < Entries.Count; i++) + { + var before = problems.Count; + Entries[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"entries[{i}]"); + } + } +} + +/// +/// Something the parser could not fully handle, with where it happened. Blockers stop a +/// scope from being published; warnings are quality notes a human may act on. Structured +/// (rather than log lines) so review tooling can group and count them. +/// +public sealed record TriageDiagnostic +{ + /// Whether this finding blocks publication or is only worth a look. + public required TriageSeverity Severity { get; init; } + + /// Plain-English description of what the parser could not handle and why. + public required string Message { get; init; } + + /// The product the finding belongs to, when known. + public string? Product { get; init; } + + /// The release target the finding belongs to, when known. + public string? Target { get; init; } + + /// Where in the source the problem sits, so a human can jump straight to it. + public SourceLocation? Location { get; init; } + + /// Adds a plain-English description of every problem in this diagnostic to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Message)) + problems.Add("A triage diagnostic needs a non-empty message."); + Location?.Validate(problems); + } +} + +/// +/// The release notes reduced to their meaning, with formatting stripped away. Written by +/// the parser from expanded source Markdown (or native artifacts); read by planning and +/// by the semantic fidelity gate, which compares two of these instead of diffing rendered +/// output. Carries triage diagnostics for everything the parser could not confidently handle. +/// +public sealed record SemanticModelDocument : IBackfillDocument +{ + /// + public static BackfillArtifactKind Kind => BackfillArtifactKind.SemanticModel; + + /// The parsed releases. + public required IReadOnlyList Releases { get; init; } + + /// Everything the parser could not fully handle. Any blocker here stops the scope from being published. + public IReadOnlyList Diagnostics { get; init; } = []; + + /// + public void Validate(IList problems) + { + for (var i = 0; i < Releases.Count; i++) + { + var before = problems.Count; + Releases[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"releases[{i}]"); + } + for (var i = 0; i < Diagnostics.Count; i++) + { + var before = problems.Count; + Diagnostics[i].Validate(problems); + ValidationProblems.PrefixNew(problems, before, $"diagnostics[{i}]"); + } + } +} diff --git a/src/services/Elastic.Changelog/Backfill/SourceReferences.cs b/src/services/Elastic.Changelog/Backfill/SourceReferences.cs new file mode 100644 index 000000000..f84f37f9f --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/SourceReferences.cs @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// A GitHub repository, named by its owner (organization or user) and repository name. +public sealed record GitRepository +{ + /// The GitHub owner, e.g. elastic. + public required string Owner { get; init; } + + /// The repository name, e.g. elasticsearch. + public required string Name { get; init; } + + /// Adds a plain-English description of every problem in this repository name to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Owner)) + problems.Add("A repository needs a non-empty owner."); + if (string.IsNullOrWhiteSpace(Name)) + problems.Add("A repository needs a non-empty name."); + } +} + +/// +/// A repository pinned to one exact commit. Plans and ledgers record these so a run can +/// be reproduced (or audited) later even after the branch or tag has moved on: the +/// human-friendly ref says what was asked for, the commit says exactly what was used. +/// +public sealed record PinnedSource +{ + /// The repository the content came from. + public required GitRepository Repository { get; init; } + + /// The ref that was requested, e.g. main or v9.0.0. + public required string GitRef { get; init; } + + /// The full 40-character commit SHA the ref pointed at when the document was produced. + public required string Commit { get; init; } + + /// Adds a plain-English description of every problem in this pinned source to . + public void Validate(IList problems) + { + Repository.Validate(problems); + if (string.IsNullOrWhiteSpace(GitRef)) + problems.Add("A pinned source needs a non-empty git ref."); + if (!IsFullCommitSha(Commit)) + problems.Add($"A pinned source needs a full 40-character commit SHA, but found '{Commit}'."); + } + + private static bool IsFullCommitSha(string? value) + { + if (value is null || value.Length != 40) + return false; + + foreach (var c in value) + { + if (c is (< '0' or > '9') and (< 'a' or > 'f')) + return false; + } + return true; + } +} + +/// +/// Where in a source file something was found: the file path (relative to its repository +/// root) and, when known, the line range. Lets a human jump from a parsed entry or a +/// triage message straight to the text it came from. +/// +public sealed record SourceLocation +{ + /// File path relative to the repository root, e.g. docs/release-notes/index.md. + public required string Path { get; init; } + + /// First line of the relevant text (1-based), when known. + public int? StartLine { get; init; } + + /// Last line of the relevant text (1-based, inclusive), when known. + public int? EndLine { get; init; } + + /// Adds a plain-English description of every problem in this location to . + public void Validate(IList problems) + { + if (string.IsNullOrWhiteSpace(Path)) + problems.Add("A source location needs a non-empty path."); + if (StartLine is < 1) + problems.Add($"A source location's start line must be 1 or greater, but found {StartLine}."); + if (EndLine is < 1) + problems.Add($"A source location's end line must be 1 or greater, but found {EndLine}."); + if (StartLine is { } start && EndLine is { } end && end < start) + problems.Add($"A source location's end line ({end}) cannot come before its start line ({start})."); + } +} diff --git a/src/services/Elastic.Changelog/Backfill/ValidationProblems.cs b/src/services/Elastic.Changelog/Backfill/ValidationProblems.cs new file mode 100644 index 000000000..7262261df --- /dev/null +++ b/src/services/Elastic.Changelog/Backfill/ValidationProblems.cs @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Backfill; + +/// +/// Helper for building readable validation messages. Nested parts (an entry inside a +/// release inside a document) validate themselves without knowing where they sit; the +/// containing part then prefixes the new messages with its position, so the final message +/// reads like releases[2]: entries[0]: An entry needs a non-empty title. +/// +internal static class ValidationProblems +{ + /// Prefixes every problem added at or after with . + public static void PrefixNew(IList problems, int firstNewIndex, string prefix) + { + for (var i = firstNewIndex; i < problems.Count; i++) + problems[i] = $"{prefix}: {problems[i]}"; + } +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/BackfillFixtures.cs b/tests/Elastic.Changelog.Tests/Backfill/BackfillFixtures.cs new file mode 100644 index 000000000..52ad7e3c7 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/BackfillFixtures.cs @@ -0,0 +1,293 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +/// +/// Representative, valid instances of all six backfill document families. Each fixture +/// fills every field at least once so round-trip tests exercise the full shape. +/// +public static class BackfillFixtures +{ + public const string SampleHash = "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"; + + public static InventoryDocument Inventory() => new() + { + Sources = + [ + new InventorySource + { + SourceRepository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + GitRef = "main", + Docset = "docs-content", + Paths = ["docs/release-notes/index.md", "docs/release-notes/breaking-changes.md"], + ProductIds = ["elasticsearch"], + TargetScheme = TargetScheme.Semver, + Cutoff = new BackfillCutoff { Kind = CutoffKind.Version, Value = "9.0.0", Notes = "Stack products start at 9.0" }, + Substitutions = new Dictionary { ["es"] = "Elasticsearch", ["kib"] = "Kibana" }, + LinkMappings = new Dictionary { ["./breaking-changes.md"] = "https://www.elastic.co/docs/release-notes/elasticsearch/breaking-changes" }, + AttributedRepositories = + [ + new AttributedRepository + { + Repository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + OnScrubberAllowlist = true + } + ], + DefaultRepository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + BundleFilenameConvention = "{repo}-{target}.yaml", + AdoptionState = AdoptionState.NotAdopted, + Classification = SourceClassification.PublishedHistoryFound, + AppliedOverrideIds = ["fix-9-0-1-release-date"], + UnresolvedItems = ["The 9.0.2 section mixes known issues with fixes; needs a human decision."] + }, + new InventorySource + { + SourceRepository = new GitRepository { Owner = "elastic", Name = "cloud" }, + GitRef = "master", + ProductIds = ["cloud-hosted"], + TargetScheme = TargetScheme.Monthly, + AdoptionState = AdoptionState.PartiallyAdopted, + Classification = SourceClassification.HybridPage + } + ] + }; + + public static OverridesDocument Overrides() => new() + { + Overrides = + [ + new BackfillOverride + { + Id = "fix-9-0-1-release-date", + Scope = new OverrideScope + { + Product = "elasticsearch", + Repository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + Target = "9.0.1" + }, + Path = "release_date", + Operation = OverrideOperation.Set, + Value = "2025-05-06", + Reason = "The published page has no date; the GitHub release for v9.0.1 says 2025-05-06." + }, + new BackfillOverride + { + Id = "drop-duplicated-entry", + Scope = new OverrideScope { Product = "elasticsearch", Target = "9.0.0" }, + Path = "entries[12]", + Operation = OverrideOperation.Remove, + Reason = "Duplicate of entries[4]; same PR, same text, listed under two areas." + } + ] + }; + + public static SemanticModelDocument SemanticModel() => new() + { + Releases = + [ + new ProductRelease + { + Product = "elasticsearch", + Target = "9.0.0", + ReleaseDate = new DateOnly(2025, 4, 8), + Description = "First release of the 9.x series.", + Entries = + [ + new ReleaseEntry + { + CategoryFamily = EntryCategoryFamily.FeaturesAndEnhancements, + PreciseType = PreciseEntryType.Feature, + Title = "Add better binary quantization to dense vectors", + Description = "Dense vector fields now support the `bbq_hnsw` index type.", + Highlight = true, + ProductReferences = + [ + new ReleaseProductReference { Product = "elasticsearch", Target = "9.0.0", Lifecycle = ReleaseLifecycle.Ga } + ], + Links = ["https://www.elastic.co/docs/reference/elasticsearch/dense-vector"], + Prs = ["https://github.com/elastic/elasticsearch/pull/12345"], + Issues = ["https://github.com/elastic/elasticsearch/issues/12000"], + Areas = ["Vector Search"], + Identity = EntryIdentity.ForPullRequest("elastic", "elasticsearch", 12345), + SourceLocation = new SourceLocation { Path = "docs/release-notes/index.md", StartLine = 42, EndLine = 45 } + }, + new ReleaseEntry + { + CategoryFamily = EntryCategoryFamily.BreakingChanges, + Subtype = BreakingChangeSubtype.Configuration, + Title = "Remove the `node.attr` legacy setting", + Impact = "Clusters configured with `node.attr` will not start.", + Action = "Move node attributes to the new `node.attributes` block.", + Identity = EntryIdentity.ForFile("backfill-elasticsearch-9.0.0-0002.yaml", "0e5751c026e543b2e8ab2eb06099daa1d1e5df47778f7787faab45cdf12fe3a8"), + SourceLocation = new SourceLocation { Path = "docs/release-notes/breaking-changes.md", StartLine = 10, EndLine = 18 } + } + ] + } + ], + Diagnostics = + [ + new TriageDiagnostic + { + Severity = TriageSeverity.Warning, + Message = "The 'Fixes' section merges bug fixes and security fixes; entries carry the family only.", + Product = "elasticsearch", + Target = "9.0.0", + Location = new SourceLocation { Path = "docs/release-notes/index.md", StartLine = 60 } + } + ] + }; + + public static PlanDocument Plan() => new() + { + Scope = new PlanScope + { + Product = "elasticsearch", + Repository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + TargetRange = "9.0.0..9.0.2" + }, + SourceRefs = + [ + new PinnedSource + { + Repository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + GitRef = "main", + Commit = "0123456789abcdef0123456789abcdef01234567" + } + ], + InventoryHash = SampleHash, + SemanticModelHash = SampleHash, + OverridesHash = SampleHash, + EnrichmentSnapshotHash = SampleHash, + ScrubberAllowlist = new ScrubberAllowlist + { + Sha256 = SampleHash, + DeploymentCommit = "89abcdef0123456789abcdef0123456789abcdef" + }, + CurrentState = + [ + new RemoteObject + { + Key = "bundle/elasticsearch/elasticsearch-9.0.2.yaml", + ETag = "\"d41d8cd98f00b204e9800998ecf8427e\"", + Sha256 = SampleHash + } + ], + Actions = + [ + new PlanAction + { + Kind = PlanActionKind.CreateBundle, + Product = "elasticsearch", + Target = "9.0.0", + Key = "bundle/elasticsearch/elasticsearch-9.0.0.yaml", + ContentSha256 = SampleHash + }, + new PlanAction + { + Kind = PlanActionKind.CreateAmend, + Product = "elasticsearch", + Target = "9.0.2", + Key = "bundle/elasticsearch/elasticsearch-9.0.2.amend-1.yaml", + ContentSha256 = SampleHash, + ParentKey = "bundle/elasticsearch/elasticsearch-9.0.2.yaml" + }, + new PlanAction + { + Kind = PlanActionKind.SkipExisting, + Product = "elasticsearch", + Target = "9.0.1", + Key = "bundle/elasticsearch/elasticsearch-9.0.1.yaml", + Reason = "The key already exists with exactly the bytes this plan would write." + }, + new PlanAction + { + Kind = PlanActionKind.ManualReview, + Product = "elasticsearch", + Target = "9.0.3", + Reason = "Two bundles claim this target and neither is an unambiguous amend parent." + } + ] + }; + + public static ProvenanceDocument Provenance() => new() + { + Records = + [ + new ProvenanceRecord + { + Product = "elasticsearch", + Target = "9.0.0", + Entry = EntryIdentity.ForPullRequest("elastic", "elasticsearch", 12345), + Field = "precise_type", + Value = "feature", + Source = EvidenceSource.GithubMetadata, + Confidence = EvidenceConfidence.High, + Evidence = "PR label '>feature' via changelog.yml label mapping" + }, + new ProvenanceRecord + { + Product = "elasticsearch", + Target = "9.0.0", + Field = "release_date", + Value = "2025-04-08", + Source = EvidenceSource.ReleaseNoteSource, + Confidence = EvidenceConfidence.Medium, + Evidence = "Date printed under the release heading", + Location = new SourceLocation { Path = "docs/release-notes/index.md", StartLine = 40 } + } + ] + }; + + public static LedgerDocument Ledger() => new() + { + PlanHash = SampleHash, + InputRefs = + [ + new PinnedSource + { + Repository = new GitRepository { Owner = "elastic", Name = "elasticsearch" }, + GitRef = "main", + Commit = "0123456789abcdef0123456789abcdef01234567" + } + ], + CreatedObjectHashes = new Dictionary + { + ["bundle/elasticsearch/elasticsearch-9.0.0.yaml"] = SampleHash + }, + Actions = + [ + new LedgerAction + { + PlannedKind = PlanActionKind.CreateBundle, + Key = "bundle/elasticsearch/elasticsearch-9.0.0.yaml", + Outcome = LedgerActionOutcome.Created + }, + new LedgerAction + { + PlannedKind = PlanActionKind.CreateAmend, + Key = "bundle/elasticsearch/elasticsearch-9.0.2.amend-1.yaml", + Outcome = LedgerActionOutcome.Conflict, + Detail = "The key appeared between planning and apply, with different bytes." + } + ], + RegistryState = + [ + new RegistryRefresh + { + Key = "bundle/elasticsearch/registry.json", + Outcome = RegistryRefreshOutcome.Updated + } + ], + Verification = new VerificationResult + { + Outcome = VerificationOutcome.Failed, + Details = ["9.0.2: one planned entry is missing from the public bundle (conflict above was never applied)."] + }, + StartedAt = new DateTimeOffset(2026, 7, 20, 12, 0, 0, TimeSpan.Zero), + FinishedAt = new DateTimeOffset(2026, 7, 20, 12, 5, 30, TimeSpan.Zero) + }; +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/BackfillHashTests.cs b/tests/Elastic.Changelog.Tests/Backfill/BackfillHashTests.cs new file mode 100644 index 000000000..12dbad60d --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/BackfillHashTests.cs @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +/// +/// The determinism guarantees: the same content always hashes the same, no matter how it +/// was formatted or assembled, and any change in meaning changes the hash. +/// +public class BackfillHashTests +{ + [Fact] + public void ComputeHash_SameDocument_AlwaysProducesSameHash() + { + var first = BackfillDocuments.ComputeHash(BackfillFixtures.Plan()); + var second = BackfillDocuments.ComputeHash(BackfillFixtures.Plan()); + + second.Should().Be(first); + } + + [Fact] + public void ComputeHash_AllFamilies_ProduceWellFormedHashes() + { + var hashes = new[] + { + BackfillDocuments.ComputeHash(BackfillFixtures.Inventory()), + BackfillDocuments.ComputeHash(BackfillFixtures.Overrides()), + BackfillDocuments.ComputeHash(BackfillFixtures.SemanticModel()), + BackfillDocuments.ComputeHash(BackfillFixtures.Plan()), + BackfillDocuments.ComputeHash(BackfillFixtures.Provenance()), + BackfillDocuments.ComputeHash(BackfillFixtures.Ledger()) + }; + + foreach (var hash in hashes) + _ = BackfillHash.IsWellFormed(hash).Should().BeTrue($"'{hash}' should be sha256: plus 64 lower-case hex characters"); + + hashes.Distinct().Should().HaveCount(hashes.Length, "different documents must not collide on the same hash"); + } + + [Fact] + public void ComputeHash_DictionaryInsertionOrder_DoesNotChangeHash() + { + var source = BackfillFixtures.Inventory().Sources[0]; + + var oneWay = source with + { + Substitutions = new Dictionary { ["es"] = "Elasticsearch", ["kib"] = "Kibana" } + }; + var otherWay = source with + { + Substitutions = new Dictionary { ["kib"] = "Kibana", ["es"] = "Elasticsearch" } + }; + + var oneHash = BackfillDocuments.ComputeHash(new InventoryDocument { Sources = [oneWay] }); + var otherHash = BackfillDocuments.ComputeHash(new InventoryDocument { Sources = [otherWay] }); + + otherHash.Should().Be(oneHash); + } + + [Fact] + public void ComputeHash_LedgerDictionaryInsertionOrder_DoesNotChangeHash() + { + var ledger = BackfillFixtures.Ledger(); + + var oneWay = ledger with + { + CreatedObjectHashes = new Dictionary + { + ["bundle/es/a.yaml"] = BackfillFixtures.SampleHash, + ["bundle/es/b.yaml"] = BackfillFixtures.SampleHash + } + }; + var otherWay = ledger with + { + CreatedObjectHashes = new Dictionary + { + ["bundle/es/b.yaml"] = BackfillFixtures.SampleHash, + ["bundle/es/a.yaml"] = BackfillFixtures.SampleHash + } + }; + + BackfillDocuments.ComputeHash(otherWay).Should().Be(BackfillDocuments.ComputeHash(oneWay)); + } + + [Fact] + public void ComputeHash_FormattingOfPersistedFile_DoesNotChangeHash() + { + var document = BackfillFixtures.Plan(); + var indented = BackfillDocuments.Serialize(document); + // Simulate a file that was reformatted (e.g. by an editor) without changing content. + var compact = CanonicalJson.Canonicalize(indented); + + BackfillDocuments.ComputeHash(compact).Should().Be(BackfillDocuments.ComputeHash(indented)); + BackfillDocuments.ComputeHash(indented).Should().Be(BackfillDocuments.ComputeHash(document)); + } + + [Fact] + public void ComputeHash_LineEndingDifferencesInsideText_DoNotChangeHash() + { + var release = BackfillFixtures.SemanticModel().Releases[0]; + + var withUnixEndings = new SemanticModelDocument + { + Releases = [release with { Description = "line one\nline two" }] + }; + var withWindowsEndings = new SemanticModelDocument + { + Releases = [release with { Description = "line one\r\nline two" }] + }; + + BackfillDocuments.ComputeHash(withWindowsEndings).Should().Be(BackfillDocuments.ComputeHash(withUnixEndings)); + } + + [Fact] + public void ComputeHash_SemanticChange_ChangesHash() + { + var plan = BackfillFixtures.Plan(); + var changed = plan with + { + Actions = + [ + .. plan.Actions.Take(plan.Actions.Count - 1), + plan.Actions[^1] with { Reason = "A different reason." } + ] + }; + + BackfillDocuments.ComputeHash(changed).Should().NotBe(BackfillDocuments.ComputeHash(plan)); + } + + [Fact] + public void ComputeHash_ListOrder_IsPartOfTheMeaning() + { + var ledger = BackfillFixtures.Ledger(); + var reversed = ledger with { Actions = [.. ledger.Actions.Reverse()] }; + + // Ledger actions record the order steps ran in, so reordering them is a real change. + BackfillDocuments.ComputeHash(reversed).Should().NotBe(BackfillDocuments.ComputeHash(ledger)); + } + + [Fact] + public void Compute_KnownInput_MatchesKnownSha256() + { + // SHA-256 of the ASCII bytes of "{}" — pins the algorithm and the output format. + BackfillHash.Compute("{}").Should() + .Be("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"); + } + + [Theory] + [InlineData(null, false)] + [InlineData("", false)] + [InlineData("sha256:", false)] + [InlineData("sha256:abc", false)] + [InlineData("md5:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", false)] + [InlineData("sha256:44136FA355B3678A1146AD16F7E8649E94FB4FC21FE77E8310C060F61CAAFF8A", false)] + [InlineData("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", true)] + public void IsWellFormed_RecognizesOnlyLowercaseSha256(string? value, bool expected) => + BackfillHash.IsWellFormed(value).Should().Be(expected); +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/BackfillRoundTripTests.cs b/tests/Elastic.Changelog.Tests/Backfill/BackfillRoundTripTests.cs new file mode 100644 index 000000000..c4d525378 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/BackfillRoundTripTests.cs @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using AwesomeAssertions; +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +/// +/// Every document family must survive serialize → deserialize unchanged, and the +/// serialized form must carry the envelope header (artifact name + schema version). +/// +public class BackfillRoundTripTests +{ + [Fact] + public void Serialize_InventoryDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.Inventory(), "inventory"); + + [Fact] + public void Serialize_OverridesDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.Overrides(), "overrides"); + + [Fact] + public void Serialize_SemanticModelDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.SemanticModel(), "semantic-model"); + + [Fact] + public void Serialize_PlanDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.Plan(), "plan"); + + [Fact] + public void Serialize_ProvenanceDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.Provenance(), "provenance"); + + [Fact] + public void Serialize_LedgerDocument_RoundTripsUnchanged() => + AssertRoundTrip(BackfillFixtures.Ledger(), "ledger"); + + private static void AssertRoundTrip(T document, string expectedArtifactName) + where T : class, IBackfillDocument + { + var json = BackfillDocuments.Serialize(document); + + using (var parsed = JsonDocument.Parse(json)) + { + parsed.RootElement.GetProperty("artifact").GetString().Should().Be(expectedArtifactName); + parsed.RootElement.GetProperty("schema_version").GetInt32().Should().Be(BackfillSchemaVersions.Current(T.Kind)); + _ = parsed.RootElement.TryGetProperty("payload", out _).Should().BeTrue(); + } + + var roundTripped = BackfillDocuments.Deserialize(json); + roundTripped.Should().BeEquivalentTo(document); + + // The hash must also survive the trip: same content, same identity. + BackfillDocuments.ComputeHash(roundTripped).Should().Be(BackfillDocuments.ComputeHash(document)); + } + + [Fact] + public void Serialize_SemanticModelEnums_UseKebabCaseNames() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.SemanticModel()); + + json.Should().Contain("\"features-and-enhancements\""); + json.Should().Contain("\"breaking-changes\""); + json.Should().Contain("\"pull-request\""); + json.Should().Contain("\"synthetic-file\""); + } + + [Fact] + public void Serialize_PlanActionKinds_UseKebabCaseNames() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Plan()); + + json.Should().Contain("\"create-bundle\""); + json.Should().Contain("\"create-amend\""); + json.Should().Contain("\"skip-existing\""); + json.Should().Contain("\"manual-review\""); + } + + [Fact] + public void Serialize_ReleaseDate_UsesPlainIsoDate() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.SemanticModel()); + + json.Should().Contain("\"release_date\": \"2025-04-08\""); + } + + [Fact] + public void Serialize_NullOptionalFields_AreOmitted() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Overrides()); + + // The second override has no value (it is a removal); the property must be absent, not null. + json.Should().NotContain("\"value\": null"); + } + + [Fact] + public void ArtifactKindNames_RoundTripThroughTryParse() + { + foreach (var kind in Enum.GetValues()) + { + var name = BackfillArtifactKinds.Name(kind); + _ = BackfillArtifactKinds.TryParse(name, out var parsed).Should().BeTrue(); + parsed.Should().Be(kind); + } + } +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/BackfillValidationTests.cs b/tests/Elastic.Changelog.Tests/Backfill/BackfillValidationTests.cs new file mode 100644 index 000000000..06cbf4ef0 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/BackfillValidationTests.cs @@ -0,0 +1,314 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +/// +/// Documents with missing or invalid fields must be rejected with messages that say +/// where the problem is and what a valid value looks like. +/// +public class BackfillValidationTests +{ + [Fact] + public void Deserialize_MissingRequiredJsonField_FailsWithParseError() + { + // A release without its required 'target' field. + var json = /*lang=json,strict*/ """ + { + "artifact": "semantic-model", + "schema_version": 1, + "payload": { + "releases": [ { "product": "elasticsearch", "entries": [] } ], + "diagnostics": [] + } + } + """; + + var act = () => BackfillDocuments.Deserialize(json); + + act.Should().Throw() + .WithMessage("*'semantic-model' document could not be parsed*"); + } + + [Fact] + public void Serialize_EmptyTitle_FailsWithLocationAndProblem() + { + var model = BackfillFixtures.SemanticModel(); + var release = model.Releases[0]; + var invalid = model with + { + Releases = [release with { Entries = [release.Entries[0] with { Title = " " }] }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*releases[0]*entries[0]*non-empty title*"); + } + + [Fact] + public void Serialize_BarePrNumber_IsRejectedWithExampleOfCanonicalUrl() + { + var model = BackfillFixtures.SemanticModel(); + var release = model.Releases[0]; + var invalid = model with + { + Releases = [release with { Entries = [release.Entries[0] with { Prs = ["12345"] }] }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*full canonical URLs*https://github.com/*pull*'12345'*"); + } + + [Fact] + public void Serialize_SetOverrideWithoutValue_IsRejected() + { + var invalid = new OverridesDocument + { + Overrides = + [ + new BackfillOverride + { + Id = "broken", + Scope = new OverrideScope { Product = "elasticsearch" }, + Path = "release_date", + Operation = OverrideOperation.Set, + Reason = "Testing" + } + ] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*overrides[0]*sets a value needs the value*"); + } + + [Fact] + public void Serialize_DuplicateOverrideIds_AreRejected() + { + var overrides = BackfillFixtures.Overrides(); + var invalid = overrides with + { + Overrides = [overrides.Overrides[0], overrides.Overrides[1] with { Id = overrides.Overrides[0].Id }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*used by more than one override*"); + } + + [Fact] + public void Serialize_CreateActionWithoutContentHash_IsRejected() + { + var plan = BackfillFixtures.Plan(); + var invalid = plan with + { + Actions = [plan.Actions[0] with { ContentSha256 = null }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*actions[0]*create-bundle*hash of the content*"); + } + + [Fact] + public void Serialize_AmendWithoutParentKey_IsRejected() + { + var plan = BackfillFixtures.Plan(); + var amend = plan.Actions.First(a => a.Kind == PlanActionKind.CreateAmend); + var invalid = plan with { Actions = [amend with { ParentKey = null }] }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*needs the key of the parent bundle*"); + } + + [Fact] + public void Serialize_MalformedPlanInputHash_IsRejected() + { + var invalid = BackfillFixtures.Plan() with { InventoryHash = "sha256:nope" }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*inventory hash*sha256:*64 lower-case hex*"); + } + + [Fact] + public void Serialize_AllowlistWithNeitherHashNorCommit_IsRejected() + { + var invalid = BackfillFixtures.Plan() with { ScrubberAllowlist = new ScrubberAllowlist() }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*scrubber_allowlist*content hash*deployment commit*"); + } + + [Fact] + public void Serialize_TruncatedCommitSha_IsRejected() + { + var plan = BackfillFixtures.Plan(); + var invalid = plan with + { + SourceRefs = [plan.SourceRefs[0] with { Commit = "0123456" }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*source_refs[0]*full 40-character commit SHA*"); + } + + [Fact] + public void Serialize_NonUtcLedgerTimestamp_IsRejected() + { + var invalid = BackfillFixtures.Ledger() with + { + StartedAt = new DateTimeOffset(2026, 7, 20, 9, 0, 0, TimeSpan.FromHours(-3)) + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*must be in UTC*"); + } + + [Fact] + public void Serialize_LedgerFinishingBeforeItStarted_IsRejected() + { + var ledger = BackfillFixtures.Ledger(); + var invalid = ledger with { FinishedAt = ledger.StartedAt.AddMinutes(-1) }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*cannot finish before it started*"); + } + + [Fact] + public void Serialize_FailedLedgerActionWithoutDetail_IsRejected() + { + var ledger = BackfillFixtures.Ledger(); + var invalid = ledger with + { + Actions = + [ + new LedgerAction + { + PlannedKind = PlanActionKind.CreateBundle, + Key = "bundle/elasticsearch/elasticsearch-9.0.0.yaml", + Outcome = LedgerActionOutcome.Failed + } + ] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*failed ledger action needs a detail*"); + } + + [Fact] + public void Serialize_InventorySourceWithoutProducts_IsRejected() + { + var inventory = BackfillFixtures.Inventory(); + var invalid = inventory with + { + Sources = [inventory.Sources[0] with { ProductIds = [] }] + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + act.Should().Throw() + .WithMessage("*sources[0]*at least one product ID*"); + } + + [Fact] + public void Serialize_ReportsAllProblemsAtOnce() + { + var plan = BackfillFixtures.Plan(); + var invalid = plan with + { + InventoryHash = "bad", + SemanticModelHash = "also-bad" + }; + + var act = () => BackfillDocuments.Serialize(invalid); + + // Both problems in one error, so a human fixes everything in one pass. + act.Should().Throw() + .WithMessage("*inventory hash*") + .WithMessage("*semantic-model hash*"); + } + + [Fact] + public void Validate_PullRequestIdentityWithFileBlock_IsRejected() + { + var identity = EntryIdentity.ForPullRequest("elastic", "elasticsearch", 1) with + { + File = new SyntheticFileIdentity { Name = "x.yaml", Checksum = "abc" } + }; + + var problems = new List(); + identity.Validate(problems); + + problems.Should().ContainSingle().Which.Should().Contain("must not carry a file block"); + } + + [Fact] + public void Validate_IdentityFactories_ProduceValidIdentities() + { + var identities = new[] + { + EntryIdentity.ForPullRequest("elastic", "elasticsearch", 12345), + EntryIdentity.ForIssue("elastic", "apm-agent-dotnet", 42), + EntryIdentity.ForFile("backfill-elasticsearch-9.0.0-0001.yaml", "deadbeef") + }; + + foreach (var identity in identities) + { + var problems = new List(); + identity.Validate(problems); + problems.Should().BeEmpty(); + } + + identities[0].Url.Should().Be("https://github.com/elastic/elasticsearch/pull/12345"); + identities[1].Url.Should().Be("https://github.com/elastic/apm-agent-dotnet/issues/42"); + } + + [Fact] + public void Validate_SourceLocationWithBackwardsRange_IsRejected() + { + var location = new SourceLocation { Path = "docs/index.md", StartLine = 10, EndLine = 5 }; + + var problems = new List(); + location.Validate(problems); + + problems.Should().ContainSingle().Which.Should().Contain("cannot come before its start line"); + } + + [Fact] + public void Fixtures_AreAllValid() + { + var problems = new List(); + BackfillFixtures.Inventory().Validate(problems); + BackfillFixtures.Overrides().Validate(problems); + BackfillFixtures.SemanticModel().Validate(problems); + BackfillFixtures.Plan().Validate(problems); + BackfillFixtures.Provenance().Validate(problems); + BackfillFixtures.Ledger().Validate(problems); + + problems.Should().BeEmpty(); + } +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/BackfillVersionTests.cs b/tests/Elastic.Changelog.Tests/Backfill/BackfillVersionTests.cs new file mode 100644 index 000000000..3a43760e9 --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/BackfillVersionTests.cs @@ -0,0 +1,117 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +/// +/// A document a reader does not fully understand must fail loudly and helpfully — +/// wrong schema version, wrong document kind, or a missing envelope header. +/// +public class BackfillVersionTests +{ + [Fact] + public void Deserialize_NewerSchemaVersion_FailsWithActionableError() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Inventory()) + .Replace("\"schema_version\": 1", "\"schema_version\": 2", StringComparison.Ordinal); + + var act = () => BackfillDocuments.Deserialize(json); + + act.Should().Throw() + .WithMessage("*written with schema version 2*only understands version 1*") + .WithMessage("*Regenerate the document*"); + } + + [Fact] + public void Deserialize_OlderSchemaVersion_AlsoFails() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Plan()) + .Replace("\"schema_version\": 1", "\"schema_version\": 0", StringComparison.Ordinal); + + var act = () => BackfillDocuments.Deserialize(json); + + act.Should().Throw() + .WithMessage("*schema version 0*"); + } + + [Fact] + public void Deserialize_WrongDocumentKind_SaysWhatItFoundAndWhatWasRequested() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Plan()); + + var act = () => BackfillDocuments.Deserialize(json); + + act.Should().Throw() + .WithMessage("*contains a 'plan' document*'ledger' document was requested*"); + } + + [Fact] + public void Deserialize_UnknownArtifactName_ListsTheValidKinds() + { + var json = BackfillDocuments.Serialize(BackfillFixtures.Inventory()) + .Replace("\"artifact\": \"inventory\"", "\"artifact\": \"census\"", StringComparison.Ordinal); + + var act = () => BackfillDocuments.Deserialize(json); + + act.Should().Throw() + .WithMessage("*Unknown document kind 'census'*inventory, overrides, semantic-model, plan, provenance, ledger*"); + } + + [Fact] + public void Deserialize_MissingArtifactField_FailsExplicitly() + { + var act = () => BackfillDocuments.Deserialize(/*lang=json,strict*/ """{"schema_version":1,"payload":{"sources":[]}}"""); + + act.Should().Throw() + .WithMessage("*'artifact' field*missing*"); + } + + [Fact] + public void Deserialize_MissingSchemaVersionField_FailsExplicitly() + { + var act = () => BackfillDocuments.Deserialize(/*lang=json,strict*/ """{"artifact":"inventory","payload":{"sources":[]}}"""); + + act.Should().Throw() + .WithMessage("*missing the 'schema_version' field*"); + } + + [Fact] + public void Deserialize_NonObjectDocument_FailsExplicitly() + { + var act = () => BackfillDocuments.Deserialize("[1,2,3]"); + + act.Should().Throw() + .WithMessage("*top level must be a JSON object*"); + } + + [Fact] + public void Deserialize_InvalidJson_FailsExplicitly() + { + var act = () => BackfillDocuments.Deserialize("{ this is not json"); + + act.Should().Throw() + .WithMessage("*not valid JSON*"); + } + + [Fact] + public void Deserialize_OversizedDocument_IsRefused() + { + var oversized = new string(' ', BackfillDocuments.MaxDocumentCharacters + 1); + + var act = () => BackfillDocuments.Deserialize(oversized); + + act.Should().Throw() + .WithMessage("*safety limit*"); + } + + [Fact] + public void SchemaVersions_CoverEveryFamily() + { + foreach (var kind in Enum.GetValues()) + BackfillSchemaVersions.Current(kind).Should().BePositive(); + } +} diff --git a/tests/Elastic.Changelog.Tests/Backfill/CanonicalJsonTests.cs b/tests/Elastic.Changelog.Tests/Backfill/CanonicalJsonTests.cs new file mode 100644 index 000000000..f022b1acf --- /dev/null +++ b/tests/Elastic.Changelog.Tests/Backfill/CanonicalJsonTests.cs @@ -0,0 +1,105 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Changelog.Backfill; + +namespace Elastic.Changelog.Tests.Backfill; + +public class CanonicalJsonTests +{ + [Fact] + public void Canonicalize_SortsObjectKeysByOrdinal() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"b":1,"a":2,"Z":3}"""); + + // Ordinal order puts upper-case 'Z' before lower-case letters. + canonical.Should().Be(/*lang=json,strict*/ """{"Z":3,"a":2,"b":1}"""); + } + + [Fact] + public void Canonicalize_RemovesInsignificantWhitespace() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ "{\n \"a\" : [ 1 , 2 ]\n}"); + + canonical.Should().Be(/*lang=json,strict*/ """{"a":[1,2]}"""); + } + + [Fact] + public void Canonicalize_DropsNullObjectProperties() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"a":1,"b":null}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"a":1}"""); + } + + [Fact] + public void Canonicalize_KeepsNullArrayItems() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"a":[1,null,2]}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"a":[1,null,2]}"""); + } + + [Fact] + public void Canonicalize_PreservesArrayOrder() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"a":[3,1,2]}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"a":[3,1,2]}"""); + } + + [Fact] + public void Canonicalize_NormalizesLineEndingsInsideStrings() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"text":"line one\r\nline two\rline three"}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"text":"line one\nline two\nline three"}"""); + } + + [Fact] + public void Canonicalize_SortsNestedObjectsToo() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"outer":{"b":1,"a":{"d":4,"c":3}}}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"outer":{"a":{"c":3,"d":4},"b":1}}"""); + } + + [Fact] + public void Canonicalize_KeepsNumberTextVerbatim() + { + var canonical = CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"a":10,"b":0}"""); + + canonical.Should().Be(/*lang=json,strict*/ """{"a":10,"b":0}"""); + } + + [Fact] + public void Canonicalize_DuplicateKeys_FailsWithClearError() + { + var act = () => CanonicalJson.Canonicalize(/*lang=json,strict*/ """{"a":1,"a":2}"""); + + act.Should().Throw() + .WithMessage("*'a' appears more than once*"); + } + + [Fact] + public void Canonicalize_InvalidJson_FailsWithClearError() + { + var act = () => CanonicalJson.Canonicalize("not json at all"); + + act.Should().Throw() + .WithMessage("*not valid JSON*"); + } + + [Fact] + public void Canonicalize_IsIdempotent() + { + const string input = /*lang=json,strict*/ """{"b":{"y":2,"x":1},"a":[true,false,null]}"""; + + var once = CanonicalJson.Canonicalize(input); + var twice = CanonicalJson.Canonicalize(once); + + twice.Should().Be(once); + } +}