Add versioned v1 backfill artifact contracts

src/services/Elastic.Changelog/Backfill/BackfillArtifactKind.cs

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Changelog.Backfill;
+
+/// <summary>
+/// The kinds of document the backfill pipeline passes between its stages.
+/// Each persisted file contains exactly one of these, named in its envelope
+/// (e.g. <c>"artifact": "semantic-model"</c>) so a reader knows what it is
+/// looking at before parsing the payload. See the README in this folder for
+/// a tour of who writes and reads each one.
+/// </summary>
+public enum BackfillArtifactKind
+{
+	/// <summary>The census: which products and release-note sources exist, where they came from, and what we decided about each.</summary>
+	Inventory,
+
+	/// <summary>Manual corrections an operator feeds into planning, each with a reason attached.</summary>
+	Overrides,
+
+	/// <summary>The release notes reduced to their meaning, with formatting stripped away.</summary>
+	SemanticModel,
+
+	/// <summary>Exactly what we intend to create in S3, pinned to all of its inputs.</summary>
+	Plan,
+
+	/// <summary>The evidence trail: why we believe each recovered fact about an entry or release.</summary>
+	Provenance,
+
+	/// <summary>What actually happened when a plan was applied: every attempted step and its outcome.</summary>
+	Ledger
+}
+
+/// <summary>
+/// Converts between <see cref="BackfillArtifactKind"/> values and the names used in
+/// files (<c>inventory</c>, <c>overrides</c>, <c>semantic-model</c>, <c>plan</c>,
+/// <c>provenance</c>, <c>ledger</c>). Kept as explicit code, not reflection, so the
+/// on-disk names can never drift by accident and the library stays AOT-friendly.
+/// </summary>
+public static class BackfillArtifactKinds
+{
+	/// <summary>The name written to files for <paramref name="kind"/>, e.g. <c>semantic-model</c>.</summary>
+	public static string Name(BackfillArtifactKind kind) => kind switch
+	{
+		BackfillArtifactKind.Inventory => "inventory",
+		BackfillArtifactKind.Overrides => "overrides",
+		BackfillArtifactKind.SemanticModel => "semantic-model",
+		BackfillArtifactKind.Plan => "plan",
+		BackfillArtifactKind.Provenance => "provenance",
+		BackfillArtifactKind.Ledger => "ledger",
+		_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unknown backfill artifact kind")
+	};
+
+	/// <summary>
+	/// Looks up the kind for a name read from a file. Returns false for anything unknown.
+	/// Case-sensitive on purpose: these files are written by tools, so a case mismatch
+	/// means something is wrong and should surface rather than be papered over.
+	/// </summary>
+	public static bool TryParse(string? name, out BackfillArtifactKind kind)
+	{
+		switch (name)
+		{
+			case "inventory":
+				kind = BackfillArtifactKind.Inventory;
+				return true;
+			case "overrides":
+				kind = BackfillArtifactKind.Overrides;
+				return true;
+			case "semantic-model":
+				kind = BackfillArtifactKind.SemanticModel;
+				return true;
+			case "plan":
+				kind = BackfillArtifactKind.Plan;
+				return true;
+			case "provenance":
+				kind = BackfillArtifactKind.Provenance;
+				return true;
+			case "ledger":
+				kind = BackfillArtifactKind.Ledger;
+				return true;
+			default:
+				kind = default;
+				return false;
+		}
+	}
+}

src/services/Elastic.Changelog/Backfill/BackfillDocumentException.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Changelog.Backfill;
+
+/// <summary>
+/// Thrown when a backfill document cannot be safely read or written: the file is not
+/// valid JSON, it is not the kind of document the caller asked for, it was written
+/// with a schema version this code does not support, or a required field is missing
+/// or invalid. The message always says what is wrong in plain terms and, where
+/// possible, what to do about it — callers should surface it rather than swallow it,
+/// because a half-understood document must never flow further down the pipeline.
+/// </summary>
+public sealed class BackfillDocumentException : Exception
+{
+	/// <summary>Creates the exception with a plain-English description of the problem.</summary>
+	public BackfillDocumentException(string message) : base(message) { }
+
+	/// <summary>Creates the exception, keeping the lower-level parse error as the inner exception.</summary>
+	public BackfillDocumentException(string message, Exception innerException) : base(message, innerException) { }
+}

src/services/Elastic.Changelog/Backfill/BackfillDocuments.cs

@@ -0,0 +1,176 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.Json;
+using System.Text.Json.Serialization.Metadata;
+
+namespace Elastic.Changelog.Backfill;
+
+/// <summary>
+/// The one place to read, write, and hash backfill documents. Writing wraps the document
+/// in its envelope (artifact name + schema version) and validates it first; reading checks
+/// the envelope <em>before</em> parsing the payload and fails with a clear
+/// <see cref="BackfillDocumentException"/> on anything unexpected — a wrong document kind,
+/// an unsupported schema version, invalid JSON, or a missing/invalid field. There is no
+/// silent best-effort mode: a document either reads cleanly or not at all.
+/// </summary>
+public static class BackfillDocuments
+{
+	/// <summary>
+	/// Safety limit on document size (in characters). Backfill documents are review-sized
+	/// artifacts; anything this large is a bug or garbage input, and refusing it early
+	/// keeps a corrupt file from exhausting memory.
+	/// </summary>
+	public const int MaxDocumentCharacters = 64 * 1024 * 1024;
+
+	/// <summary>
+	/// Validates <paramref name="document"/>, wraps it in its envelope, and returns
+	/// indented JSON ready to persist. The indentation is for human review only — it has
+	/// no effect on the document's hash, which is computed over the canonical form.
+	/// </summary>
+	public static string Serialize<T>(T document) where T : class, IBackfillDocument
+	{
+		ArgumentNullException.ThrowIfNull(document);
+		ThrowIfInvalid(document);
+
+		var envelope = new BackfillEnvelope<T>
+		{
+			Artifact = BackfillArtifactKinds.Name(T.Kind),
+			SchemaVersion = BackfillSchemaVersions.Current(T.Kind),
+			Payload = document
+		};
+		return JsonSerializer.Serialize(envelope, EnvelopeTypeInfo<T>());
+	}
+
+	/// <summary>
+	/// Reads a document of type <typeparamref name="T"/> from <paramref name="json"/>.
+	/// Throws <see cref="BackfillDocumentException"/> — never returns a half-parsed
+	/// document — when the text is not valid JSON, contains a different kind of document,
+	/// was written with a schema version this code does not support, or fails validation.
+	/// </summary>
+	public static T Deserialize<T>(string json) where T : class, IBackfillDocument
+	{
+		ArgumentNullException.ThrowIfNull(json);
+		ThrowIfTooLarge(json);
+
+		var expectedName = BackfillArtifactKinds.Name(T.Kind);
+		CheckEnvelopeHeader(json, expectedName, BackfillSchemaVersions.Current(T.Kind));
+
+		BackfillEnvelope<T>? envelope;
+		try
+		{
+			envelope = JsonSerializer.Deserialize(json, EnvelopeTypeInfo<T>());
+		}
+		catch (JsonException e)
+		{
+			throw new BackfillDocumentException($"The '{expectedName}' document could not be parsed: {e.Message}", e);
+		}
+
+		if (envelope?.Payload is not { } document)
+			throw new BackfillDocumentException($"The '{expectedName}' document has no payload.");
+
+		ThrowIfInvalid(document);
+		return document;
+	}
+
+	/// <summary>
+	/// The document's stable identity: SHA-256 over the canonical form of its envelope
+	/// (see <see cref="CanonicalJson"/>), as <c>sha256:</c> + 64 hex characters. The same
+	/// content always produces the same hash, regardless of formatting or the order
+	/// fields were assembled in — this is what makes plans content-addressed.
+	/// </summary>
+	public static string ComputeHash<T>(T document) where T : class, IBackfillDocument =>
+		BackfillHash.Compute(CanonicalJson.Canonicalize(Serialize(document)));
+
+	/// <summary>
+	/// Hashes an already-serialized document exactly as <see cref="ComputeHash{T}(T)"/>
+	/// would. Useful for hashing a file as it sits on disk without knowing its type.
+	/// </summary>
+	public static string ComputeHash(string json)
+	{
+		ArgumentNullException.ThrowIfNull(json);
+		ThrowIfTooLarge(json);
+		return BackfillHash.Compute(CanonicalJson.Canonicalize(json));
+	}
+
+	private static void ThrowIfTooLarge(string json)
+	{
+		if (json.Length > MaxDocumentCharacters)
+			throw new BackfillDocumentException(
+				$"The document is {json.Length} characters long, over the {MaxDocumentCharacters} character safety limit for backfill documents.");
+	}
+
+	private static void ThrowIfInvalid<T>(T document) where T : class, IBackfillDocument
+	{
+		var problems = new List<string>();
+		document.Validate(problems);
+		if (problems.Count == 0)
+			return;
+
+		var name = BackfillArtifactKinds.Name(T.Kind);
+		throw new BackfillDocumentException(
+			$"The '{name}' document is invalid:\n - {string.Join("\n - ", problems)}");
+	}
+
+	/// <summary>
+	/// Checks the two header fields before any payload parsing, so the errors for "wrong
+	/// file" and "wrong version" are specific instead of a generic parse failure.
+	/// </summary>
+	private static void CheckEnvelopeHeader(string json, string expectedName, int expectedVersion)
+	{
+		JsonDocument parsed;
+		try
+		{
+			parsed = JsonDocument.Parse(json);
+		}
+		catch (JsonException e)
+		{
+			throw new BackfillDocumentException($"The document is not valid JSON: {e.Message}", e);
+		}
+
+		using (parsed)
+		{
+			var root = parsed.RootElement;
+			if (root.ValueKind != JsonValueKind.Object)
+				throw new BackfillDocumentException(
+					"This is not a backfill document: the top level must be a JSON object with 'artifact' and 'schema_version' fields.");
+
+			CheckArtifactField(root, expectedName);
+			CheckSchemaVersionField(root, expectedName, expectedVersion);
+		}
+	}
+
+	private static void CheckArtifactField(JsonElement root, string expectedName)
+	{
+		if (!root.TryGetProperty("artifact", out var artifact) || artifact.ValueKind != JsonValueKind.String)
+			throw new BackfillDocumentException(
+				"This is not a backfill document: the 'artifact' field naming the document kind is missing.");
+
+		var name = artifact.GetString() ?? "";
+		if (!BackfillArtifactKinds.TryParse(name, out _))
+			throw new BackfillDocumentException(
+				$"Unknown document kind '{name}'. Expected one of: inventory, overrides, semantic-model, plan, provenance, ledger.");
+
+		if (!string.Equals(name, expectedName, StringComparison.Ordinal))
+			throw new BackfillDocumentException(
+				$"This file contains a '{name}' document, but a '{expectedName}' document was requested. Check that the right file is being read.");
+	}
+
+	private static void CheckSchemaVersionField(JsonElement root, string expectedName, int expectedVersion)
+	{
+		if (!root.TryGetProperty("schema_version", out var version) || version.ValueKind != JsonValueKind.Number || !version.TryGetInt32(out var value))
+			throw new BackfillDocumentException(
+				$"This '{expectedName}' document is missing the 'schema_version' field, so it cannot be read safely.");
+
+		if (value != expectedVersion)
+			throw new BackfillDocumentException(
+				$"This '{expectedName}' document was written with schema version {value}, but this code only understands version {expectedVersion}. " +
+				"Regenerate the document with matching tooling, or upgrade to a version of this code that understands it.");
+	}
+
+	private static JsonTypeInfo<BackfillEnvelope<T>> EnvelopeTypeInfo<T>() where T : IBackfillDocument =>
+		BackfillJsonContext.Default.GetTypeInfo(typeof(BackfillEnvelope<T>)) as JsonTypeInfo<BackfillEnvelope<T>>
+		?? throw new InvalidOperationException(
+			$"BackfillEnvelope<{typeof(T).Name}> is not registered on BackfillJsonContext; add a [JsonSerializable] attribute for it.");
+}

src/services/Elastic.Changelog/Backfill/BackfillEnvelope.cs

@@ -0,0 +1,42 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Changelog.Backfill;
+
+/// <summary>
+/// Implemented by the root type of each backfill document family (inventory, overrides,
+/// semantic model, plan, provenance, ledger). Ties the type to its artifact name so the
+/// reading and writing helpers in <see cref="BackfillDocuments"/> can check "is this file
+/// really the kind of document you asked for?" before parsing the payload.
+/// </summary>
+public interface IBackfillDocument
+{
+	/// <summary>Which of the six document families this type is the root of.</summary>
+	static abstract BackfillArtifactKind Kind { get; }
+
+	/// <summary>
+	/// Adds a plain-English description of every problem found in this document to
+	/// <paramref name="problems"/>. An empty list afterwards means the document is valid.
+	/// Run automatically on every read and write.
+	/// </summary>
+	void Validate(IList<string> problems);
+}
+
+/// <summary>
+/// The small header wrapper around every persisted backfill document. It records what
+/// kind of document the file contains (<see cref="Artifact"/>) and which schema version
+/// wrote it (<see cref="SchemaVersion"/>), so a reader can fail fast — with a clear
+/// error — on files it does not understand, instead of guessing at a payload shape.
+/// </summary>
+public sealed record BackfillEnvelope<T>
+{
+	/// <summary>The document family name, e.g. <c>inventory</c> or <c>semantic-model</c>. See <see cref="BackfillArtifactKinds"/>.</summary>
+	public required string Artifact { get; init; }
+
+	/// <summary>The schema version of <see cref="Payload"/> at the time of writing. Compared against <see cref="BackfillSchemaVersions"/> on read.</summary>
+	public required int SchemaVersion { get; init; }
+
+	/// <summary>The document itself.</summary>
+	public required T Payload { get; init; }
+}

src/services/Elastic.Changelog/Backfill/BackfillHash.cs

@@ -0,0 +1,54 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Elastic.Changelog.Backfill;
+
+/// <summary>
+/// Computes and checks the hashes that give backfill documents a stable identity.
+/// A hash is SHA-256 over the UTF-8 bytes of a document's canonical form (see
+/// <see cref="CanonicalJson"/>), written as <c>sha256:</c> followed by 64 lower-case
+/// hex characters. Because the canonical form is stable, the same content always
+/// produces the same hash — which is what lets a plan be pinned to the exact inputs
+/// it was computed from.
+/// </summary>
+public static class BackfillHash
+{
+	/// <summary>Every hash value starts with this, so a reader can tell the algorithm at a glance.</summary>
+	public const string Prefix = "sha256:";
+
+	private const int HexLength = 64;
+
+	/// <summary>
+	/// Hashes <paramref name="canonicalText"/> and returns e.g.
+	/// <c>sha256:9f86d08…</c>. The caller is responsible for passing canonical text;
+	/// to hash a document, prefer <see cref="BackfillDocuments.ComputeHash{T}(T)"/>,
+	/// which canonicalizes first.
+	/// </summary>
+	public static string Compute(string canonicalText)
+	{
+		ArgumentNullException.ThrowIfNull(canonicalText);
+		var digest = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalText));
+		return Prefix + Convert.ToHexStringLower(digest);
+	}
+
+	/// <summary>True when <paramref name="value"/> is a well-formed hash: <c>sha256:</c> plus 64 lower-case hex characters.</summary>
+	public static bool IsWellFormed(string? value)
+	{
+		if (value is null || value.Length != Prefix.Length + HexLength)
+			return false;
+		if (!value.StartsWith(Prefix, StringComparison.Ordinal))
+			return false;
+
+		for (var i = Prefix.Length; i < value.Length; i++)
+		{
+			var c = value[i];
+			if (c is (< '0' or > '9') and (< 'a' or > 'f'))
+				return false;
+		}
+		return true;
+	}
+}