From 2403a36d7e96d24d284023b74546c09b811f4621 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 12:57:13 +0000 Subject: [PATCH 1/9] feat: add curation desk gateway routes Five public GETs and eight signed POSTs under /private-api/curation-desk/, proxied to curation/desk/ upstream. Every desk call carries a shared secret header and the routes answer 503 while it is unconfigured, so the backend never sees an unauthenticated read or write through this service. Public reads whitelist, clamp and order their query so equivalent requests share one memo entry and one cache key, are memoized as bytes for the s-maxage of their Cache-Control (single-flight per key, last-good fallback on an upstream error) and have curator-identity keys stripped before they are stored. Writes resolve the caller from the signed code, memoize a successful validation briefly, and forward only whitelisted body fields under the validated username. BytesCache entries gain an optional tag so a memoized body keeps its content type; UpstreamResponse keeps the raw body bytes for the same reason. --- dotnet/EcencyApi/Config.cs | 10 + .../Handlers/PrivateApi.CurationDesk.cs | 846 ++++++++++++++++++ dotnet/EcencyApi/Handlers/Routes.cs | 17 + dotnet/EcencyApi/Infrastructure/BytesCache.cs | 21 +- .../EcencyApi/Infrastructure/CachePolicy.cs | 38 + dotnet/EcencyApi/Infrastructure/Upstream.cs | 12 +- 6 files changed, 938 insertions(+), 6 deletions(-) create mode 100644 dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs diff --git a/dotnet/EcencyApi/Config.cs b/dotnet/EcencyApi/Config.cs index de3eddaf..5b09b5ad 100644 --- a/dotnet/EcencyApi/Config.cs +++ b/dotnet/EcencyApi/Config.cs @@ -21,6 +21,16 @@ public static class Config public static string EnotifyInternalToken { get; } = Env("ENOTIFY_INTERNAL_TOKEN") ?? ""; + /// + /// Shared secret presented to the curation desk backend on every desk call, + /// public reads included: the desk answers only requests that carry it, so + /// the memo and rate limits in front of it cannot be bypassed by going + /// around this service. No default: when unset the desk routes fail closed + /// (503) rather than forward an empty secret. + /// + public static string DeskInternalToken { get; } = + Env("DESK_INTERNAL_TOKEN") ?? ""; + public static string HsClientSecret { get; } = Env("HIVESIGNER_SECRET") ?? "hivesignerclientsecret"; diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs new file mode 100644 index 00000000..b29b5f80 --- /dev/null +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -0,0 +1,846 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using EcencyApi.Infrastructure; + +namespace EcencyApi.Handlers; + +/// +/// Curation desk gateway: /private-api/curation-desk/* -> curation/desk/* upstream. +/// +/// Five public reads and eight signed writes. This service does three things +/// for the desk that the generic pipe handlers do not: +/// +/// - every upstream call carries a shared secret header, reads included. The +/// desk backend answers nothing without it, so the memo, the cache policies +/// and the rate limits in front of this service cannot be skipped by calling +/// the backend directly. When the secret is not configured the routes fail +/// closed (503), the same way the payment routes do; +/// - public reads are whitelisted, clamped and emitted in one fixed order, so +/// every spelling of the same question collapses onto one memo entry and one +/// shared-cache key, and the answer is memoized as bytes for exactly the +/// s-maxage the response promises (single-flight per key, last-good on an +/// upstream error); +/// - writes resolve the caller from the signed code (memoized briefly, see +/// ) and forward only whitelisted +/// body fields under that username; a client-supplied username or code never +/// reaches the backend. +/// +public static partial class PrivateApi +{ + // ---- configuration seams ------------------------------------------------- + + /// Header carrying the shared secret to the desk backend. + internal const string DeskTokenHeader = "X-Desk-Internal-Token"; + + /// + /// The configured secret, or null when the desk is switched off. A static + /// field rather than a Config read so tests can flip it; production reads it + /// once at startup like every other setting. + /// + internal static string? DeskToken = string.IsNullOrWhiteSpace(Config.DeskInternalToken) + ? null + : Config.DeskInternalToken.Trim(); + + /// + /// The one upstream call every desk route goes through. Replaceable so tests + /// can observe the request (path, method, headers, payload) without a network. + /// + internal static Func>, JsonNode?, Task> + DeskUpstream = (endpoint, method, headers, payload) => ApiClient.ApiRequest(endpoint, method, headers, payload); + + /// Signed-code validation, replaceable for tests (no chain RPC). + internal static Func> DeskValidateCode = ValidateCode; + + /// + /// How long a successful code validation is remembered. A validation costs + /// one uncached account lookup per call, and a curator on the desk sends a + /// write every few seconds; 90 s keeps that to one lookup per curator per + /// window while a posting-key rotation lags by at most this much. Failures + /// are never remembered. + /// + internal static double DeskAuthMemoSeconds = 90; + + private const string DeskAuthMemoPrefix = "desk-auth:"; + + private const string DeskNotConfigured = "curation desk not configured"; + + // ---- public reads -------------------------------------------------------- + + // GET /private-api/curation-desk/feed + public static Task CurationDeskFeed(HttpContext ctx) => + ServeDeskRead(ctx, + CurationDeskQuery.Endpoint("curation/desk/feed", CurationDeskQuery.NormalizeFeed(RawQuery(ctx))), + CachePolicy.CurationDeskFeed); + + // GET /private-api/curation-desk/status + public static Task CurationDeskStatus(HttpContext ctx) => + ServeDeskRead(ctx, "curation/desk/status", CachePolicy.CurationDeskStatus); + + // GET /private-api/curation-desk/roster + public static Task CurationDeskRoster(HttpContext ctx) => + ServeDeskRead(ctx, "curation/desk/roster", CachePolicy.CurationDeskRoster); + + // GET /private-api/curation-desk/recommendations + public static Task CurationDeskRecommendations(HttpContext ctx) => + ServeDeskRead(ctx, + CurationDeskQuery.Endpoint("curation/desk/recommendations", + CurationDeskQuery.NormalizeRecommendations(RawQuery(ctx))), + CachePolicy.CurationDeskRecommendations); + + // GET /private-api/curation-desk/post/{author}/{permlink} + public static async Task CurationDeskPost(HttpContext ctx) + { + var author = ctx.Request.RouteValues["author"]?.ToString() ?? ""; + var permlink = ctx.Request.RouteValues["permlink"]?.ToString() ?? ""; + var path = CurationDeskPostPath(author, permlink); + if (path == null) + { + await ctx.SendText(400, "Invalid author or permlink"); + return; + } + await ServeDeskRead(ctx, path, CachePolicy.CurationDeskPost); + } + + private static readonly Regex DeskAuthorPattern = new("^[a-z0-9.-]{3,16}$", RegexOptions.Compiled); + private static readonly Regex DeskPermlinkPattern = new("^[a-z0-9-]{1,255}$", RegexOptions.Compiled); + + /// + /// Upstream path for a single post, or null when either value is not a plain + /// Hive name or permlink. Same reasoning as : route + /// values arrive percent-decoded, and a `/`, `?` or `#` left in place would be + /// re-parsed as URL structure once the string becomes a Uri, with the desk + /// secret attached to wherever it then points. The character classes already + /// exclude every structural character; the escaping stays as a second fence + /// and the dot-segment check as the one case escaping cannot fix. + /// + public static string? CurationDeskPostPath(string author, string permlink) + { + if (author is "." or ".." || permlink is "." or "..") + { + return null; + } + if (!DeskAuthorPattern.IsMatch(author) || !DeskPermlinkPattern.IsMatch(permlink)) + { + return null; + } + return $"curation/desk/post/{Uri.EscapeDataString(author)}/{Uri.EscapeDataString(permlink)}"; + } + + private static IEnumerable> RawQuery(HttpContext ctx) + { + foreach (var kv in ctx.Request.Query) + { + // A repeated key takes its first value, like Express `req.query` read + // as a scalar; the normalizer sees each key once. + var first = kv.Value.Count > 0 ? kv.Value[0] : null; + if (first != null) + { + yield return new KeyValuePair(kv.Key, first); + } + } + } + + /// + /// Serve one public desk read: memo hit, or a single-flight fill of the + /// normalized endpoint. Cache-Control is attached for a 200 only. + /// + private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string policy) + { + var token = DeskToken; + if (token == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + + ctx.CacheWhenOk(policy); + + if (CurationDeskMemo.TryGetFresh(endpoint, out var hit, out var hitType)) + { + await WriteBytes(ctx, 200, hitType, hit); + return; + } + + var gate = CurationDeskMemo.GateFor(endpoint); + if (!await gate.WaitAsync(CurationDeskMemo.FillWait)) + { + // Someone else's fill is taking longer than a whole upstream timeout. + // Do not stack another one behind it; answer from what is known. + await ServeLastGoodOr(ctx, endpoint, 504, "Upstream Timeout"); + return; + } + + try + { + // The fill that held the gate may have landed while this one queued. + if (CurationDeskMemo.TryGetFresh(endpoint, out hit, out hitType)) + { + await WriteBytes(ctx, 200, hitType, hit); + return; + } + + UpstreamResponse r; + try + { + r = await DeskUpstream(endpoint, HttpMethod.Get, DeskHeaders(token), null); + } + catch (UpstreamTimeoutException) + { + await ServeLastGoodOr(ctx, endpoint, 504, "Upstream Timeout"); + return; + } + catch (Exception) + { + await ServeLastGoodOr(ctx, endpoint, 500, "Server Error"); + return; + } + + if (r.Status == 200 && r.Json is JsonObject or JsonArray) + { + var bytes = CurationDeskPublicPayload.ToPublicBytes(r); + CurationDeskMemo.Store(endpoint, bytes, JsonContentType, CachePolicy.SharedMaxAge(policy)); + await WriteBytes(ctx, 200, JsonContentType, bytes); + return; + } + + if (r.Status >= 500) + { + // The backend is unwell; a body it last answered with is better + // than its error page, and the error is not worth memoizing. + if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + { + await WriteBytes(ctx, 200, staleType, stale); + return; + } + } + + // 4xx (an unknown post, a rejected token), a 200 that is not JSON, or + // a 5xx with nothing to fall back on: pass through unmemoized, the + // way Pipe would, so the client sees what the backend said. + await Upstream.SendLikeExpress(ctx, r.Status, r.Json, r.RawText); + } + finally + { + gate.Release(); + CurationDeskMemo.ReleaseGate(endpoint, gate); + } + } + + private const string JsonContentType = "application/json; charset=utf-8"; + + private static async Task ServeLastGoodOr(HttpContext ctx, string endpoint, int status, string text) + { + if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + { + await WriteBytes(ctx, 200, staleType, stale); + return; + } + await ctx.SendText(status, text); + } + + private static async Task WriteBytes(HttpContext ctx, int status, string contentType, byte[] bytes) + { + ctx.Response.StatusCode = status; + ctx.Response.ContentType = contentType; + await ctx.Response.Body.WriteAsync(bytes); + } + + private static List> DeskHeaders(string token, string? clientIp = null) + { + var headers = new List> { new(DeskTokenHeader, token) }; + if (clientIp != null) + { + headers.Add(new KeyValuePair("X-Real-IP-V", clientIp)); + } + return headers; + } + + // ---- signed writes ------------------------------------------------------- + + // POST /private-api/curation-desk/roster-feed + public static Task CurationDeskRosterFeed(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RosterFeed); + + // POST /private-api/curation-desk/tick + public static Task CurationDeskTick(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Tick); + + // POST /private-api/curation-desk/mark + public static Task CurationDeskMark(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Mark); + + // POST /private-api/curation-desk/mark-clear + public static Task CurationDeskMarkClear(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.MarkClear); + + // POST /private-api/curation-desk/marks + public static Task CurationDeskMarks(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Marks); + + // POST /private-api/curation-desk/cursor + public static Task CurationDeskCursor(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Cursor); + + // POST /private-api/curation-desk/recommend-meta + public static Task CurationDeskRecommendMeta(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RecommendMeta); + + // POST /private-api/curation-desk/recommendation-dismiss + public static Task CurationDeskRecommendationDismiss(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RecommendationDismiss); + + /// + /// Signed write: authenticate, fail closed when unconfigured, build the + /// whitelisted payload under the validated username, pipe. Never cacheable. + /// + private static async Task ServeDeskWrite(HttpContext ctx, CurationDeskWrites.Route route) + { + var body = await ctx.ReadBody(); + var username = await RequireAuthedUsernameCached(ctx, body); + if (username == null) + { + return; + } + + var token = DeskToken; + if (token == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + + var (payload, error) = CurationDeskWrites.Build(route, username, body); + if (payload == null) + { + await ctx.SendText(400, error ?? "Invalid request"); + return; + } + + // The client address rides along only where the backend uses it (the + // recommendation meta ping). Same source as the signup path: the + // proxy-set header, never a forwarded-for chain a client can extend. + var headers = DeskHeaders(token, route.ForwardClientAddress ? SignupClientIp(ctx) : null); + + ctx.Response.Headers.CacheControl = "no-store"; + await Upstream.Pipe(DeskUpstream(route.UpstreamPath, HttpMethod.Post, headers, payload), ctx); + } + + /// + /// with a short memo of successful + /// validations, keyed by the SHA-256 of the code. The validation itself is + /// unchanged and still decides every miss; only its positive answer is + /// remembered, for . A failed validation is + /// never stored, so a probe costs the same as before and a code that stops + /// validating is refused on its next miss. Desk routes only. + /// + public static async Task RequireAuthedUsernameCached(HttpContext ctx, JsonObject body) + { + var code = body.Str("code"); + var memoKey = string.IsNullOrEmpty(code) ? null : DeskAuthMemoPrefix + Sha256Hex(code); + + if (memoKey != null && MemCache.Get(memoKey) is { Length: > 0 } remembered) + { + return remembered; + } + + var username = await DeskValidateCode(body); + if (string.IsNullOrEmpty(username)) + { + await ctx.SendText(401, "Unauthorized"); + return null; + } + + if (memoKey != null) + { + MemCache.Set(memoKey, username, DeskAuthMemoSeconds); + } + return username; + } + + private static string Sha256Hex(string value) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); +} + +/// +/// Whitelist, clamp and order the query of the public desk reads. +/// +/// Every accepted parameter has a fixed position and a default that is dropped, +/// so `?limit=25&sort=newest&x=1` and an empty query are the same +/// upstream URL, the same memo entry and the same shared-cache key. Unknown +/// names and unusable values are dropped, never errors: a public read should +/// answer the nearest sensible question rather than 400 on a typo. +/// +public static class CurationDeskQuery +{ + public const int DefaultLimit = 25; + public const int MaxLimit = 50; + public const int MaxWords = 50000; + + private static readonly Regex CursorPattern = new("^[A-Za-z0-9_.:-]{1,80}$", RegexOptions.Compiled); + private static readonly Regex CommunityPattern = new(@"^hive-\d{5,6}$", RegexOptions.Compiled); + + public static readonly IReadOnlySet FeedSorts = new HashSet { "queue", "newest", "unique" }; + public static readonly IReadOnlySet RecommendationSorts = new HashSet { "unique", "newest" }; + public static readonly IReadOnlySet Views = + new HashSet { "queue", "latest", "new-authors", "recommended", "curated", "all" }; + public static readonly IReadOnlySet Apps = new HashSet { "all", "ecency", "peakd", "other" }; + public static readonly IReadOnlySet Windows = new HashSet { "full", "half", "eighth", "locked", "all" }; + + /// + /// Emission order of the feed parameters. Fixed so that the memo key and + /// the shared-cache key are stable regardless of how a client orders them. + /// + public static readonly string[] FeedOrder = + { + "cursor", "limit", "sort", "view", "app", "community", "window", "rep_min", "rep_max", + "min_words", "max_words", "has_images", "new_authors", "recommended", "hide_curated", + }; + + /// Route 1 (feed). + public static List> NormalizeFeed(IEnumerable> raw) + { + var q = First(raw); + var kept = new Dictionary(); + + if (q.TryGetValue("cursor", out var cursor) && CursorPattern.IsMatch(cursor)) + { + kept["cursor"] = cursor; + } + + var limit = ClampInt(q, "limit", 1, MaxLimit); + if (limit is { } l && l != DefaultLimit) + { + kept["limit"] = l.ToString(); + } + + // The public default is newest; an unknown sort (random and anything + // else the roster feed may accept) falls back to it and so disappears. + var sort = q.TryGetValue("sort", out var s) && FeedSorts.Contains(s) ? s : "newest"; + if (sort != "newest") + { + kept["sort"] = sort; + } + + if (q.TryGetValue("view", out var view) && Views.Contains(view)) + { + kept["view"] = view; + } + + if (q.TryGetValue("app", out var app) && Apps.Contains(app) && app != "all") + { + kept["app"] = app; + } + + if (q.TryGetValue("community", out var community) && CommunityPattern.IsMatch(community)) + { + kept["community"] = community; + } + + if (q.TryGetValue("window", out var window) && Windows.Contains(window) && window != "all") + { + kept["window"] = window; + } + + // Range floors at their minimum and ceilings at their maximum select + // everything, so they are the same question as leaving them out. + if (ClampInt(q, "rep_min", 0, 100) is { } repMin && repMin != 0) + { + kept["rep_min"] = repMin.ToString(); + } + if (ClampInt(q, "rep_max", 0, 100) is { } repMax && repMax != 100) + { + kept["rep_max"] = repMax.ToString(); + } + if (ClampInt(q, "min_words", 0, MaxWords) is { } minWords && minWords != 0) + { + kept["min_words"] = minWords.ToString(); + } + if (ClampInt(q, "max_words", 0, MaxWords) is { } maxWords && maxWords != MaxWords) + { + kept["max_words"] = maxWords.ToString(); + } + + if (Flag(q, "has_images") == true) kept["has_images"] = "1"; + if (Flag(q, "new_authors") == true) kept["new_authors"] = "1"; + + // sort=unique already means "recommended posts only", so the explicit + // flag adds nothing there and would only split the memo. + if (Flag(q, "recommended") == true && sort != "unique") kept["recommended"] = "1"; + + // hide_curated defaults to on; only switching it off says anything. + if (Flag(q, "hide_curated") == false) kept["hide_curated"] = "0"; + + return Ordered(kept, FeedOrder); + } + + /// Route 4 (recommendations): cursor, limit, sort in unique|newest. + public static List> NormalizeRecommendations(IEnumerable> raw) + { + var q = First(raw); + var kept = new Dictionary(); + + if (q.TryGetValue("cursor", out var cursor) && CursorPattern.IsMatch(cursor)) + { + kept["cursor"] = cursor; + } + + var limit = ClampInt(q, "limit", 1, MaxLimit); + if (limit is { } l && l != DefaultLimit) + { + kept["limit"] = l.ToString(); + } + + if (q.TryGetValue("sort", out var sort) && RecommendationSorts.Contains(sort)) + { + kept["sort"] = sort; + } + + return Ordered(kept, new[] { "cursor", "limit", "sort" }); + } + + /// The upstream endpoint with the normalized query appended. + public static string Endpoint(string path, IEnumerable> query) => + Upstream.AppendQuery(path, query); + + private static Dictionary First(IEnumerable> raw) + { + var q = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in raw) + { + q.TryAdd(key, value); + } + return q; + } + + private static List> Ordered(Dictionary kept, string[] order) + { + var list = new List>(kept.Count); + foreach (var name in order) + { + if (kept.TryGetValue(name, out var value)) + { + list.Add(new KeyValuePair(name, value)); + } + } + return list; + } + + /// Integer within [min, max], clamped; null when absent or not an integer. + private static int? ClampInt(Dictionary q, string key, int min, int max) + { + if (!q.TryGetValue(key, out var raw)) return null; + if (!int.TryParse(raw, System.Globalization.NumberStyles.AllowLeadingSign, + System.Globalization.CultureInfo.InvariantCulture, out var value)) + { + return null; + } + return Math.Clamp(value, min, max); + } + + /// "1" -> true, "0" -> false, anything else -> null (dropped). + private static bool? Flag(Dictionary q, string key) => + q.TryGetValue(key, out var raw) ? raw switch { "1" => true, "0" => false, _ => null } : null; +} + +/// +/// Payload rules for the signed desk writes: which body keys reach the backend +/// and the few values this service refuses outright rather than forward. +/// +public static class CurationDeskWrites +{ + public sealed record Route(string UpstreamPath, string[] Keys, bool ForwardClientAddress = false); + + public static readonly IReadOnlySet MarkStates = + new HashSet { "reviewed", "snoozed", "flagged", "noted" }; + public static readonly IReadOnlySet CursorActions = new HashSet { "advance", "rewind" }; + public static readonly IReadOnlySet DismissActions = new HashSet { "dismiss", "restore" }; + public static readonly IReadOnlySet UaClasses = new HashSet { "web", "mobile" }; + public static readonly IReadOnlySet RosterSorts = new HashSet { "queue", "newest", "unique", "random" }; + + private static readonly Regex TrxIdPattern = new("^[0-9a-f]{40}$", RegexOptions.Compiled); + + public static readonly Route RosterFeed = new("curation/desk/roster-feed", new[] + { + "cursor", "limit", "view", "app", "community", "min_words", "sort", "seed", "window", "rep_min", + "rep_max", "max_words", "has_images", "new_authors", "recommended", "flagged", "hide_curated", + "hide_reviewed", "hide_snoozed", + }); + + public static readonly Route Tick = new("curation/desk/tick", new[] { "since", "need", "visible" }); + + public static readonly Route Mark = new("curation/desk/marks", + new[] { "author", "permlink", "state", "reason", "note", "snooze_until" }); + + public static readonly Route MarkClear = new("curation/desk/marks/clear", new[] { "author", "permlink" }); + + public static readonly Route Marks = new("curation/desk/marks/list", new[] { "state", "cursor", "limit" }); + + public static readonly Route Cursor = new("curation/desk/cursors", new[] { "post_id", "action", "reason" }); + + public static readonly Route RecommendMeta = new("curation/desk/recommendations/meta", + new[] { "author", "permlink", "trx_id", "ua_class" }, ForwardClientAddress: true); + + public static readonly Route RecommendationDismiss = new("curation/desk/recommendations/dismiss", + new[] { "author", "permlink", "action" }); + + /// + /// The upstream body: the validated username plus the route's whitelisted + /// keys copied as the client sent them. `username` and `code` are never in + /// a whitelist, so a forged identity cannot ride along. Returns no payload + /// and a reason when a value the backend would only reject is caught here. + /// + public static (JsonObject? Payload, string? Error) Build(Route route, string username, JsonObject body) + { + var error = Validate(route, body); + if (error != null) + { + return (null, error); + } + + var payload = new JsonObject { ["username"] = username }; + foreach (var key in route.Keys) + { + if (key is "username" or "code") + { + continue; + } + CopyIfPresent(payload, body, key); + } + + if (ReferenceEquals(route, RosterFeed)) + { + // An unknown sort is not an error for a read: drop it and let the + // backend apply its default, as the public feed does. + var sort = body.Str("sort"); + if (sort == null || !RosterSorts.Contains(sort)) + { + payload.Remove("sort"); + } + // seed only means something to the random order; for any other sort + // it is noise that would make two identical feeds look different. + if (sort != "random") + { + payload.Remove("seed"); + } + if (payload.Remove("limit") && body.Field("limit") is JsonValue limitValue + && limitValue.TryGetValue(out var limit)) + { + payload["limit"] = Math.Clamp((int)limit, 1, CurationDeskQuery.MaxLimit); + } + } + + if (ReferenceEquals(route, RecommendMeta) && payload.ContainsKey("ua_class") + && !(body.Str("ua_class") is { } ua && UaClasses.Contains(ua))) + { + payload.Remove("ua_class"); + } + + return (payload, null); + } + + private static string? Validate(Route route, JsonObject body) + { + if (ReferenceEquals(route, Mark)) + { + return RequireAuthorPermlink(body) ?? RequireOneOf(body, "state", MarkStates); + } + if (ReferenceEquals(route, MarkClear)) + { + return RequireAuthorPermlink(body); + } + if (ReferenceEquals(route, Marks)) + { + return body.ContainsKey("state") ? RequireOneOf(body, "state", MarkStates) : null; + } + if (ReferenceEquals(route, Cursor)) + { + if (body.Field("post_id") is not JsonValue idValue + || idValue.GetValueKind() is not (JsonValueKind.Number or JsonValueKind.String)) + { + return "post_id required"; + } + return RequireOneOf(body, "action", CursorActions); + } + if (ReferenceEquals(route, RecommendMeta)) + { + var missing = RequireAuthorPermlink(body); + if (missing != null) return missing; + if (body.ContainsKey("trx_id")) + { + // Optional and informational, but a value that is not a + // transaction id is a client bug worth surfacing, not storing. + if (body.Str("trx_id") is not { } trx || !TrxIdPattern.IsMatch(trx)) + { + return "invalid trx_id"; + } + } + return null; + } + if (ReferenceEquals(route, RecommendationDismiss)) + { + return RequireAuthorPermlink(body) ?? RequireOneOf(body, "action", DismissActions); + } + return null; + } + + /// + /// Copy a body field only when the key is present (absent == undefined, which + /// JSON.stringify omits; a present null is kept), as the other passthroughs do. + /// + private static void CopyIfPresent(JsonObject target, JsonObject body, string key) + { + if (body.TryGetPropertyValue(key, out var value)) + { + target[key] = value?.DeepClone(); + } + } + + private static string? RequireAuthorPermlink(JsonObject body) => + RequireNonEmpty(body, "author") ?? RequireNonEmpty(body, "permlink"); + + private static string? RequireNonEmpty(JsonObject body, string key) => + body.Str(key) is { Length: > 0 } ? null : $"{key} required"; + + private static string? RequireOneOf(JsonObject body, string key, IReadOnlySet allowed) => + body.Str(key) is { } value && allowed.Contains(value) ? null : $"invalid {key}"; +} + +/// +/// What a public desk response may carry. The backend is specified to omit +/// these already; this is the fence on this side of the boundary, so a backend +/// change that starts leaking a curator's identity or a hashed address into a +/// publicly cached body is stopped here rather than served for its s-maxage. +/// +public static class CurationDeskPublicPayload +{ + public static readonly IReadOnlySet PrivateKeys = new HashSet(StringComparer.Ordinal) + { + "set_by", "set_at", "active_curators", "trail_alerts", "note", "excluded_reason", "ip_hash", "key_id", + }; + + /// + /// Remove every private key anywhere in the tree. Returns whether anything + /// was removed, so an untouched body can be served as the bytes it came in. + /// + public static bool Strip(JsonNode? node) + { + var removed = false; + switch (node) + { + case JsonObject obj: + foreach (var key in obj.Select(kv => kv.Key).ToArray()) + { + if (PrivateKeys.Contains(key)) + { + obj.Remove(key); + removed = true; + } + else + { + removed |= Strip(obj[key]); + } + } + break; + case JsonArray arr: + foreach (var item in arr) + { + removed |= Strip(item); + } + break; + } + return removed; + } + + /// + /// The body to memoize and serve: the upstream bytes as received when they + /// were already clean, otherwise the stripped tree re-serialized once. + /// + public static byte[] ToPublicBytes(UpstreamResponse r) + { + if (!Strip(r.Json)) + { + return r.Bytes.Length > 0 ? r.Bytes : Encoding.UTF8.GetBytes(JsJson.Stringify(r.Json)); + } + return Encoding.UTF8.GetBytes(JsJson.Stringify(r.Json)); + } +} + +/// +/// Byte memo for the public desk reads, keyed by the normalized upstream +/// endpoint. Two bounded stores: the fresh one holds a body for the s-maxage of +/// its route, the last-good one holds the most recent 200 for longer so an +/// upstream error answers with something recent rather than an error page. +/// Bytes, not trees: a hit is a lookup and a write, whatever the read rate. +/// +public static class CurationDeskMemo +{ + /// Budget of each store. A feed page is tens of KB; this is thousands of them. + internal const long BudgetBytes = 64L * 1024 * 1024; + + /// + /// How long a last-good body stays eligible as a fallback. Long enough to + /// ride out a backend restart, short enough that a stale feed does not + /// outlive an outage by much. + /// + internal const int LastGoodTtlMs = 10 * 60 * 1000; + + /// + /// How long a request waits for another request's fill of the same key. A + /// fill is one upstream call, so this only bounds the case where that call + /// is itself timing out; the waiter then answers from last-good or 504. + /// + internal static readonly TimeSpan FillWait = TimeSpan.FromMilliseconds(Upstream.DefaultTimeoutMs + 1000); + + internal static BytesCache Fresh = new(BudgetBytes); + internal static BytesCache LastGood = new(BudgetBytes); + + /// + /// One fill per key at a time. Gates are created on demand and dropped once + /// released with nobody holding them, so a scan over many distinct keys does + /// not leave a semaphore per key behind. A request that read a gate just + /// before it was dropped can start a second fill; that costs one duplicate + /// upstream call, not correctness. + /// + private static readonly ConcurrentDictionary Gates = new(); + + public static bool TryGetFresh(string key, out byte[] bytes, out string contentType) + { + var hit = Fresh.TryGet(key, out bytes, out var tag); + contentType = tag ?? "application/json; charset=utf-8"; + return hit; + } + + public static bool TryGetLastGood(string key, out byte[] bytes, out string contentType) + { + var hit = LastGood.TryGet(key, out bytes, out var tag); + contentType = tag ?? "application/json; charset=utf-8"; + return hit; + } + + public static void Store(string key, byte[] bytes, string contentType, int ttlSeconds) + { + Fresh.Set(key, bytes, ttlSeconds * 1000, contentType); + LastGood.Set(key, bytes, LastGoodTtlMs, contentType); + } + + internal static SemaphoreSlim GateFor(string key) => Gates.GetOrAdd(key, _ => new SemaphoreSlim(1, 1)); + + internal static void ReleaseGate(string key, SemaphoreSlim gate) + { + if (gate.CurrentCount == 1) + { + Gates.TryRemove(new KeyValuePair(key, gate)); + } + } + + internal static void ResetForTests() + { + Fresh = new BytesCache(BudgetBytes); + LastGood = new BytesCache(BudgetBytes); + Gates.Clear(); + } +} diff --git a/dotnet/EcencyApi/Handlers/Routes.cs b/dotnet/EcencyApi/Handlers/Routes.cs index 9348c0a7..33fb4683 100644 --- a/dotnet/EcencyApi/Handlers/Routes.cs +++ b/dotnet/EcencyApi/Handlers/Routes.cs @@ -173,6 +173,23 @@ public static void Map(WebApplication app) app.MapPost("/private-api/chats-update", PrivateApi.ChatsUpdate); app.MapPost("/private-api/channel-add", PrivateApi.ChannelAdd); + // ---- Curation desk (see PrivateApi.CurationDesk.cs) ---- + // Literal segments under curation-desk/: a different first segment from + // the curation/{duration} route above, so the two can never collide. + app.MapGet("/private-api/curation-desk/feed", PrivateApi.CurationDeskFeed); + app.MapGet("/private-api/curation-desk/status", PrivateApi.CurationDeskStatus); + app.MapGet("/private-api/curation-desk/roster", PrivateApi.CurationDeskRoster); + app.MapGet("/private-api/curation-desk/recommendations", PrivateApi.CurationDeskRecommendations); + app.MapGet("/private-api/curation-desk/post/{author}/{permlink}", PrivateApi.CurationDeskPost); + app.MapPost("/private-api/curation-desk/roster-feed", PrivateApi.CurationDeskRosterFeed); + app.MapPost("/private-api/curation-desk/tick", PrivateApi.CurationDeskTick); + app.MapPost("/private-api/curation-desk/mark", PrivateApi.CurationDeskMark); + app.MapPost("/private-api/curation-desk/mark-clear", PrivateApi.CurationDeskMarkClear); + app.MapPost("/private-api/curation-desk/marks", PrivateApi.CurationDeskMarks); + app.MapPost("/private-api/curation-desk/cursor", PrivateApi.CurationDeskCursor); + app.MapPost("/private-api/curation-desk/recommend-meta", PrivateApi.CurationDeskRecommendMeta); + app.MapPost("/private-api/curation-desk/recommendation-dismiss", PrivateApi.CurationDeskRecommendationDismiss); + // ---- SSR RPC cache (internal, header-gated; see SsrRpc.cs) ---- app.MapPost("/private-api/ssr/rpc", SsrRpc.Rpc); app.MapGet("/private-api/ssr/stats", SsrRpc.Stats); diff --git a/dotnet/EcencyApi/Infrastructure/BytesCache.cs b/dotnet/EcencyApi/Infrastructure/BytesCache.cs index 26fce2df..3ecf3704 100644 --- a/dotnet/EcencyApi/Infrastructure/BytesCache.cs +++ b/dotnet/EcencyApi/Infrastructure/BytesCache.cs @@ -20,6 +20,9 @@ private sealed class Entry public required byte[] Bytes; public required long ExpiresAtMs; public required LinkedListNode Node; + // Small caller-defined label stored beside the bytes (a content type, + // say), so a value that is not self-describing can be served as it came. + public string? Tag; } private readonly Dictionary _map = new(); @@ -55,7 +58,13 @@ public BytesCache(long budgetBytes) private static long NowMs => Environment.TickCount64; /// Fresh entry or nothing; an expired entry is dropped on the way. - public bool TryGet(string key, out byte[] bytes) + public bool TryGet(string key, out byte[] bytes) => TryGet(key, out bytes, out _); + + /// + /// that also returns the tag the + /// entry was stored with (null when it had none). + /// + public bool TryGet(string key, out byte[] bytes, out string? tag) { lock (_lock) { @@ -66,20 +75,24 @@ public bool TryGet(string key, out byte[] bytes) _lru.Remove(entry.Node); _lru.AddLast(entry.Node); bytes = entry.Bytes; + tag = entry.Tag; return true; } RemoveLocked(key, entry); } } bytes = Array.Empty(); + tag = null; return false; } /// /// Store a value for . A value larger than the whole - /// budget is not stored (it would evict everything for one reader). + /// budget is not stored (it would evict everything for one reader). The + /// optional travels with the bytes and comes back from + /// . /// - public void Set(string key, byte[] bytes, int ttlMs) + public void Set(string key, byte[] bytes, int ttlMs, string? tag = null) { if (ttlMs <= 0 || bytes.Length > Budget) return; lock (_lock) @@ -89,7 +102,7 @@ public void Set(string key, byte[] bytes, int ttlMs) RemoveLocked(key, existing); } var node = _lru.AddLast(key); - var entry = new Entry { Bytes = bytes, ExpiresAtMs = NowMs + ttlMs, Node = node }; + var entry = new Entry { Bytes = bytes, ExpiresAtMs = NowMs + ttlMs, Node = node, Tag = tag }; _map[key] = entry; _byExpiry.Add((entry.ExpiresAtMs, key)); _bytes += bytes.Length; diff --git a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs index b8057b70..77f814be 100644 --- a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs +++ b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs @@ -36,6 +36,44 @@ public static class CachePolicy /// public const string PostTips = "public, max-age=60, stale-while-revalidate=600"; + /// + /// Curation desk public reads. `max-age=0` makes browsers revalidate on every + /// poll while `s-maxage` lets shared caches absorb the polling; the in-process + /// memo of each route uses the same s-maxage as its TTL (see + /// ), so the two layers never disagree on freshness. + /// The feed and the recommendations list move with every new post; status is + /// the poll target and stays short; the roster changes when a curator is added + /// or promoted, so it can stay put for minutes; a single post's recommenders + /// are optimistic on the client and confirmed from here. + /// + public const string CurationDeskFeed = "public, max-age=0, s-maxage=30"; + public const string CurationDeskStatus = "public, max-age=0, s-maxage=15"; + public const string CurationDeskRoster = "public, max-age=0, s-maxage=600"; + public const string CurationDeskRecommendations = "public, max-age=0, s-maxage=30"; + public const string CurationDeskPost = "public, max-age=0, s-maxage=15"; + + /// + /// The `s-maxage` of a policy in seconds, or its `max-age` when it has no + /// shared-cache directive. Handlers that memoize a response derive their TTL + /// from this so the memo can never outlive what the policy promises. + /// + public static int SharedMaxAge(string policy) + { + var tokens = policy.Split(',', StringSplitOptions.TrimEntries); + foreach (var name in new[] { "s-maxage=", "max-age=" }) + { + foreach (var token in tokens) + { + if (token.StartsWith(name, StringComparison.Ordinal) + && int.TryParse(token.AsSpan(name.Length), out var seconds) && seconds >= 0) + { + return seconds; + } + } + } + throw new ArgumentException("policy carries no max-age", nameof(policy)); + } + /// /// A policy applies only to a successful response. Handlers attach it before /// the upstream call resolves, and can still turn diff --git a/dotnet/EcencyApi/Infrastructure/Upstream.cs b/dotnet/EcencyApi/Infrastructure/Upstream.cs index e5add5d0..4accea0e 100644 --- a/dotnet/EcencyApi/Infrastructure/Upstream.cs +++ b/dotnet/EcencyApi/Infrastructure/Upstream.cs @@ -21,6 +21,14 @@ public sealed class UpstreamResponse /// Set when the body wasn't parseable JSON (axios keeps the raw string). public string? RawText { get; init; } + /// + /// The body exactly as received, before parsing. Handlers that memoize a + /// response keep these bytes rather than the parsed tree, so a hit is a + /// dictionary lookup and a write instead of a clone and a re-serialization. + /// Empty when a caller built the response without a body. + /// + public byte[] Bytes { get; init; } = Array.Empty(); + public required HttpResponseHeaders2 Headers { get; init; } public bool BodyIsJson => RawText == null; @@ -204,11 +212,11 @@ private static async Task ReadUpstreamResponse(HttpResponseMes { AllowTrailingCommas = false, }); - return new UpstreamResponse { Status = status, Json = node, Headers = respHeaders }; + return new UpstreamResponse { Status = status, Json = node, Headers = respHeaders, Bytes = bytes }; } catch (JsonException) { - return new UpstreamResponse { Status = status, RawText = text, Headers = respHeaders }; + return new UpstreamResponse { Status = status, RawText = text, Headers = respHeaders, Bytes = bytes }; } } From 8e2142dec084911fe7d7da3cebdaa26288126a72 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 13:04:37 +0000 Subject: [PATCH 2/9] test: cover the curation desk gateway Payload rules (validated username only, whitelists, state and action allowlists, route 5 path grammar), query normalization (whitelist, clamps, dropped defaults, fixed order, public random sort falls back), the public payload fence (private keys stripped at any depth on every read), and the handler behaviour: token on every upstream call, 503 while unconfigured, validation memo hit and expiry with failures never remembered, byte memo with single flight and last-good fallback, Cache-Control on 200 only and no-store on writes. CachePolicyTests cover the five desk policies and SharedMaxAge. --- dotnet/EcencyApi.Tests/CachePolicyTests.cs | 36 +- .../EcencyApi.Tests/CurationDeskAuthTests.cs | 428 ++++++++++++++++++ .../CurationDeskPayloadTests.cs | 258 +++++++++++ .../CurationDeskPublicPayloadTests.cs | 109 +++++ .../EcencyApi.Tests/CurationDeskQueryTests.cs | 178 ++++++++ .../CurationDeskTestSupport.cs | 188 ++++++++ 6 files changed, 1196 insertions(+), 1 deletion(-) create mode 100644 dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs create mode 100644 dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs create mode 100644 dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs create mode 100644 dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs create mode 100644 dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs diff --git a/dotnet/EcencyApi.Tests/CachePolicyTests.cs b/dotnet/EcencyApi.Tests/CachePolicyTests.cs index 0f591eb4..5d6c61f7 100644 --- a/dotnet/EcencyApi.Tests/CachePolicyTests.cs +++ b/dotnet/EcencyApi.Tests/CachePolicyTests.cs @@ -11,7 +11,22 @@ namespace EcencyApi.Tests; public class CachePolicyTests { public static TheoryData AllPolicies() => - new() { CachePolicy.ProMembers, CachePolicy.Announcements, CachePolicy.PostTips }; + new() + { + CachePolicy.ProMembers, CachePolicy.Announcements, CachePolicy.PostTips, + CachePolicy.CurationDeskFeed, CachePolicy.CurationDeskStatus, CachePolicy.CurationDeskRoster, + CachePolicy.CurationDeskRecommendations, CachePolicy.CurationDeskPost, + }; + + public static TheoryData DeskPolicies() => + new() + { + { CachePolicy.CurationDeskFeed, 30 }, + { CachePolicy.CurationDeskStatus, 15 }, + { CachePolicy.CurationDeskRoster, 600 }, + { CachePolicy.CurationDeskRecommendations, 30 }, + { CachePolicy.CurationDeskPost, 15 }, + }; [Theory] [MemberData(nameof(AllPolicies))] @@ -49,6 +64,25 @@ public void AnnouncementsOutliveTheOtherPolicies() Assert.True(MaxAge(CachePolicy.ProMembers) > MaxAge(CachePolicy.PostTips)); } + [Theory] + [MemberData(nameof(DeskPolicies))] + public void DeskPoliciesRevalidateInTheBrowserAndAreSharedForTheirSMaxAge(string policy, int sMaxAge) + { + // max-age=0 makes every browser poll revalidate; s-maxage is what shared + // caches and the in-process memo hold the body for. + Assert.Equal(0, MaxAge(policy)); + Assert.Equal(sMaxAge, CachePolicy.SharedMaxAge(policy)); + Assert.DoesNotContain("stale-while-revalidate", policy); + } + + [Fact] + public void SharedMaxAgeFallsBackToMaxAgeWhenAPolicyHasNoSharedDirective() + { + Assert.Equal(600, CachePolicy.SharedMaxAge(CachePolicy.ProMembers)); + Assert.Equal(60, CachePolicy.SharedMaxAge(CachePolicy.PostTips)); + Assert.Throws(() => CachePolicy.SharedMaxAge("public")); + } + private static int MaxAge(string policy) { var token = policy.Split(',').Select(p => p.Trim()) diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs new file mode 100644 index 00000000..deba9b32 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -0,0 +1,428 @@ +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using EcencyApi.Infrastructure; +using Xunit; +using static EcencyApi.Tests.CurationDeskTestSupport; + +namespace EcencyApi.Tests; + +/// +/// Handler-level behaviour of the desk routes: the shared secret on every +/// upstream call, the fail-closed 503, the validation memo, the byte memo with +/// its single flight and last-good fallback, and the Cache-Control rules. +/// +[Collection("curation-desk")] +public class CurationDeskAuthTests +{ + // ---- the token ----------------------------------------------------------- + + [Fact] + public async Task EveryUpstreamCallCarriesTheDeskToken() + { + var upstream = Install(); + + foreach (var (name, handler, request, _) in PublicReads()) + { + await handler(request()); + var call = Assert.Single(upstream.Calls); + Assert.Equal(Token, call.Header(PrivateApi.DeskTokenHeader)); + Assert.Equal(HttpMethod.Get, call.Method); + Assert.StartsWith("curation/desk/", call.Endpoint); + Assert.Null(call.Payload); + upstream.Calls.Clear(); + CurationDeskMemo.ResetForTests(); + } + + foreach (var (name, handler, body) in SignedWrites()) + { + await handler(Post("/private-api/curation-desk/" + name, body)); + var call = Assert.Single(upstream.Calls); + Assert.Equal(Token, call.Header(PrivateApi.DeskTokenHeader)); + Assert.Equal(HttpMethod.Post, call.Method); + Assert.StartsWith("curation/desk/", call.Endpoint); + Assert.Equal("alice", call.Payload!["username"]!.GetValue()); + Assert.False(((JsonObject)call.Payload).ContainsKey("code")); + upstream.Calls.Clear(); + } + } + + [Fact] + public void TheTokenComesFromItsOwnEnvironmentVariable() + { + // Not set in the test environment: the desk is switched off by default. + Assert.Equal("", Config.DeskInternalToken); + } + + // ---- fail closed --------------------------------------------------------- + + [Fact] + public async Task WithoutTheTokenReadsAnswer503BeforeAnyUpstreamCall() + { + var upstream = Install(token: null); + + foreach (var (name, handler, request, _) in PublicReads()) + { + var ctx = request(); + await handler(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task WithoutTheTokenWritesAuthenticateThenAnswer503() + { + var upstream = Install(token: null); + + foreach (var (name, handler, body) in SignedWrites()) + { + var ok = Post("/private-api/curation-desk/" + name, body); + await handler(ok); + Assert.Equal(503, ok.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ok)); + + // An unauthenticated caller learns nothing about the configuration. + var anon = Post("/private-api/curation-desk/" + name, "{}"); + await handler(anon); + Assert.Equal(401, anon.Response.StatusCode); + Assert.Equal("Unauthorized", Body(anon)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task InvalidSignedCodesAre401WithTheRealValidator() + { + var upstream = Install(); + PrivateApi.DeskValidateCode = PrivateApi.ValidateCode; + + foreach (var (name, handler, _) in SignedWrites()) + { + var empty = Post("/private-api/curation-desk/" + name, "{}"); + await handler(empty); + Assert.Equal(401, empty.Response.StatusCode); + + // The parity probe: decodes to {"not":"valid"}, fails on structure + // before any account lookup. + var probe = Post("/private-api/curation-desk/" + name, "{\"code\":\"eyJub3QiOiJ2YWxpZCJ9\"}"); + await handler(probe); + Assert.Equal(401, probe.Response.StatusCode); + Assert.Equal("Unauthorized", Body(probe)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task ARejectedPayloadIs400AndNeverReachesUpstream() + { + var upstream = Install(); + + var mark = Post("/private-api/curation-desk/mark", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"deleted\"}"); + await PrivateApi.CurationDeskMark(mark); + Assert.Equal(400, mark.Response.StatusCode); + Assert.Equal("invalid state", Body(mark)); + + var meta = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":\"nope\"}"); + await PrivateApi.CurationDeskRecommendMeta(meta); + Assert.Equal(400, meta.Response.StatusCode); + Assert.Equal("invalid trx_id", Body(meta)); + + Assert.Empty(upstream.Calls); + } + + // ---- the validation memo ------------------------------------------------- + + [Fact] + public async Task ASuccessfulValidationIsRememberedWithinTheTtlAndForgottenAfter() + { + var upstream = Install(); + var validations = 0; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult("alice"); }; + PrivateApi.DeskAuthMemoSeconds = 0.2; + var code = "memo-" + Guid.NewGuid().ToString("N"); + var body = "{\"code\":\"" + code + "\",\"author\":\"bob\",\"permlink\":\"p\"}"; + + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(1, validations); + Assert.Equal(2, upstream.Calls.Count); + Assert.All(upstream.Calls, c => Assert.Equal("alice", c.Payload!["username"]!.GetValue())); + + // A different code is a different memo entry. + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body.Replace(code, code + "x"))); + Assert.Equal(2, validations); + + await Task.Delay(350); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(3, validations); + } + + [Fact] + public async Task AFailedValidationIsNeverRemembered() + { + var upstream = Install(); + var validations = 0; + string? answer = null; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult(answer); }; + var code = "fail-" + Guid.NewGuid().ToString("N"); + var body = "{\"code\":\"" + code + "\",\"author\":\"bob\",\"permlink\":\"p\"}"; + + for (var i = 0; i < 3; i++) + { + var ctx = Post("/private-api/curation-desk/mark-clear", body); + await PrivateApi.CurationDeskMarkClear(ctx); + Assert.Equal(401, ctx.Response.StatusCode); + } + Assert.Equal(3, validations); + Assert.Empty(upstream.Calls); + + // Once the code validates it is remembered from that point, not before. + answer = "alice"; + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(4, validations); + Assert.Equal(2, upstream.Calls.Count); + } + + [Fact] + public async Task AMemoizedIdentityIsStillTheValidatedOneNotTheBodysUsername() + { + var upstream = Install(); + var body = "{\"code\":\"as:alice\",\"username\":\"victim\",\"author\":\"bob\",\"permlink\":\"p\"}"; + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.All(upstream.Calls, c => Assert.Equal("alice", c.Payload!["username"]!.GetValue())); + } + + // ---- client address ------------------------------------------------------ + + [Fact] + public async Task OnlyRecommendMetaForwardsTheProxySetClientAddress() + { + var upstream = Install(); + + foreach (var (name, handler, body) in SignedWrites()) + { + var ctx = Post("/private-api/curation-desk/" + name, body); + ctx.Request.Headers["X-Real-IP"] = "198.51.100.7"; + ctx.Request.Headers["X-Forwarded-For"] = "203.0.113.9, 198.51.100.7"; + await handler(ctx); + var call = Assert.Single(upstream.Calls); + Assert.Equal(name == "recommend-meta" ? "198.51.100.7" : null, call.Header("X-Real-IP-V")); + upstream.Calls.Clear(); + } + + // No proxy header: an empty value, never the forwarded-for chain. + var bare = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\"}"); + bare.Request.Headers["X-Forwarded-For"] = "203.0.113.9"; + await PrivateApi.CurationDeskRecommendMeta(bare); + Assert.Equal("", Assert.Single(upstream.Calls).Header("X-Real-IP-V")); + } + + [Fact] + public async Task RecommendMetaAcceptsABodyWithoutATrxId() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(202, "{\"ok\":true}")); + var ctx = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\"}"); + await PrivateApi.CurationDeskRecommendMeta(ctx); + Assert.Equal(202, ctx.Response.StatusCode); + Assert.Equal("{\"ok\":true}", Body(ctx)); + Assert.Equal("curation/desk/recommendations/meta", Assert.Single(upstream.Calls).Endpoint); + } + + // ---- Cache-Control ------------------------------------------------------- + + [Fact] + public async Task ReadsCarryTheirPolicyOnlyOnA200() + { + var upstream = Install(); + + foreach (var (name, handler, request, policy) in PublicReads()) + { + var ok = request(); + await handler(ok); + await Start(ok); + Assert.Equal(200, ok.Response.StatusCode); + Assert.Equal(policy, CacheControl(ok)); + Assert.StartsWith("application/json", ok.Response.ContentType); + + CurationDeskMemo.ResetForTests(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, "{\"error\":\"not found\"}")); + var missing = request(); + await handler(missing); + await Start(missing); + Assert.Equal(404, missing.Response.StatusCode); + Assert.Null(CacheControl(missing)); + Assert.Equal("{\"error\":\"not found\"}", Body(missing)); + + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + CurationDeskMemo.ResetForTests(); + var timeout = request(); + await handler(timeout); + await Start(timeout); + Assert.Equal(504, timeout.Response.StatusCode); + Assert.Equal("Upstream Timeout", Body(timeout)); + Assert.Null(CacheControl(timeout)); + + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{}")); + CurationDeskMemo.ResetForTests(); + } + } + + [Fact] + public async Task WritesAreNeverCacheable() + { + Install(); + foreach (var (name, handler, body) in SignedWrites()) + { + var ctx = Post("/private-api/curation-desk/" + name, body); + await handler(ctx); + await Start(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + Assert.Equal("no-store", CacheControl(ctx)); + } + } + + [Fact] + public void EachPolicySharedMaxAgeIsTheMemoTtl() + { + Assert.Equal(30, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskFeed)); + Assert.Equal(15, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskStatus)); + Assert.Equal(600, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskRoster)); + Assert.Equal(30, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskRecommendations)); + Assert.Equal(15, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskPost)); + } + + // ---- the byte memo ------------------------------------------------------- + + [Fact] + public async Task ASecondReadWithinTheTtlIsServedFromTheMemoAsBytes() + { + var upstream = Install(); + const string body = "{\"items\":[{\"post_id\":1}],\"feed_version\":\"v1\"}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + + var first = Get("/private-api/curation-desk/feed", "limit=10&sort=queue"); + await PrivateApi.CurationDeskFeed(first); + var second = Get("/private-api/curation-desk/feed", "sort=queue&limit=10&x=1"); + await PrivateApi.CurationDeskFeed(second); + + Assert.Single(upstream.Calls); + Assert.Equal("curation/desk/feed?limit=10&sort=queue", upstream.Calls[0].Endpoint); + Assert.Equal(body, Body(first)); + Assert.Equal(body, Body(second)); + + // Stored as the bytes that were served, keyed by the normalized endpoint. + Assert.True(CurationDeskMemo.Fresh.TryGet("curation/desk/feed?limit=10&sort=queue", out var stored, out var tag)); + Assert.IsType(stored); + Assert.Equal(body, System.Text.Encoding.UTF8.GetString(stored)); + Assert.Equal("application/json; charset=utf-8", tag); + Assert.False(CurationDeskMemo.Fresh.TryGet("curation/desk/feed", out _)); + } + + [Fact] + public async Task DifferentQuestionsAreDifferentMemoEntries() + { + var upstream = Install(); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "sort=queue")); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "sort=unique")); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "window=full")); + Assert.Equal(3, upstream.Calls.Count); + Assert.Equal(3, CurationDeskMemo.Fresh.Count); + } + + [Fact] + public async Task ConcurrentReadsOfOneKeyMakeOneUpstreamCall() + { + var upstream = Install(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => release.Task; + + var requests = Enumerable.Range(0, 8).Select(_ => Get("/private-api/curation-desk/status")).ToArray(); + var pending = requests.Select(r => PrivateApi.CurationDeskStatus(r)).ToArray(); + + // Give every request time to reach the gate before the fill completes. + await Task.Delay(100); + Assert.Single(upstream.Calls); + release.SetResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await Task.WhenAll(pending); + + Assert.Single(upstream.Calls); + Assert.All(requests, r => + { + Assert.Equal(200, r.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(r)); + }); + } + + [Fact] + public async Task AnUpstreamErrorAnswersWithTheLastGoodBody() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"curators\":[{\"username\":\"alice\"}]}")); + await PrivateApi.CurationDeskRoster(Get("/private-api/curation-desk/roster")); + + // The fresh entry lapses (simulated), the last-good one is still there. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + var stale = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal("{\"curators\":[{\"username\":\"alice\"}]}", Body(stale)); + + upstream.Answer = _ => Task.FromResult(TextResponse(502, "bad gateway")); + var down = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(down); + Assert.Equal(200, down.Response.StatusCode); + Assert.Equal("{\"curators\":[{\"username\":\"alice\"}]}", Body(down)); + + // With nothing good to fall back on, the backend's answer passes through. + CurationDeskMemo.ResetForTests(); + var bare = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(bare); + Assert.Equal(502, bare.Response.StatusCode); + Assert.Equal("bad gateway", Body(bare)); + + // Errors are never memoized: the next read tries upstream again. + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"curators\":[]}")); + var recovered = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(recovered); + Assert.Equal(200, recovered.Response.StatusCode); + Assert.Equal("{\"curators\":[]}", Body(recovered)); + } + + [Fact] + public async Task ANon200IsPipedThroughAndNotMemoized() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, "{\"error\":\"unknown post\"}")); + var ctx = Get("/private-api/curation-desk/post/good-karma/nope", "", new[] { ("author", "good-karma"), ("permlink", "nope") }); + await PrivateApi.CurationDeskPost(ctx); + Assert.Equal(404, ctx.Response.StatusCode); + Assert.Equal("{\"error\":\"unknown post\"}", Body(ctx)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + Assert.Equal(0, CurationDeskMemo.LastGood.Count); + Assert.Equal("curation/desk/post/good-karma/nope", Assert.Single(upstream.Calls).Endpoint); + } + + [Fact] + public async Task AnInvalidPostPathIs400BeforeAnyUpstreamCall() + { + var upstream = Install(); + foreach (var (author, permlink) in new[] { ("..", "p"), ("good-karma", "a/b"), ("good-karma", "p?x=1"), ("x", "p") }) + { + var ctx = Get("/private-api/curation-desk/post/x/y", "", new[] { ("author", author), ("permlink", permlink) }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(400, ctx.Response.StatusCode); + Assert.Equal("Invalid author or permlink", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs new file mode 100644 index 00000000..b385fd90 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -0,0 +1,258 @@ +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The desk writes forward a body built here, never the client's. What matters +/// is who the backend believes is acting (the validated username, always) and +/// that a value the backend would only reject is refused before it travels. +/// +public class CurationDeskPayloadTests +{ + private static readonly CurationDeskWrites.Route[] AllRoutes = + { + CurationDeskWrites.RosterFeed, CurationDeskWrites.Tick, CurationDeskWrites.Mark, + CurationDeskWrites.MarkClear, CurationDeskWrites.Marks, CurationDeskWrites.Cursor, + CurationDeskWrites.RecommendMeta, CurationDeskWrites.RecommendationDismiss, + }; + + private static JsonObject Body(string json) => (JsonObject)JsonNode.Parse(json)!; + + private static JsonObject Ok(CurationDeskWrites.Route route, string json) + { + var (payload, error) = CurationDeskWrites.Build(route, "alice", Body(json)); + Assert.Null(error); + Assert.NotNull(payload); + return payload!; + } + + private static string Rejected(CurationDeskWrites.Route route, string json) + { + var (payload, error) = CurationDeskWrites.Build(route, "alice", Body(json)); + Assert.Null(payload); + Assert.NotNull(error); + return error!; + } + + /// A body that passes each route's validation, with a forged identity attached. + private static string ValidBodyFor(CurationDeskWrites.Route route) + { + const string forged = "\"username\":\"victim\",\"code\":\"as:victim\","; + if (ReferenceEquals(route, CurationDeskWrites.Mark)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"flagged\",\"reason\":\"farming\"}"; + if (ReferenceEquals(route, CurationDeskWrites.MarkClear) || ReferenceEquals(route, CurationDeskWrites.RecommendMeta)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\"}"; + if (ReferenceEquals(route, CurationDeskWrites.Cursor)) + return "{" + forged + "\"post_id\":7,\"action\":\"advance\"}"; + if (ReferenceEquals(route, CurationDeskWrites.RecommendationDismiss)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"restore\"}"; + return "{" + forged + "\"limit\":5}"; + } + + [Fact] + public void TheValidatedUsernameIsTheOnlyIdentityForwarded() + { + foreach (var route in AllRoutes) + { + var payload = Ok(route, ValidBodyFor(route)); + Assert.Equal("alice", payload["username"]!.GetValue()); + Assert.False(payload.ContainsKey("code"), route.UpstreamPath); + + // And no whitelist can ever be widened to include them. + Assert.DoesNotContain("username", route.Keys); + Assert.DoesNotContain("code", route.Keys); + } + } + + [Fact] + public void OnlyWhitelistedKeysTravel() + { + var payload = Ok(CurationDeskWrites.Mark, + "{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"noted\",\"note\":\"hi\",\"admin\":true,\"weight\":10000}"); + Assert.Equal(new[] { "username", "author", "permlink", "state", "note" }, payload.Select(kv => kv.Key).ToArray()); + + var tick = Ok(CurationDeskWrites.Tick, "{\"since\":\"t\",\"need\":[1],\"visible\":[2],\"curator\":\"x\"}"); + Assert.Equal(new[] { "username", "since", "need", "visible" }, tick.Select(kv => kv.Key).ToArray()); + } + + [Theory] + [InlineData("{\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":\"\",\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":\"bob\",\"state\":\"reviewed\"}", "permlink required")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"\",\"state\":\"reviewed\"}", "permlink required")] + [InlineData("{\"author\":7,\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":null,\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + public void MarkRequiresANonEmptyAuthorAndPermlink(string body, string error) + { + Assert.Equal(error, Rejected(CurationDeskWrites.Mark, body)); + } + + [Fact] + public void EveryPostAddressedRouteRequiresAuthorAndPermlink() + { + Assert.Equal("author required", Rejected(CurationDeskWrites.MarkClear, "{\"permlink\":\"p\"}")); + Assert.Equal("permlink required", Rejected(CurationDeskWrites.MarkClear, "{\"author\":\"bob\",\"permlink\":\"\"}")); + Assert.Equal("author required", Rejected(CurationDeskWrites.RecommendMeta, "{\"permlink\":\"p\"}")); + Assert.Equal("permlink required", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"action\":\"dismiss\"}")); + } + + [Theory] + [InlineData("reviewed")] + [InlineData("snoozed")] + [InlineData("flagged")] + [InlineData("noted")] + public void MarkAcceptsEachKnownState(string state) + { + var payload = Ok(CurationDeskWrites.Mark, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"{state}\"}}"); + Assert.Equal(state, payload["state"]!.GetValue()); + } + + [Theory] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"deleted\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"Reviewed\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":1}")] + public void MarkRefusesAnUnknownState(string body) + { + Assert.Equal("invalid state", Rejected(CurationDeskWrites.Mark, body)); + } + + [Fact] + public void MarksListStateIsOptionalButMustBeKnownWhenGiven() + { + Assert.Equal(new[] { "username" }, Ok(CurationDeskWrites.Marks, "{}").Select(kv => kv.Key).ToArray()); + Assert.Equal("snoozed", Ok(CurationDeskWrites.Marks, "{\"state\":\"snoozed\",\"limit\":10}")["state"]!.GetValue()); + Assert.Equal("invalid state", Rejected(CurationDeskWrites.Marks, "{\"state\":\"all\"}")); + } + + [Theory] + [InlineData("advance")] + [InlineData("rewind")] + public void CursorAcceptsEachKnownAction(string action) + { + var payload = Ok(CurationDeskWrites.Cursor, $"{{\"post_id\":42,\"action\":\"{action}\",\"reason\":\"oops\"}}"); + Assert.Equal(action, payload["action"]!.GetValue()); + Assert.Equal(42, payload["post_id"]!.GetValue()); + Assert.Equal("oops", payload["reason"]!.GetValue()); + } + + [Fact] + public void CursorRefusesAnUnknownActionOrAMissingPostId() + { + Assert.Equal("invalid action", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":42,\"action\":\"jump\"}")); + Assert.Equal("invalid action", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":42}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"action\":\"advance\"}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":null,\"action\":\"advance\"}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":{\"id\":1},\"action\":\"advance\"}")); + } + + [Fact] + public void DismissAcceptsDismissAndRestoreOnly() + { + Ok(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"dismiss\"}"); + Ok(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"restore\"}"); + Assert.Equal("invalid action", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"delete\"}")); + Assert.Equal("invalid action", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\"}")); + } + + [Fact] + public void RecommendMetaTrxIdIsOptionalAndStrictWhenPresent() + { + var without = Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\"}"); + Assert.False(without.ContainsKey("trx_id")); + + var trx = new string('a', 40); + var with = Ok(CurationDeskWrites.RecommendMeta, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":\"{trx}\"}}"); + Assert.Equal(trx, with["trx_id"]!.GetValue()); + + foreach (var bad in new[] { "\"abc\"", "\"" + new string('A', 40) + "\"", "\"" + new string('a', 39) + "\"", "null", "42" }) + { + Assert.Equal("invalid trx_id", + Rejected(CurationDeskWrites.RecommendMeta, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":{bad}}}")); + } + } + + [Fact] + public void RecommendMetaForwardsOnlyAKnownUaClass() + { + Assert.Equal("mobile", + Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"mobile\"}")["ua_class"]!.GetValue()); + Assert.False( + Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"bot\"}").ContainsKey("ua_class")); + Assert.True(CurationDeskWrites.RecommendMeta.ForwardClientAddress); + Assert.All(AllRoutes.Where(r => !ReferenceEquals(r, CurationDeskWrites.RecommendMeta)), + r => Assert.False(r.ForwardClientAddress, r.UpstreamPath)); + } + + [Fact] + public void RosterFeedKeepsSeedOnlyForTheRandomOrderAndClampsLimit() + { + var random = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":\"abcd1234\",\"limit\":500}"); + Assert.Equal("abcd1234", random["seed"]!.GetValue()); + Assert.Equal(50, random["limit"]!.GetValue()); + + var queue = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"queue\",\"seed\":\"abcd1234\",\"limit\":0}"); + Assert.False(queue.ContainsKey("seed")); + Assert.Equal(1, queue["limit"]!.GetValue()); + + var unknown = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"payout\",\"seed\":\"abcd1234\",\"view\":\"excluded\",\"hide_reviewed\":false}"); + Assert.False(unknown.ContainsKey("sort")); + Assert.False(unknown.ContainsKey("seed")); + Assert.Equal("excluded", unknown["view"]!.GetValue()); + Assert.False(unknown["hide_reviewed"]!.GetValue()); + } + + // ---- route 5 path -------------------------------------------------------- + + [Fact] + public void RealAuthorsAndPermlinksMapToTheUpstreamPathUnchanged() + { + Assert.Equal("curation/desk/post/good-karma/my-post-title-2026", + PrivateApi.CurationDeskPostPath("good-karma", "my-post-title-2026")); + Assert.Equal("curation/desk/post/user.name/re-a-b-c-20260905t101010z", + PrivateApi.CurationDeskPostPath("user.name", "re-a-b-c-20260905t101010z")); + } + + [Theory] + // Dot segments would resolve upward once the string becomes a Uri. + [InlineData("..", "p")] + [InlineData("good-karma", "..")] + [InlineData(".", "p")] + // Route values arrive percent-decoded, so a slash is a slash; and the + // still-encoded spelling is not a name character either. + [InlineData("a/b", "p")] + [InlineData("good-karma", "p/q")] + [InlineData("a%2Fb", "p")] + [InlineData("good-karma", "p%2Fq")] + // A question mark or hash would truncate the path. + [InlineData("a?x=1", "p")] + [InlineData("good-karma", "p?x=1")] + [InlineData("good-karma", "p#f")] + // Outside the Hive name grammar. + [InlineData("ab", "p")] + [InlineData("Good-Karma", "p")] + [InlineData("good_karma", "p")] + [InlineData("good-karma", "")] + [InlineData("good-karma", "P")] + [InlineData("good-karma", "a_b")] + [InlineData("", "p")] + [InlineData("undefined-undefined", "p")] + public void AnythingOutsideTheNameGrammarIsRejected(string author, string permlink) + { + Assert.Null(PrivateApi.CurationDeskPostPath(author, permlink)); + } + + [Fact] + public void ThePermlinkLengthBoundIsEnforced() + { + Assert.NotNull(PrivateApi.CurationDeskPostPath("good-karma", new string('a', 255))); + Assert.Null(PrivateApi.CurationDeskPostPath("good-karma", new string('a', 256))); + Assert.NotNull(PrivateApi.CurationDeskPostPath(new string('a', 16), "p")); + Assert.Null(PrivateApi.CurationDeskPostPath(new string('a', 17), "p")); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs new file mode 100644 index 00000000..55ec621e --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs @@ -0,0 +1,109 @@ +using System.Text; +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using Xunit; +using static EcencyApi.Tests.CurationDeskTestSupport; + +namespace EcencyApi.Tests; + +/// +/// A public desk body is memoized and shared-cached for its s-maxage, so a key +/// that names a curator or carries a hashed address must never get in. The +/// backend is specified to omit them; this is the fence on this side, and it +/// has to hold for every public route and at any depth of the tree. +/// +[Collection("curation-desk")] +public class CurationDeskPublicPayloadTests +{ + private const string Leaky = + "{\"team_cursor\":{\"post_id\":1,\"created\":\"t\",\"set_by\":\"alice\",\"set_at\":\"t\"}," + + "\"active_curators\":[{\"username\":\"alice\"}],\"trail_alerts\":[]," + + "\"items\":[{\"post_id\":1,\"excluded_reason\":\"abuser\",\"marks\":[{\"curator\":\"alice\",\"note\":\"secret\"}]," + + "\"recommenders\":[{\"username\":\"bob\",\"ip_hash\":\"ab\",\"key_id\":3}]}],\"generated_at\":\"t\"}"; + + private static void AssertClean(JsonNode? node) + { + switch (node) + { + case JsonObject obj: + foreach (var kv in obj) + { + Assert.DoesNotContain(kv.Key, CurationDeskPublicPayload.PrivateKeys); + AssertClean(kv.Value); + } + break; + case JsonArray arr: + foreach (var item in arr) AssertClean(item); + break; + } + } + + [Fact] + public void TheFenceNamesEveryRosterOnlyKey() + { + Assert.Equal( + new[] { "active_curators", "excluded_reason", "ip_hash", "key_id", "note", "set_at", "set_by", "trail_alerts" }, + CurationDeskPublicPayload.PrivateKeys.OrderBy(k => k, StringComparer.Ordinal).ToArray()); + } + + [Fact] + public void StripRemovesPrivateKeysAtEveryDepthAndKeepsTheRest() + { + var node = JsonNode.Parse(Leaky); + Assert.True(CurationDeskPublicPayload.Strip(node)); + AssertClean(node); + + var obj = (JsonObject)node!; + Assert.Equal(1, obj["team_cursor"]!["post_id"]!.GetValue()); + Assert.Equal("t", obj["team_cursor"]!["created"]!.GetValue()); + Assert.Equal("t", obj["generated_at"]!.GetValue()); + Assert.Equal("alice", obj["items"]![0]!["marks"]![0]!["curator"]!.GetValue()); + Assert.Equal("bob", obj["items"]![0]!["recommenders"]![0]!["username"]!.GetValue()); + } + + [Fact] + public void ACleanBodyIsServedAsTheBytesItArrivedIn() + { + var r = JsonResponse(200, "{\"items\":[{\"post_id\":1,\"author\":\"bob\",\"trailed_by\":{\"curator\":\"alice\"}}],\"feed_version\":\"v\"}"); + Assert.False(CurationDeskPublicPayload.Strip(r.Json)); + Assert.Same(r.Bytes, CurationDeskPublicPayload.ToPublicBytes(r)); + } + + [Fact] + public void ALeakyBodyIsReserializedWithoutTheKeys() + { + var r = JsonResponse(200, Leaky); + var bytes = CurationDeskPublicPayload.ToPublicBytes(r); + Assert.NotSame(r.Bytes, bytes); + var text = Encoding.UTF8.GetString(bytes); + foreach (var key in CurationDeskPublicPayload.PrivateKeys) + { + Assert.DoesNotContain("\"" + key + "\"", text); + } + Assert.Contains("\"generated_at\":\"t\"", text); + } + + [Fact] + public async Task EveryPublicRouteServesAndMemoizesTheStrippedBody() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, Leaky)); + + foreach (var (name, handler, request, _) in PublicReads()) + { + var ctx = request(); + await handler(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + var served = JsonNode.Parse(Body(ctx)); + AssertClean(served); + Assert.Equal("t", served!["generated_at"]!.GetValue()); + + // The memo holds the same clean bytes, so a hit cannot leak either. + var again = request(); + await handler(again); + AssertClean(JsonNode.Parse(Body(again))); + } + + Assert.Equal(5, upstream.Calls.Count); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs new file mode 100644 index 00000000..fa6ced8c --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs @@ -0,0 +1,178 @@ +using EcencyApi.Handlers; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The public desk reads are memoized and shared-cached by their normalized +/// upstream URL. Whitelisting decides what a stranger can make the backend +/// compute; the fixed order and dropped defaults decide how many distinct keys +/// a burst of equivalent requests turns into. +/// +public class CurationDeskQueryTests +{ + private static KeyValuePair[] Q(params (string, string)[] pairs) => + pairs.Select(p => new KeyValuePair(p.Item1, p.Item2)).ToArray(); + + private static string Feed(params (string, string)[] pairs) => + CurationDeskQuery.Endpoint("curation/desk/feed", CurationDeskQuery.NormalizeFeed(Q(pairs))); + + private static string Recommendations(params (string, string)[] pairs) => + CurationDeskQuery.Endpoint("curation/desk/recommendations", CurationDeskQuery.NormalizeRecommendations(Q(pairs))); + + [Fact] + public void AnEmptyQueryIsTheBarePath() + { + Assert.Equal("curation/desk/feed", Feed()); + Assert.Equal("curation/desk/recommendations", Recommendations()); + } + + [Fact] + public void UnknownParametersAreDropped() + { + Assert.Equal("curation/desk/feed", + Feed(("x", "1"), ("order", "asc"), ("username", "alice"), ("code", "abc"), ("hide_reviewed", "0"), ("flagged", "1"))); + } + + [Fact] + public void DefaultsAreDroppedSoTheyCollapseOntoTheBarePath() + { + Assert.Equal("curation/desk/feed", + Feed(("limit", "25"), ("sort", "newest"), ("app", "all"), ("window", "all"), ("hide_curated", "1"), + ("rep_min", "0"), ("rep_max", "100"), ("min_words", "0"), ("max_words", "50000"), + ("has_images", "0"), ("new_authors", "0"), ("recommended", "0"))); + } + + [Theory] + [InlineData("0", "limit=1")] + [InlineData("-3", "limit=1")] + [InlineData("1", "limit=1")] + [InlineData("50", "limit=50")] + [InlineData("999", "limit=50")] + [InlineData("25", "")] + [InlineData("abc", "")] + [InlineData("1e1", "")] + [InlineData("", "")] + public void LimitIsClampedToItsRange(string given, string expected) + { + var url = Feed(("limit", given)); + Assert.Equal(expected.Length == 0 ? "curation/desk/feed" : "curation/desk/feed?" + expected, url); + } + + [Fact] + public void RangesAreClampedAndTheirNoOpBoundsDropped() + { + Assert.Equal("curation/desk/feed?rep_min=100", Feed(("rep_min", "150"))); + Assert.Equal("curation/desk/feed?rep_max=0", Feed(("rep_max", "-1"))); + Assert.Equal("curation/desk/feed?rep_min=40&rep_max=70", Feed(("rep_max", "70"), ("rep_min", "40"))); + Assert.Equal("curation/desk/feed", Feed(("min_words", "-5"))); + Assert.Equal("curation/desk/feed?min_words=50000", Feed(("min_words", "999999"))); + Assert.Equal("curation/desk/feed", Feed(("max_words", "999999"))); + Assert.Equal("curation/desk/feed?max_words=300", Feed(("max_words", "300"))); + Assert.Equal("curation/desk/feed", Feed(("rep_min", "high"))); + } + + [Fact] + public void FlagsAreZeroOrOneOnly() + { + Assert.Equal("curation/desk/feed?has_images=1&new_authors=1&recommended=1", + Feed(("has_images", "1"), ("new_authors", "1"), ("recommended", "1"))); + Assert.Equal("curation/desk/feed", + Feed(("has_images", "true"), ("new_authors", "yes"), ("recommended", "2"))); + Assert.Equal("curation/desk/feed?hide_curated=0", Feed(("hide_curated", "0"))); + Assert.Equal("curation/desk/feed", Feed(("hide_curated", "false"))); + } + + [Fact] + public void EnumsAcceptOnlyTheirAllowlist() + { + Assert.Equal("curation/desk/feed?sort=queue", Feed(("sort", "queue"))); + Assert.Equal("curation/desk/feed?sort=unique", Feed(("sort", "unique"))); + Assert.Equal("curation/desk/feed?view=new-authors", Feed(("view", "new-authors"))); + Assert.Equal("curation/desk/feed", Feed(("view", "excluded"))); + Assert.Equal("curation/desk/feed?app=peakd", Feed(("app", "peakd"))); + Assert.Equal("curation/desk/feed", Feed(("app", "hive"))); + Assert.Equal("curation/desk/feed?window=locked", Feed(("window", "locked"))); + Assert.Equal("curation/desk/feed", Feed(("window", "week"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "Queue"))); + } + + [Fact] + public void APublicRandomSortFallsBackToNewestAndItsSeedIsDropped() + { + Assert.Equal("curation/desk/feed", Feed(("sort", "random"), ("seed", "abcd1234"))); + Assert.Equal("curation/desk/feed", Feed(("seed", "abcd1234"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "payout"), ("order", "desc"))); + } + + [Fact] + public void UniqueImpliesRecommendedSoTheFlagIsRedundantThere() + { + Assert.Equal("curation/desk/feed?sort=unique", Feed(("sort", "unique"), ("recommended", "1"))); + Assert.Equal(Feed(("sort", "unique")), Feed(("sort", "unique"), ("recommended", "1"))); + } + + [Fact] + public void CursorMustMatchTheOpaqueCursorGrammar() + { + Assert.Equal("curation/desk/feed?cursor=2026-09-05T10%3A00%3A00Z%3A123", Feed(("cursor", "2026-09-05T10:00:00Z:123"))); + Assert.Equal("curation/desk/feed?cursor=s%3Aabc_def.1%3A25", Feed(("cursor", "s:abc_def.1:25"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", "a b"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", "a/b"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", ""))); + Assert.Equal("curation/desk/feed", Feed(("cursor", new string('a', 81)))); + Assert.Equal("curation/desk/feed?cursor=" + new string('a', 80), Feed(("cursor", new string('a', 80)))); + } + + [Fact] + public void CommunityMustBeAHiveCommunityName() + { + Assert.Equal("curation/desk/feed?community=hive-125125", Feed(("community", "hive-125125"))); + Assert.Equal("curation/desk/feed?community=hive-12512", Feed(("community", "hive-12512"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-1234"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-1234567"))); + Assert.Equal("curation/desk/feed", Feed(("community", "photography"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-125125/x"))); + } + + [Fact] + public void ParametersAreEmittedInOneFixedOrderWhateverTheClientSent() + { + var a = Feed(("recommended", "1"), ("community", "hive-125125"), ("limit", "10"), ("view", "queue"), + ("cursor", "c1"), ("window", "full"), ("app", "ecency"), ("sort", "queue"), ("has_images", "1"), + ("rep_min", "30"), ("max_words", "800"), ("hide_curated", "0"), ("new_authors", "1"), ("rep_max", "75"), + ("min_words", "100")); + var b = Feed(("min_words", "100"), ("rep_max", "75"), ("new_authors", "1"), ("hide_curated", "0"), + ("max_words", "800"), ("rep_min", "30"), ("has_images", "1"), ("sort", "queue"), ("app", "ecency"), + ("window", "full"), ("cursor", "c1"), ("view", "queue"), ("limit", "10"), ("community", "hive-125125"), + ("recommended", "1")); + Assert.Equal(a, b); + Assert.Equal( + "curation/desk/feed?cursor=c1&limit=10&sort=queue&view=queue&app=ecency&community=hive-125125&window=full" + + "&rep_min=30&rep_max=75&min_words=100&max_words=800&has_images=1&new_authors=1&recommended=1&hide_curated=0", + a); + + // The order is the whitelist itself: nothing else can appear, and the + // shared-cache key upstream of this service lists the same names. + Assert.Equal(15, CurationDeskQuery.FeedOrder.Length); + Assert.Equal(CurationDeskQuery.FeedOrder.Length, CurationDeskQuery.FeedOrder.Distinct().Count()); + } + + [Fact] + public void ARepeatedKeyTakesItsFirstValue() + { + Assert.Equal("curation/desk/feed?limit=10", Feed(("limit", "10"), ("limit", "40"))); + } + + [Fact] + public void RecommendationsAcceptOnlyCursorLimitAndTheirTwoSorts() + { + Assert.Equal("curation/desk/recommendations?sort=unique", Recommendations(("sort", "unique"))); + Assert.Equal("curation/desk/recommendations?sort=newest", Recommendations(("sort", "newest"))); + Assert.Equal("curation/desk/recommendations", Recommendations(("sort", "queue"))); + Assert.Equal("curation/desk/recommendations?cursor=abc&limit=50", + Recommendations(("view", "all"), ("limit", "60"), ("cursor", "abc"), ("seed", "x"))); + Assert.Equal("curation/desk/recommendations?limit=10&sort=unique", + Recommendations(("sort", "unique"), ("limit", "10"))); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs new file mode 100644 index 00000000..c5826054 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs @@ -0,0 +1,188 @@ +using System.Text; +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using EcencyApi.Infrastructure; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The desk handler tests replace static seams (the configured token, the +/// upstream call, code validation) and share the static memo, so they must not +/// run alongside each other. One collection serializes them. +/// +[CollectionDefinition("curation-desk", DisableParallelization = true)] +public class CurationDeskCollection { } + +/// +/// Handler-level scaffolding: a request context whose OnStarting callbacks can be +/// run (DefaultHttpContext drops them, and CacheWhenOk relies on them), a +/// recording upstream, and a reset of every seam between tests. +/// +internal static class CurationDeskTestSupport +{ + public const string Token = "test-desk-token"; + + /// HttpResponseFeature that keeps OnStarting callbacks so a test can fire them. + public sealed class StartingAwareResponseFeature : HttpResponseFeature + { + private readonly List<(Func Callback, object State)> _starting = new(); + + public override void OnStarting(Func callback, object state) => _starting.Add((callback, state)); + + public async Task RunStarting() + { + foreach (var (callback, state) in _starting) + { + await callback(state); + } + } + } + + public sealed record Call(string Endpoint, HttpMethod Method, List> Headers, JsonNode? Payload) + { + public string? Header(string name) => + Headers.FirstOrDefault(h => h.Key.Equals(name, StringComparison.OrdinalIgnoreCase)).Value; + } + + /// Records every upstream call and answers from a script. + public sealed class Recorder + { + public readonly List Calls = new(); + public Func> Answer = _ => Task.FromResult(JsonResponse(200, "{}")); + + public Task Handle(string endpoint, HttpMethod method, + IEnumerable> headers, JsonNode? payload) + { + var call = new Call(endpoint, method, headers.ToList(), payload?.DeepClone()); + lock (Calls) Calls.Add(call); + return Answer(call); + } + } + + /// + /// Fresh seams: token configured, validation accepts any non-empty code as + /// the account named in it (`code` "as:alice" -> "alice"), empty memo. + /// + public static Recorder Install(string? token = Token) + { + PrivateApi.DeskToken = token; + PrivateApi.DeskAuthMemoSeconds = 90; + PrivateApi.DeskValidateCode = body => + { + var code = body["code"]?.GetValue(); + return Task.FromResult(code != null && code.StartsWith("as:", StringComparison.Ordinal) ? code[3..] : null); + }; + CurationDeskMemo.ResetForTests(); + var recorder = new Recorder(); + PrivateApi.DeskUpstream = recorder.Handle; + return recorder; + } + + public static UpstreamResponse JsonResponse(int status, string json) + { + var bytes = Encoding.UTF8.GetBytes(json); + return new UpstreamResponse + { + Status = status, + Json = JsonNode.Parse(json), + Headers = new HttpResponseHeaders2(new HttpResponseMessage()), + Bytes = bytes, + }; + } + + public static UpstreamResponse TextResponse(int status, string text) + { + return new UpstreamResponse + { + Status = status, + RawText = text, + Headers = new HttpResponseHeaders2(new HttpResponseMessage()), + Bytes = Encoding.UTF8.GetBytes(text), + }; + } + + public static DefaultHttpContext Get(string path, string query = "", (string Name, string Value)[]? routeValues = null) + { + var ctx = NewContext(); + ctx.Request.Method = "GET"; + ctx.Request.Path = path; + if (query.Length > 0) + { + ctx.Request.QueryString = new QueryString(query.StartsWith('?') ? query : "?" + query); + } + if (routeValues != null) + { + foreach (var (name, value) in routeValues) + { + ctx.Request.RouteValues[name] = value; + } + } + return ctx; + } + + public static DefaultHttpContext Post(string path, string body) + { + var ctx = NewContext(); + ctx.Request.Method = "POST"; + ctx.Request.Path = path; + ctx.Request.ContentType = "application/json"; + var bytes = Encoding.UTF8.GetBytes(body); + ctx.Request.Body = new MemoryStream(bytes); + ctx.Request.ContentLength = bytes.Length; + return ctx; + } + + private static DefaultHttpContext NewContext() + { + var ctx = new DefaultHttpContext(); + ctx.Features.Set(new StartingAwareResponseFeature()); + ctx.Response.Body = new MemoryStream(); + return ctx; + } + + /// Fire the OnStarting callbacks, as the server would before the first byte. + public static Task Start(HttpContext ctx) => + ((StartingAwareResponseFeature)ctx.Features.Get()!).RunStarting(); + + public static string Body(HttpContext ctx) + { + ctx.Response.Body.Position = 0; + return new StreamReader(ctx.Response.Body, Encoding.UTF8).ReadToEnd(); + } + + public static string? CacheControl(HttpContext ctx) => + ctx.Response.Headers.TryGetValue("Cache-Control", out var v) ? v.ToString() : null; + + /// Every public read, as (handler, request factory, policy). + public static IEnumerable<(string Name, Func Handler, Func Request, string Policy)> PublicReads() + { + yield return ("feed", PrivateApi.CurationDeskFeed, + () => Get("/private-api/curation-desk/feed", "limit=10"), CachePolicy.CurationDeskFeed); + yield return ("status", PrivateApi.CurationDeskStatus, + () => Get("/private-api/curation-desk/status"), CachePolicy.CurationDeskStatus); + yield return ("roster", PrivateApi.CurationDeskRoster, + () => Get("/private-api/curation-desk/roster"), CachePolicy.CurationDeskRoster); + yield return ("recommendations", PrivateApi.CurationDeskRecommendations, + () => Get("/private-api/curation-desk/recommendations", "sort=unique"), CachePolicy.CurationDeskRecommendations); + yield return ("post", PrivateApi.CurationDeskPost, + () => Get("/private-api/curation-desk/post/good-karma/hello-world", "", + new[] { ("author", "good-karma"), ("permlink", "hello-world") }), CachePolicy.CurationDeskPost); + } + + /// Every signed write, as (handler, a body that passes validation). + public static IEnumerable<(string Name, Func Handler, string Body)> SignedWrites() + { + const string code = "\"code\":\"as:alice\""; + yield return ("roster-feed", PrivateApi.CurationDeskRosterFeed, "{" + code + ",\"limit\":10}"); + yield return ("tick", PrivateApi.CurationDeskTick, "{" + code + ",\"since\":\"2026-09-05T00:00:00Z\",\"need\":[1],\"visible\":[1,2]}"); + yield return ("mark", PrivateApi.CurationDeskMark, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"reviewed\"}"); + yield return ("mark-clear", PrivateApi.CurationDeskMarkClear, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\"}"); + yield return ("marks", PrivateApi.CurationDeskMarks, "{" + code + ",\"state\":\"flagged\"}"); + yield return ("cursor", PrivateApi.CurationDeskCursor, "{" + code + ",\"post_id\":42,\"action\":\"advance\"}"); + yield return ("recommend-meta", PrivateApi.CurationDeskRecommendMeta, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"web\"}"); + yield return ("recommendation-dismiss", PrivateApi.CurationDeskRecommendationDismiss, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"dismiss\"}"); + } +} From ac441c694f3d0d3a684fcd0e5f9b154b99726f27 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 13:04:37 +0000 Subject: [PATCH 3/9] chore: parity divergences and env doc for the curation desk routes One KNOWN_DIVERGENCES entry per generated catalog case (::get for the five reads, ::min ::pop ::badcode for the eight writes) and the DESK_INTERNAL_TOKEN row in the environment table. --- README.md | 1 + dotnet/parity/driver.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/README.md b/README.md index 3ef0e10c..1b12f0bb 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ docker run -it --rm -p 4000:4000 \ | `SEARCH_API_ADDR` | hivesearcher api endpoint | | `SEARCH_API_SECRET` | hivesearcher api auth token | | `STRIPE_INTERNAL_SECRET` | shared secret for the Stripe money endpoints (unset = they fail closed) | +| `DESK_INTERNAL_TOKEN` | shared secret sent on every curation desk call (`/private-api/curation-desk/*`), public reads included (unset = the desk routes answer 503) | | `TURNSTILE_SECRET` | Cloudflare Turnstile secret for account-create captcha | | `CAPTCHA_MODE` | `hard` (default) or `off` (operator break-glass) | | `BLOCKSTREAM_CLIENT_ID` / `BLOCKSTREAM_CLIENT_SECRET` | optional Blockstream Enterprise esplora auth (BTC fallback) | diff --git a/dotnet/parity/driver.py b/dotnet/parity/driver.py index 2ee86618..0001b40d 100644 --- a/dotnet/parity/driver.py +++ b/dotnet/parity/driver.py @@ -241,6 +241,27 @@ def norm_body(text): "no behavior the reference ever had is changing." ) +CURATION_DESK_DIVERGENCE = ( + "Curation desk gateway route added after the port. The reference build has no such " + "route and answers 404 (POST) or the unmatched-GET template page; this one answers " + "503 while its shared secret is unconfigured, 401 for a missing or invalid signed " + "code, 400 for a rejected path or body, and otherwise proxies. Deterministic and " + "additive -- no behavior the reference ever had is changing." +) + +CURATION_DESK_ROUTES = [ + "/private-api/curation-desk/feed::get", + "/private-api/curation-desk/status::get", + "/private-api/curation-desk/roster::get", + "/private-api/curation-desk/recommendations::get", + "/private-api/curation-desk/post/x/x::get", +] + [ + f"/private-api/curation-desk/{route}::{case}" + for route in ("roster-feed", "tick", "mark", "mark-clear", "marks", "cursor", + "recommend-meta", "recommendation-dismiss") + for case in ("min", "pop", "badcode") +] + # Cases where the C# port intentionally differs from Node (Node bugs the port fixes). KNOWN_DIVERGENCES = { "/auth-api/hs-token-refresh::min": @@ -308,6 +329,9 @@ def norm_body(text): FAVORITE_TAGS_DIVERGENCE, "/private-api/favorite-tags-delete::badcode": FAVORITE_TAGS_DIVERGENCE, + # Curation desk routes, added after the port: five public reads and eight signed + # writes. Same shape as the entries above; one entry per generated case. + **{case: CURATION_DESK_DIVERGENCE for case in CURATION_DESK_ROUTES}, } # Deliberately NOT listed above: /wallet-api/portfolio-v2::pop, whose HP action list From 0cd287124b009d0247b971a798971b98fa18be01 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 13:32:10 +0000 Subject: [PATCH 4/9] fix: keep desk reads off the gate and off the cache when the body is not ours The single-flight gate exists to share one upstream call per key, so a fill now only computes the public bytes and stores them: the response is written after the gate is released, and a reader on a slow connection can no longer hold every other reader of that key behind its own socket. Cache-Control is attached only where a JSON object or array this service holds is served (memo hit, fresh fill, last-good). A 200 whose body is neither is treated like a 5xx: last-good first, then piped uncached and unmemoized, so a gateway error page can no longer be stored publicly for the s-maxage. A JSON error body now passes through the same private-key fence as a served one. Writes answer the unconfigured 503 before validating, so a dark desk costs no chain lookup whoever is asking. The roster feed body reuses the public feed's value rules (clamped ranges, the view, app and window allowlists, the community and cursor patterns, a seed the backend can hash with) and the tick truncates its id lists to the documented 100. Validation patterns anchor with \A and \z and spell community digits as ASCII, so a trailing newline or an Arabic-Indic digit is no longer the value. Numeric values parse as doubles before clamping, so limit=99999999999 asks for 50 rather than falling back to the default. The memo budget moves to Config as DESK_MEMO_BYTES, same 64 MiB default. --- README.md | 1 + dotnet/EcencyApi/Config.cs | 11 + .../Handlers/PrivateApi.CurationDesk.cs | 355 ++++++++++++++---- dotnet/parity/driver.py | 5 +- 4 files changed, 288 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 1b12f0bb..da21f441 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ docker run -it --rm -p 4000:4000 \ | `SEARCH_API_SECRET` | hivesearcher api auth token | | `STRIPE_INTERNAL_SECRET` | shared secret for the Stripe money endpoints (unset = they fail closed) | | `DESK_INTERNAL_TOKEN` | shared secret sent on every curation desk call (`/private-api/curation-desk/*`), public reads included (unset = the desk routes answer 503) | +| `DESK_MEMO_BYTES` | byte budget of each curation desk memo store (fresh and last-good), LRU beyond it (default 64 MiB) | | `TURNSTILE_SECRET` | Cloudflare Turnstile secret for account-create captcha | | `CAPTCHA_MODE` | `hard` (default) or `off` (operator break-glass) | | `BLOCKSTREAM_CLIENT_ID` / `BLOCKSTREAM_CLIENT_SECRET` | optional Blockstream Enterprise esplora auth (BTC fallback) | diff --git a/dotnet/EcencyApi/Config.cs b/dotnet/EcencyApi/Config.cs index 5b09b5ad..b8e609d3 100644 --- a/dotnet/EcencyApi/Config.cs +++ b/dotnet/EcencyApi/Config.cs @@ -31,6 +31,17 @@ public static class Config public static string DeskInternalToken { get; } = Env("DESK_INTERNAL_TOKEN") ?? ""; + /// + /// Byte budget of each curation desk memo store (the fresh one and the + /// last-good one), LRU beyond it. A feed page is tens of KB, so the default + /// holds thousands of distinct questions; it is exposed so a deployment can + /// shrink it without a rebuild if the process is short of memory. + /// + public static long DeskMemoBytes { get; } = + long.TryParse(Env("DESK_MEMO_BYTES"), out var deskBytes) && deskBytes >= 0 + ? deskBytes + : 64L * 1024 * 1024; + public static string HsClientSecret { get; } = Env("HIVESIGNER_SECRET") ?? "hivesignerclientsecret"; diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs index b29b5f80..0f6eaf1b 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -70,6 +70,11 @@ public static partial class PrivateApi // ---- public reads -------------------------------------------------------- + // The one-line handlers here and under "signed writes" return the delegate's + // Task instead of awaiting it: they do nothing after the call, so an async + // state machine per request would be pure overhead and Routes.cs only needs + // a Task back. Handlers that do work of their own stay `async`. + // GET /private-api/curation-desk/feed public static Task CurationDeskFeed(HttpContext ctx) => ServeDeskRead(ctx, @@ -105,8 +110,11 @@ public static async Task CurationDeskPost(HttpContext ctx) await ServeDeskRead(ctx, path, CachePolicy.CurationDeskPost); } - private static readonly Regex DeskAuthorPattern = new("^[a-z0-9.-]{3,16}$", RegexOptions.Compiled); - private static readonly Regex DeskPermlinkPattern = new("^[a-z0-9-]{1,255}$", RegexOptions.Compiled); + // \A and \z, not ^ and $: in .NET `$` also matches before a trailing + // newline, so "good-karma\n" would pass a `$`-anchored name check and + // travel into the upstream path. + private static readonly Regex DeskAuthorPattern = new(@"\A[a-z0-9.-]{3,16}\z", RegexOptions.Compiled); + private static readonly Regex DeskPermlinkPattern = new(@"\A[a-z0-9-]{1,255}\z", RegexOptions.Compiled); /// /// Upstream path for a single post, or null when either value is not a plain @@ -146,7 +154,15 @@ private static IEnumerable> RawQuery(HttpContext ct /// /// Serve one public desk read: memo hit, or a single-flight fill of the - /// normalized endpoint. Cache-Control is attached for a 200 only. + /// normalized endpoint. + /// + /// Nothing is written to the client while the per-key gate is held. The gate + /// exists to collapse concurrent fills of one key onto one upstream call, so + /// a fill only computes the public bytes and stores them; what to send is + /// kept in locals, the gate is released in the finally, and the response is + /// written after it. Writing under the gate would make every reader of a key + /// wait for the slowest reader's socket to drain rather than for the upstream + /// call, which is the one thing the gate is meant to share. /// private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string policy) { @@ -157,86 +173,125 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string return; } - ctx.CacheWhenOk(policy); - if (CurationDeskMemo.TryGetFresh(endpoint, out var hit, out var hitType)) { - await WriteBytes(ctx, 200, hitType, hit); + await SendPublicJson(ctx, policy, hitType, hit); return; } + // Filled under the gate, sent after it: bytes to serve, an error to + // answer with, or the upstream response to pass through. + byte[]? bytes = null; + string? bytesType = null; + var errorStatus = 0; + string? errorText = null; + UpstreamResponse? passthrough = null; + var gate = CurationDeskMemo.GateFor(endpoint); if (!await gate.WaitAsync(CurationDeskMemo.FillWait)) { // Someone else's fill is taking longer than a whole upstream timeout. // Do not stack another one behind it; answer from what is known. - await ServeLastGoodOr(ctx, endpoint, 504, "Upstream Timeout"); + await ServeLastGoodOr(ctx, endpoint, policy, 504, "Upstream Timeout"); return; } try { // The fill that held the gate may have landed while this one queued. - if (CurationDeskMemo.TryGetFresh(endpoint, out hit, out hitType)) + if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType)) { - await WriteBytes(ctx, 200, hitType, hit); - return; + bytes = fresh; + bytesType = freshType; } - - UpstreamResponse r; - try - { - r = await DeskUpstream(endpoint, HttpMethod.Get, DeskHeaders(token), null); - } - catch (UpstreamTimeoutException) + else { - await ServeLastGoodOr(ctx, endpoint, 504, "Upstream Timeout"); - return; - } - catch (Exception) - { - await ServeLastGoodOr(ctx, endpoint, 500, "Server Error"); - return; - } - - if (r.Status == 200 && r.Json is JsonObject or JsonArray) - { - var bytes = CurationDeskPublicPayload.ToPublicBytes(r); - CurationDeskMemo.Store(endpoint, bytes, JsonContentType, CachePolicy.SharedMaxAge(policy)); - await WriteBytes(ctx, 200, JsonContentType, bytes); - return; - } + UpstreamResponse? r = null; + try + { + r = await DeskUpstream(endpoint, HttpMethod.Get, DeskHeaders(token), null); + } + catch (UpstreamTimeoutException) + { + (errorStatus, errorText) = (504, "Upstream Timeout"); + } + catch (Exception) + { + (errorStatus, errorText) = (500, "Server Error"); + } - if (r.Status >= 500) - { - // The backend is unwell; a body it last answered with is better - // than its error page, and the error is not worth memoizing. - if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + if (r != null && r.Status == 200 && r.Json is JsonObject or JsonArray) + { + bytes = CurationDeskPublicPayload.ToPublicBytes(r); + bytesType = JsonContentType; + CurationDeskMemo.Store(endpoint, bytes, JsonContentType, CachePolicy.SharedMaxAge(policy)); + } + else { - await WriteBytes(ctx, 200, staleType, stale); - return; + passthrough = r; } } - - // 4xx (an unknown post, a rejected token), a 200 that is not JSON, or - // a 5xx with nothing to fall back on: pass through unmemoized, the - // way Pipe would, so the client sees what the backend said. - await Upstream.SendLikeExpress(ctx, r.Status, r.Json, r.RawText); } finally { gate.Release(); CurationDeskMemo.ReleaseGate(endpoint, gate); } + + if (bytes != null) + { + await SendPublicJson(ctx, policy, bytesType!, bytes); + return; + } + + if (errorText != null) + { + await ServeLastGoodOr(ctx, endpoint, policy, errorStatus, errorText); + return; + } + + var response = passthrough!; + + // A 5xx, or a 200 whose body is not a JSON object or array (an error + // page, a redirect body, a bare string): either way the backend is not + // answering the question this route asks, so a body it did answer with + // is better than passing that on, and neither is worth memoizing. + if ((response.Status >= 500 || response.Status == 200) + && CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + { + await SendPublicJson(ctx, policy, staleType, stale); + return; + } + + // 4xx (an unknown post, a rejected token), or nothing to fall back on: + // pass through the way Pipe would, unmemoized and with no Cache-Control + // of ours, so the client sees what the backend said. An error body is + // still a public body this service emits, so it goes through the same + // fence as a served one. + CurationDeskPublicPayload.Strip(response.Json); + await Upstream.SendLikeExpress(ctx, response.Status, response.Json, response.RawText); } private const string JsonContentType = "application/json; charset=utf-8"; - private static async Task ServeLastGoodOr(HttpContext ctx, string endpoint, int status, string text) + /// + /// Send a JSON body this service holds (a memo hit, a fresh fill or a + /// last-good fallback) with the route's cache policy. Only these bodies are + /// publicly cacheable: an upstream passthrough carries whatever the backend + /// answered, which may be an error page or a body meant for one caller, so + /// it never gets a Cache-Control of ours. + /// + private static async Task SendPublicJson(HttpContext ctx, string policy, string contentType, byte[] bytes) + { + ctx.CacheWhenOk(policy); + await WriteBytes(ctx, 200, contentType, bytes); + } + + private static async Task ServeLastGoodOr(HttpContext ctx, string endpoint, string policy, int status, string text) { if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) { - await WriteBytes(ctx, 200, staleType, stale); + await SendPublicJson(ctx, policy, staleType, stale); return; } await ctx.SendText(status, text); @@ -299,17 +354,21 @@ public static Task CurationDeskRecommendationDismiss(HttpContext ctx) => /// private static async Task ServeDeskWrite(HttpContext ctx, CurationDeskWrites.Route route) { - var body = await ctx.ReadBody(); - var username = await RequireAuthedUsernameCached(ctx, body); - if (username == null) + // Before anything else: a dark desk answers 503 whoever is asking, so + // validating first would spend one chain lookup per request on a route + // that cannot do any work. The answer reveals nothing a reader of the + // public routes cannot see, which answer 503 unauthenticated too. + var token = DeskToken; + if (token == null) { + await ctx.SendText(503, DeskNotConfigured); return; } - var token = DeskToken; - if (token == null) + var body = await ctx.ReadBody(); + var username = await RequireAuthedUsernameCached(ctx, body); + if (username == null) { - await ctx.SendText(503, DeskNotConfigured); return; } @@ -380,8 +439,15 @@ public static class CurationDeskQuery public const int MaxLimit = 50; public const int MaxWords = 50000; - private static readonly Regex CursorPattern = new("^[A-Za-z0-9_.:-]{1,80}$", RegexOptions.Compiled); - private static readonly Regex CommunityPattern = new(@"^hive-\d{5,6}$", RegexOptions.Compiled); + // Anchored with \A and \z (a `$` would also match before a trailing + // newline) and written with explicit digit classes: .NET's `\d` matches + // every Unicode decimal digit, so `hive-\d{5,6}` accepts Arabic-Indic or + // Devanagari digits that name no community here. + private static readonly Regex CursorPattern = new(@"\A[A-Za-z0-9_.:-]{1,80}\z", RegexOptions.Compiled); + private static readonly Regex CommunityPattern = new(@"\Ahive-[0-9]{5,6}\z", RegexOptions.Compiled); + + /// Random-order seed: one per browser session, roster feed only. + private static readonly Regex SeedPattern = new(@"\A[a-z0-9]{8,16}\z", RegexOptions.Compiled); public static readonly IReadOnlySet FeedSorts = new HashSet { "queue", "newest", "unique" }; public static readonly IReadOnlySet RecommendationSorts = new HashSet { "unique", "newest" }; @@ -529,18 +595,43 @@ private static Dictionary First(IEnumerableInteger within [min, max], clamped; null when absent or not an integer. - private static int? ClampInt(Dictionary q, string key, int min, int max) + /// Integer within [min, max], clamped; null when absent or not a number. + private static int? ClampInt(Dictionary q, string key, int min, int max) => + q.TryGetValue(key, out var raw) ? ClampValue(raw, min, max) : null; + + /// + /// for a value that did not come from a query string: + /// the roster feed reads the same names out of a JSON body and must clamp + /// them the same way, or the two feeds answer different questions. + /// + /// Parsed as a double rather than an int so that a value outside the int + /// range still clamps into the range: `limit=99999999999` asks for as many + /// rows as there are, and 50 is the right answer to that, while dropping it + /// would silently serve the default page size instead. Only text that is not + /// a plain signed integer is refused, so `1e6`, `12.5` and `abc` are dropped + /// and fall back to the default the same way they did before. + /// + public static int? ClampValue(string? raw, int min, int max) { - if (!q.TryGetValue(key, out var raw)) return null; - if (!int.TryParse(raw, System.Globalization.NumberStyles.AllowLeadingSign, - System.Globalization.CultureInfo.InvariantCulture, out var value)) + if (raw == null + || !double.TryParse(raw, System.Globalization.NumberStyles.AllowLeadingSign, + System.Globalization.CultureInfo.InvariantCulture, out var value) + || double.IsNaN(value)) { return null; } - return Math.Clamp(value, min, max); + return (int)Math.Clamp(value, min, max); } + /// Opaque paging cursor grammar; shared with the roster feed body. + public static bool IsCursor(string? value) => value != null && CursorPattern.IsMatch(value); + + /// A `hive-NNNNN` community name; shared with the roster feed body. + public static bool IsCommunity(string? value) => value != null && CommunityPattern.IsMatch(value); + + /// A random-order seed; the roster feed is the only route that takes one. + public static bool IsSeed(string? value) => value != null && SeedPattern.IsMatch(value); + /// "1" -> true, "0" -> false, anything else -> null (dropped). private static bool? Flag(Dictionary q, string key) => q.TryGetValue(key, out var raw) ? raw switch { "1" => true, "0" => false, _ => null } : null; @@ -561,7 +652,19 @@ public sealed record Route(string UpstreamPath, string[] Keys, bool ForwardClien public static readonly IReadOnlySet UaClasses = new HashSet { "web", "mobile" }; public static readonly IReadOnlySet RosterSorts = new HashSet { "queue", "newest", "unique", "random" }; - private static readonly Regex TrxIdPattern = new("^[0-9a-f]{40}$", RegexOptions.Compiled); + /// + /// Views the roster feed takes: the public ones plus `excluded`, which is + /// the only place an excluded row is ever listed. + /// + public static readonly IReadOnlySet RosterViews = + new HashSet(CurationDeskQuery.Views, StringComparer.Ordinal) { "excluded" }; + + /// How many post ids one tick may name per list. + public const int MaxTickIds = 100; + + // \A and \z for the same reason as the name patterns above: `$` would let + // a trailing newline through. + private static readonly Regex TrxIdPattern = new(@"\A[0-9a-f]{40}\z", RegexOptions.Compiled); public static readonly Route RosterFeed = new("curation/desk/roster-feed", new[] { @@ -613,24 +716,16 @@ public static (JsonObject? Payload, string? Error) Build(Route route, string use if (ReferenceEquals(route, RosterFeed)) { - // An unknown sort is not an error for a read: drop it and let the - // backend apply its default, as the public feed does. - var sort = body.Str("sort"); - if (sort == null || !RosterSorts.Contains(sort)) - { - payload.Remove("sort"); - } - // seed only means something to the random order; for any other sort - // it is noise that would make two identical feeds look different. - if (sort != "random") - { - payload.Remove("seed"); - } - if (payload.Remove("limit") && body.Field("limit") is JsonValue limitValue - && limitValue.TryGetValue(out var limit)) - { - payload["limit"] = Math.Clamp((int)limit, 1, CurationDeskQuery.MaxLimit); - } + NormalizeRosterFeed(payload, body); + } + + if (ReferenceEquals(route, Tick)) + { + // The backend caps both lists at this many ids; truncating here + // keeps a client bug from turning one tick into thousands of + // primary-key probes on the way to the same 400. + Truncate(payload, "need", MaxTickIds); + Truncate(payload, "visible", MaxTickIds); } if (ReferenceEquals(route, RecommendMeta) && payload.ContainsKey("ua_class") @@ -642,6 +737,102 @@ public static (JsonObject? Payload, string? Error) Build(Route route, string use return (payload, null); } + /// + /// The roster feed carries the filter names of the public feed, so it gets + /// the public feed's value rules: an out-of-range number is clamped rather + /// than forwarded, and a value outside an allowlist or a pattern is dropped + /// so the backend applies its default. Both feeds then answer the same + /// question for the same request, and a typo cannot ask for a query plan + /// nobody sized. + /// + private static void NormalizeRosterFeed(JsonObject payload, JsonObject body) + { + // An unknown sort is not an error for a read: drop it and let the + // backend apply its default, as the public feed does. + var sort = body.Str("sort"); + if (sort == null || !RosterSorts.Contains(sort)) + { + payload.Remove("sort"); + } + + // seed only means something to the random order; for any other sort it + // is noise that would make two identical feeds look different, and a + // seed outside the grammar is not a seed the backend can hash with. + if (sort != "random" || !CurationDeskQuery.IsSeed(body.Str("seed"))) + { + payload.Remove("seed"); + } + + KeepAllowed(payload, "view", RosterViews); + KeepAllowed(payload, "app", CurationDeskQuery.Apps); + KeepAllowed(payload, "window", CurationDeskQuery.Windows); + KeepMatching(payload, "cursor", CurationDeskQuery.IsCursor); + KeepMatching(payload, "community", CurationDeskQuery.IsCommunity); + + Clamp(payload, "limit", 1, CurationDeskQuery.MaxLimit); + Clamp(payload, "rep_min", 0, 100); + Clamp(payload, "rep_max", 0, 100); + Clamp(payload, "min_words", 0, CurationDeskQuery.MaxWords); + Clamp(payload, "max_words", 0, CurationDeskQuery.MaxWords); + } + + /// Drop a field whose value is not one of . + private static void KeepAllowed(JsonObject payload, string key, IReadOnlySet allowed) + { + if (payload.ContainsKey(key) && !(JsVal.AsString(payload[key]) is { } value && allowed.Contains(value))) + { + payload.Remove(key); + } + } + + /// Drop a field whose value does not match . + private static void KeepMatching(JsonObject payload, string key, Func matches) + { + if (payload.ContainsKey(key) && !matches(JsVal.AsString(payload[key]))) + { + payload.Remove(key); + } + } + + /// + /// Clamp a numeric field into [min, max], accepting the number or its string + /// spelling; a value that is neither is dropped rather than forwarded. + /// + private static void Clamp(JsonObject payload, string key, int min, int max) + { + if (!payload.ContainsKey(key)) + { + return; + } + var node = payload[key]; + int? value = JsVal.AsNumber(node) is { } number + ? (int)Math.Clamp(number, min, max) + : CurationDeskQuery.ClampValue(JsVal.AsString(node), min, max); + if (value is { } clamped) + { + payload[key] = clamped; + } + else + { + payload.Remove(key); + } + } + + /// Keep at most elements of an array field. + private static void Truncate(JsonObject payload, string key, int max) + { + if (payload[key] is not JsonArray array || array.Count <= max) + { + return; + } + var kept = new JsonArray(); + for (var i = 0; i < max; i++) + { + kept.Add(array[i]?.DeepClone()); + } + payload[key] = kept; + } + private static string? Validate(Route route, JsonObject body) { if (ReferenceEquals(route, Mark)) @@ -778,8 +969,8 @@ public static byte[] ToPublicBytes(UpstreamResponse r) /// public static class CurationDeskMemo { - /// Budget of each store. A feed page is tens of KB; this is thousands of them. - internal const long BudgetBytes = 64L * 1024 * 1024; + /// Budget of each store (DESK_MEMO_BYTES; 64 MiB by default). + internal static readonly long BudgetBytes = Config.DeskMemoBytes; /// /// How long a last-good body stays eligible as a fallback. Long enough to diff --git a/dotnet/parity/driver.py b/dotnet/parity/driver.py index 0001b40d..c03a52bb 100644 --- a/dotnet/parity/driver.py +++ b/dotnet/parity/driver.py @@ -244,8 +244,9 @@ def norm_body(text): CURATION_DESK_DIVERGENCE = ( "Curation desk gateway route added after the port. The reference build has no such " "route and answers 404 (POST) or the unmatched-GET template page; this one answers " - "503 while its shared secret is unconfigured, 401 for a missing or invalid signed " - "code, 400 for a rejected path or body, and otherwise proxies. Deterministic and " + "503 while its shared secret is unconfigured (before it validates anything), 401 " + "for a missing or invalid signed code, 400 for a rejected path or body, and " + "otherwise proxies. Deterministic and " "additive -- no behavior the reference ever had is changing." ) From 238d1251ed7c34cf5183bc96a27cfee93f8cd099 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 13:32:17 +0000 Subject: [PATCH 5/9] test: pin the desk gateway review findings A response writer stalled on one key must not hold a second read of that key past the fill; an HTML 200 and a scalar JSON 200 carry no Cache-Control and are not memoized, while a last-good answer to one does; an error body keeps the private-key fence; a write answers 503 while unconfigured for a signed body and an anonymous one alike, without validating either; a trailing newline or a non-ASCII digit fails the patterns; a limit past the int range clamps; the roster feed body and the tick id lists follow the public value rules; a lone surrogate survives the strip and the memo. Cache policy theory rows are keyed by route so no two rows share a test id, and the validation memo TTL is wide enough that a loaded runner cannot expire it between two back-to-back calls. --- dotnet/EcencyApi.Tests/CachePolicyTests.cs | 56 ++++---- .../EcencyApi.Tests/CurationDeskAuthTests.cs | 133 ++++++++++++++++-- .../CurationDeskPayloadTests.cs | 73 ++++++++++ .../CurationDeskPublicPayloadTests.cs | 38 +++++ .../EcencyApi.Tests/CurationDeskQueryTests.cs | 25 ++++ .../CurationDeskTestSupport.cs | 42 ++++++ 6 files changed, 329 insertions(+), 38 deletions(-) diff --git a/dotnet/EcencyApi.Tests/CachePolicyTests.cs b/dotnet/EcencyApi.Tests/CachePolicyTests.cs index 5d6c61f7..fcf70b88 100644 --- a/dotnet/EcencyApi.Tests/CachePolicyTests.cs +++ b/dotnet/EcencyApi.Tests/CachePolicyTests.cs @@ -10,49 +10,55 @@ namespace EcencyApi.Tests; /// public class CachePolicyTests { - public static TheoryData AllPolicies() => + // Keyed by route: several routes share a policy string, and a theory row + // that repeats its arguments is a duplicate test id that xUnit reports as a + // skip rather than running. + public static TheoryData AllPolicies() => new() { - CachePolicy.ProMembers, CachePolicy.Announcements, CachePolicy.PostTips, - CachePolicy.CurationDeskFeed, CachePolicy.CurationDeskStatus, CachePolicy.CurationDeskRoster, - CachePolicy.CurationDeskRecommendations, CachePolicy.CurationDeskPost, + { "pro-members", CachePolicy.ProMembers }, + { "announcements", CachePolicy.Announcements }, + { "post-tips", CachePolicy.PostTips }, + { "desk-feed", CachePolicy.CurationDeskFeed }, + { "desk-status", CachePolicy.CurationDeskStatus }, + { "desk-roster", CachePolicy.CurationDeskRoster }, + { "desk-recommendations", CachePolicy.CurationDeskRecommendations }, + { "desk-post", CachePolicy.CurationDeskPost }, }; - public static TheoryData DeskPolicies() => + public static TheoryData DeskPolicies() => new() { - { CachePolicy.CurationDeskFeed, 30 }, - { CachePolicy.CurationDeskStatus, 15 }, - { CachePolicy.CurationDeskRoster, 600 }, - { CachePolicy.CurationDeskRecommendations, 30 }, - { CachePolicy.CurationDeskPost, 15 }, + { "desk-feed", CachePolicy.CurationDeskFeed, 30 }, + { "desk-status", CachePolicy.CurationDeskStatus, 15 }, + { "desk-roster", CachePolicy.CurationDeskRoster, 600 }, + { "desk-recommendations", CachePolicy.CurationDeskRecommendations, 30 }, + { "desk-post", CachePolicy.CurationDeskPost, 15 }, }; [Theory] [MemberData(nameof(AllPolicies))] - public void EveryPolicyIsPubliclyCacheableWithAMaxAge(string policy) + public void EveryPolicyIsPubliclyCacheableWithAMaxAge(string route, string policy) { - Assert.StartsWith("public, max-age=", policy); + Assert.True(policy.StartsWith("public, max-age=", StringComparison.Ordinal), route + ": " + policy); Assert.DoesNotContain("no-store", policy); Assert.DoesNotContain("private", policy); } [Theory] [MemberData(nameof(AllPolicies))] - public void APolicyOnlyAppliesToASuccessfulResponse(string policy) + public void APolicyOnlyAppliesToASuccessfulResponse(string route, string policy) { Assert.Equal(policy, CachePolicy.ForStatus(200, policy)); - // Pipe() turns upstream transport failures into these after the handler - // has already attached the policy. Caching one would keep a healthy + // Pipe() turns upstream transport failures into 504/500 after the + // handler has already attached the policy, and an upstream error + // passthrough keeps its own status. Caching either would keep a healthy // endpoint broken for the whole max-age. - Assert.Null(CachePolicy.ForStatus(504, policy)); - Assert.Null(CachePolicy.ForStatus(500, policy)); - - // Upstream error passthroughs must not be cached either. - Assert.Null(CachePolicy.ForStatus(404, policy)); - Assert.Null(CachePolicy.ForStatus(401, policy)); - Assert.Null(CachePolicy.ForStatus(429, policy)); + foreach (var status in new[] { 504, 500, 404, 401, 429 }) + { + Assert.True(CachePolicy.ForStatus(status, policy) == null, route + " " + status); + } } [Fact] @@ -66,12 +72,12 @@ public void AnnouncementsOutliveTheOtherPolicies() [Theory] [MemberData(nameof(DeskPolicies))] - public void DeskPoliciesRevalidateInTheBrowserAndAreSharedForTheirSMaxAge(string policy, int sMaxAge) + public void DeskPoliciesRevalidateInTheBrowserAndAreSharedForTheirSMaxAge(string route, string policy, int sMaxAge) { // max-age=0 makes every browser poll revalidate; s-maxage is what shared // caches and the in-process memo hold the body for. - Assert.Equal(0, MaxAge(policy)); - Assert.Equal(sMaxAge, CachePolicy.SharedMaxAge(policy)); + Assert.True(MaxAge(policy) == 0, route); + Assert.True(CachePolicy.SharedMaxAge(policy) == sMaxAge, route); Assert.DoesNotContain("stale-while-revalidate", policy); } diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs index deba9b32..395c54e6 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -73,23 +73,28 @@ public async Task WithoutTheTokenReadsAnswer503BeforeAnyUpstreamCall() } [Fact] - public async Task WithoutTheTokenWritesAuthenticateThenAnswer503() + public async Task WithoutTheTokenWritesAnswer503BeforeValidatingAnything() { var upstream = Install(token: null); + var validations = 0; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult("alice"); }; foreach (var (name, handler, body) in SignedWrites()) { - var ok = Post("/private-api/curation-desk/" + name, body); - await handler(ok); - Assert.Equal(503, ok.Response.StatusCode); - Assert.Equal("curation desk not configured", Body(ok)); - - // An unauthenticated caller learns nothing about the configuration. - var anon = Post("/private-api/curation-desk/" + name, "{}"); - await handler(anon); - Assert.Equal(401, anon.Response.StatusCode); - Assert.Equal("Unauthorized", Body(anon)); + // A dark desk answers the same to a signed body and to an anonymous + // one, and neither costs a chain lookup: there is no work behind the + // route to authorize. + foreach (var payload in new[] { body, "{}" }) + { + var ctx = Post("/private-api/curation-desk/" + name, payload); + await handler(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } } + Assert.Equal(0, validations); Assert.Empty(upstream.Calls); } @@ -141,7 +146,9 @@ public async Task ASuccessfulValidationIsRememberedWithinTheTtlAndForgottenAfter var upstream = Install(); var validations = 0; PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult("alice"); }; - PrivateApi.DeskAuthMemoSeconds = 0.2; + // Wide enough that two back-to-back in-process calls cannot straddle it + // on a loaded runner; the delay below is what expires it. + PrivateApi.DeskAuthMemoSeconds = 2; var code = "memo-" + Guid.NewGuid().ToString("N"); var body = "{\"code\":\"" + code + "\",\"author\":\"bob\",\"permlink\":\"p\"}"; @@ -155,7 +162,7 @@ public async Task ASuccessfulValidationIsRememberedWithinTheTtlAndForgottenAfter await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body.Replace(code, code + "x"))); Assert.Equal(2, validations); - await Task.Delay(350); + await Task.Delay(2300); await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); Assert.Equal(3, validations); } @@ -359,6 +366,106 @@ public async Task ConcurrentReadsOfOneKeyMakeOneUpstreamCall() }); } + [Fact] + public async Task ASlowReaderDoesNotHoldTheGateOfItsKey() + { + var upstream = Install(); + var fill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => fill.Task; + + // A reader whose socket never drains takes the gate and starts the fill. + var slowBody = new BlockingBody(); + var slow = Get("/private-api/curation-desk/status"); + slow.Response.Body = slowBody; + var slowRequest = PrivateApi.CurationDeskStatus(slow); + await Task.Delay(100); + Assert.Single(upstream.Calls); + + // A second reader of the same key arrives during the fill, so it is + // queued on the gate rather than served from the memo. + var fast = Get("/private-api/curation-desk/status"); + var fastRequest = PrivateApi.CurationDeskStatus(fast); + await Task.Delay(100); + Assert.False(fastRequest.IsCompleted); + + fill.SetResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await slowBody.WriteReached; + + // The fill is done and the slow reader is stuck in its write. The bound + // is far under CurationDeskMemo.FillWait on purpose: waiting for the + // gate to time out would also "answer", just seconds later. + await fastRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal(200, fast.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(fast)); + Assert.False(slowRequest.IsCompleted); + + slowBody.Release(); + await slowRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal("{\"behind_seconds\":3}", slowBody.Text); + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task A200ThatIsNotAJsonBodyIsNeitherCachedNorMemoized() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(TextResponse(200, "gateway login")); + + var page = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(page); + await Start(page); + Assert.Equal(200, page.Response.StatusCode); + Assert.Equal("gateway login", Body(page)); + Assert.StartsWith("text/html", page.Response.ContentType); + Assert.Null(CacheControl(page)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + Assert.Equal(0, CurationDeskMemo.LastGood.Count); + + // A JSON body that is not an object or an array is the same case. + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "\"maintenance\"")); + var scalar = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(scalar); + await Start(scalar); + Assert.Equal(200, scalar.Response.StatusCode); + Assert.Null(CacheControl(scalar)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + + // And an answer the desk did give is preferred over that page, with the + // route's policy on it because it is a body this service holds. + CurationDeskMemo.ResetForTests(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await PrivateApi.CurationDeskStatus(Get("/private-api/curation-desk/status")); + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + upstream.Answer = _ => Task.FromResult(TextResponse(200, "gateway login")); + var stale = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(stale); + await Start(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(stale)); + Assert.StartsWith("application/json", stale.Response.ContentType); + Assert.Equal(CachePolicy.CurationDeskStatus, CacheControl(stale)); + } + + [Fact] + public async Task AJsonErrorBodyPassesThroughTheSameFenceAsAServedOne() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, + "{\"error\":\"unknown post\",\"excluded_reason\":\"abuser\",\"detail\":{\"set_by\":\"alice\"}}")); + + var ctx = Get("/private-api/curation-desk/post/good-karma/nope", "", + new[] { ("author", "good-karma"), ("permlink", "nope") }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(404, ctx.Response.StatusCode); + Assert.Null(CacheControl(ctx)); + var body = Body(ctx); + Assert.Contains("\"error\":\"unknown post\"", body); + Assert.DoesNotContain("excluded_reason", body); + Assert.DoesNotContain("set_by", body); + } + [Fact] public async Task AnUpstreamErrorAnswersWithTheLastGoodBody() { diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs index b385fd90..201fe04c 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -203,10 +203,80 @@ public void RosterFeedKeepsSeedOnlyForTheRandomOrderAndClampsLimit() var unknown = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"payout\",\"seed\":\"abcd1234\",\"view\":\"excluded\",\"hide_reviewed\":false}"); Assert.False(unknown.ContainsKey("sort")); Assert.False(unknown.ContainsKey("seed")); + // The roster is the only feed that lists excluded rows, so its view + // allowlist is the public one plus that. Assert.Equal("excluded", unknown["view"]!.GetValue()); Assert.False(unknown["hide_reviewed"]!.GetValue()); } + [Fact] + public void RosterFeedTakesOnlyASeedTheBackendCanHashWith() + { + foreach (var seed in new[] { "\"abc\"", "\"" + new string('a', 17) + "\"", "\"ABCD1234\"", "\"abcd 1234\"", "\"abcd1234\\n\"", "42", "null" }) + { + var payload = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":" + seed + "}"); + Assert.False(payload.ContainsKey("seed"), seed); + } + Assert.Equal(new string('a', 16), + Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":\"" + new string('a', 16) + "\"}")["seed"]!.GetValue()); + } + + [Fact] + public void RosterFeedFiltersFollowThePublicFeedsValueRules() + { + // Allowlists: a value the public feed drops is dropped here too, so the + // two feeds answer the same question for the same request. + var enums = Ok(CurationDeskWrites.RosterFeed, + "{\"view\":\"secret\",\"app\":\"hive\",\"window\":\"week\",\"community\":\"photography\",\"cursor\":\"a b\"}"); + Assert.Equal(new[] { "username" }, enums.Select(kv => kv.Key).ToArray()); + + var kept = Ok(CurationDeskWrites.RosterFeed, + "{\"view\":\"queue\",\"app\":\"ecency\",\"window\":\"full\",\"community\":\"hive-125125\",\"cursor\":\"s:abc.1:25\"}"); + Assert.Equal("queue", kept["view"]!.GetValue()); + Assert.Equal("ecency", kept["app"]!.GetValue()); + Assert.Equal("full", kept["window"]!.GetValue()); + Assert.Equal("hive-125125", kept["community"]!.GetValue()); + Assert.Equal("s:abc.1:25", kept["cursor"]!.GetValue()); + + // Trailing newlines and non-ASCII digits are not the value either. + var newline = Ok(CurationDeskWrites.RosterFeed, "{\"community\":\"hive-125125\\n\",\"cursor\":\"abc\\n\",\"view\":\"queue\\n\"}"); + Assert.Equal(new[] { "username" }, newline.Select(kv => kv.Key).ToArray()); + Assert.False(Ok(CurationDeskWrites.RosterFeed, "{\"community\":\"hive-\\u0661\\u0662\\u0663\\u0664\\u0665\"}").ContainsKey("community")); + + // Ranges clamp instead of travelling as sent, from a number or its + // string spelling; something that is neither is dropped. + var ranges = Ok(CurationDeskWrites.RosterFeed, + "{\"rep_min\":150,\"rep_max\":-1,\"min_words\":999999,\"max_words\":\"300\",\"limit\":\"99999999999\"}"); + Assert.Equal(100, ranges["rep_min"]!.GetValue()); + Assert.Equal(0, ranges["rep_max"]!.GetValue()); + Assert.Equal(50000, ranges["min_words"]!.GetValue()); + Assert.Equal(300, ranges["max_words"]!.GetValue()); + Assert.Equal(50, ranges["limit"]!.GetValue()); + + var unusable = Ok(CurationDeskWrites.RosterFeed, "{\"limit\":\"lots\",\"rep_min\":true,\"max_words\":null}"); + Assert.Equal(new[] { "username" }, unusable.Select(kv => kv.Key).ToArray()); + } + + [Fact] + public void TheTickNamesAtMost100IdsPerList() + { + var need = string.Join(",", Enumerable.Range(1, 150)); + var visible = string.Join(",", Enumerable.Range(1000, 101)); + var payload = Ok(CurationDeskWrites.Tick, + "{\"since\":\"t\",\"need\":[" + need + "],\"visible\":[" + visible + "]}"); + + Assert.Equal(CurationDeskWrites.MaxTickIds, ((JsonArray)payload["need"]!).Count); + Assert.Equal(CurationDeskWrites.MaxTickIds, ((JsonArray)payload["visible"]!).Count); + Assert.Equal(1, payload["need"]![0]!.GetValue()); + Assert.Equal(100, payload["need"]![99]!.GetValue()); + Assert.Equal(1000, payload["visible"]![0]!.GetValue()); + + // A list that already fits travels unchanged. + var short_ = Ok(CurationDeskWrites.Tick, "{\"since\":\"t\",\"need\":[1,2,3],\"visible\":[]}"); + Assert.Equal(3, ((JsonArray)short_["need"]!).Count); + Assert.Empty((JsonArray)short_["visible"]!); + } + // ---- route 5 path -------------------------------------------------------- [Fact] @@ -242,6 +312,9 @@ public void RealAuthorsAndPermlinksMapToTheUpstreamPathUnchanged() [InlineData("good-karma", "a_b")] [InlineData("", "p")] [InlineData("undefined-undefined", "p")] + // `$` would match before a trailing newline; these anchor with \A and \z. + [InlineData("good-karma\n", "p")] + [InlineData("good-karma", "p\n")] public void AnythingOutsideTheNameGrammarIsRejected(string author, string permlink) { Assert.Null(PrivateApi.CurationDeskPostPath(author, permlink)); diff --git a/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs index 55ec621e..653eedbf 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs @@ -83,6 +83,44 @@ public void ALeakyBodyIsReserializedWithoutTheKeys() Assert.Contains("\"generated_at\":\"t\"", text); } + [Fact] + public void ALoneSurrogateSurvivesTheStrip() + { + // JavaScript strings are arbitrary UTF-16 and a title can carry half a + // surrogate pair. Stripping a key next to one re-serializes the tree, so + // that path has to go through JsJson.Stringify: System.Text.Json's writer + // throws on a lone surrogate and would turn one odd title into a 500 for + // the whole feed page. + var r = JsonResponse(200, + "{\"items\":[{\"title\":\"lone \\ud83d end\",\"note\":\"secret\"}],\"generated_at\":\"t\"}"); + + var bytes = CurationDeskPublicPayload.ToPublicBytes(r); + var text = Encoding.UTF8.GetString(bytes); + Assert.DoesNotContain("\"note\"", text); + Assert.Contains("\\ud83d", text); + Assert.Contains("\"generated_at\":\"t\"", text); + } + + [Fact] + public async Task ALoneSurrogateIsServedAndMemoizedTheSameWay() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, + "{\"items\":[{\"title\":\"lone \\ud83d end\",\"note\":\"secret\"}],\"generated_at\":\"t\"}")); + + var ctx = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + var served = Body(ctx); + Assert.DoesNotContain("\"note\"", served); + Assert.Contains("\\ud83d", served); + + var again = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(again); + Assert.Equal(served, Body(again)); + Assert.Single(upstream.Calls); + } + [Fact] public async Task EveryPublicRouteServesAndMemoizesTheStrippedBody() { diff --git a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs index fa6ced8c..db70509e 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs @@ -49,6 +49,9 @@ public void DefaultsAreDroppedSoTheyCollapseOntoTheBarePath() [InlineData("1", "limit=1")] [InlineData("50", "limit=50")] [InlineData("999", "limit=50")] + // Beyond int.MaxValue: "as many as you can", not "give me the default". + [InlineData("99999999999", "limit=50")] + [InlineData("-99999999999", "limit=1")] [InlineData("25", "")] [InlineData("abc", "")] [InlineData("1e1", "")] @@ -135,6 +138,28 @@ public void CommunityMustBeAHiveCommunityName() Assert.Equal("curation/desk/feed", Feed(("community", "hive-125125/x"))); } + [Fact] + public void AValueWithATrailingNewlineIsNotTheValue() + { + // `$` in .NET also matches before a trailing newline, so every pattern + // here anchors with \A and \z instead. + Assert.Equal("curation/desk/feed", Feed(("cursor", "abc\n"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-125125\n"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "queue\n"))); + Assert.Equal("curation/desk/feed", Feed(("view", "queue\n"))); + Assert.Equal("curation/desk/recommendations", Recommendations(("cursor", "abc\n"))); + } + + [Fact] + public void CommunityDigitsAreAsciiDigitsOnly() + { + // .NET's `\d` matches every Unicode decimal digit; a community name is + // five or six ASCII digits and nothing else. + Assert.Equal("curation/desk/feed", Feed(("community", "hive-\u0661\u0662\u0663\u0664\u0665"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-\u09E7\u09E8\u09E9\u09EA\u09EB\u09EC"))); + Assert.Equal("curation/desk/feed?community=hive-125125", Feed(("community", "hive-125125"))); + } + [Fact] public void ParametersAreEmittedInOneFixedOrderWhateverTheClientSent() { diff --git a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs index c5826054..515cc08b 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs @@ -41,6 +41,48 @@ public async Task RunStarting() } } + /// + /// A response body whose writes block until the test releases them, so one + /// request can be held inside its response write while another runs. Stands + /// in for a reader on a slow connection. + /// + public sealed class BlockingBody : Stream + { + private readonly MemoryStream _inner = new(); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _started = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Completes once the handler has reached its first write. + public Task WriteReached => _started.Task; + + public void Release() => _release.TrySetResult(); + + public string Text => Encoding.UTF8.GetString(_inner.ToArray()); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + _started.TrySetResult(); + await _release.Task; + await _inner.WriteAsync(buffer, cancellationToken); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override void Write(byte[] buffer, int offset, int count) => + WriteAsync(buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + } + public sealed record Call(string Endpoint, HttpMethod Method, List> Headers, JsonNode? Payload) { public string? Header(string name) => From 426524470e561f994fd8c985a32856a150a15b70 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 14:26:54 +0000 Subject: [PATCH 6/9] fix: count desk gate users, clamp body numbers, fail closed on route 5 The single-flight gate was dropped from its table on the semaphore count alone, so a reader handed the gate before that cleanup could later wait on an entry nobody owns while the next reader filled the same key through a replacement. Two fills of one key then finish in any order and the older answer can be stored last. Users are counted under the table lock instead, and the entry is dropped only when the last one leaves and the key still maps to that same gate. The roster feed body cast any finite JSON number to an int, so 1.9 travelled as 1 while the query string dropped it. Only whole numbers are limits, reputations and word counts now: a fraction is dropped, a string still has to spell a plain signed integer, and a number is judged by its value so 1e6 and 1000000 clamp alike in a body and in a query string. Route 5 read its path values before the shared unconfigured check, so a dark desk answered 400 there and 503 everywhere else. It answers the documented 503 first, before it looks at anything the caller sent. --- .../EcencyApi.Tests/CurationDeskAuthTests.cs | 114 ++++++++++++++- .../CurationDeskPayloadTests.cs | 30 ++++ .../EcencyApi.Tests/CurationDeskQueryTests.cs | 9 ++ .../Handlers/PrivateApi.CurationDesk.cs | 132 ++++++++++++++---- 4 files changed, 260 insertions(+), 25 deletions(-) diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs index 395c54e6..80a91c72 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -366,6 +366,94 @@ public async Task ConcurrentReadsOfOneKeyMakeOneUpstreamCall() }); } + [Fact] + public async Task AKeysGateIsKeptWhileAnyReaderStillHoldsIt() + { + Install(); + const string key = "curation/desk/feed?limit=10"; + + // Two readers take the key's gate. The second stands for a request that + // has been handed the gate and has not reached its wait yet. + var first = CurationDeskMemo.GateFor(key); + var late = CurationDeskMemo.GateFor(key); + Assert.Same(first, late); + + // The first reader fills and hands the gate back. + Assert.True(await first.Semaphore.WaitAsync(TimeSpan.Zero)); + first.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, first); + + // A third reader arrives after that. It must land on the gate the late + // reader is about to wait on, or the two of them fill one key at once + // and the fill that finishes second stores its answer over the first. + var newcomer = CurationDeskMemo.GateFor(key); + Assert.Same(late, newcomer); + Assert.True(await newcomer.Semaphore.WaitAsync(TimeSpan.Zero)); + Assert.False(await late.Semaphore.WaitAsync(TimeSpan.Zero)); + + newcomer.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, newcomer); + Assert.Equal(1, CurationDeskMemo.GateCount); + + // With the last reader gone the entry is dropped, so a scan over many + // distinct keys leaves no semaphore per key behind. + CurationDeskMemo.ReleaseGate(key, late); + Assert.Equal(0, CurationDeskMemo.GateCount); + var afterwards = CurationDeskMemo.GateFor(key); + Assert.NotSame(late, afterwards); + CurationDeskMemo.ReleaseGate(key, afterwards); + Assert.Equal(0, CurationDeskMemo.GateCount); + } + + [Fact] + public async Task ALateWaiterNeverFillsAKeyBesideTheReaderFillingIt() + { + var upstream = Install(); + const string key = "curation/desk/status"; + + // A reader that has been handed the key's gate and has not waited on it + // yet: the fill below must not be able to drop the gate under it. + var late = CurationDeskMemo.GateFor(key); + + var firstFill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => firstFill.Task; + var first = Get("/private-api/curation-desk/status"); + var firstRequest = PrivateApi.CurationDeskStatus(first); + await Task.Delay(100); + Assert.Single(upstream.Calls); + firstFill.SetResult(JsonResponse(200, "{\"behind_seconds\":1}")); + await firstRequest; + Assert.Equal("{\"behind_seconds\":1}", Body(first)); + + // That entry lapses (simulated), so the next reader fills again. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + var secondFill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => secondFill.Task; + var second = Get("/private-api/curation-desk/status"); + var secondRequest = PrivateApi.CurationDeskStatus(second); + await Task.Delay(100); + Assert.Equal(2, upstream.Calls.Count); + + // The late reader reaches its wait now and must queue behind the fill in + // flight instead of being admitted beside it. + Assert.False(await late.Semaphore.WaitAsync(TimeSpan.Zero)); + + secondFill.SetResult(JsonResponse(200, "{\"behind_seconds\":2}")); + await secondRequest; + Assert.Equal("{\"behind_seconds\":2}", Body(second)); + + // Once admitted it finds the answer memoized, so the key was filled once + // per reader that needed it and never twice at a time. + Assert.True(await late.Semaphore.WaitAsync(TimeSpan.FromSeconds(3))); + Assert.True(CurationDeskMemo.TryGetFresh(key, out var memoized, out _)); + Assert.Equal("{\"behind_seconds\":2}", System.Text.Encoding.UTF8.GetString(memoized)); + late.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, late); + Assert.Equal(2, upstream.Calls.Count); + Assert.Equal(0, CurationDeskMemo.GateCount); + } + [Fact] public async Task ASlowReaderDoesNotHoldTheGateOfItsKey() { @@ -521,7 +609,7 @@ public async Task ANon200IsPipedThroughAndNotMemoized() public async Task AnInvalidPostPathIs400BeforeAnyUpstreamCall() { var upstream = Install(); - foreach (var (author, permlink) in new[] { ("..", "p"), ("good-karma", "a/b"), ("good-karma", "p?x=1"), ("x", "p") }) + foreach (var (author, permlink) in MalformedPostPaths) { var ctx = Get("/private-api/curation-desk/post/x/y", "", new[] { ("author", author), ("permlink", permlink) }); await PrivateApi.CurationDeskPost(ctx); @@ -532,4 +620,28 @@ public async Task AnInvalidPostPathIs400BeforeAnyUpstreamCall() } Assert.Empty(upstream.Calls); } + + [Fact] + public async Task WithoutTheTokenAMalformedPostPathIs503LikeEveryOtherRoute() + { + var upstream = Install(token: null); + + // While the desk is dark every route answers the same, so this one does + // not single itself out by reporting on the path it was given. + foreach (var (author, permlink) in MalformedPostPaths) + { + var ctx = Get("/private-api/curation-desk/post/x/y", "", new[] { ("author", author), ("permlink", permlink) }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } + + private static readonly (string Author, string Permlink)[] MalformedPostPaths = + { + ("..", "p"), ("good-karma", "a/b"), ("good-karma", "p?x=1"), ("x", "p"), + }; } diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs index 201fe04c..2c032026 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -257,6 +257,36 @@ public void RosterFeedFiltersFollowThePublicFeedsValueRules() Assert.Equal(new[] { "username" }, unusable.Select(kv => kv.Key).ToArray()); } + [Fact] + public void RosterFeedNumbersAreWholeNumbersOrNothing() + { + // These names count rows, reputations and words. A fraction is none of + // them: truncating 1.9 to 1 would forward a filter nobody asked for, so + // it is dropped and the backend applies its default, exactly as the + // query string does with `limit=1.9`. + var fractions = Ok(CurationDeskWrites.RosterFeed, + "{\"limit\":1.9,\"rep_min\":10.5,\"rep_max\":99.9,\"min_words\":0.5,\"max_words\":300.25}"); + Assert.Equal(new[] { "username" }, fractions.Select(kv => kv.Key).ToArray()); + + // A whole number is kept, as a number or as its plain spelling. + Assert.Equal(12, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":12}")["limit"]!.GetValue()); + Assert.Equal(12, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":\"12\"}")["limit"]!.GetValue()); + Assert.Equal(40, Ok(CurationDeskWrites.RosterFeed, "{\"rep_min\":40}")["rep_min"]!.GetValue()); + + // JSON keeps no spelling of a number, so 1e6 is the number 1000000 and + // clamps to the bound the same way that value does in a query string. + Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1e6}")["limit"]!.GetValue()); + Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1000000}")["limit"]!.GetValue()); + + // A string is read by the query string's rule, so only a plain signed + // integer is a number there. + foreach (var spelling in new[] { "\"1e6\"", "\"1.9\"", "\"12.0\"", "\" 12\"", "\"0x0c\"" }) + { + Assert.False( + Ok(CurationDeskWrites.RosterFeed, "{\"limit\":" + spelling + "}").ContainsKey("limit"), spelling); + } + } + [Fact] public void TheTickNamesAtMost100IdsPerList() { diff --git a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs index db70509e..78bc8590 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs @@ -52,9 +52,18 @@ public void DefaultsAreDroppedSoTheyCollapseOntoTheBarePath() // Beyond int.MaxValue: "as many as you can", not "give me the default". [InlineData("99999999999", "limit=50")] [InlineData("-99999999999", "limit=1")] + // A plain whole number past the range clamps; the body path reads the same + // value out of JSON and answers the same. + [InlineData("1000000", "limit=50")] [InlineData("25", "")] [InlineData("abc", "")] + // Text is a number only when it spells a plain signed integer, here and for + // a string in a roster feed body: a fraction or an exponent is dropped + // rather than truncated. [InlineData("1e1", "")] + [InlineData("1e6", "")] + [InlineData("1.9", "")] + [InlineData("12.0", "")] [InlineData("", "")] public void LimitIsClampedToItsRange(string given, string expected) { diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs index 0f6eaf1b..bb3e670a 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -99,6 +98,16 @@ public static Task CurationDeskRecommendations(HttpContext ctx) => // GET /private-api/curation-desk/post/{author}/{permlink} public static async Task CurationDeskPost(HttpContext ctx) { + // The unconfigured answer comes first, before this route looks at + // anything the caller sent. A dark desk answers 503 on every route the + // same way; answering 400 here instead would make this one route report + // on its own path grammar while the other four report nothing. + if (DeskToken == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + var author = ctx.Request.RouteValues["author"]?.ToString() ?? ""; var permlink = ctx.Request.RouteValues["permlink"]?.ToString() ?? ""; var path = CurationDeskPostPath(author, permlink); @@ -187,19 +196,24 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string string? errorText = null; UpstreamResponse? passthrough = null; + // Held for the whole block, so the gate cannot be dropped and replaced + // between being handed out and being waited on. var gate = CurationDeskMemo.GateFor(endpoint); - if (!await gate.WaitAsync(CurationDeskMemo.FillWait)) - { - // Someone else's fill is taking longer than a whole upstream timeout. - // Do not stack another one behind it; answer from what is known. - await ServeLastGoodOr(ctx, endpoint, policy, 504, "Upstream Timeout"); - return; - } + var entered = false; + var gateTimedOut = false; try { + entered = await gate.Semaphore.WaitAsync(CurationDeskMemo.FillWait); + if (!entered) + { + // Someone else's fill is taking longer than a whole upstream + // timeout. Do not stack another one behind it; answer from what + // is known, once the gate has been handed back. + gateTimedOut = true; + } // The fill that held the gate may have landed while this one queued. - if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType)) + else if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType)) { bytes = fresh; bytesType = freshType; @@ -234,10 +248,19 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string } finally { - gate.Release(); + if (entered) + { + gate.Semaphore.Release(); + } CurationDeskMemo.ReleaseGate(endpoint, gate); } + if (gateTimedOut) + { + await ServeLastGoodOr(ctx, endpoint, policy, 504, "Upstream Timeout"); + return; + } + if (bytes != null) { await SendPublicJson(ctx, policy, bytesType!, bytes); @@ -795,8 +818,16 @@ private static void KeepMatching(JsonObject payload, string key, Func - /// Clamp a numeric field into [min, max], accepting the number or its string - /// spelling; a value that is neither is dropped rather than forwarded. + /// Clamp a whole-number field into [min, max], accepting the number or its + /// string spelling; a value that is neither is dropped rather than forwarded. + /// + /// These names count rows, reputations and words, so only an integral value + /// is one of them: `1.9` is not "1", it is a client sending something else, + /// and truncating it would forward a filter nobody asked for. Strings go + /// through the query-string parser, so a body and a query string accept the + /// same spellings. A number is judged by its value, not its spelling, since + /// JSON parsing keeps no spelling: `1e6` and `1000000` are the same number + /// and clamp to the same bound, exactly as `1000000` does in a query string. /// private static void Clamp(JsonObject payload, string key, int min, int max) { @@ -806,7 +837,7 @@ private static void Clamp(JsonObject payload, string key, int min, int max) } var node = payload[key]; int? value = JsVal.AsNumber(node) is { } number - ? (int)Math.Clamp(number, min, max) + ? (double.IsInteger(number) ? (int?)Math.Clamp(number, min, max) : null) : CurationDeskQuery.ClampValue(JsVal.AsString(node), min, max); if (value is { } clamped) { @@ -990,13 +1021,27 @@ public static class CurationDeskMemo internal static BytesCache LastGood = new(BudgetBytes); /// - /// One fill per key at a time. Gates are created on demand and dropped once - /// released with nobody holding them, so a scan over many distinct keys does - /// not leave a semaphore per key behind. A request that read a gate just - /// before it was dropped can start a second fill; that costs one duplicate - /// upstream call, not correctness. + /// One fill of one key at a time. A reader takes the key's gate, waits on + /// its semaphore, fills and hands the gate back. + /// + /// Users are counted rather than inferred from the semaphore: a reader that + /// has been handed the gate but has not reached its wait yet is a user, and + /// dropping the entry under it (the semaphore looks free, because that + /// reader has not taken it) would let the next reader create a replacement + /// and fill the same key beside it. Two fills of one key can then finish out + /// of order and store the older answer last. /// - private static readonly ConcurrentDictionary Gates = new(); + internal sealed class Gate + { + /// Admission to the fill; one holder at a time. + internal readonly SemaphoreSlim Semaphore = new(1, 1); + + /// Readers holding this gate. Guarded by . + internal int Users; + } + + private static readonly Dictionary Gates = new(StringComparer.Ordinal); + private static readonly object GateLock = new(); public static bool TryGetFresh(string key, out byte[] bytes, out string contentType) { @@ -1018,20 +1063,59 @@ public static void Store(string key, byte[] bytes, string contentType, int ttlSe LastGood.Set(key, bytes, LastGoodTtlMs, contentType); } - internal static SemaphoreSlim GateFor(string key) => Gates.GetOrAdd(key, _ => new SemaphoreSlim(1, 1)); + /// + /// The gate of a key, counting this caller as one of its users. Every call + /// must be paired with a , whether or not the + /// caller went on to take the semaphore. + /// + internal static Gate GateFor(string key) + { + lock (GateLock) + { + if (!Gates.TryGetValue(key, out var gate)) + { + gate = new Gate(); + Gates[key] = gate; + } + gate.Users++; + return gate; + } + } - internal static void ReleaseGate(string key, SemaphoreSlim gate) + /// + /// Give a gate back. The entry is dropped only when the last user leaves and + /// the key still maps to this same gate, so a scan over many distinct keys + /// leaves no semaphore per key behind while no in-flight reader ever loses + /// the gate it is about to wait on. + /// + internal static void ReleaseGate(string key, Gate gate) { - if (gate.CurrentCount == 1) + lock (GateLock) { - Gates.TryRemove(new KeyValuePair(key, gate)); + if (--gate.Users > 0) + { + return; + } + if (Gates.TryGetValue(key, out var current) && ReferenceEquals(current, gate)) + { + Gates.Remove(key); + } } } + /// Gates currently held, for the tests that pin the cleanup. + internal static int GateCount + { + get { lock (GateLock) { return Gates.Count; } } + } + internal static void ResetForTests() { Fresh = new BytesCache(BudgetBytes); LastGood = new BytesCache(BudgetBytes); - Gates.Clear(); + lock (GateLock) + { + Gates.Clear(); + } } } From fc3469f1cd090f929f41415084cd1ce804e57e53 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 14:35:23 +0000 Subject: [PATCH 7/9] test: bound the late-waiter gate test and pin negative body numbers --- dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs | 4 ++-- dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs index 80a91c72..f4f43620 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -422,7 +422,7 @@ public async Task ALateWaiterNeverFillsAKeyBesideTheReaderFillingIt() await Task.Delay(100); Assert.Single(upstream.Calls); firstFill.SetResult(JsonResponse(200, "{\"behind_seconds\":1}")); - await firstRequest; + await firstRequest.WaitAsync(TimeSpan.FromSeconds(3)); Assert.Equal("{\"behind_seconds\":1}", Body(first)); // That entry lapses (simulated), so the next reader fills again. @@ -440,7 +440,7 @@ public async Task ALateWaiterNeverFillsAKeyBesideTheReaderFillingIt() Assert.False(await late.Semaphore.WaitAsync(TimeSpan.Zero)); secondFill.SetResult(JsonResponse(200, "{\"behind_seconds\":2}")); - await secondRequest; + await secondRequest.WaitAsync(TimeSpan.FromSeconds(3)); Assert.Equal("{\"behind_seconds\":2}", Body(second)); // Once admitted it finds the answer memoized, so the key was filled once diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs index 2c032026..7c718d12 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -277,6 +277,8 @@ public void RosterFeedNumbersAreWholeNumbersOrNothing() // clamps to the bound the same way that value does in a query string. Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1e6}")["limit"]!.GetValue()); Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1000000}")["limit"]!.GetValue()); + Assert.Equal(1, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":-5}")["limit"]!.GetValue()); + Assert.Equal(new[] { "username" }, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":-1.5}").Select(kv => kv.Key).ToArray()); // A string is read by the query string's rule, so only a plain signed // integer is a number there. From 3d2afa485944b60a4daa650a91bbe37fb36b2239 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 16:19:50 +0000 Subject: [PATCH 8/9] fix: age desk memo hits instead of restarting their shared window A body served from the in-process memo went out with the route's full s-maxage, so a roster answered from the memo just before its 600 s TTL lapsed could be held downstream for another 600 s: nearly twenty minutes of one answer across the two layers. Memo entries now record when they were filled. A hit sends the rest of its window, floored at one second, plus an Age header so a shared cache that computes freshness from Age expires it at the same moment as one that only reads s-maxage. A fresh fill is unchanged (full window, Age 0). A last-good body served after an upstream failure is handled separately: it carries a short five second window rather than the route's own, so a backend that recovers is picked up within a poll or two. --- dotnet/EcencyApi.Tests/CachePolicyTests.cs | 47 ++++++++ .../EcencyApi.Tests/CurationDeskAuthTests.cs | 94 +++++++++++++++- .../CurationDeskTestSupport.cs | 27 +++++ .../Handlers/PrivateApi.CurationDesk.cs | 104 +++++++++++++----- .../EcencyApi/Infrastructure/CachePolicy.cs | 58 ++++++++++ .../Infrastructure/HttpContextExtensions.cs | 28 +++++ 6 files changed, 325 insertions(+), 33 deletions(-) diff --git a/dotnet/EcencyApi.Tests/CachePolicyTests.cs b/dotnet/EcencyApi.Tests/CachePolicyTests.cs index fcf70b88..d1e5c392 100644 --- a/dotnet/EcencyApi.Tests/CachePolicyTests.cs +++ b/dotnet/EcencyApi.Tests/CachePolicyTests.cs @@ -81,6 +81,53 @@ public void DeskPoliciesRevalidateInTheBrowserAndAreSharedForTheirSMaxAge(string Assert.DoesNotContain("stale-while-revalidate", policy); } + [Theory] + [MemberData(nameof(DeskPolicies))] + public void AnAgedPolicyOffersOnlyTheRestOfTheSharedWindow(string route, string policy, int sMaxAge) + { + // A body served from the in-process memo has already spent part of its + // life there; handing a shared cache a fresh window would let the two + // layers serve one answer for a lifetime each, in series. + Assert.Equal(policy, CachePolicy.Aged(policy, 0)); + Assert.Equal(sMaxAge - 1, CachePolicy.SharedMaxAge(CachePolicy.Aged(policy, 1))); + + foreach (var age in new[] { 1, sMaxAge - 1, sMaxAge, sMaxAge + 1, sMaxAge * 10 }) + { + var aged = CachePolicy.Aged(policy, age); + var remaining = CachePolicy.SharedMaxAge(aged); + + // Floored at a second, never longer than what is left, and still a + // policy of the same shape. + Assert.True(remaining >= 1, route + " " + age); + Assert.True(remaining <= Math.Max(1, sMaxAge - age), route + " " + age); + Assert.StartsWith("public, max-age=0, s-maxage=", aged); + Assert.Null(CachePolicy.ForStatus(504, aged)); + } + } + + [Theory] + [MemberData(nameof(DeskPolicies))] + public void TheStalePolicyIsShortAndNeverLongerThanTheRouteItself(string route, string policy, int sMaxAge) + { + // Served because the upstream call failed: still cacheable, so a burst + // does not all queue behind a struggling backend, but only for seconds. + var stale = CachePolicy.Stale(policy); + Assert.Equal("public, max-age=0, s-maxage=" + CachePolicy.StaleSharedMaxAge, stale); + Assert.True(CachePolicy.SharedMaxAge(stale) < sMaxAge, route); + Assert.Null(CachePolicy.ForStatus(500, stale)); + } + + [Fact] + public void ShorteningAPolicyLeavesItsOtherDirectivesAlone() + { + // A policy whose shared window is its max-age gains an s-maxage rather + // than having what browsers were told rewritten under them. + var aged = CachePolicy.Aged(CachePolicy.ProMembers, 100); + Assert.StartsWith(CachePolicy.ProMembers, aged); + Assert.Contains("stale-while-revalidate=3600", aged); + Assert.Equal(500, CachePolicy.SharedMaxAge(aged)); + } + [Fact] public void SharedMaxAgeFallsBackToMaxAgeWhenAPolicyHasNoSharedDirective() { diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs index f4f43620..ce15be4e 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -255,6 +255,8 @@ public async Task ReadsCarryTheirPolicyOnlyOnA200() await Start(ok); Assert.Equal(200, ok.Response.StatusCode); Assert.Equal(policy, CacheControl(ok)); + // Filled by this request, so the whole window and no age spent yet. + Assert.Equal("0", Age(ok)); Assert.StartsWith("application/json", ok.Response.ContentType); CurationDeskMemo.ResetForTests(); @@ -264,6 +266,7 @@ public async Task ReadsCarryTheirPolicyOnlyOnA200() await Start(missing); Assert.Equal(404, missing.Response.StatusCode); Assert.Null(CacheControl(missing)); + Assert.Null(Age(missing)); Assert.Equal("{\"error\":\"not found\"}", Body(missing)); upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); @@ -274,6 +277,7 @@ public async Task ReadsCarryTheirPolicyOnlyOnA200() Assert.Equal(504, timeout.Response.StatusCode); Assert.Equal("Upstream Timeout", Body(timeout)); Assert.Null(CacheControl(timeout)); + Assert.Null(Age(timeout)); upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{}")); CurationDeskMemo.ResetForTests(); @@ -294,6 +298,84 @@ public async Task WritesAreNeverCacheable() } } + [Fact] + public async Task AMemoHitAdvertisesOnlyWhatIsLeftOfTheSharedWindow() + { + var upstream = Install(); + var clock = UseTestClock(); + const string body = "{\"curators\":[{\"username\":\"alice\"}]}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + + var fill = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(fill); + await Start(fill); + Assert.Equal(CachePolicy.CurationDeskRoster, CacheControl(fill)); + Assert.Equal("0", Age(fill)); + + // 590 s into the roster's 600 s window. Sending the whole window again + // here would let a shared cache hold this body for another 600 s on top + // of the 590 it has already lived in the memo. + clock.Advance(TimeSpan.FromSeconds(590)); + var hit = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(hit); + await Start(hit); + Assert.Equal(200, hit.Response.StatusCode); + Assert.Equal(body, Body(hit)); + Assert.Equal("public, max-age=0, s-maxage=10", CacheControl(hit)); + Assert.Equal("590", Age(hit)); + + // Both readers were answered from one upstream call. + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task AMemoHitAtTheEndOfItsWindowStaysCacheableForOneSecond() + { + var upstream = Install(); + var clock = UseTestClock(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await PrivateApi.CurationDeskStatus(Get("/private-api/curation-desk/status")); + + // Past the 15 s the status policy promises. The memo entry is about to + // lapse and refill, so the floor keeps this answer cacheable rather than + // sending s-maxage=0 to every reader in the last moment of a window. + clock.Advance(TimeSpan.FromSeconds(20)); + var hit = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(hit); + await Start(hit); + Assert.Equal("public, max-age=0, s-maxage=1", CacheControl(hit)); + Assert.Equal("20", Age(hit)); + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task TheLastGoodBodyCarriesTheShortWindowAndItsRealAge() + { + var upstream = Install(); + var clock = UseTestClock(); + const string body = "{\"curators\":[{\"username\":\"alice\"}]}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + await PrivateApi.CurationDeskRoster(Get("/private-api/curation-desk/roster")); + + // The fresh entry lapses (simulated) and the backend stops answering, so + // the next read falls back to the last good body, now minutes old. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + clock.Advance(TimeSpan.FromSeconds(120)); + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + + var stale = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(stale); + await Start(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal(body, Body(stale)); + + // Never the route's own window: a backend that recovers has to reach + // readers within a poll or two, not ten minutes later. + Assert.Equal("public, max-age=0, s-maxage=5", CacheControl(stale)); + Assert.Equal(CachePolicy.Stale(CachePolicy.CurationDeskRoster), CacheControl(stale)); + Assert.Equal("120", Age(stale)); + } + [Fact] public void EachPolicySharedMaxAgeIsTheMemoTtl() { @@ -324,10 +406,12 @@ public async Task ASecondReadWithinTheTtlIsServedFromTheMemoAsBytes() Assert.Equal(body, Body(second)); // Stored as the bytes that were served, keyed by the normalized endpoint. - Assert.True(CurationDeskMemo.Fresh.TryGet("curation/desk/feed?limit=10&sort=queue", out var stored, out var tag)); + Assert.True(CurationDeskMemo.TryGetFresh("curation/desk/feed?limit=10&sort=queue", + out var stored, out var storedType, out var storedAge)); Assert.IsType(stored); Assert.Equal(body, System.Text.Encoding.UTF8.GetString(stored)); - Assert.Equal("application/json; charset=utf-8", tag); + Assert.Equal("application/json; charset=utf-8", storedType); + Assert.Equal(0, storedAge); Assert.False(CurationDeskMemo.Fresh.TryGet("curation/desk/feed", out _)); } @@ -446,7 +530,7 @@ public async Task ALateWaiterNeverFillsAKeyBesideTheReaderFillingIt() // Once admitted it finds the answer memoized, so the key was filled once // per reader that needed it and never twice at a time. Assert.True(await late.Semaphore.WaitAsync(TimeSpan.FromSeconds(3))); - Assert.True(CurationDeskMemo.TryGetFresh(key, out var memoized, out _)); + Assert.True(CurationDeskMemo.TryGetFresh(key, out var memoized, out _, out _)); Assert.Equal("{\"behind_seconds\":2}", System.Text.Encoding.UTF8.GetString(memoized)); late.Semaphore.Release(); CurationDeskMemo.ReleaseGate(key, late); @@ -532,7 +616,9 @@ public async Task A200ThatIsNotAJsonBodyIsNeitherCachedNorMemoized() Assert.Equal(200, stale.Response.StatusCode); Assert.Equal("{\"behind_seconds\":3}", Body(stale)); Assert.StartsWith("application/json", stale.Response.ContentType); - Assert.Equal(CachePolicy.CurationDeskStatus, CacheControl(stale)); + + // A last-good body, so the short window rather than the route's own. + Assert.Equal(CachePolicy.Stale(CachePolicy.CurationDeskStatus), CacheControl(stale)); } [Fact] diff --git a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs index 515cc08b..6a7da1a4 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs @@ -198,6 +198,33 @@ public static string Body(HttpContext ctx) public static string? CacheControl(HttpContext ctx) => ctx.Response.Headers.TryGetValue("Cache-Control", out var v) ? v.ToString() : null; + public static string? Age(HttpContext ctx) => + ctx.Response.Headers.TryGetValue("Age", out var v) ? v.ToString() : null; + + /// + /// A clock the test moves by hand. The memo reads its fill times from this, + /// so a test can put an entry near the end of its window without sleeping + /// through it (and without a wall-clock bound that turns into a flaky test + /// on a loaded machine). Reset by . + /// + public sealed class TestClock + { + // An arbitrary fixed epoch: only differences matter. + private long _ms = 1_700_000_000_000; + + public long NowMs() => _ms; + + public void Advance(TimeSpan by) => _ms += (long)by.TotalMilliseconds; + } + + /// Hand the memo a movable clock and return it. + public static TestClock UseTestClock() + { + var clock = new TestClock(); + CurationDeskMemo.NowMs = clock.NowMs; + return clock; + } + /// Every public read, as (handler, request factory, policy). public static IEnumerable<(string Name, Func Handler, Func Request, string Policy)> PublicReads() { diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs index bb3e670a..a9728907 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -22,7 +23,9 @@ namespace EcencyApi.Handlers; /// every spelling of the same question collapses onto one memo entry and one /// shared-cache key, and the answer is memoized as bytes for exactly the /// s-maxage the response promises (single-flight per key, last-good on an -/// upstream error); +/// upstream error). A body served from that memo says how old it is and +/// offers shared caches only the rest of its window, so the two layers do +/// not each hold it for a full lifetime in series; /// - writes resolve the caller from the signed code (memoized briefly, see /// ) and forward only whitelisted /// body fields under that username; a client-supplied username or code never @@ -182,9 +185,9 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string return; } - if (CurationDeskMemo.TryGetFresh(endpoint, out var hit, out var hitType)) + if (CurationDeskMemo.TryGetFresh(endpoint, out var hit, out var hitType, out var hitAge)) { - await SendPublicJson(ctx, policy, hitType, hit); + await SendPublicJson(ctx, policy, hitType, hit, hitAge); return; } @@ -192,6 +195,8 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string // answer with, or the upstream response to pass through. byte[]? bytes = null; string? bytesType = null; + // How long this service has held those bytes; a fill made here is new. + var bytesAge = 0; var errorStatus = 0; string? errorText = null; UpstreamResponse? passthrough = null; @@ -213,10 +218,11 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string gateTimedOut = true; } // The fill that held the gate may have landed while this one queued. - else if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType)) + else if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType, out var freshAge)) { bytes = fresh; bytesType = freshType; + bytesAge = freshAge; } else { @@ -263,7 +269,7 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string if (bytes != null) { - await SendPublicJson(ctx, policy, bytesType!, bytes); + await SendPublicJson(ctx, policy, bytesType!, bytes, bytesAge); return; } @@ -280,9 +286,9 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string // answering the question this route asks, so a body it did answer with // is better than passing that on, and neither is worth memoizing. if ((response.Status >= 500 || response.Status == 200) - && CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + && CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType, out var staleAge)) { - await SendPublicJson(ctx, policy, staleType, stale); + await SendStaleJson(ctx, policy, staleType, stale, staleAge); return; } @@ -298,23 +304,39 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string private const string JsonContentType = "application/json; charset=utf-8"; /// - /// Send a JSON body this service holds (a memo hit, a fresh fill or a - /// last-good fallback) with the route's cache policy. Only these bodies are - /// publicly cacheable: an upstream passthrough carries whatever the backend - /// answered, which may be an error page or a body meant for one caller, so - /// it never gets a Cache-Control of ours. + /// Send a JSON body this service holds (a fresh fill, or a memo hit that is + /// already old) with the route's cache policy. + /// Only these bodies are publicly cacheable: an upstream passthrough carries + /// whatever the backend answered, which may be an error page or a body meant + /// for one caller, so it never gets a Cache-Control of ours. + /// + /// A hit goes out with the rest of its window rather than a new one, so the + /// memo and the caches downstream expire the same body at the same moment + /// instead of holding it for one lifetime each in series. + /// + private static async Task SendPublicJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) + { + ctx.CacheWhenOk(CachePolicy.Aged(policy, ageSeconds), ageSeconds); + await WriteBytes(ctx, 200, contentType, bytes); + } + + /// + /// Send a last-good body: a real answer from the backend, but one kept + /// because the call that should have replaced it failed. It carries a short + /// window instead of the route's own, so an upstream that comes back is + /// picked up within a poll or two rather than at the end of a full one. /// - private static async Task SendPublicJson(HttpContext ctx, string policy, string contentType, byte[] bytes) + private static async Task SendStaleJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) { - ctx.CacheWhenOk(policy); + ctx.CacheWhenOk(CachePolicy.Stale(policy), ageSeconds); await WriteBytes(ctx, 200, contentType, bytes); } private static async Task ServeLastGoodOr(HttpContext ctx, string endpoint, string policy, int status, string text) { - if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType)) + if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType, out var staleAge)) { - await SendPublicJson(ctx, policy, staleType, stale); + await SendStaleJson(ctx, policy, staleType, stale, staleAge); return; } await ctx.SendText(status, text); @@ -1020,6 +1042,15 @@ public static class CurationDeskMemo internal static BytesCache Fresh = new(BudgetBytes); internal static BytesCache LastGood = new(BudgetBytes); + /// Wall clock in milliseconds behind the fill times below. + internal static readonly Func SystemClock = () => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + /// + /// The clock the fill times are read from. Replaceable so a test can age an + /// entry rather than wait for it; production never moves it. + /// + internal static Func NowMs = SystemClock; + /// /// One fill of one key at a time. A reader takes the key's gate, waits on /// its semaphore, fills and hands the gate back. @@ -1043,24 +1074,38 @@ internal sealed class Gate private static readonly Dictionary Gates = new(StringComparer.Ordinal); private static readonly object GateLock = new(); - public static bool TryGetFresh(string key, out byte[] bytes, out string contentType) - { - var hit = Fresh.TryGet(key, out bytes, out var tag); - contentType = tag ?? "application/json; charset=utf-8"; - return hit; - } + /// + /// A stored entry carries when it was filled as well as what it is, so a hit + /// can say how much of its shared window is left. Both travel in the one tag + /// the byte cache keeps beside the bytes, so they are evicted together and + /// no side table can outlive or contradict an entry. + /// + private const char TagSeparator = '|'; + + private const string DefaultContentType = "application/json; charset=utf-8"; + + public static bool TryGetFresh(string key, out byte[] bytes, out string contentType, out int ageSeconds) => + Read(Fresh, key, out bytes, out contentType, out ageSeconds); - public static bool TryGetLastGood(string key, out byte[] bytes, out string contentType) + public static bool TryGetLastGood(string key, out byte[] bytes, out string contentType, out int ageSeconds) => + Read(LastGood, key, out bytes, out contentType, out ageSeconds); + + public static void Store(string key, byte[] bytes, string contentType, int ttlSeconds) { - var hit = LastGood.TryGet(key, out bytes, out var tag); - contentType = tag ?? "application/json; charset=utf-8"; - return hit; + var tag = NowMs().ToString(CultureInfo.InvariantCulture) + TagSeparator + contentType; + Fresh.Set(key, bytes, ttlSeconds * 1000, tag); + LastGood.Set(key, bytes, LastGoodTtlMs, tag); } - public static void Store(string key, byte[] bytes, string contentType, int ttlSeconds) + private static bool Read(BytesCache cache, string key, out byte[] bytes, out string contentType, out int ageSeconds) { - Fresh.Set(key, bytes, ttlSeconds * 1000, contentType); - LastGood.Set(key, bytes, LastGoodTtlMs, contentType); + var hit = cache.TryGet(key, out bytes, out var tag); + var split = tag?.IndexOf(TagSeparator) ?? -1; + contentType = split >= 0 ? tag![(split + 1)..] : tag ?? DefaultContentType; + ageSeconds = split > 0 && long.TryParse(tag.AsSpan(0, split), NumberStyles.None, CultureInfo.InvariantCulture, out var filledAtMs) + ? (int)Math.Clamp((NowMs() - filledAtMs) / 1000, 0, int.MaxValue) + : 0; + return hit; } /// @@ -1113,6 +1158,7 @@ internal static void ResetForTests() { Fresh = new BytesCache(BudgetBytes); LastGood = new BytesCache(BudgetBytes); + NowMs = SystemClock; lock (GateLock) { Gates.Clear(); diff --git a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs index 77f814be..5733adef 100644 --- a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs +++ b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs @@ -1,3 +1,5 @@ +using System.Globalization; + namespace EcencyApi.Infrastructure; /// @@ -41,6 +43,9 @@ public static class CachePolicy /// poll while `s-maxage` lets shared caches absorb the polling; the in-process /// memo of each route uses the same s-maxage as its TTL (see /// ), so the two layers never disagree on freshness. + /// A body served from that memo goes out through , which + /// hands the shared cache only what is left of the window rather than a fresh + /// one, and a last-good body through . /// The feed and the recommendations list move with every new post; status is /// the poll target and stays short; the roster changes when a curator is added /// or promoted, so it can stay put for minutes; a single post's recommenders @@ -74,6 +79,59 @@ public static int SharedMaxAge(string policy) throw new ArgumentException("policy carries no max-age", nameof(policy)); } + /// + /// Seconds a shared cache may keep a body that was served after the upstream + /// failed. It is still an answer the backend gave, so it stays publicly + /// cacheable rather than being refetched by every reader at once, but a + /// recovered backend has to reach readers within a poll or two rather than at + /// the end of a full window. + /// + public const int StaleSharedMaxAge = 5; + + /// + /// The same policy with its shared window reduced to what is left of it after + /// . + /// + /// A memoized body is served for the rest of its TTL, not for a fresh one: + /// without this a roster read from the memo a second before it lapses would + /// license a shared cache to hold that body for another whole window, so the + /// two layers together could serve one answer for nearly twice its TTL. The + /// floor of one second keeps the response cacheable at all, since the memo is + /// about to refill anyway. + /// + public static string Aged(string policy, int ageSeconds) => + ageSeconds <= 0 ? policy : WithSharedMaxAge(policy, Math.Max(1, SharedMaxAge(policy) - ageSeconds)); + + /// + /// The same policy cut down to , for a body + /// served because the upstream call failed. Never longer than the policy + /// itself, so a route with an even shorter window keeps its own. + /// + public static string Stale(string policy) => + WithSharedMaxAge(policy, Math.Min(StaleSharedMaxAge, SharedMaxAge(policy))); + + /// + /// Rewrite the `s-maxage` of a policy, leaving every other directive alone. + /// A policy that carries none gains one: its shared window was its `max-age` + /// (see ), and this caps that without touching what + /// browsers were told. + /// + private static string WithSharedMaxAge(string policy, int seconds) + { + var value = "s-maxage=" + seconds.ToString(CultureInfo.InvariantCulture); + var tokens = policy.Split(',', StringSplitOptions.TrimEntries); + var replaced = false; + for (var i = 0; i < tokens.Length; i++) + { + if (tokens[i].StartsWith("s-maxage=", StringComparison.Ordinal)) + { + tokens[i] = value; + replaced = true; + } + } + return string.Join(", ", replaced ? tokens : tokens.Append(value)); + } + /// /// A policy applies only to a successful response. Handlers attach it before /// the upstream call resolves, and can still turn diff --git a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs index 4be22624..64d15854 100644 --- a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs +++ b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; @@ -118,4 +119,31 @@ public static void CacheWhenOk(this HttpContext ctx, string policy) return Task.CompletedTask; }); } + + /// + /// for a body this service has + /// been holding for before sending it. + /// + /// The age goes out as an `Age` header so a shared cache that computes + /// freshness from it reaches the same expiry as one that only reads + /// `s-maxage`; without it a body handed out at the end of its in-process + /// lifetime would start a full downstream window of its own. Shortening the + /// window is the caller's job (, + /// ): what is left of it differs between a + /// memo hit and a body served because the upstream call failed. + /// + public static void CacheWhenOk(this HttpContext ctx, string policy, int ageSeconds) + { + var age = Math.Max(0, ageSeconds).ToString(CultureInfo.InvariantCulture); + ctx.Response.OnStarting(() => + { + var value = CachePolicy.ForStatus(ctx.Response.StatusCode, policy); + if (value != null) + { + ctx.Response.Headers.CacheControl = value; + ctx.Response.Headers.Age = age; + } + return Task.CompletedTask; + }); + } } From 743f1270a484ec5df9a01072b4a0d3c83f4b9979 Mon Sep 17 00:00:00 2001 From: feruzm Date: Sat, 5 Sep 2026 17:23:50 +0000 Subject: [PATCH 9/9] fix: send only the remaining lifetime on desk memo hits, no Age header A shortened s-maxage together with an Age header is subtracted twice by a cache that honours both, so a memo hit or a last-good body arrived stale. The remaining window (or the short stale window) is now the only freshness signal, and the Age-emitting overload is gone. --- .../EcencyApi.Tests/CurationDeskAuthTests.cs | 12 ++++----- .../Handlers/PrivateApi.CurationDesk.cs | 12 ++++++--- .../Infrastructure/HttpContextExtensions.cs | 27 ------------------- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs index ce15be4e..3d3bf188 100644 --- a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -256,7 +256,7 @@ public async Task ReadsCarryTheirPolicyOnlyOnA200() Assert.Equal(200, ok.Response.StatusCode); Assert.Equal(policy, CacheControl(ok)); // Filled by this request, so the whole window and no age spent yet. - Assert.Equal("0", Age(ok)); + Assert.Null(Age(ok)); Assert.StartsWith("application/json", ok.Response.ContentType); CurationDeskMemo.ResetForTests(); @@ -310,7 +310,7 @@ public async Task AMemoHitAdvertisesOnlyWhatIsLeftOfTheSharedWindow() await PrivateApi.CurationDeskRoster(fill); await Start(fill); Assert.Equal(CachePolicy.CurationDeskRoster, CacheControl(fill)); - Assert.Equal("0", Age(fill)); + Assert.Null(Age(fill)); // 590 s into the roster's 600 s window. Sending the whole window again // here would let a shared cache hold this body for another 600 s on top @@ -322,7 +322,7 @@ public async Task AMemoHitAdvertisesOnlyWhatIsLeftOfTheSharedWindow() Assert.Equal(200, hit.Response.StatusCode); Assert.Equal(body, Body(hit)); Assert.Equal("public, max-age=0, s-maxage=10", CacheControl(hit)); - Assert.Equal("590", Age(hit)); + Assert.Null(Age(hit)); // Both readers were answered from one upstream call. Assert.Single(upstream.Calls); @@ -344,12 +344,12 @@ public async Task AMemoHitAtTheEndOfItsWindowStaysCacheableForOneSecond() await PrivateApi.CurationDeskStatus(hit); await Start(hit); Assert.Equal("public, max-age=0, s-maxage=1", CacheControl(hit)); - Assert.Equal("20", Age(hit)); + Assert.Null(Age(hit)); Assert.Single(upstream.Calls); } [Fact] - public async Task TheLastGoodBodyCarriesTheShortWindowAndItsRealAge() + public async Task TheLastGoodBodyCarriesTheShortWindowAndNoAgeHeader() { var upstream = Install(); var clock = UseTestClock(); @@ -373,7 +373,7 @@ public async Task TheLastGoodBodyCarriesTheShortWindowAndItsRealAge() // readers within a poll or two, not ten minutes later. Assert.Equal("public, max-age=0, s-maxage=5", CacheControl(stale)); Assert.Equal(CachePolicy.Stale(CachePolicy.CurationDeskRoster), CacheControl(stale)); - Assert.Equal("120", Age(stale)); + Assert.Null(Age(stale)); } [Fact] diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs index a9728907..568171f8 100644 --- a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -312,11 +312,14 @@ private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string /// /// A hit goes out with the rest of its window rather than a new one, so the /// memo and the caches downstream expire the same body at the same moment - /// instead of holding it for one lifetime each in series. + /// instead of holding it for one lifetime each in series. The remaining + /// lifetime is the only freshness signal sent: an Age header on top of an + /// already shortened s-maxage would be subtracted a second time by a cache + /// that honours both, leaving the body stale on arrival. /// private static async Task SendPublicJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) { - ctx.CacheWhenOk(CachePolicy.Aged(policy, ageSeconds), ageSeconds); + ctx.CacheWhenOk(CachePolicy.Aged(policy, ageSeconds)); await WriteBytes(ctx, 200, contentType, bytes); } @@ -328,7 +331,10 @@ private static async Task SendPublicJson(HttpContext ctx, string policy, string /// private static async Task SendStaleJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) { - ctx.CacheWhenOk(CachePolicy.Stale(policy), ageSeconds); + // the body's real age is not advertised: the short window alone is the + // freshness, and an Age older than it would make the answer stale at once + _ = ageSeconds; + ctx.CacheWhenOk(CachePolicy.Stale(policy)); await WriteBytes(ctx, 200, contentType, bytes); } diff --git a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs index 64d15854..9919e1fb 100644 --- a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs +++ b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs @@ -119,31 +119,4 @@ public static void CacheWhenOk(this HttpContext ctx, string policy) return Task.CompletedTask; }); } - - /// - /// for a body this service has - /// been holding for before sending it. - /// - /// The age goes out as an `Age` header so a shared cache that computes - /// freshness from it reaches the same expiry as one that only reads - /// `s-maxage`; without it a body handed out at the end of its in-process - /// lifetime would start a full downstream window of its own. Shortening the - /// window is the caller's job (, - /// ): what is left of it differs between a - /// memo hit and a body served because the upstream call failed. - /// - public static void CacheWhenOk(this HttpContext ctx, string policy, int ageSeconds) - { - var age = Math.Max(0, ageSeconds).ToString(CultureInfo.InvariantCulture); - ctx.Response.OnStarting(() => - { - var value = CachePolicy.ForStatus(ctx.Response.StatusCode, policy); - if (value != null) - { - ctx.Response.Headers.CacheControl = value; - ctx.Response.Headers.Age = age; - } - return Task.CompletedTask; - }); - } }