From 35d0f2ac47a8f027026cf2498fa3fafdcfa024bb Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 5 Aug 2026 00:14:53 -0700 Subject: [PATCH] feat: enable shadowing for untracked caches --- README.md | 37 +++++++------ src/dialcache.ts | 13 ++--- src/internal/redis-cache.ts | 12 +--- test/dialcache-shadow-confirmation.test.ts | 50 ++++++++++++++--- test/dialcache-shadow-validation.test.ts | 11 ++-- test/redis-real.integration.test.ts | 64 ++++++++++++++++------ 6 files changed, 124 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 2b8ec06..d782262 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ request-local cache -> process-local cache -> Redis cache -> fallback function - Process-local hits return immediately. - Process-local misses try Redis and populate the process-local cache on a Redis hit. - Redis misses call the fallback and attempt to populate Redis and, when active, the process-local cache. Tracked invalidation may suppress both publications. -- Selected tracked Redis keys can execute non-serving [shadow work](#shadow-validation) that validates hits and fills clean misses, even before Redis is allowed to serve callers. +- Selected Redis keys can execute non-serving [shadow work](#shadow-validation) that validates hits and fills clean misses, even before Redis is allowed to serve callers. - Redis read failures and timeouts are logged, counted in metrics, and fail open without attempting a second Redis operation. Redis write failures also fail open. `invalidateRemote` requires a configured Redis client; missing configuration and Redis failures are logged, counted, and rethrown so callers do not assume invalidation succeeded. - Cache-key construction and config-provider failures also fail open and run the fallback uncached. - A missing effective process-local/Redis TTL disables that layer by policy; a configured TTL with no ramp defaults to 100%. Disabled layers record a disabled reason and fall through to the next layer/fallback. @@ -236,7 +236,7 @@ const dialcache = new DialCache({ return new DialCacheKeyConfig({ // Sparse override: inherit both TTLs and the local ramp from defaultConfig. ramp: { [CacheLayer.REMOTE]: 25 }, - // Independently sample tracked Redis keys for detached validation/fill. + // Independently sample Redis keys for detached validation/fill. shadow: { ramp: 5, // Emit one warning with a bounded key and native-JSON value strings. @@ -263,7 +263,7 @@ const getUser = dialcache.cached((userId: string) => db.fetchUser(userId), { `ramp` values are percentages from 0 to 100. `0` disables the layer, `100` enables it, and intermediate values are deterministically sampled by cache key and layer, so the same key is consistently sampled in or out of a partial rollout across calls and instances. The assignment algorithm is owned by DialCache and remains stable across releases. Applications that need an externally coordinated cohort can use `cacheConfigProvider` to return a sparse per-key ramp override of `0` or `100`. DialCache fetches and resolves one config snapshot per enabled invocation. Provider errors do not activate defaults: they fail open, record `config_error`, and execute the fallback function uncached. -`shadow.ramp` uses the same inclusive 0–100 percentage domain but is independent of cache-layer serving ramps. Omission and `0` disable shadow work; `100` selects every eligible tracked Redis key; intermediate values assign each exact cache key to a stable shadow cohort across calls and instances. A valid remote policy can therefore use `ramp.remote: 0` with a nonzero `shadow.ramp` to exercise and populate Redis without serving from it. A nonzero value explicitly authorizes detached tracked writes after clean shadow-only misses. It does not create another `CacheLayer`, activate Redis without a valid remote TTL, or make a request-local/process-local hit continue to Redis. +`shadow.ramp` uses the same inclusive 0–100 percentage domain but is independent of cache-layer serving ramps. Omission and `0` disable shadow work; `100` selects every eligible Redis key; intermediate values assign each exact cache key to a stable shadow cohort across calls and instances. A valid remote policy can therefore use `ramp.remote: 0` with a nonzero `shadow.ramp` to exercise and populate Redis without serving from it. A nonzero value explicitly authorizes detached writes after clean shadow-only misses: tracked keys use their watermark-aware write, while untracked keys use their ordinary TTL-based last-writer-wins write. It does not create another `CacheLayer`, activate Redis without a valid remote TTL, or make a request-local/process-local hit continue to Redis. Remote serving and shadow sampling use independent deterministic cohorts. Equal partial percentages do not imply the same keys, so a partial shadow cohort does not guarantee that every key admitted by a later partial serving ramp was warmed or validated. Use `shadow: { ramp: 100 }` when every otherwise eligible invocation must exercise the non-serving Redis path before a serving-ramp increase. @@ -459,7 +459,7 @@ This guard is deliberately conservative and is not a proof of runtime data. Type #### Shadow validation -Shadow mode runs a sampled, detached Redis path for tracked keys without allowing that path to serve the caller. A Redis hit is compared with the source of truth (SoT); a clean Redis miss can be filled from the caller-accepted SoT value. Redis serving and shadow execution are independent, and shadowing is opt-in per use case through `shadow.ramp`: +Shadow mode runs a sampled, detached Redis path for tracked or untracked keys without allowing that path to serve the caller. A Redis hit is compared with the source of truth (SoT); a clean Redis miss can be filled from the caller-accepted SoT value. Redis serving and shadow execution are independent, and shadowing is opt-in per use case through `shadow.ramp`: ```ts import { CacheLayer, DialCache, DialCacheKeyConfig } from "dialcache"; @@ -479,6 +479,7 @@ const getUser = dialcache.cached( keyType: "user_id", useCase: "GetUser", cacheKey: (userId) => userId, + // Optional for shadowing; adds watermark fencing to Redis reads and fills. trackForInvalidation: true, // Optional: override strict deep equality with use-case semantics. shadowComparator: (cached, source) => @@ -498,28 +499,28 @@ const getUser = dialcache.cached( ); ``` -Shadow work is eligible only when the operation sets `trackForInvalidation: true`, a valid remote TTL/policy exists, its effective `shadow.ramp` selects the exact cache key, a configured metrics adapter implements `shadowValidation`, and capacity is available. Logging is supplemental to that metric; enabling `logMismatches` does not activate shadow work without the metrics hook. The bundled Prometheus and Datadog adapters implement that hook. There are two paths: +Shadow work is eligible only when a valid remote TTL/policy exists, its effective `shadow.ramp` selects the exact cache key, a configured metrics adapter implements `shadowValidation`, and capacity is available. Tracked and untracked Redis keys are both eligible; each keeps its existing read and write mode. Logging is supplemental to the metric; enabling `logMismatches` does not activate shadow work without the metrics hook. The bundled Prometheus and Datadog adapters implement that hook. There are two paths: -- When remote serving is enabled and produces a tracked Redis hit, DialCache retains the exact serialized payload that supplied the caller as `C0`. -- When the remote policy is valid but disabled specifically by `ramped_down`, DialCache starts a detached tracked Redis read for `C0`. Its result can be validated or used to decide whether a clean miss may be filled, but can never supply the caller or populate an in-memory layer. +- When remote serving is enabled and produces a Redis hit, DialCache retains the exact serialized payload that supplied the caller as `C0`. +- When the remote policy is valid but disabled specifically by `ramped_down`, DialCache starts a detached Redis read for `C0` using the key's existing tracked or untracked mode. Its result can be validated or used to decide whether a clean miss may be filled, but can never supply the caller or populate an in-memory layer. -A missing or invalid remote policy, config-provider failure, absent Redis client, disabled call, untracked key, omitted metrics hook, zero/omitted shadow ramp, cohort exclusion, capacity rejection, or earlier request-local/process-local hit does not launch a shadow-only Redis path. Shadow work begins only if normal traversal reaches the Redis layer. A normally enabled remote miss already follows the caller's ordinary fallback-and-fill path and does not launch a duplicate shadow fill. +A missing or invalid remote policy, config-provider failure, absent Redis client, disabled call, omitted metrics hook, zero/omitted shadow ramp, cohort exclusion, capacity rejection, or earlier request-local/process-local hit does not launch a shadow-only Redis path. Shadow work begins only if normal traversal reaches the Redis layer. A normally enabled remote miss already follows the caller's ordinary fallback-and-fill path and does not launch a duplicate shadow fill. On a served hit, DialCache returns the already-decoded cached value before starting the SoT read or any confirmation work. On a ramped-down path, the caller invokes and awaits its normal configured fallback exactly once and receives only that result; detached work reuses the same accepted `S` instead of calling the loader again. The caller never awaits shadow `C0`, comparison, confirmation `C1`, shadow serialization, or fill. Slow, failed, or timed-out shadow work cannot delay, reject, or change the caller result. The detached job uses this bounded algorithm: -1. Obtain the original tracked Redis payload as `C0`. -2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary and attempt one normal tracked Redis write using the resolved TTL and watermark machinery. Before the whole-job deadline, emit `filled` when Redis accepts it, `fill_blocked` when the invalidation watermark rejects it, or `fill_error` when serialization or the write fails. +1. Obtain the original Redis payload as `C0` using the key's existing tracked or untracked read mode. +2. If `C0` is missing, wait for the caller's successfully accepted `S` after its configured fallback boundary and attempt one normal Redis write in the same mode using the resolved TTL. Before the whole-job deadline, emit `filled` when Redis accepts it, `fill_blocked` when a tracked invalidation watermark rejects it, or `fill_error` when serialization or the write fails. `fill_blocked` is not produced by compliant untracked writes. 3. If `C0` is non-null, obtain `S`, deserialize an isolated snapshot of `C0`, and run the default or custom semantic comparator. Any non-null `C0` is observation-only: DialCache never repairs or overwrites it, including when deserialization fails. 4. If `C0` and `S` match semantically, emit `match` without another Redis read. -5. Otherwise, reread tracked Redis directly as `C1`, bypassing request-local and process-local cache. +5. Otherwise, reread Redis directly in the same mode as `C1`, bypassing request-local and process-local cache. 6. If `C1` is missing or differs byte-for-byte from `C0`, emit `superseded`; if it is identical, emit `mismatch`. 7. If the confirmation read fails or reaches its Redis-read deadline, emit `confirmation_error`. -Here a clean miss means the tracked semantic Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. +Here a clean miss means the semantic Redis read returned `null`; it does not include a non-null payload that later fails deserialization. A caller fallback rejection or timeout never becomes accepted `S` and never starts the fill. -Both detached Redis reads use the existing watermark-aware tracked-read protocol, primary-read behavior, and effective `remoteReadTimeoutMs`. The clean-miss fill uses the same serializer, TTL, Redis-time timestamp, and invalidation watermark as an ordinary tracked fill. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. +Both detached Redis reads use the effective `remoteReadTimeoutMs` and the key's normal protocol. Every clean-miss fill uses the same serializer, TTL, and Redis-time timestamp as an ordinary fill. Tracked `C0` and `C1` reads remain watermark-aware and are routed to primaries by the bundled adapters, while tracked fills also retain the ordinary invalidation watermark. Untracked reads use the ordinary one-key read route, which has no shadow-specific primary guarantee, and untracked fills use the ordinary TTL write without a watermark. Strings compare exactly, Buffers compare by bytes, and string/Buffer pairs compare by their UTF-8 bytes. DialCache does not deserialize `C1`, compare it with `S`, or chase another version. The detached scheduler, Redis-read deadline timers, and overall shadow deadline timer are unreferenced, so they do not keep an otherwise idle process alive. Detachment is asynchronous work on the Node event loop, not a worker thread: synchronous source, serializer, or comparator work can still occupy the event loop after the request path has been released. @@ -541,7 +542,7 @@ DialCache retains the semantic `string | Buffer` returned by the Redis client bu The effective serializer's `load` method therefore runs a second time for a sampled served hit and once in detached work for a shadow-only hit. It must be repeatable, non-mutating, and return independently usable values. On a clean miss, its `dump` method may run after the caller has received `S`, so `S` must remain immutable through detached serialization. A custom `DialCacheRedisClient` must return an operation-owned payload whose string/Buffer contents remain stable after `read()` settles. Comparing the deserialized cached snapshot with the raw source value intentionally detects lossy serialization; use a custom comparator only when such normalization or ignored fields are valid use-case semantics. -Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read; `confirmation_error` applies to `C1`. A clean `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. +Shadow metrics use the bounded outcomes `match`, `mismatch`, `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, `source_error`, `deserialization_error`, `comparison_error`, `confirmation_error`, `timeout`, and `dropped`. `redis_error` applies to the initial shadow-only `C0` read; `confirmation_error` applies to `C1`; `fill_blocked` applies only when a tracked watermark rejects the write. A clean `C0` miss is an ordinary `miss{layer="remote_shadow"}` and terminates with a fill, source, or timeout outcome rather than a second shadow outcome for the miss itself. Labels never contain cache ids, values, payloads, Redis keys, or raw exception text. Confirmed-mismatch logging is separately opt-in through `shadow.logMismatches`; it never replaces the required `shadowValidation` metric or emits for a mismatch candidate that becomes `superseded`. The single warning contains `cacheNamespace`, `useCase`, `keyType`, `outcome: "mismatch"`, `cacheKey`, `cachedValueJson`, and `sourceValueJson`. `cacheKey` is the logical DialCache URN capped at 2 KiB, not the physical Redis storage key. DialCache independently applies native `JSON.stringify` to the deserialized cached snapshot and raw source value supplied to the comparator, then caps each resulting string at 8 KiB. A byte-clipped field ends in `...[truncated]`, counted inside its cap. If native JSON throws or returns `undefined`, the corresponding JSON field is `null`; the other side is still attempted. DialCache does not compute a textual diff or call the configured serializer again for logging. @@ -551,13 +552,13 @@ The byte caps apply before logger framing or escaping, so they do not guarantee Detached Redis reads, serializer loads/dumps, payload sizes, and read/write errors use the existing layer label with `layer="remote_shadow"`. This distinguishes non-serving Redis cost from caller-path `layer="remote"` telemetry without adding a metric or label. The established `observeGet{layer="remote"}` boundary includes caller-path deserialization, while `observeGet{layer="remote_shadow"}` ends when the deadline-bounded Redis read result settles; detached serializer work and any later raw-client settlement are outside that timer. The request-path read that supplied a served `C0` keeps `layer="remote"`, and a ramped-down caller keeps `disabled{layer="remote", reason="ramped_down"}`. No `disabled{layer="remote_shadow"}` event is emitted for ineligible or dropped work; `dropped` remains the terminal shadow outcome. Confirmation reads use the same `remote_shadow` value, with `superseded` or `confirmation_error` describing their role. -The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one tracked fill. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived a tracked Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. +The command amplification is bounded: a selected served hit adds one SoT read and adds `C1` only for a semantic mismatch candidate; a selected ramped-down hit adds detached `C0`, reuses the caller's existing SoT read, and likewise adds `C1` only for a candidate; a selected ramped-down miss adds detached `C0` and at most one fill in the key's existing mode. `superseded` means only that the original observation could not be confirmed. `mismatch` means the exact `C0` payload survived another Redis read after the SoT disagreement; it is not a cross-system atomic snapshot or a guarantee that the mismatch persists. For an untracked key it is also not proof of primary freshness or invalidation safety. -The initial `C0` read and later fill are not atomic. The fill is a normal tracked overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the clean miss and be overwritten by the shadow fill. Invalidation watermarks still fence writes using Redis time, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, and write interval when stale-publication protection matters. Shadow mode never repairs a non-null `C0`, refreshes its TTL, invalidates, evicts local state, or changes the value returned to the caller. +The initial `C0` read and later fill are not atomic. The fill is a normal overwrite, not a compare-and-set or write-if-still-missing operation: another writer can populate Redis after the clean miss and be overwritten by the shadow fill. Tracked invalidation watermarks still fence tracked writes using Redis time, so size `futureBufferMs` to cover the complete SoT, serialization, client queue, network, and write interval when stale-publication protection matters. An untracked shadow fill has no such fence and retains the ordinary TTL-based last-writer-wins contract; because it is detached, an older accepted source value may be written after a concurrent source mutation and remain until expiry. Shadow mode never repairs a non-null `C0`, refreshes its TTL, invalidates, evicts local state, or changes the value returned to the caller. A served-hit sample invokes the wrapped function or inline loader as an additional source read, so that loader must be safe to call for observation. A ramped-down sample reuses the caller's ordinary invocation and does not add another SoT call. -For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The clean-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the tracked Redis write described above. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. +For valid policies, shadow-specific source calls, cache-path Redis traffic, returned values, and metrics are unchanged when `shadow` is omitted or `shadow.ramp` is `0`. Shadow policy is grouped under `DialCacheKeyConfig.shadow`; consumers of the former flat ramp field must migrate to `shadow: { ramp }`. The public constructor, static defaults, and runtime provider results reject the removed field. `DialCacheKeyConfig.disabled()` explicitly disables the shadow ramp and mismatch logging. `shadowComparator` remains a typed `cached()` / `getOrLoad()` option because it defines stable use-case equality, while `shadowMaxInFlight` remains a per-instance concurrency limit. The clean-miss bootstrap adds no additional ramp knob, Redis protocol operation, metric instrument, or label key; enabling shadowing authorizes the same-mode Redis write described above. Untracked keys now participate when they have a nonzero effective shadow ramp and an observable metrics hook, so deployments that previously supplied such a ramp while relying on the tracked-only eligibility rule must set it to `0` before upgrading if they do not want the added SoT reads, Redis traffic, possible fills, and opted-in mismatch logs. Exported unions include `remote_shadow` in `MetricLayer` and `superseded`, `filled`, `fill_blocked`, `fill_error`, `redis_error`, and `confirmation_error` in `ShadowValidationOutcome`. TypeScript consumers with exhaustive switches or `Record` values must include those cases, and dashboards restricted to `layer="remote"` intentionally exclude detached traffic. ## Cached-value ownership @@ -614,7 +615,7 @@ Invalidation writes a Redis watermark at `{encodedNamespace:encodedKeyType:encod The internal `:dialcache-frame-v1` suffix identifies values written with DialCache's binary protocol. Watermarks are stored as decimal timestamps. -A cached Redis value whose Redis-created timestamp is older than or equal to the watermark is treated as stale and refreshed through fallback. `invalidateRemote(keyType, id, futureBufferMs)` sets the watermark to the greater of its existing value and Redis's current time plus the buffer. While that future window is active, an invocation that reaches the tracked Redis read treats the covered value as a miss. If its fallback then reaches the tracked Redis write, Redis rejects the write and DialCache also suppresses the corresponding process-local population; the fallback value still returns to its caller. Request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult the watermark; a selected shadow path does consult it for `C0` and any clean-miss fill, although caller-path request-local/process-local publication remains independent. +A cached Redis value whose Redis-created timestamp is older than or equal to the watermark is treated as stale and refreshed through fallback. `invalidateRemote(keyType, id, futureBufferMs)` sets the watermark to the greater of its existing value and Redis's current time plus the buffer. While that future window is active, an invocation that reaches the tracked Redis read treats the covered value as a miss. If its fallback then reaches the tracked Redis write, Redis rejects the write and DialCache also suppresses the corresponding process-local population; the fallback value still returns to its caller. Request-local memoization remains unconditional. A ramped-out invocation without shadow work does not consult the watermark; a selected shadow path for that tracked key does consult it for `C0`, `C1` when needed, and any clean-miss fill, although caller-path request-local/process-local publication remains independent. The bundled timestamp protocol assumes that system clocks are synchronized across every Redis node eligible for primary promotion. Redis does not guarantee that `TIME` is monotonic across nodes, and DialCache does not detect or compensate for cross-node clock skew. If this deployment assumption is violated, failover can temporarily suppress tracked cache fills or allow a pre-invalidation value to remain readable until it expires or a later invalidation advances the watermark past its timestamp. diff --git a/src/dialcache.ts b/src/dialcache.ts index 4c670c1..e094072 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -516,9 +516,10 @@ export class DialCache { * The watermark fences only invocations that reach the tracked Redis write. * A rejected caller-path write also suppresses the corresponding process-local * population. Request-local memoization remains unconditional. A ramped-out - * invocation without shadow work does not consult the watermark; a selected - * shadow path consults it for its tracked read and any clean-miss fill, while - * caller-path request-local and process-local publication remains independent. + * invocation without shadow work does not consult the watermark. A selected + * shadow path for a tracked key consults it for Redis reads and any clean-miss + * fill; untracked shadow work does not. Caller-path request-local and + * process-local publication remains independent. * * @param futureBufferMs Nonnegative safe integer no greater than * 31,536,000,000 (365 days); defaults to zero for backward compatibility. @@ -799,10 +800,6 @@ export class DialCache { validation: ShadowValidationPlan, readTimeoutMs: number, ): void { - if (!key.trackForInvalidation) { - return; - } - const shadowConfig: unknown = keyConfig?.shadow; if ( shadowConfig === null @@ -938,7 +935,7 @@ export class DialCache { maybeRelease(); }; const readShadowPayload = (): Promise => { - const read = redisCache.startTrackedPayloadReadForShadow(key, readTimeoutMs); + const read = redisCache.startPayloadReadForShadow(key, readTimeoutMs); pendingRedisReads.add(read.settled); void read.settled.then(() => { pendingRedisReads.delete(read.settled); diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index 3241a5d..e5f8709 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -148,18 +148,15 @@ export class RedisCache { } /** - * Start a measured tracked Redis read for detached shadow work. + * Start a measured Redis read for detached shadow work. * * The bounded result may reject before the semantic client operation settles, * so callers must retain shadow capacity until `settled` fulfills. */ - startTrackedPayloadReadForShadow( + startPayloadReadForShadow( key: DialCacheKey, readTimeoutMs: number, ): StartedRedisRead { - if (!key.trackForInvalidation) { - throw new Error("DialCache shadow Redis reads require tracked keys"); - } return this.startMeasuredPayloadRead( key, readTimeoutMs, @@ -176,16 +173,13 @@ export class RedisCache { return await this.putWithLayer(key, value, ttlSec, CacheLayer.REMOTE); } - /** Populate a definitive detached tracked miss using the caller's resolved policy snapshot. */ + /** Populate a clean detached Redis miss using the caller's resolved policy snapshot. */ async putForShadow( key: DialCacheKey, value: T, config: { readonly ttlSec: number }, shouldWrite: () => boolean, ): Promise { - if (!key.trackForInvalidation) { - throw new Error("DialCache shadow Redis writes require tracked keys"); - } return await this.putWithLayer( key, value, diff --git a/test/dialcache-shadow-confirmation.test.ts b/test/dialcache-shadow-confirmation.test.ts index 95e7e45..c024aef 100644 --- a/test/dialcache-shadow-confirmation.test.ts +++ b/test/dialcache-shadow-confirmation.test.ts @@ -226,6 +226,11 @@ function expectTrackedReads( } } +function expectUntrackedReads(redis: ScriptedRedis, count: number): void { + expect(redis.requests).toHaveLength(count); + expect(redis.requests.every((request) => !Object.hasOwn(request, "watermarkKey"))).toBe(true); +} + describe("DialCache Redis shadow confirmation", () => { it("skips confirmation when the served Redis payload semantically matches SoT", async () => { const payload = JSON.stringify({ id: "123", version: 1 }); @@ -916,14 +921,23 @@ describe("DialCache Redis shadow confirmation", () => { } }); - it("fills a definitive dark Redis miss and attributes the read and write to remote_shadow", async () => { + it.each([ + { name: "tracked", tracked: true }, + { name: "untracked", tracked: false }, + ])("fills a clean $name dark Redis miss and attributes the read and write to remote_shadow", async ({ + name, + tracked, + }) => { const redis = new ScriptedRedis([() => null]); const metrics = new RecordingMetrics(); const source = vi.fn(async () => ({ id: "123" })); const dialcache = createCache(redis, metrics); const getUser = dialcache.cached(source, { - ...trackedOptions("ShadowDarkMissFill", remoteConfig(0)), + keyType: "user_id", + useCase: `ShadowDarkMissFill${name}`, cacheKey: () => "123", + trackForInvalidation: tracked, + defaultConfig: remoteConfig(0), }); await expect(dialcache.enable(async () => await getUser())).resolves.toEqual({ id: "123" }); @@ -931,11 +945,17 @@ describe("DialCache Redis shadow confirmation", () => { expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["filled"]); expect(source).toHaveBeenCalledOnce(); + if (tracked) { + expectTrackedReads(redis, 1); + } else { + expectUntrackedReads(redis, 1); + } expect(redis.write).toHaveBeenCalledOnce(); expect(redis.write).toHaveBeenCalledWith(expect.objectContaining({ cacheTtlMs: 60_000, - watermarkKey: expect.any(String), + value: JSON.stringify({ id: "123" }), })); + expect(Object.hasOwn(redis.write.mock.calls[0]?.[0] ?? {}, "watermarkKey")).toBe(tracked); expect(metrics.ordinaryEvents.filter(({ name, labels }) => name === "request" && labels.layer === REMOTE_SHADOW_CACHE_LAYER )).toHaveLength(1); @@ -1269,7 +1289,7 @@ describe("DialCache Redis shadow confirmation", () => { )).toHaveLength(0); }); - it("dark-reads only an otherwise valid tracked remote policy with observable shadowing", async () => { + it("dark-reads only an otherwise valid remote policy with observable shadowing", async () => { const cases = [ { name: "missing remote policy", @@ -1278,10 +1298,19 @@ describe("DialCache Redis shadow confirmation", () => { config: new DialCacheKeyConfig({ shadow: { ramp: 100 } }), }, { - name: "untracked", + name: "untracked omitted shadow ramp", tracked: false, metrics: new RecordingMetrics(), - config: remoteConfig(0), + config: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.REMOTE]: 60 }, + ramp: { [CacheLayer.REMOTE]: 0 }, + }), + }, + { + name: "untracked zero shadow ramp", + tracked: false, + metrics: new RecordingMetrics(), + config: remoteConfig(0, 0), }, { name: "missing shadow hook", @@ -1308,8 +1337,9 @@ describe("DialCache Redis shadow confirmation", () => { for (const testCase of cases) { const redis = new ScriptedRedis([]); + const source = vi.fn(async () => ({ id: testCase.name })); const dialcache = createCache(redis, testCase.metrics); - const getUser = dialcache.cached(async () => ({ id: testCase.name }), { + const getUser = dialcache.cached(source, { keyType: "user_id", useCase: `ShadowDarkIneligible${testCase.name}`, cacheKey: () => "123", @@ -1319,7 +1349,13 @@ describe("DialCache Redis shadow confirmation", () => { await dialcache.enable(async () => await getUser()); await nextImmediate(); + expect(source, testCase.name).toHaveBeenCalledOnce(); expect(redis.requests, testCase.name).toHaveLength(0); + expect(redis.write, testCase.name).not.toHaveBeenCalled(); + expect(redis.invalidate, testCase.name).not.toHaveBeenCalled(); + if (testCase.metrics instanceof RecordingMetrics) { + expect(testCase.metrics.shadowEvents, testCase.name).toHaveLength(0); + } } }); diff --git a/test/dialcache-shadow-validation.test.ts b/test/dialcache-shadow-validation.test.ts index 34698d2..dafbf25 100644 --- a/test/dialcache-shadow-validation.test.ts +++ b/test/dialcache-shadow-validation.test.ts @@ -294,7 +294,7 @@ describe("DialCache Redis shadow validation", () => { expect(metrics.shadowEvents[0]?.outcome).toBe("match"); }); - it("does not validate an untracked Redis hit", async () => { + it("validates an untracked Redis hit without consulting a watermark", async () => { const redis = new FakeRedis(); const metrics = new RecordingMetrics(); const useCase = "ShadowUntracked"; @@ -314,10 +314,13 @@ describe("DialCache Redis shadow validation", () => { }); expect(await dialcache.enable(async () => await getUser())).toEqual({ id: "123", source: "cache" }); - await nextImmediate(); + await waitForShadowEvents(metrics, 1); - expect(source).not.toHaveBeenCalled(); - expect(metrics.shadowEvents).toHaveLength(0); + expect(source).toHaveBeenCalledOnce(); + expect(metrics.shadowEvents.map(({ outcome }) => outcome)).toEqual(["mismatch"]); + expect(redis.getCalls).toBe(2); + expect(redis.mGetCalls).toBe(0); + expect(redis.setCalls).toBe(0); }); it("does not validate a tracked Redis miss", async () => { diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index 4ca5df9..eafe6d9 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -458,19 +458,30 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { expect(await admin.get(commandOptions({ returnBuffers: true }), valueKey)).toBeNull(); }); - it("shadow-reads tracked Redis without serving or repairing a warm hit when the remote ramp is zero", async () => { + it.each([ + { name: "tracked", tracked: true }, + { name: "untracked", tracked: false }, + ])("shadow-reads $name Redis without serving or repairing a warm hit when the remote ramp is zero", async ({ + name, + tracked, + }) => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } - const namespace = "real-dark-shadow"; - const useCase = "RealDarkShadowPayload"; - const valueKey = `{${namespace}:item_id:dark}#${useCase}:dialcache-frame-v1`; + const namespace = `real-dark-shadow-${name}`; + const useCase = `RealDarkShadowPayload${name}`; + const rawPrefix = `${namespace}:item_id:dark`; + const valueKey = tracked + ? `{${rawPrefix}}#${useCase}:dialcache-frame-v1` + : `${rawPrefix}#${useCase}:dialcache-frame-v1`; const watermarkKey = `{${namespace}:item_id:dark}#watermark`; const cachedValue = { id: "dark", version: 1 }; const sourceValue = { id: "dark", version: 2 }; const storedFrame = encodeFrame(JSON.stringify(cachedValue), 0); await admin.set(valueKey, storedFrame, { PX: 60_000 }); - await admin.set(watermarkKey, "0", { PX: 60_000 }); + if (tracked) { + await admin.set(watermarkKey, "0", { PX: 60_000 }); + } const read = vi.fn(client.adapter.read); const write = vi.fn(client.adapter.write); @@ -504,7 +515,7 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { keyType: "item_id", useCase, cacheKey: () => "dark", - trackForInvalidation: true, + trackForInvalidation: tracked, defaultConfig: new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 60 }, ramp: { [CacheLayer.REMOTE]: 0 }, @@ -519,7 +530,10 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { expect(source).toHaveBeenCalledOnce(); expect(read).toHaveBeenCalledTimes(2); expect(read.mock.calls.every(([request]) => - request.valueKey === valueKey && request.watermarkKey === watermarkKey + request.valueKey === valueKey + && (tracked + ? request.watermarkKey === watermarkKey + : !Object.hasOwn(request, "watermarkKey")) )).toBe(true); expect(metrics.shadowValidation).toHaveBeenCalledOnce(); expect(metrics.shadowValidation).toHaveBeenCalledWith({ @@ -577,13 +591,22 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { expect(await admin.get(commandOptions({ returnBuffers: true }), valueKey)).toEqual(storedFrame); }); - it("fills a clean tracked shadow miss asynchronously and serves it after Redis ramps up", async () => { + it.each([ + { name: "tracked", tracked: true }, + { name: "untracked", tracked: false }, + ])("fills a clean $name shadow miss asynchronously and serves it after Redis ramps up", async ({ + name, + tracked, + }) => { if (client === undefined || admin === undefined) { throw new Error("Redis test clients did not start"); } - const namespace = "real-dark-shadow-fill"; - const useCase = "RealDarkShadowFill"; - const valueKey = `{${namespace}:item_id:cold}#${useCase}:dialcache-frame-v1`; + const namespace = `real-dark-shadow-fill-${name}`; + const useCase = `RealDarkShadowFill${name}`; + const rawPrefix = `${namespace}:item_id:cold`; + const valueKey = tracked + ? `{${rawPrefix}}#${useCase}:dialcache-frame-v1` + : `${rawPrefix}#${useCase}:dialcache-frame-v1`; const watermarkKey = `{${namespace}:item_id:cold}#watermark`; const sourceValue = { id: "cold", version: 1 }; const writeStarted = deferred(); @@ -631,7 +654,7 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { keyType: "item_id", useCase, cacheKey: () => "cold", - trackForInvalidation: true, + trackForInvalidation: tracked, }); const result = await dialcache.enable(async () => await getPayload()); @@ -646,16 +669,23 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { expect(write).toHaveBeenCalledOnce(); expect(write).toHaveBeenCalledWith({ valueKey, - watermarkKey, cacheTtlMs: 60_000, value: JSON.stringify(sourceValue), + ...(tracked ? { watermarkKey } : {}), }); - expect(await client.adapter.read({ valueKey, watermarkKey })).toBe(JSON.stringify(sourceValue)); - expect(await admin.get(watermarkKey)).toBe("0"); + expect(await client.adapter.read({ + valueKey, + ...(tracked ? { watermarkKey } : {}), + })).toBe(JSON.stringify(sourceValue)); expect(await admin.pTTL(valueKey)).toBeGreaterThan(55_000); expect(await admin.pTTL(valueKey)).toBeLessThanOrEqual(60_000); - expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(115_000); - expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(120_000); + if (tracked) { + expect(await admin.get(watermarkKey)).toBe("0"); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(115_000); + expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(120_000); + } else { + expect(await admin.exists(watermarkKey)).toBe(0); + } expect(metrics.shadowValidation).toHaveBeenCalledOnce(); expect(metrics.shadowValidation).toHaveBeenCalledWith({ cacheNamespace: namespace,