✨ feat(registries): 429 retry with Retry-After + per-host token bucket + GH-token fallback

- New withRetry helper (app/registries/http-retry.ts) wraps registry HTTP
  calls. Retries 429/503 only, honors Retry-After (seconds and HTTP-date
  forms), falls back to exponential backoff (1s/2s/4s, cap 60s), max 3
  retries. Non-retryable statuses and non-Axios errors throw immediately.
- New per-host token bucket (app/registries/token-bucket.ts) prevents the
  watcher from self-inflicting 429s by bursting too fast within a cron
  cycle. Conservative defaults per host: GHCR/Hub at 2/s burst-5,
  api.github.com at 1/s burst-3, other hosts 5/s burst-10.
- callRegistry() now acquires a token bucket slot before calling axios,
  then wraps the call in withRetry. The full AxiosResponse is captured in
  a closure so resolveWithFullResponse callers still get response headers.
- GithubProvider falls back to the first configured GHCR PAT
  (ghcr-token-fallback.ts) when DD_RELEASE_NOTES_GITHUB_TOKEN is unset.
  GitHub PATs work for both ghcr.io and api.github.com, so users who
  configured GHCR auth get release notes for free.
- GithubProvider also wrapped in withRetry so 429s from api.github.com
  are absorbed instead of cascading into spurious 'rate-limited' warnings.

Fixes: GH-342

Author: s-b-e-n-s-o-n

app/registries/Registry.test.ts

@@ -8,6 +8,19 @@ vi.mock('../prometheus/registry', () => ({
   }),
 }));
 
+// withRetry: pass-through by default (calls the request fn once and returns its data).
+// acquireToken: no-op (token bucket has no effect on unit tests).
+vi.mock('./http-retry.js', () => ({
+  withRetry: vi.fn(async (requestFn) => {
+    const response = await requestFn();
+    return response.data;
+  }),
+}));
+vi.mock('./token-bucket.js', () => ({
+  acquireToken: vi.fn(() => Promise.resolve()),
+  getBucketForUrl: vi.fn(() => ({ key: 'mock-host', ratePerSec: 10, burst: 10 })),
+}));
+
 import Registry from './Registry.js';
 
 // --- Factory helpers (not used inside vi.mock, safe to define here) ---
@@ -1204,4 +1217,74 @@ describe('callRegistry', () => {
     });
     expect(result).toBe(mockResponse);
   });
+
+  test('acquires a token bucket token before each request', async () => {
+    const { default: axios } = await import('axios');
+    const { acquireToken } = await import('./token-bucket.js');
+    vi.clearAllMocks();
+    axios.mockResolvedValue({ data: 'ok', headers: {} });
+    const registryMocked = createMockedRegistry();
+    await registryMocked.callRegistry({
+      image: {},
+      url: 'https://ghcr.io/v2/img/tags/list',
+      method: 'get',
+    });
+    expect(acquireToken).toHaveBeenCalledTimes(1);
+    expect(acquireToken).toHaveBeenCalledWith(expect.objectContaining({ key: expect.any(String) }));
+  });
+
+  test('delegates 429 retry to withRetry and rethrows after exhaustion', async () => {
+    const { withRetry } = await import('./http-retry.js');
+    // Override withRetry to simulate exhausted retries throwing the 429
+    const err429 = new Error('Request failed with status code 429');
+    (err429 as any).response = { status: 429, headers: {} };
+    (withRetry as ReturnType<typeof vi.fn>).mockRejectedValueOnce(err429);
+
+    const registryMocked = createMockedRegistry();
+    registryMocked.type = 'hub';
+    registryMocked.name = 'test';
+
+    await expect(
+      registryMocked.callRegistry({ image: {}, url: 'url', method: 'get' }),
+    ).rejects.toThrow('status code 429');
+  });
+
+  test('delegates 503 retry to withRetry and rethrows after exhaustion', async () => {
+    const { withRetry } = await import('./http-retry.js');
+    const err503 = new Error('Request failed with status code 503');
+    (err503 as any).response = { status: 503, headers: {} };
+    (withRetry as ReturnType<typeof vi.fn>).mockRejectedValueOnce(err503);
+
+    const registryMocked = createMockedRegistry();
+    registryMocked.type = 'hub';
+    registryMocked.name = 'test';
+
+    await expect(
+      registryMocked.callRegistry({ image: {}, url: 'url', method: 'get' }),
+    ).rejects.toThrow('status code 503');
+  });
+
+  test('passes logger and requestLabel to withRetry', async () => {
+    const { default: axios } = await import('axios');
+    const { withRetry } = await import('./http-retry.js');
+    axios.mockResolvedValue({ data: 'payload', headers: {} });
+
+    const registryMocked = createMockedRegistry();
+    registryMocked.type = 'hub';
+    registryMocked.name = 'myname';
+
+    await registryMocked.callRegistry({
+      image: {},
+      url: 'https://registry.io/v2/img/tags/list',
+      method: 'get',
+    });
+
+    expect(withRetry).toHaveBeenCalledWith(
+      expect.any(Function),
+      expect.objectContaining({
+        logger: expect.objectContaining({ debug: expect.any(Function) }),
+        requestLabel: expect.stringContaining('https://registry.io/v2/img/tags/list'),
+      }),
+    );
+  });
 });

app/registries/Registry.ts

@@ -6,6 +6,8 @@ import { getSummaryTags } from '../prometheus/registry.js';
 import Component, { type ComponentConfiguration } from '../registry/Component.js';
 import { getErrorMessage } from '../util/error.js';
 import { getRegistryRequestTimeoutMs } from './configuration.js';
+import { withRetry } from './http-retry.js';
+import { acquireToken, getBucketForUrl } from './token-bucket.js';
 
 interface RegistryManifest {
   digest?: string;
@@ -465,10 +467,34 @@ class Registry<
     };
 
     try {
-      const response = await axios<T>(axiosOptionsWithConnectionReuse);
+      // Rate-limit ourselves before hitting the registry
+      await acquireToken(getBucketForUrl(url));
+
+      // Capture the full axios response so we can return headers when needed.
+      let lastAxiosResponse: AxiosResponse<T> | undefined;
+
+      await withRetry<T>(
+        () =>
+          axios<T>(axiosOptionsWithConnectionReuse).then((r) => {
+            lastAxiosResponse = r;
+            return {
+              status: r.status,
+              headers: r.headers as Record<string, string | undefined>,
+              data: r.data,
+            };
+          }),
+        {
+          logger: this.log,
+          requestLabel: `${this.getId()} ${method} ${url}`,
+        },
+      );
+
       const end = Date.now();
       getSummaryTags()?.observe({ type: this.type, name: this.name }, (end - start) / 1000);
-      return resolveWithFullResponse ? response : response.data;
+      // lastAxiosResponse is always set when withRetry resolves
+      return resolveWithFullResponse
+        ? (lastAxiosResponse as AxiosResponse<T>)
+        : lastAxiosResponse!.data;
     } catch (error) {
       const end = Date.now();
       getSummaryTags()?.observe({ type: this.type, name: this.name }, (end - start) / 1000);

app/registries/ghcr-token-fallback.ts

@@ -0,0 +1,23 @@
+/**
+ * Returns the PAT token from the first credentialed GHCR registry instance,
+ * or undefined if none is configured.
+ *
+ * GHCR tokens (github.com PATs) work for both the container registry and the
+ * GitHub REST API, so the release-notes provider can reuse them when no
+ * dedicated DD_RELEASE_NOTES_GITHUB_TOKEN is set.
+ */
+
+import { getState } from '../registry/index.js';
+
+export function getGhcrTokenFallback(): string | undefined {
+  const registryState = getState().registry;
+  for (const instance of Object.values(registryState)) {
+    // Duck-type: we only need the provider type and the token field.
+    const cfg = (instance as { type?: string; configuration?: { token?: string } }).configuration;
+    const type = (instance as { type?: string }).type;
+    if (type === 'ghcr' && typeof cfg?.token === 'string' && cfg.token.length > 0) {
+      return cfg.token;
+    }
+  }
+  return undefined;
+}