feat(http): Cleanup for HTTP cache (#28381)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

docs/usage/self-hosted-configuration.md

@@ -666,6 +666,16 @@ Use the `extends` field instead of this if, for example, you need the ability fo
     When Renovate resolves `globalExtends` it does not fully process the configuration.
     This means that Renovate does not have the authentication it needs to fetch private things.
 
+## httpCacheTtlDays
+
+This option sets the number of days that Renovate will cache HTTP responses.
+The default value is 90 days.
+Value of `0` means no caching.
+
+<!-- prettier-ignore -->
+!!! warning
+    When you set `httpCacheTtlDays` to `0`, Renovate will remove the cached HTTP data.
+
 ## includeMirrors
 
 By default, Renovate does not autodiscover repositories that are mirrors.

lib/config/global.ts

@@ -32,6 +32,7 @@ export class GlobalConfig {
     'gitTimeout',
     'platform',
     'endpoint',
+    'httpCacheTtlDays',
   ];
 
   private static config: RepoGlobalConfig = {};

lib/config/options/index.ts

@@ -2981,6 +2981,14 @@ const options: RenovateOptions[] = [
     default: null,
     supportedPlatforms: ['github'],
   },
+  {
+    name: 'httpCacheTtlDays',
+    description: 'Maximum duration in days to keep HTTP cache entries.',
+    type: 'integer',
+    stage: 'repository',
+    default: 90,
+    globalOnly: true,
+  },
 ];
 
 export function getOptions(): RenovateOptions[] {

lib/config/types.ts

@@ -158,6 +158,7 @@ export interface RepoGlobalConfig {
   presetCachePersistence?: boolean;
   privateKey?: string;
   privateKeyOld?: string;
+  httpCacheTtlDays?: number;
 }
 
 export interface LegacyAdminConfig {

lib/util/cache/repository/http-cache.spec.ts

@@ -0,0 +1,75 @@
+import { DateTime, Settings } from 'luxon';
+import { GlobalConfig } from '../../../config/global';
+import { cleanupHttpCache } from './http-cache';
+
+describe('util/cache/repository/http-cache', () => {
+  beforeEach(() => {
+    const now = DateTime.fromISO('2024-04-12T12:00:00.000Z').valueOf();
+    Settings.now = () => now;
+    GlobalConfig.reset();
+  });
+
+  it('should not throw if cache is not a valid HttpCache', () => {
+    expect(() => cleanupHttpCache('not a valid cache')).not.toThrow();
+  });
+
+  it('should remove expired items from the cache', () => {
+    const now = DateTime.now();
+    const expiredItemTimestamp = now.minus({ days: 91 }).toISO();
+    const cache = {
+      httpCache: {
+        'http://example.com/foo': {
+          timestamp: expiredItemTimestamp,
+          etag: 'abc',
+          lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT',
+          httpResponse: {},
+        },
+        'http://example.com/bar': {
+          timestamp: now.toISO(),
+          etag: 'abc',
+          lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT',
+          httpResponse: {},
+        },
+      },
+    };
+
+    cleanupHttpCache(cache);
+
+    expect(cache).toEqual({
+      httpCache: {
+        'http://example.com/bar': {
+          timestamp: now.toISO(),
+          etag: 'abc',
+          httpResponse: {},
+          lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT',
+        },
+      },
+    });
+  });
+
+  it('should remove all items if ttlDays is not configured', () => {
+    GlobalConfig.set({ httpCacheTtlDays: 0 });
+
+    const now = DateTime.now();
+    const cache = {
+      httpCache: {
+        'http://example.com/foo': {
+          timestamp: now.toISO(),
+          etag: 'abc',
+          lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT',
+          httpResponse: {},
+        },
+        'http://example.com/bar': {
+          timestamp: now.toISO(),
+          etag: 'abc',
+          lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT',
+          httpResponse: {},
+        },
+      },
+    };
+
+    cleanupHttpCache(cache);
+
+    expect(cache).toEqual({});
+  });
+});

lib/util/cache/repository/http-cache.ts

@@ -0,0 +1,33 @@
+import is from '@sindresorhus/is';
+import { DateTime } from 'luxon';
+import { GlobalConfig } from '../../../config/global';
+import { logger } from '../../../logger';
+import { HttpCacheSchema } from '../../http/cache/schema';
+
+export function cleanupHttpCache(cacheData: unknown): void {
+  if (!is.plainObject(cacheData) || !is.plainObject(cacheData['httpCache'])) {
+    logger.warn('cleanupHttpCache: invalid cache data');
+    return;
+  }
+  const httpCache = cacheData['httpCache'];
+
+  const ttlDays = GlobalConfig.get('httpCacheTtlDays', 90);
+  if (ttlDays === 0) {
+    logger.trace('cleanupHttpCache: zero value received, removing the cache');
+    delete cacheData['httpCache'];
+    return;
+  }
+
+  const now = DateTime.now();
+  for (const [url, item] of Object.entries(httpCache)) {
+    const parsed = HttpCacheSchema.safeParse(item);
+    if (parsed.success && parsed.data) {
+      const item = parsed.data;
+      const expiry = DateTime.fromISO(item.timestamp).plus({ days: ttlDays });
+      if (expiry < now) {
+        logger.debug(`http cache: removing expired cache for ${url}`);
+        delete httpCache[url];
+      }
+    }
+  }
+}

lib/util/cache/repository/impl/base.ts

@@ -5,6 +5,7 @@ import { compressToBase64, decompressFromBase64 } from '../../../compress';
 import { hash } from '../../../hash';
 import { safeStringify } from '../../../stringify';
 import { CACHE_REVISION } from '../common';
+import { cleanupHttpCache } from '../http-cache';
 import { RepoCacheRecord, RepoCacheV13 } from '../schema';
 import type { RepoCache, RepoCacheData } from '../types';
 
@@ -70,6 +71,8 @@ export abstract class RepoCacheBase implements RepoCache {
   }
 
   async save(): Promise<void> {
+    cleanupHttpCache(this.data);
+
     const jsonStr = safeStringify(this.data);
     const hashedJsonStr = hash(jsonStr);
     if (hashedJsonStr === this.oldHash) {