Skip to content

Commit

Permalink
refactor(rubygems): Use Result type for caching control flow (#23266)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zharinov
zharinov committed Jul 11, 2023
1 parent 5c32fa7 commit 37ebff7
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 96 deletions.
12 changes: 5 additions & 7 deletions lib/modules/datasource/rubygems/index.ts
View file
Expand Up @@ -41,13 +41,11 @@ export class RubyGemsDatasource extends Datasource {
}

try {
const cachedVersions = await this.versionsEndpointCache.getVersions(
registryUrl,
packageName
);
const { res: versionsResult } =
await this.versionsEndpointCache.getVersions(registryUrl, packageName);

if (cachedVersions.type === 'success') {
const { versions } = cachedVersions;
if (versionsResult.success) {
const { value: versions } = versionsResult;
const result = await this.metadataCache.getRelease(
registryUrl,
packageName,
Expand All @@ -58,7 +56,7 @@ export class RubyGemsDatasource extends Datasource {

const registryHostname = parseUrl(registryUrl)?.hostname;
if (
cachedVersions.type === 'not-supported' &&
versionsResult.error === 'unsupported-api' &&
registryHostname !== 'rubygems.org'
) {
if (
Expand Down
70 changes: 28 additions & 42 deletions lib/modules/datasource/rubygems/versions-endpoint-cache.spec.ts
View file
Expand Up @@ -30,14 +30,12 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
const baz = await rubygems.getVersions(registryUrl, 'baz');
const qux = await rubygems.getVersions(registryUrl, 'qux');

expect(foo).toEqual({ type: 'success', versions: ['1.1.1'] });
expect(bar).toEqual({ type: 'success', versions: ['2.2.2'] });
expect(baz).toEqual({ type: 'success', versions: ['3.3.3'] });
expect(qux).toEqual({ type: 'not-found' });

expect(
memCache.get('rubygems-versions-cache:https://rubygems.org')
).toMatchObject({
expect(foo.value).toEqual(['1.1.1']);
expect(bar.value).toEqual(['2.2.2']);
expect(baz.value).toEqual(['3.3.3']);
expect(qux.error).toBe('package-not-found');

expect(memCache.get('https://rubygems.org')?.value).toMatchObject({
contentTail: '33333333333333333333333333333333\n',
});
});
Expand All @@ -51,21 +49,19 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
rubygems.getVersions(registryUrl, 'baz'),
]);

expect(foo).toEqual({ type: 'success', versions: ['1.1.1'] });
expect(bar).toEqual({ type: 'success', versions: ['2.2.2'] });
expect(baz).toEqual({ type: 'success', versions: ['3.3.3'] });
expect(foo.value).toEqual(['1.1.1']);
expect(bar.value).toEqual(['2.2.2']);
expect(baz.value).toEqual(['3.3.3']);
});

it('handles 404', async () => {
httpMock.scope(registryUrl).get('/versions').reply(404);

expect(await rubygems.getVersions(registryUrl, 'foo')).toEqual({
type: 'not-supported',
});
const res1 = await rubygems.getVersions(registryUrl, 'foo');
expect(res1.error).toBe('unsupported-api');

expect(await rubygems.getVersions(registryUrl, 'foo')).toEqual({
type: 'not-supported',
});
const res2 = await rubygems.getVersions(registryUrl, 'foo');
expect(res2.error).toBe('unsupported-api');

expect(memCache.size).toBe(1);
});
Expand Down Expand Up @@ -96,7 +92,7 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
httpMock.scope(registryUrl).get('/versions').reply(200, fullBody);

const res1 = await rubygems.getVersions(registryUrl, 'foo');
expect(res1).toEqual({ type: 'success', versions: ['1.1.1'] });
expect(res1.value).toEqual(['1.1.1']);

jest.advanceTimersByTime(15 * 60 * 1000);
httpMock
Expand All @@ -111,11 +107,9 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
);

const res2 = await rubygems.getVersions(registryUrl, 'foo');
expect(res2).toEqual({ type: 'success', versions: ['1.2.3'] });
expect(res2.value).toEqual(['1.2.3']);

expect(
memCache.get('rubygems-versions-cache:https://rubygems.org')
).toMatchObject({
expect(memCache.get('https://rubygems.org')?.value).toMatchObject({
contentTail: '44444444444444444444444444444444\n',
});
});
Expand All @@ -124,7 +118,7 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
httpMock.scope(registryUrl).get('/versions').reply(200, fullBody);

const res1 = await rubygems.getVersions(registryUrl, 'foo');
expect(res1).toEqual({ type: 'success', versions: ['1.1.1'] });
expect(res1.value).toEqual(['1.1.1']);

jest.advanceTimersByTime(15 * 60 * 1000);
httpMock
Expand All @@ -150,11 +144,9 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
);

const res2 = await rubygems.getVersions(registryUrl, 'foo');
expect(res2).toEqual({ type: 'success', versions: ['1.2.3'] });
expect(res2.value).toEqual(['1.2.3']);

expect(
memCache.get('rubygems-versions-cache:https://rubygems.org')
).toMatchObject({
expect(memCache.get('https://rubygems.org')?.value).toMatchObject({
contentTail: '01010101010101010101010101010101\n',
});
});
Expand All @@ -163,7 +155,7 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
httpMock.scope(registryUrl).get('/versions').reply(200, fullBody);

const res1 = await rubygems.getVersions(registryUrl, 'foo');
expect(res1).toEqual({ type: 'success', versions: ['1.1.1'] });
expect(res1.value).toEqual(['1.1.1']);

jest.advanceTimersByTime(15 * 60 * 1000);
httpMock
Expand All @@ -175,11 +167,9 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
);

const res2 = await rubygems.getVersions(registryUrl, 'foo');
expect(res2).toEqual({ type: 'success', versions: ['1.2.3'] });
expect(res2.value).toEqual(['1.2.3']);

expect(
memCache.get('rubygems-versions-cache:https://rubygems.org')
).toMatchObject({
expect(memCache.get('https://rubygems.org')?.value).toMatchObject({
contentTail: '44444444444444444444444444444444\n',
});
});
Expand All @@ -196,13 +186,11 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
it('handles 404', async () => {
httpMock.scope(registryUrl).get('/versions').reply(404);

expect(await rubygems.getVersions(registryUrl, 'foo')).toEqual({
type: 'not-supported',
});
const res1 = await rubygems.getVersions(registryUrl, 'foo');
expect(res1.error).toBe('unsupported-api');

expect(await rubygems.getVersions(registryUrl, 'foo')).toEqual({
type: 'not-supported',
});
const res2 = await rubygems.getVersions(registryUrl, 'foo');
expect(res2.error).toBe('unsupported-api');
});

it('handles 416', async () => {
Expand All @@ -222,7 +210,7 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {

const res = await rubygems.getVersions(registryUrl, 'foo');

expect(res).toEqual({ type: 'success', versions: ['9.9.9'] });
expect(res.value).toEqual(['9.9.9']);
});

it('handles unknown errors', async () => {
Expand All @@ -235,9 +223,7 @@ describe('modules/datasource/rubygems/versions-endpoint-cache', () => {
'Unknown error'
);

expect(
memCache.get('rubygems-versions-cache:https://rubygems.org')
).toBeUndefined();
expect(memCache.get('https://rubygems.org')).toBeUndefined();
});
});
});
Expand Down
93 changes: 46 additions & 47 deletions lib/modules/datasource/rubygems/versions-endpoint-cache.ts
View file
Expand Up @@ -4,18 +4,14 @@ import { getElapsedMinutes } from '../../../util/date';
import { Http, HttpError } from '../../../util/http';
import type { HttpOptions } from '../../../util/http/types';
import { newlineRegex } from '../../../util/regex';
import { Result } from '../../../util/result';
import { LooseArray } from '../../../util/schema-utils';
import { copystr } from '../../../util/string';
import { parseUrl } from '../../../util/url';

interface VersionsEndpointUnsupported {
versionsEndpointSupported: false;
}

type PackageVersions = Map<string, string[]>;

interface VersionsEndpointData {
versionsEndpointSupported: true;
packageVersions: PackageVersions;
syncedAt: Date;
contentLength: number;
Expand Down Expand Up @@ -68,8 +64,7 @@ function reconcilePackageVersions(
return packageVersions;
}

function parseFullBody(body: string): VersionsEndpointData {
const versionsEndpointSupported = true;
function parseFullBody(body: string): VersionsEndpointResult {
const packageVersions = reconcilePackageVersions(
new Map<string, string[]>(),
VersionLines.parse(body)
Expand All @@ -78,21 +73,28 @@ function parseFullBody(body: string): VersionsEndpointData {
const contentLength = body.length;
const contentTail = getContentTail(body);

return {
versionsEndpointSupported,
return Result.ok({
packageVersions,
syncedAt,
contentLength,
contentTail,
};
});
}

type VersionsEndpointResult =
| VersionsEndpointUnsupported
| VersionsEndpointData;
type VersionsEndpointResult = Result<VersionsEndpointData, 'unsupported-api'>;

export const memCache = new Map<string, VersionsEndpointResult>();

function cacheResult(
registryUrl: string,
result: VersionsEndpointResult
): void {
const registryHostname = parseUrl(registryUrl)?.hostname;
if (registryHostname === 'rubygems.org') {
memCache.set(registryUrl, result);
}
}

const VersionLines = z
.string()
.transform((x) => x.split(newlineRegex))
Expand Down Expand Up @@ -126,10 +128,10 @@ function isStale(regCache: VersionsEndpointData): boolean {
return getElapsedMinutes(regCache.syncedAt) >= 15;
}

export type VersionsResult =
| { type: 'success'; versions: string[] }
| { type: 'not-supported' }
| { type: 'not-found' };
export type VersionsResult = Result<
string[],
'unsupported-api' | 'package-not-found'
>;

export class VersionsEndpointCache {
constructor(private readonly http: Http) {}
Expand All @@ -140,28 +142,26 @@ export class VersionsEndpointCache {
* At any given time, there should only be one request for a given registryUrl.
*/
private async getCache(registryUrl: string): Promise<VersionsEndpointResult> {
const cacheKey = `rubygems-versions-cache:${registryUrl}`;
const oldResult = memCache.get(registryUrl);

const oldCache = memCache.get(cacheKey);
memCache.delete(cacheKey); // If no error is thrown, we'll re-set the cache

let newCache: VersionsEndpointResult;
if (!oldResult) {
const newResult = await this.fullSync(registryUrl);
cacheResult(registryUrl, newResult);
return newResult;
}

if (!oldCache) {
newCache = await this.fullSync(registryUrl);
} else if (oldCache.versionsEndpointSupported === false) {
newCache = oldCache;
} else if (isStale(oldCache)) {
newCache = await this.deltaSync(oldCache, registryUrl);
} else {
newCache = oldCache;
if (!oldResult.res.success) {
return oldResult;
}

const registryHostname = parseUrl(registryUrl)?.hostname;
if (registryHostname === 'rubygems.org') {
memCache.set(cacheKey, newCache);
if (isStale(oldResult.res.value)) {
memCache.delete(registryUrl); // If no error is thrown, we'll re-set the cache
const newResult = await this.deltaSync(oldResult.res.value, registryUrl);
cacheResult(registryUrl, newResult);
return newResult;
}
return newCache;

return oldResult;
}

async getVersions(
Expand All @@ -176,31 +176,32 @@ export class VersionsEndpointCache {
cacheRequest = this.getCache(registryUrl);
this.cacheRequests.set(registryUrl, cacheRequest);
}
let cache: VersionsEndpointResult;
let cachedResult: VersionsEndpointResult;
try {
cache = await cacheRequest;
cachedResult = await cacheRequest;
} finally {
this.cacheRequests.delete(registryUrl);
}
const { res } = cachedResult;

if (cache.versionsEndpointSupported === false) {
if (!res.success) {
logger.debug(
{ packageName, registryUrl },
'Rubygems: endpoint not supported'
);
return { type: 'not-supported' };
return Result.err('unsupported-api');
}

const versions = cache.packageVersions.get(packageName);
const versions = res.value.packageVersions.get(packageName);
if (!versions?.length) {
logger.debug(
{ packageName, registryUrl },
'Rubygems: versions not found'
);
return { type: 'not-found' };
return Result.err('package-not-found');
}

return { type: 'success', versions };
return Result.ok(versions);
}

private async fullSync(registryUrl: string): Promise<VersionsEndpointResult> {
Expand All @@ -211,7 +212,7 @@ export class VersionsEndpointCache {
return parseFullBody(body);
} catch (err) {
if (err instanceof HttpError && err.response?.statusCode === 404) {
return { versionsEndpointSupported: false };
return Result.err('unsupported-api');
}

throw err;
Expand Down Expand Up @@ -255,7 +256,6 @@ export class VersionsEndpointCache {
/**
* Update the cache with the new data.
*/
const versionsEndpointSupported = true;
const delta = stripContentHead(body);
const packageVersions = reconcilePackageVersions(
oldCache.packageVersions,
Expand All @@ -265,13 +265,12 @@ export class VersionsEndpointCache {
const contentLength = oldCache.contentLength + delta.length;
const contentTail = getContentTail(body);

return {
versionsEndpointSupported,
return Result.ok({
packageVersions,
syncedAt,
contentLength,
contentTail,
};
});
} catch (err) {
if (err instanceof HttpError) {
const responseStatus = err.response?.statusCode;
Expand All @@ -289,7 +288,7 @@ export class VersionsEndpointCache {
* This is unlikely to happen in real life, but still.
*/
if (responseStatus === 404) {
return { versionsEndpointSupported: false };
return Result.err('unsupported-api');
}
}

Expand Down

0 comments on commit 37ebff7

Please sign in to comment.