From 5917c548a6f70c7d5b7546e9e9110a4dc3921b8b Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Fri, 13 Jan 2023 12:05:32 +0100 Subject: [PATCH] fix(datasource/packagist): Revert "refactor(composer): Simplify Packagist lookups" (#19824) --- .../datasource/packagist/index.spec.ts | 35 ++-------- lib/modules/datasource/packagist/index.ts | 40 +++++++---- lib/modules/datasource/packagist/schema.ts | 67 ------------------- 3 files changed, 33 insertions(+), 109 deletions(-) delete mode 100644 lib/modules/datasource/packagist/schema.ts diff --git a/lib/modules/datasource/packagist/index.spec.ts b/lib/modules/datasource/packagist/index.spec.ts index 4ef75442e11e33..a0c7b09fc9f92b 100644 --- a/lib/modules/datasource/packagist/index.spec.ts +++ b/lib/modules/datasource/packagist/index.spec.ts @@ -82,12 +82,7 @@ describe('modules/datasource/packagist/index', () => { .scope('https://composer.renovatebot.com') .get('/packages.json') .replyWithError({ code: 'ETIMEDOUT' }); - httpMock - .scope(baseUrl) - .get('/p2/vendor/package-name2.json') - .reply(200) - .get('/p2/vendor/package-name2~dev.json') - .reply(200); + httpMock.scope(baseUrl).get('/p2/vendor/package-name2.json').reply(200); const res = await getPkgReleases({ ...config, datasource, @@ -102,12 +97,7 @@ describe('modules/datasource/packagist/index', () => { .scope('https://composer.renovatebot.com') .get('/packages.json') .reply(403); - httpMock - .scope(baseUrl) - .get('/p2/vendor/package-name.json') - .reply(200) - .get('/p2/vendor/package-name~dev.json') - .reply(200); + httpMock.scope(baseUrl).get('/p2/vendor/package-name.json').reply(200); const res = await getPkgReleases({ ...config, datasource, @@ -122,12 +112,7 @@ describe('modules/datasource/packagist/index', () => { .scope('https://composer.renovatebot.com') .get('/packages.json') .reply(404); - httpMock - .scope(baseUrl) - .get('/p2/drewm/mailchimp-api.json') - .reply(200) - .get('/p2/drewm/mailchimp-api~dev.json') - .reply(200); + httpMock.scope(baseUrl).get('/p2/drewm/mailchimp-api.json').reply(200); const res = await getPkgReleases({ ...config, datasource, @@ -281,12 +266,7 @@ describe('modules/datasource/packagist/index', () => { '/p/providers-2018-09$14346045d7a7261cb3a12a6b7a1a7c4151982530347b115e5e277d879cad1942.json' ) .reply(200, fileJson); - httpMock - .scope(baseUrl) - .get('/p2/some/other.json') - .reply(200, beytJson) - .get('/p2/some/other~dev.json') - .reply(200, beytJson); + httpMock.scope(baseUrl).get('/p2/some/other.json').reply(200, beytJson); const res = await getPkgReleases({ ...config, datasource, @@ -377,12 +357,7 @@ describe('modules/datasource/packagist/index', () => { .scope('https://composer.renovatebot.com') .get('/packages.json') .reply(200, packagesJson); - httpMock - .scope(baseUrl) - .get('/p2/some/other.json') - .reply(200, beytJson) - .get('/p2/some/other~dev.json') - .reply(200, beytJson); + httpMock.scope(baseUrl).get('/p2/some/other.json').reply(200, beytJson); const res = await getPkgReleases({ ...config, datasource, diff --git a/lib/modules/datasource/packagist/index.ts b/lib/modules/datasource/packagist/index.ts index 45271363604cdd..877592193d9c4f 100644 --- a/lib/modules/datasource/packagist/index.ts +++ b/lib/modules/datasource/packagist/index.ts @@ -1,6 +1,7 @@ import URL from 'url'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; +import * as packageCache from '../../../util/cache/package'; import { cache } from '../../../util/cache/package/decorator'; import * as hostRules from '../../../util/host-rules'; import type { HttpOptions } from '../../../util/http/types'; @@ -10,7 +11,6 @@ import { ensureTrailingSlash, joinUrlParts } from '../../../util/url'; import * as composerVersioning from '../../versioning/composer'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; -import * as schema from './schema'; import type { AllPackages, PackageMeta, @@ -199,19 +199,35 @@ export class PackagistDatasource extends Datasource { return allPackages; } - @cache({ - namespace: `datasource-${PackagistDatasource.id}-org`, - key: (regUrl: string) => regUrl, - ttlMinutes: 10, - }) async packagistOrgLookup(name: string): Promise { - const regUrl = 'https://packagist.org'; - const pkgUrl = joinUrlParts(regUrl, `/p2/${name}.json`); - const devUrl = joinUrlParts(regUrl, `/p2/${name}~dev.json`); - const results = await p.map([pkgUrl, devUrl], (url) => - this.http.getJson(url).then(({ body }) => body) + const cacheNamespace = 'datasource-packagist-org'; + const cachedResult = await packageCache.get( + cacheNamespace, + name ); - return schema.ComposerV2ReleaseResult.parse(results); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + let dep: ReleaseResult | null = null; + const regUrl = 'https://packagist.org'; + const pkgUrl = [ + joinUrlParts(regUrl, `/p2/${name}.json`), + joinUrlParts(regUrl, `/p2/${name}~dev.json`), + ]; + // TODO: fix types (#9610) + let res = (await this.http.getJson(pkgUrl[0])).body.packages[name]; + res = [ + ...res, + ...(await this.http.getJson(pkgUrl[1])).body.packages[name], + ]; + if (res) { + dep = PackagistDatasource.extractDepReleases(res); + logger.trace({ dep }, 'dep'); + } + const cacheMinutes = 10; + await packageCache.set(cacheNamespace, name, dep, cacheMinutes); + return dep; } public override async getReleases({ diff --git a/lib/modules/datasource/packagist/schema.ts b/lib/modules/datasource/packagist/schema.ts deleted file mode 100644 index c78032f80f8892..00000000000000 --- a/lib/modules/datasource/packagist/schema.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { z } from 'zod'; -import { api as versioning } from '../../versioning/composer'; -import type { Release, ReleaseResult } from '../types'; - -export const PackageName = z - .string() - .refine((s) => s.split('/').length === 2, 'Invalid package name'); - -export const ComposerV2Release = z.object({ - version: z.string(), - homepage: z.optional(z.string().url()), - source: z.optional( - z.object({ - url: z.string().url(), - }) - ), - time: z.string().datetime({ offset: true }), -}); - -export const ComposerV2PackageResponse = z.object({ - packages: z.record(PackageName, z.array(ComposerV2Release)), -}); - -export const ComposerV2ReleaseResult = z - .array(ComposerV2PackageResponse) - .transform((responses): ReleaseResult => { - const releases: Release[] = []; - let maxVersion: string | undefined; - let homepage: string | undefined = undefined; - let sourceUrl: string | undefined = undefined; - - for (const response of responses) { - for (const responsePackage of Object.values(response.packages)) { - for (const composerV2Release of responsePackage) { - const { version, time: releaseTimestamp } = composerV2Release; - const dep: Release = { - version: version.replace(/^v/, ''), - gitRef: version, - releaseTimestamp, - }; - releases.push(dep); - - if (!versioning.isValid(version)) { - continue; - } - - if (!maxVersion || versioning.isGreaterThan(version, maxVersion)) { - maxVersion = version; - homepage = composerV2Release.homepage; - sourceUrl = composerV2Release.source?.url; - } - } - } - } - - const result: ReleaseResult = { releases }; - - if (homepage) { - result.homepage = homepage; - } - - if (sourceUrl) { - result.sourceUrl = sourceUrl; - } - - return result; - });