diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts index 6d684fd115d3df..386241a255e838 100644 --- a/lib/modules/datasource/api.ts +++ b/lib/modules/datasource/api.ts @@ -6,6 +6,7 @@ import { CdnJsDatasource } from './cdnjs'; import { ClojureDatasource } from './clojure'; import { ConanDatasource } from './conan'; import { CondaDatasource } from './conda'; +import { CpanDatasource } from './cpan'; import { CrateDatasource } from './crate'; import { DartDatasource } from './dart'; import { DockerDatasource } from './docker'; @@ -55,6 +56,7 @@ api.set(CdnJsDatasource.id, new CdnJsDatasource()); api.set(ClojureDatasource.id, new ClojureDatasource()); api.set(ConanDatasource.id, new ConanDatasource()); api.set(CondaDatasource.id, new CondaDatasource()); +api.set(CpanDatasource.id, new CpanDatasource()); api.set(CrateDatasource.id, new CrateDatasource()); api.set(DartDatasource.id, new DartDatasource()); api.set(DockerDatasource.id, new DockerDatasource()); diff --git a/lib/modules/datasource/cpan/__fixtures__/Plack.json b/lib/modules/datasource/cpan/__fixtures__/Plack.json new file mode 100644 index 00000000000000..9fb33a17fd6236 --- /dev/null +++ b/lib/modules/datasource/cpan/__fixtures__/Plack.json @@ -0,0 +1,235 @@ +{ + "took" : 3, + "_shards" : { + "total" : 3, + "successful" : 3, + "failed" : 0 + }, + "hits" : { + "total" : 169, + "hits" : [ + { + "_type" : "file", + "_id" : "iGmpDWEqEQB4FO1lXb0j3tLgIDw", + "_index" : "cpan_v1_01", + "sort" : [ + 1606695696000 + ], + "_score" : null, + "_source" : { + "deprecated" : false, + "module" : [ + { + "name" : "Plack", + "version" : "1.0048" + } + ], + "date" : "2020-11-30T00:21:36", + "maturity" : "released", + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0048.tar.gz" + } + }, + { + "_index" : "cpan_v1_01", + "_id" : "rV2b5IZa2PG_JbjbYRdzPXZPFw4", + "sort" : [ + 1518254730000 + ], + "_score" : null, + "_source" : { + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0047.tar.gz", + "deprecated" : false, + "module" : [ + { + "version" : "1.0047", + "name" : "Plack" + } + ], + "date" : "2018-02-10T09:25:30", + "maturity" : "released" + }, + "_type" : "file" + }, + { + "_source" : { + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0046.tar.gz", + "maturity" : "released", + "module" : [ + { + "version" : "1.0046", + "name" : "Plack" + } + ], + "date" : "2018-02-10T07:52:31", + "deprecated" : false + }, + "sort" : [ + 1518249151000 + ], + "_score" : null, + "_index" : "cpan_v1_01", + "_id" : "K3Pl0SIFDJ2K3kt5HxLnT2gVIxc", + "_type" : "file" + }, + { + "_source" : { + "deprecated" : false, + "maturity" : "released", + "module" : [ + { + "version" : "1.0045", + "name" : "Plack" + } + ], + "date" : "2017-12-31T20:42:50", + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0045.tar.gz" + }, + "_index" : "cpan_v1_01", + "_id" : "mMVzdQVT1y8stbntvAQQc87mGeM", + "sort" : [ + 1514752970000 + ], + "_score" : null, + "_type" : "file" + }, + { + "_type" : "file", + "_source" : { + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0044.tar.gz", + "distribution" : "Plack", + "module" : [ + { + "name" : "Plack", + "version" : "1.0044" + } + ], + "date" : "2017-04-27T17:48:20", + "maturity" : "released", + "deprecated" : false + }, + "_score" : null, + "sort" : [ + 1493315300000 + ], + "_index" : "cpan_v1_01", + "_id" : "FCsCemrqDXJgxmIF1MYaf3yxHtk" + }, + { + "_type" : "file", + "_index" : "cpan_v1_01", + "_id" : "Noy_4t07AY__XEYnpJ9efRqKkNY", + "_score" : null, + "sort" : [ + 1487732525000 + ], + "_source" : { + "deprecated" : false, + "module" : [ + { + "name" : "Plack", + "version" : "1.0043" + } + ], + "date" : "2017-02-22T03:02:05", + "maturity" : "released", + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0043.tar.gz" + } + }, + { + "_type" : "file", + "_score" : null, + "sort" : [ + 1475127522000 + ], + "_index" : "cpan_v1_01", + "_id" : "xC2vIHM6w4Bruk0EsQCeVm25d2c", + "_source" : { + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0042.tar.gz", + "maturity" : "released", + "module" : [ + { + "name" : "Plack", + "version" : "1.0042" + } + ], + "date" : "2016-09-29T05:38:42", + "deprecated" : false + } + }, + { + "_source" : { + "deprecated" : false, + "maturity" : "released", + "module" : [ + { + "name" : "Plack", + "version" : "1.0041" + } + ], + "date" : "2016-09-25T21:25:47", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0041.tar.gz", + "distribution" : "Plack" + }, + "_index" : "cpan_v1_01", + "_id" : "JBBPOsgzGyMPb5CkrBY8GnrCocs", + "_score" : null, + "sort" : [ + 1474838747000 + ], + "_type" : "file" + }, + { + "_source" : { + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0040-TRIAL.tar.gz", + "deprecated" : false, + "module" : [ + { + "version" : "1.0040", + "name" : "Plack" + } + ], + "date" : "2016-04-01T16:58:21", + "maturity" : "developer" + }, + "sort" : [ + 1459529901000 + ], + "_score" : null, + "_index" : "cpan_v1_01", + "_id" : "NkAJcXogQmuVpc46K4jm07QqUS8", + "_type" : "file" + }, + { + "_type" : "file", + "_source" : { + "deprecated" : false, + "maturity" : "released", + "module" : [ + { + "version" : "1.0039", + "name" : "Plack" + } + ], + "date" : "2015-12-06T11:29:40", + "distribution" : "Plack", + "download_url" : "https://cpan.metacpan.org/authors/id/M/MI/MIYAGAWA/Plack-1.0039.tar.gz" + }, + "_index" : "cpan_v1_01", + "_id" : "Y7WlIYOZjk3rh9O05F3UZE6WwGo", + "_score" : null, + "sort" : [ + 1449401380000 + ] + } + ], + "max_score" : null + }, + "timed_out" : false +} diff --git a/lib/modules/datasource/cpan/__fixtures__/empty.json b/lib/modules/datasource/cpan/__fixtures__/empty.json new file mode 100644 index 00000000000000..8b2e7fe6f30195 --- /dev/null +++ b/lib/modules/datasource/cpan/__fixtures__/empty.json @@ -0,0 +1,14 @@ +{ + "took" : 3, + "_shards" : { + "total" : 3, + "successful" : 3, + "failed" : 0 + }, + "hits" : { + "max_score" : null, + "hits" : [], + "total" : 0 + }, + "timed_out" : false +} diff --git a/lib/modules/datasource/cpan/index.spec.ts b/lib/modules/datasource/cpan/index.spec.ts new file mode 100644 index 00000000000000..1f5f31882c5df3 --- /dev/null +++ b/lib/modules/datasource/cpan/index.spec.ts @@ -0,0 +1,91 @@ +import { getPkgReleases } from '..'; +import { Fixtures } from '../../../../test/fixtures'; +import * as httpMock from '../../../../test/http-mock'; +import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages'; +import { CpanDatasource } from '.'; + +const baseUrl = 'https://fastapi.metacpan.org/'; + +describe('modules/datasource/cpan/index', () => { + describe('getReleases', () => { + it('returns null for empty result', async () => { + httpMock + .scope(baseUrl) + .post( + '/v1/file/_search', + (body) => + body.query.filtered.filter.and[0].term['module.name'] === 'FooBar' + ) + .reply(200, Fixtures.get('empty.json')); + expect( + await getPkgReleases({ + datasource: CpanDatasource.id, + depName: 'FooBar', + }) + ).toBeNull(); + }); + + it('returns null for 404', async () => { + httpMock.scope(baseUrl).post('/v1/file/_search').reply(404); + expect( + await getPkgReleases({ + datasource: CpanDatasource.id, + depName: 'Plack', + }) + ).toBeNull(); + }); + + it('throws for 5xx', async () => { + httpMock.scope(baseUrl).post('/v1/file/_search').reply(502); + await expect( + getPkgReleases({ + datasource: CpanDatasource.id, + depName: 'Plack', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + }); + + it('returns null for unknown error', async () => { + httpMock.scope(baseUrl).post('/v1/file/_search').replyWithError(''); + expect( + await getPkgReleases({ + datasource: CpanDatasource.id, + depName: 'Plack', + }) + ).toBeNull(); + }); + + it('processes real data', async () => { + httpMock + .scope(baseUrl) + .post( + '/v1/file/_search', + (body) => + body.query.filtered.filter.and[0].term['module.name'] === 'Plack' + ) + .reply(200, Fixtures.get('Plack.json')); + const res = await getPkgReleases({ + datasource: CpanDatasource.id, + depName: 'Plack', + }); + expect(res).toMatchObject({ + changelogUrl: 'https://metacpan.org/dist/Plack/changes', + homepage: 'https://metacpan.org/pod/Plack', + registryUrl: 'https://fastapi.metacpan.org/', + releases: expect.toBeArrayOfSize(10), + }); + expect(res?.releases[1]).toMatchObject({ + isDeprecated: false, + isStable: false, + releaseTimestamp: '2016-04-01T16:58:21.000Z', + version: '1.0040', + }); + expect(res?.releases[9]).toMatchObject({ + isDeprecated: false, + isStable: true, + releaseTimestamp: '2020-11-30T00:21:36.000Z', + version: '1.0048', + }); + }); + }); +}); diff --git a/lib/modules/datasource/cpan/index.ts b/lib/modules/datasource/cpan/index.ts new file mode 100644 index 00000000000000..6fb3f934de72fb --- /dev/null +++ b/lib/modules/datasource/cpan/index.ts @@ -0,0 +1,109 @@ +import { cache } from '../../../util/cache/package/decorator'; +import { joinUrlParts } from '../../../util/url'; +import * as perlVersioning from '../../versioning/perl'; +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; +import type { MetaCpanApiFile, MetaCpanApiFileSearchResult } from './types'; + +export class CpanDatasource extends Datasource { + static readonly id = 'cpan'; + + constructor() { + super(CpanDatasource.id); + } + + override readonly customRegistrySupport = false; + + override readonly defaultRegistryUrls = ['https://fastapi.metacpan.org/']; + + override readonly defaultVersioning = perlVersioning.id; + + @cache({ + namespace: `datasource-${CpanDatasource.id}`, + key: ({ packageName }: GetReleasesConfig) => `${packageName}`, + }) + override async getReleases({ + packageName, + registryUrl, + }: GetReleasesConfig): Promise { + // istanbul ignore if + if (!registryUrl) { + return null; + } + + let result: ReleaseResult | null = null; + const searchUrl = joinUrlParts(registryUrl, 'v1/file/_search'); + + let hits: MetaCpanApiFile[] | null = null; + try { + const body = { + query: { + filtered: { + query: { match_all: {} }, + filter: { + and: [ + { term: { 'module.name': packageName } }, + { exists: { field: 'module.associated_pod' } }, + ], + }, + }, + }, + _source: [ + 'module.name', + 'module.version', + 'distribution', + 'date', + 'deprecated', + 'maturity', + ], + sort: [{ date: 'desc' }], + }; + const res = await this.http.postJson( + searchUrl, + { body } + ); + hits = res.body?.hits?.hits?.map(({ _source }) => _source); + } catch (err) { + this.handleGenericErrors(err); + } + + let latestDistribution: string | null = null; + if (hits) { + const releases: Release[] = []; + for (const hit of hits) { + const { + module, + distribution, + date: releaseTimestamp, + deprecated: isDeprecated, + maturity, + } = hit; + const version = module.find( + ({ name }) => name === packageName + )?.version; + if (version) { + // https://metacpan.org/pod/CPAN::DistnameInfo#maturity + const isStable = maturity === 'released'; + releases.push({ + isDeprecated, + isStable, + releaseTimestamp, + version, + }); + + if (!latestDistribution) { + latestDistribution = distribution; + } + } + } + if (releases.length > 0 && latestDistribution) { + result = { + releases, + changelogUrl: `https://metacpan.org/dist/${latestDistribution}/changes`, + homepage: `https://metacpan.org/pod/${packageName}`, + }; + } + } + return result; + } +} diff --git a/lib/modules/datasource/cpan/types.ts b/lib/modules/datasource/cpan/types.ts new file mode 100644 index 00000000000000..e6279b5b865da8 --- /dev/null +++ b/lib/modules/datasource/cpan/types.ts @@ -0,0 +1,24 @@ +/** + * https://fastapi.metacpan.org/v1/file/_mapping + */ +export interface MetaCpanApiFile { + module: { + name: string; + version?: string; + }[]; + distribution: string; + date: string; + deprecated: boolean; + maturity: string; +} + +/** + * https://github.com/metacpan/metacpan-api/blob/master/docs/API-docs.md#available-fields + */ +export interface MetaCpanApiFileSearchResult { + hits: { + hits: { + _source: MetaCpanApiFile; + }[]; + }; +} diff --git a/lib/modules/versioning/perl/readme.md b/lib/modules/versioning/perl/readme.md index 0cb3c741f247d5..623b9f17a75abc 100644 --- a/lib/modules/versioning/perl/readme.md +++ b/lib/modules/versioning/perl/readme.md @@ -1,3 +1,3 @@ Perl versioning is based on Perl's [`version` module](https://metacpan.org/pod/version). -This could be used in combination with the `regex` manager and the `repology` datasource. +This could be used in combination with the `regex` manager and the `cpan` datasource.