diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index 78ef6d90418a09..ffdfd341174b1c 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -3,6 +3,8 @@ import { Http } from '../../util/http'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'cdnjs'; +export const registryUrlRestriction = 'fixed'; +export const defaultRegistryUrls = ['https://api.cdnjs.com/']; export const caching = true; const http = new Http(id); @@ -24,10 +26,11 @@ interface CdnjsResponse { export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { // Each library contains multiple assets, so we cache at the library level instead of per-asset const library = lookupName.split('/')[0]; - const url = `https://api.cdnjs.com/libraries/${library}?fields=homepage,repository,assets`; + const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`; try { const { assets, homepage, repository } = ( await http.getJson(url) diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts index 306c8cac4f09f8..d76f8ec01545c8 100644 --- a/lib/datasource/dart/index.ts +++ b/lib/datasource/dart/index.ts @@ -3,14 +3,17 @@ import { Http, HttpResponse } from '../../util/http'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'dart'; +export const defaultRegistryUrls = ['https://pub.dartlang.org/']; +export const registryUrlRestriction = 'fixed'; const http = new Http(id); export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { let result: ReleaseResult = null; - const pkgUrl = `https://pub.dartlang.org/api/packages/${lookupName}`; + const pkgUrl = `${registryUrl}api/packages/${lookupName}`; interface DartResult { versions?: { version: string; diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts index 99fd7f5ecb4bfa..fdb8ef9cba0de7 100644 --- a/lib/datasource/galaxy/index.ts +++ b/lib/datasource/galaxy/index.ts @@ -5,11 +5,14 @@ import { Http } from '../../util/http'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; export const id = 'galaxy'; +export const defaultRegistryUrls = ['https://galaxy.ansible.com/']; +export const registryUrlRestriction = 'fixed'; const http = new Http(id); export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { const cacheNamespace = 'datasource-galaxy'; const cacheKey = lookupName; @@ -26,14 +29,13 @@ export async function getReleases({ const userName = lookUp[0]; const projectName = lookUp[1]; - const baseUrl = 'https://galaxy.ansible.com/'; const galaxyAPIUrl = - baseUrl + + registryUrl + 'api/v1/roles/?owner__username=' + userName + '&name=' + projectName; - const galaxyProjectUrl = baseUrl + userName + '/' + projectName; + const galaxyProjectUrl = registryUrl + userName + '/' + projectName; try { let res: any = await http.get(galaxyAPIUrl); if (!res || !res.body) { diff --git a/lib/datasource/git-refs/index.ts b/lib/datasource/git-refs/index.ts index 083f89474d6762..f2b8e79bc1b50a 100644 --- a/lib/datasource/git-refs/index.ts +++ b/lib/datasource/git-refs/index.ts @@ -4,6 +4,7 @@ import * as semver from '../../versioning/semver'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'git-refs'; +export const registryUrlRestriction = 'disallowed'; const cacheMinutes = 10; diff --git a/lib/datasource/git-tags/index.ts b/lib/datasource/git-tags/index.ts index 5e2ada38c420f0..b848900727d68f 100644 --- a/lib/datasource/git-tags/index.ts +++ b/lib/datasource/git-tags/index.ts @@ -3,6 +3,7 @@ import * as gitRefs from '../git-refs'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'git-tags'; +export const registryUrlRestriction = 'disallowed'; export async function getReleases({ lookupName, diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts index 37e931c6f0e4fa..275059647d594a 100644 --- a/lib/datasource/go/index.ts +++ b/lib/datasource/go/index.ts @@ -11,6 +11,7 @@ import * as gitlab from '../gitlab-tags'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'go'; +export const registryUrlRestriction = 'disallowed'; const http = new Http(id); const gitlabRegExp = /^(https:\/\/[^/]*gitlab.[^/]*)\/(.*)$/; diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts index df886608fe8f93..45e883c1cc97b5 100644 --- a/lib/datasource/hex/index.ts +++ b/lib/datasource/hex/index.ts @@ -5,6 +5,8 @@ import * as hexVersioning from '../../versioning/hex'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'hex'; +export const defaultRegistryUrls = ['https://hex.pm/']; +export const registryUrlRestriction = 'fixed'; export const defaultVersioning = hexVersioning.id; const http = new Http(id); @@ -20,6 +22,7 @@ interface HexRelease { export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { // Get dependency name from lookupName. // If the dependency is private lookupName contains organization name as following: @@ -27,7 +30,7 @@ export async function getReleases({ // hexPackageName is used to pass it in hex dep url // organizationName is used for accessing to private deps const hexPackageName = lookupName.split(':')[0]; - const hexUrl = `https://hex.pm/api/packages/${hexPackageName}`; + const hexUrl = `${registryUrl}api/packages/${hexPackageName}`; try { const response = await http.getJson(hexUrl); diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 76f06e0d1fd8ec..04b1f7c6e64f93 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -1,4 +1,4 @@ -import { getName, mocked } from '../../test/util'; +import { getName, logger, mocked } from '../../test/util'; import { EXTERNAL_HOST_ERROR, HOST_DISABLED, @@ -6,6 +6,7 @@ import { import { ExternalHostError } from '../types/errors/external-host-error'; import { loadModules } from '../util/modules'; import * as datasourceDocker from './docker'; +import * as datasourceGalaxy from './galaxy'; import * as datasourceGithubTags from './github-tags'; import * as datasourceMaven from './maven'; import * as datasourceNpm from './npm'; @@ -14,11 +15,13 @@ import type { DatasourceApi } from './types'; import * as datasource from '.'; jest.mock('./docker'); +jest.mock('./galaxy'); jest.mock('./maven'); jest.mock('./npm'); jest.mock('./packagist'); const dockerDatasource = mocked(datasourceDocker); +const galaxyDatasource = mocked(datasourceGalaxy); const mavenDatasource = mocked(datasourceMaven); const npmDatasource = mocked(datasourceNpm); const packagistDatasource = mocked(datasourcePackagist); @@ -128,6 +131,15 @@ describe(getName(__filename), () => { expect(res).toMatchSnapshot(); expect(res.sourceUrl).toBeDefined(); }); + it('ignores and warns for registryUrls', async () => { + galaxyDatasource.getReleases.mockResolvedValue(null); + await datasource.getPkgReleases({ + datasource: datasourceGalaxy.id, + depName: 'some.dep', + registryUrls: ['https://google.com/'], + }); + expect(logger.logger.warn).toHaveBeenCalled(); + }); it('warns if multiple registryUrls for registryStrategy=first', async () => { dockerDatasource.getReleases.mockResolvedValue(null); const res = await datasource.getPkgReleases({ diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 6f865853df7e1f..58b8efeaedd08a 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -171,8 +171,17 @@ function resolveRegistryUrls( datasource: DatasourceApi, extractedUrls: string[] ): string[] { - const { defaultRegistryUrls = [] } = datasource; + const { defaultRegistryUrls = [], registryUrlRestriction } = datasource; const customUrls = extractedUrls?.filter(Boolean); + if (registryUrlRestriction) { + if (is.nonEmptyArray(customUrls)) { + logger.warn( + { datasource: datasource.id, customUrls }, + 'Ignoring custom registryUrls as they cannot be overridden' + ); + } + return defaultRegistryUrls; + } let registryUrls: string[]; if (is.nonEmptyArray(customUrls)) { registryUrls = [...customUrls]; @@ -199,7 +208,10 @@ async function fetchReleases( const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); let dep: ReleaseResult = null; try { - if (datasource.registryStrategy) { + if ( + datasource.registryStrategy || + datasource.registryUrlRestriction === 'fixed' + ) { // istanbul ignore if if (!registryUrls.length) { logger.warn( @@ -214,6 +226,9 @@ async function fetchReleases( dep = await huntRegistries(config, datasource, registryUrls); } else if (datasource.registryStrategy === 'merge') { dep = await mergeRegistries(config, datasource, registryUrls); + } else { + // Default to hunting default registries if no rangeStrategy provided + dep = await huntRegistries(config, datasource, registryUrls); } } else { dep = await datasource.getReleases({ diff --git a/lib/datasource/npm/index.ts b/lib/datasource/npm/index.ts index 673b2843cd445d..2bc09a87d68612 100644 --- a/lib/datasource/npm/index.ts +++ b/lib/datasource/npm/index.ts @@ -5,3 +5,4 @@ export { getReleases } from './releases'; export { getNpmrc, setNpmrc } from './npmrc'; export { id } from './common'; export const defaultVersioning = npmVersioning.id; +export const registryUrlRestriction = 'disallowed'; diff --git a/lib/datasource/orb/index.ts b/lib/datasource/orb/index.ts index 298eebe2749dec..f33a61edbea484 100644 --- a/lib/datasource/orb/index.ts +++ b/lib/datasource/orb/index.ts @@ -4,6 +4,8 @@ import { Http } from '../../util/http'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'orb'; +export const defaultRegistryUrls = ['https://circleci.com/']; +export const registryUrlRestriction = 'fixed'; const http = new Http(id); @@ -22,6 +24,7 @@ interface OrbRelease { */ export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { logger.debug({ lookupName }, 'orb.getReleases()'); const cacheNamespace = 'orb'; @@ -34,7 +37,7 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - const url = 'https://circleci.com/graphql-unstable'; + const url = `${registryUrl}graphql-unstable`; const body = { query: `{orb(name:"${lookupName}"){name, homeUrl, versions {version, createdAt}}}`, variables: {}, diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index 5c63c03916d37a..eb85626a444680 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -7,6 +7,8 @@ import { getQueryString } from '../../util/url'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'repology'; +export const defaultRegistryUrls = ['https://repology.org/']; +export const registryStrategy = 'hunt'; const http = new Http(id); const cacheNamespace = `datasource-${id}-list`; @@ -43,6 +45,7 @@ async function queryPackages(url: string): Promise { } async function queryPackagesViaResolver( + registryUrl: string, repoName: string, packageName: string, packageType: RepologyPackageType @@ -57,19 +60,20 @@ async function queryPackagesViaResolver( // Retrieve list of packages by looking up Repology project const packages = await queryPackages( - `https://repology.org/tools/project-by?${query}` + `${registryUrl}tools/project-by?${query}` ); return packages; } async function queryPackagesViaAPI( + registryUrl: string, packageName: string ): Promise { // Directly query the package via the API. This will only work if `packageName` has the // same name as the repology project const packages = await queryPackages( - `https://repology.org/api/v1/project/${packageName}` + `${registryUrl}api/v1/project/${packageName}` ); return packages; @@ -110,6 +114,7 @@ function findPackageInResponse( } async function queryPackage( + registryUrl: string, repoName: string, pkgName: string ): Promise { @@ -123,7 +128,12 @@ async function queryPackage( // are looking for. try { for (const pkgType of packageTypes) { - response = await queryPackagesViaResolver(repoName, pkgName, pkgType); + response = await queryPackagesViaResolver( + registryUrl, + repoName, + pkgName, + pkgType + ); if (response) { pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]); @@ -144,7 +154,7 @@ async function queryPackage( // API. This will support all repositories but requires that the project name is equal to the // package name. This won't be always the case but for a good portion we might be able to resolve // the package this way. - response = await queryPackagesViaAPI(pkgName); + response = await queryPackagesViaAPI(registryUrl, pkgName); pkg = findPackageInResponse(response, repoName, pkgName, packageTypes); if (pkg) { // exit immediately if package found @@ -163,11 +173,12 @@ async function queryPackage( } async function getCachedPackage( + registryUrl: string, repoName: string, pkgName: string ): Promise { // Fetch previous result from cache if available - const cacheKey = `${repoName}/${pkgName}`; + const cacheKey = `${registryUrl}${repoName}/${pkgName}`; const cachedResult = await packageCache.get( cacheNamespace, cacheKey @@ -178,7 +189,7 @@ async function getCachedPackage( } // Attempt a package lookup and return if found non empty list - const pkg = await queryPackage(repoName, pkgName); + const pkg = await queryPackage(registryUrl, repoName, pkgName); if (pkg && pkg.length > 0) { await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes); return pkg; @@ -190,6 +201,7 @@ async function getCachedPackage( export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { // Ensure lookup name contains both repository and package const [repoName, pkgName] = lookupName.split('/', 2); @@ -204,7 +216,7 @@ export async function getReleases({ logger.trace(`repology.getReleases(${repoName}, ${pkgName})`); try { // Attempt to retrieve (cached) package information from Repology - const pkg = await getCachedPackage(repoName, pkgName); + const pkg = await getCachedPackage(registryUrl, repoName, pkgName); if (!pkg) { return null; } diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts index bb01ea6a264a0d..134e33796e0423 100644 --- a/lib/datasource/ruby-version/index.ts +++ b/lib/datasource/ruby-version/index.ts @@ -6,15 +6,15 @@ import { isVersion, id as rubyVersioningId } from '../../versioning/ruby'; import type { GetReleasesConfig, ReleaseResult } from '../types'; export const id = 'ruby-version'; +export const defaultRegistryUrls = ['https://www.ruby-lang.org/']; +export const registryUrlRestriction = 'fixed'; export const defaultVersioning = rubyVersioningId; const http = new Http(id); -const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/'; - -export async function getReleases( - _config?: GetReleasesConfig -): Promise { +export async function getReleases({ + registryUrl, +}: GetReleasesConfig): Promise { // First check the persistent cache const cacheNamespace = 'datasource-ruby-version'; const cachedResult = await packageCache.get( @@ -31,6 +31,7 @@ export async function getReleases( sourceUrl: 'https://github.com/ruby/ruby', releases: [], }; + const rubyVersionsUrl = `${registryUrl}en/downloads/releases/`; const response = await http.get(rubyVersionsUrl); const root = parse(response.body); const rows = root.querySelector('.release-list').querySelectorAll('tr'); diff --git a/lib/datasource/types.ts b/lib/datasource/types.ts index 19102ecc0f7b1d..b49045cc4cd690 100644 --- a/lib/datasource/types.ts +++ b/lib/datasource/types.ts @@ -63,6 +63,26 @@ export interface DatasourceApi { defaultRegistryUrls?: string[]; defaultVersioning?: string; defaultConfig?: Record; + + /** + * Strategy to use when multiple registryUrls are available to the datasource. + * first: only the first registryUrl will be tried and others ignored + * hunt: registryUrls will be tried in order until one returns a result + * merge: all registryUrls will be tried and the results merged if more than one returns a result + */ registryStrategy?: 'first' | 'hunt' | 'merge'; + + /** + * Whether restrictions apply on custom registryUrls. If unspecified, it means custom registryUrls are allowed (no retriction). + * fixed: the default registryUrl settings can't be overridden + * disallowed: registryUrls are not applicable to this datasource + */ + registryUrlRestriction?: 'fixed' | 'disallowed'; + + /** + * Whether to perform caching in the datasource index/wrapper or not. + * true: datasoure index wrapper should cache all results (based on registryUrl/lookupName) + * false: caching is not performed, or performed within the datasource implementation + */ caching?: boolean; }