feat(internal): fixedRegistries (#9124)

lib/datasource/cdnjs/index.ts

@@ -3,6 +3,8 @@ import { Http } from '../../util/http';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'cdnjs';
+export const registryUrlRestriction = 'fixed';
+export const defaultRegistryUrls = ['https://api.cdnjs.com/'];
 export const caching = true;
 
 const http = new Http(id);
@@ -24,10 +26,11 @@ interface CdnjsResponse {
 
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   // Each library contains multiple assets, so we cache at the library level instead of per-asset
   const library = lookupName.split('/')[0];
-  const url = `https://api.cdnjs.com/libraries/${library}?fields=homepage,repository,assets`;
+  const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`;
   try {
     const { assets, homepage, repository } = (
       await http.getJson<CdnjsResponse>(url)

lib/datasource/dart/index.ts

@@ -3,14 +3,17 @@ import { Http, HttpResponse } from '../../util/http';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'dart';
+export const defaultRegistryUrls = ['https://pub.dartlang.org/'];
+export const registryUrlRestriction = 'fixed';
 
 const http = new Http(id);
 
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   let result: ReleaseResult = null;
-  const pkgUrl = `https://pub.dartlang.org/api/packages/${lookupName}`;
+  const pkgUrl = `${registryUrl}api/packages/${lookupName}`;
   interface DartResult {
     versions?: {
       version: string;

lib/datasource/galaxy/index.ts

@@ -5,11 +5,14 @@ import { Http } from '../../util/http';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
 
 export const id = 'galaxy';
+export const defaultRegistryUrls = ['https://galaxy.ansible.com/'];
+export const registryUrlRestriction = 'fixed';
 
 const http = new Http(id);
 
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   const cacheNamespace = 'datasource-galaxy';
   const cacheKey = lookupName;
@@ -26,14 +29,13 @@ export async function getReleases({
   const userName = lookUp[0];
   const projectName = lookUp[1];
 
-  const baseUrl = 'https://galaxy.ansible.com/';
   const galaxyAPIUrl =
-    baseUrl +
+    registryUrl +
     'api/v1/roles/?owner__username=' +
     userName +
     '&name=' +
     projectName;
-  const galaxyProjectUrl = baseUrl + userName + '/' + projectName;
+  const galaxyProjectUrl = registryUrl + userName + '/' + projectName;
   try {
     let res: any = await http.get(galaxyAPIUrl);
     if (!res || !res.body) {

lib/datasource/git-refs/index.ts

@@ -4,6 +4,7 @@ import * as semver from '../../versioning/semver';
 import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'git-refs';
+export const registryUrlRestriction = 'disallowed';
 
 const cacheMinutes = 10;
 

lib/datasource/git-tags/index.ts

@@ -3,6 +3,7 @@ import * as gitRefs from '../git-refs';
 import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'git-tags';
+export const registryUrlRestriction = 'disallowed';
 
 export async function getReleases({
   lookupName,

lib/datasource/go/index.ts

@@ -11,6 +11,7 @@ import * as gitlab from '../gitlab-tags';
 import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'go';
+export const registryUrlRestriction = 'disallowed';
 
 const http = new Http(id);
 const gitlabRegExp = /^(https:\/\/[^/]*gitlab.[^/]*)\/(.*)$/;

lib/datasource/hex/index.ts

@@ -5,6 +5,8 @@ import * as hexVersioning from '../../versioning/hex';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'hex';
+export const defaultRegistryUrls = ['https://hex.pm/'];
+export const registryUrlRestriction = 'fixed';
 export const defaultVersioning = hexVersioning.id;
 
 const http = new Http(id);
@@ -20,14 +22,15 @@ interface HexRelease {
 
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   // Get dependency name from lookupName.
   // If the dependency is private lookupName contains organization name as following:
   // hexPackageName:organizationName
   // hexPackageName is used to pass it in hex dep url
   // organizationName is used for accessing to private deps
   const hexPackageName = lookupName.split(':')[0];
-  const hexUrl = `https://hex.pm/api/packages/${hexPackageName}`;
+  const hexUrl = `${registryUrl}api/packages/${hexPackageName}`;
   try {
     const response = await http.getJson<HexRelease>(hexUrl);
 

lib/datasource/index.spec.ts

@@ -1,11 +1,12 @@
-import { getName, mocked } from '../../test/util';
+import { getName, logger, mocked } from '../../test/util';
 import {
   EXTERNAL_HOST_ERROR,
   HOST_DISABLED,
 } from '../constants/error-messages';
 import { ExternalHostError } from '../types/errors/external-host-error';
 import { loadModules } from '../util/modules';
 import * as datasourceDocker from './docker';
+import * as datasourceGalaxy from './galaxy';
 import * as datasourceGithubTags from './github-tags';
 import * as datasourceMaven from './maven';
 import * as datasourceNpm from './npm';
@@ -14,11 +15,13 @@ import type { DatasourceApi } from './types';
 import * as datasource from '.';
 
 jest.mock('./docker');
+jest.mock('./galaxy');
 jest.mock('./maven');
 jest.mock('./npm');
 jest.mock('./packagist');
 
 const dockerDatasource = mocked(datasourceDocker);
+const galaxyDatasource = mocked(datasourceGalaxy);
 const mavenDatasource = mocked(datasourceMaven);
 const npmDatasource = mocked(datasourceNpm);
 const packagistDatasource = mocked(datasourcePackagist);
@@ -128,6 +131,15 @@ describe(getName(__filename), () => {
     expect(res).toMatchSnapshot();
     expect(res.sourceUrl).toBeDefined();
   });
+  it('ignores and warns for registryUrls', async () => {
+    galaxyDatasource.getReleases.mockResolvedValue(null);
+    await datasource.getPkgReleases({
+      datasource: datasourceGalaxy.id,
+      depName: 'some.dep',
+      registryUrls: ['https://google.com/'],
+    });
+    expect(logger.logger.warn).toHaveBeenCalled();
+  });
   it('warns if multiple registryUrls for registryStrategy=first', async () => {
     dockerDatasource.getReleases.mockResolvedValue(null);
     const res = await datasource.getPkgReleases({

lib/datasource/index.ts

@@ -171,8 +171,17 @@ function resolveRegistryUrls(
   datasource: DatasourceApi,
   extractedUrls: string[]
 ): string[] {
-  const { defaultRegistryUrls = [] } = datasource;
+  const { defaultRegistryUrls = [], registryUrlRestriction } = datasource;
   const customUrls = extractedUrls?.filter(Boolean);
+  if (registryUrlRestriction) {
+    if (is.nonEmptyArray(customUrls)) {
+      logger.warn(
+        { datasource: datasource.id, customUrls },
+        'Ignoring custom registryUrls as they cannot be overridden'
+      );
+    }
+    return defaultRegistryUrls;
+  }
   let registryUrls: string[];
   if (is.nonEmptyArray(customUrls)) {
     registryUrls = [...customUrls];
@@ -199,7 +208,10 @@ async function fetchReleases(
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
   let dep: ReleaseResult = null;
   try {
-    if (datasource.registryStrategy) {
+    if (
+      datasource.registryStrategy ||
+      datasource.registryUrlRestriction === 'fixed'
+    ) {
       // istanbul ignore if
       if (!registryUrls.length) {
         logger.warn(
@@ -214,6 +226,9 @@ async function fetchReleases(
         dep = await huntRegistries(config, datasource, registryUrls);
       } else if (datasource.registryStrategy === 'merge') {
         dep = await mergeRegistries(config, datasource, registryUrls);
+      } else {
+        // Default to hunting default registries if no rangeStrategy provided
+        dep = await huntRegistries(config, datasource, registryUrls);
       }
     } else {
       dep = await datasource.getReleases({

lib/datasource/npm/index.ts

@@ -5,3 +5,4 @@ export { getReleases } from './releases';
 export { getNpmrc, setNpmrc } from './npmrc';
 export { id } from './common';
 export const defaultVersioning = npmVersioning.id;
+export const registryUrlRestriction = 'disallowed';

lib/datasource/orb/index.ts

@@ -4,6 +4,8 @@ import { Http } from '../../util/http';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'orb';
+export const defaultRegistryUrls = ['https://circleci.com/'];
+export const registryUrlRestriction = 'fixed';
 
 const http = new Http(id);
 
@@ -22,6 +24,7 @@ interface OrbRelease {
  */
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   logger.debug({ lookupName }, 'orb.getReleases()');
   const cacheNamespace = 'orb';
@@ -34,7 +37,7 @@ export async function getReleases({
   if (cachedResult) {
     return cachedResult;
   }
-  const url = 'https://circleci.com/graphql-unstable';
+  const url = `${registryUrl}graphql-unstable`;
   const body = {
     query: `{orb(name:"${lookupName}"){name, homeUrl, versions {version, createdAt}}}`,
     variables: {},

lib/datasource/repology/index.ts

@@ -7,6 +7,8 @@ import { getQueryString } from '../../util/url';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'repology';
+export const defaultRegistryUrls = ['https://repology.org/'];
+export const registryStrategy = 'hunt';
 
 const http = new Http(id);
 const cacheNamespace = `datasource-${id}-list`;
@@ -43,6 +45,7 @@ async function queryPackages(url: string): Promise<RepologyPackage[]> {
 }
 
 async function queryPackagesViaResolver(
+  registryUrl: string,
   repoName: string,
   packageName: string,
   packageType: RepologyPackageType
@@ -57,19 +60,20 @@ async function queryPackagesViaResolver(
 
   // Retrieve list of packages by looking up Repology project
   const packages = await queryPackages(
-    `https://repology.org/tools/project-by?${query}`
+    `${registryUrl}tools/project-by?${query}`
   );
 
   return packages;
 }
 
 async function queryPackagesViaAPI(
+  registryUrl: string,
   packageName: string
 ): Promise<RepologyPackage[]> {
   // Directly query the package via the API. This will only work if `packageName` has the
   // same name as the repology project
   const packages = await queryPackages(
-    `https://repology.org/api/v1/project/${packageName}`
+    `${registryUrl}api/v1/project/${packageName}`
   );
 
   return packages;
@@ -110,6 +114,7 @@ function findPackageInResponse(
 }
 
 async function queryPackage(
+  registryUrl: string,
   repoName: string,
   pkgName: string
 ): Promise<RepologyPackage[]> {
@@ -123,7 +128,12 @@ async function queryPackage(
   // are looking for.
   try {
     for (const pkgType of packageTypes) {
-      response = await queryPackagesViaResolver(repoName, pkgName, pkgType);
+      response = await queryPackagesViaResolver(
+        registryUrl,
+        repoName,
+        pkgName,
+        pkgType
+      );
 
       if (response) {
         pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]);
@@ -144,7 +154,7 @@ async function queryPackage(
       // API. This will support all repositories but requires that the project name is equal to the
       // package name. This won't be always the case but for a good portion we might be able to resolve
       // the package this way.
-      response = await queryPackagesViaAPI(pkgName);
+      response = await queryPackagesViaAPI(registryUrl, pkgName);
       pkg = findPackageInResponse(response, repoName, pkgName, packageTypes);
       if (pkg) {
         // exit immediately if package found
@@ -163,11 +173,12 @@ async function queryPackage(
 }
 
 async function getCachedPackage(
+  registryUrl: string,
   repoName: string,
   pkgName: string
 ): Promise<RepologyPackage[]> {
   // Fetch previous result from cache if available
-  const cacheKey = `${repoName}/${pkgName}`;
+  const cacheKey = `${registryUrl}${repoName}/${pkgName}`;
   const cachedResult = await packageCache.get<RepologyPackage[]>(
     cacheNamespace,
     cacheKey
@@ -178,7 +189,7 @@ async function getCachedPackage(
   }
 
   // Attempt a package lookup and return if found non empty list
-  const pkg = await queryPackage(repoName, pkgName);
+  const pkg = await queryPackage(registryUrl, repoName, pkgName);
   if (pkg && pkg.length > 0) {
     await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes);
     return pkg;
@@ -190,6 +201,7 @@ async function getCachedPackage(
 
 export async function getReleases({
   lookupName,
+  registryUrl,
 }: GetReleasesConfig): Promise<ReleaseResult | null> {
   // Ensure lookup name contains both repository and package
   const [repoName, pkgName] = lookupName.split('/', 2);
@@ -204,7 +216,7 @@ export async function getReleases({
   logger.trace(`repology.getReleases(${repoName}, ${pkgName})`);
   try {
     // Attempt to retrieve (cached) package information from Repology
-    const pkg = await getCachedPackage(repoName, pkgName);
+    const pkg = await getCachedPackage(registryUrl, repoName, pkgName);
     if (!pkg) {
       return null;
     }

lib/datasource/ruby-version/index.ts

@@ -6,15 +6,15 @@ import { isVersion, id as rubyVersioningId } from '../../versioning/ruby';
 import type { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'ruby-version';
+export const defaultRegistryUrls = ['https://www.ruby-lang.org/'];
+export const registryUrlRestriction = 'fixed';
 export const defaultVersioning = rubyVersioningId;
 
 const http = new Http(id);
 
-const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/';
-
-export async function getReleases(
-  _config?: GetReleasesConfig
-): Promise<ReleaseResult> {
+export async function getReleases({
+  registryUrl,
+}: GetReleasesConfig): Promise<ReleaseResult | null> {
   // First check the persistent cache
   const cacheNamespace = 'datasource-ruby-version';
   const cachedResult = await packageCache.get<ReleaseResult>(
@@ -31,6 +31,7 @@ export async function getReleases(
       sourceUrl: 'https://github.com/ruby/ruby',
       releases: [],
     };
+    const rubyVersionsUrl = `${registryUrl}en/downloads/releases/`;
     const response = await http.get(rubyVersionsUrl);
     const root = parse(response.body);
     const rows = root.querySelector('.release-list').querySelectorAll('tr');