From 044dc0fa2807c303a7043c326cec0d8f5fc9c4d1 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Tue, 21 May 2024 19:42:56 +0545 Subject: [PATCH] feat(self-hosted): `mergeConfidenceEndpoint` and `mergeConfidenceDatasources` (#28880) Co-authored-by: Rhys Arkins Co-authored-by: Michael Kriese Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- docs/usage/self-hosted-configuration.md | 33 ++++++++++ docs/usage/self-hosted-experimental.md | 20 ------ lib/config/global.ts | 2 + lib/config/options/index.ts | 20 ++++++ lib/config/types.ts | 2 + lib/config/validation.spec.ts | 6 ++ lib/config/validation.ts | 12 ++++ lib/util/merge-confidence/index.spec.ts | 70 ++++++++++----------- lib/util/merge-confidence/index.ts | 47 +++++--------- lib/workers/global/config/parse/env.spec.ts | 11 +++- lib/workers/global/config/parse/env.ts | 6 +- 11 files changed, 136 insertions(+), 93 deletions(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 297be8da6eb406..0a0bd42b6bb530 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -755,6 +755,39 @@ If left as default (null), a random short ID will be selected. ## logFileLevel +## mergeConfidenceDatasources + +This feature is applicable only if you have an access token for Mend's Merge Confidence API. + +If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list. +Otherwise, it queries all the supported datasources (check default value). + +Example: + +```js +modules.exports = { + mergeConfidenceDatasources: ['npm'], +}; +``` + +## mergeConfidenceEndpoint + +This feature is applicable only if you have an access token for Mend's Merge Confidence API. + +If set, Renovate will retrieve Merge Confidence data by querying this API. +Otherwise, it will use the default URL, which is . + +If you use the Mend Renovate Enterprise Edition (Renovate EE) and: + +- have a static merge confidence token that you set via `MEND_RNV_MC_TOKEN` +- _or_ set `MEND_RNV_MC_TOKEN` to `auto` + +Then you must set this variable at the _server_ and the _workers_. + +But if you have specified the token as a [`matchConfidence`](configuration-options.md#matchconfidence) `hostRule`, you only need to set this variable at the _workers_. + +This feature is in private beta. + ## migratePresets Use this if you have repositories that extend from a particular preset, which has now been renamed or removed. diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index f6118f1f15659e..b1800d4b6af51b 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -90,26 +90,6 @@ Suppress the default warning when a deprecated version of Node.js is used to run Skip initializing `RE2` for regular expressions and instead use Node-native `RegExp` instead. -## `RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL` - -If set, Renovate will query this API for Merge Confidence data. - -If you use the Mend Renovate Enterprise Edition (Renovate EE) and: - -- have a static merge confidence token that you set via `MEND_RNV_MC_TOKEN` -- _or_ set `MEND_RNV_MC_TOKEN` to `auto` - -Then you must set this variable at the _server_ and the _workers_. - -But if you have specified the token as a [`matchConfidence`](configuration-options.md#matchconfidence) `hostRule`, you only need to set this variable at the _workers_. - -This feature is in private beta. - -## `RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES` - -If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list. -The expected value for this environment variable is a JSON array of strings. - ## `RENOVATE_X_NUGET_DOWNLOAD_NUPKGS` If set to any value, Renovate will download `nupkg` files for determining package metadata. diff --git a/lib/config/global.ts b/lib/config/global.ts index 01bd31a965f646..1077299e45aca7 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -35,6 +35,8 @@ export class GlobalConfig { 'httpCacheTtlDays', 'autodiscoverRepoSort', 'autodiscoverRepoOrder', + 'mergeConfidenceEndpoint', + 'mergeConfidenceDatasources', 'userAgent', ]; diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 07b0988443b97a..7803bab3db0186 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2,6 +2,7 @@ import { getManagers } from '../../modules/manager'; import { getCustomManagers } from '../../modules/manager/custom'; import { getPlatformList } from '../../modules/platform'; import { getVersioningList } from '../../modules/versioning'; +import { supportedDatasources } from '../presets/internal/merge-confidence'; import type { RenovateOptions } from '../types'; const options: RenovateOptions[] = [ @@ -68,6 +69,25 @@ const options: RenovateOptions[] = [ default: false, globalOnly: true, }, + { + name: 'mergeConfidenceEndpoint', + description: + 'If set, Renovate will query this API for Merge Confidence data.', + type: 'string', + default: 'https://developer.mend.io/', + advancedUse: true, + globalOnly: true, + }, + { + name: 'mergeConfidenceDatasources', + description: + 'If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list.', + allowedValues: supportedDatasources, + default: supportedDatasources, + type: 'array', + subType: 'string', + globalOnly: true, + }, { name: 'useCloudMetadataServices', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index 9853144a82dd29..2129fd9b7ecc2b 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -154,6 +154,8 @@ export interface RepoGlobalConfig { githubTokenWarn?: boolean; includeMirrors?: boolean; localDir?: string; + mergeConfidenceEndpoint?: string; + mergeConfidenceDatasources?: string[]; migratePresets?: Record; platform?: PlatformId; presetCachePersistence?: boolean; diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index bdd5c1d2a6c16a..db30db168173a0 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -1590,6 +1590,7 @@ describe('config/validation', () => { allowedPostUpgradeCommands: ['cmd'], checkedBranches: 'invalid-type', gitNoVerify: ['invalid'], + mergeConfidenceDatasources: [1], }; const { warnings } = await configValidation.validateConfig( 'global', @@ -1602,6 +1603,11 @@ describe('config/validation', () => { 'Configuration option `checkedBranches` should be a list (Array).', topic: 'Configuration Error', }, + { + topic: 'Configuration Error', + message: + 'Invalid value `1` for `mergeConfidenceDatasources`. The allowed values are go, maven, npm, nuget, packagist, pypi, rubygems.', + }, { message: 'Invalid value for `gitNoVerify`. The allowed values are commit, push.', diff --git a/lib/config/validation.ts b/lib/config/validation.ts index 80f007f20a224a..f1620e13a4ea6f 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -24,6 +24,7 @@ import { GlobalConfig } from './global'; import { migrateConfig } from './migration'; import { getOptions } from './options'; import { resolveConfigPresets } from './presets'; +import { supportedDatasources } from './presets/internal/merge-confidence'; import { AllowedParents, type RenovateConfig, @@ -1006,6 +1007,17 @@ async function validateGlobalConfig( } } } + if (key === 'mergeConfidenceDatasources') { + const allowedValues = supportedDatasources; + for (const value of val as string[]) { + if (!allowedValues.includes(value)) { + warnings.push({ + topic: 'Configuration Error', + message: `Invalid value \`${value}\` for \`${currentPath}\`. The allowed values are ${allowedValues.join(', ')}.`, + }); + } + } + } } else { warnings.push({ topic: 'Configuration Error', diff --git a/lib/util/merge-confidence/index.spec.ts b/lib/util/merge-confidence/index.spec.ts index 9b672ba0ef78af..9529f061e43ebf 100644 --- a/lib/util/merge-confidence/index.spec.ts +++ b/lib/util/merge-confidence/index.spec.ts @@ -1,4 +1,5 @@ import * as httpMock from '../../../test/http-mock'; +import { GlobalConfig } from '../../config/global'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { logger } from '../../logger'; import type { HostRule } from '../../types'; @@ -17,6 +18,15 @@ import { describe('util/merge-confidence/index', () => { const apiBaseUrl = 'https://www.baseurl.com/'; const defaultApiBaseUrl = 'https://developer.mend.io/'; + const supportedDatasources = [ + 'go', + 'maven', + 'npm', + 'nuget', + 'packagist', + 'pypi', + 'rubygems', + ]; describe('isActiveConfidenceLevel()', () => { it('returns false if null', () => { @@ -57,10 +67,10 @@ describe('util/merge-confidence/index', () => { }; beforeEach(() => { - process.env.RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL = apiBaseUrl; hostRules.add(hostRule); initConfig(); memCache.reset(); + GlobalConfig.set({ mergeConfidenceEndpoint: apiBaseUrl }); }); afterEach(() => { @@ -302,16 +312,13 @@ describe('util/merge-confidence/index', () => { }); it('using default base url if none is set', async () => { - delete process.env.RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL; + GlobalConfig.reset(); httpMock .scope(defaultApiBaseUrl) .get(`/api/mc/availability`) .reply(200); await expect(initMergeConfidence()).toResolve(); - expect(logger.trace).toHaveBeenCalledWith( - 'using default merge confidence API base URL', - ); expect(logger.debug).toHaveBeenCalledWith( { supportedDatasources: [ @@ -329,8 +336,7 @@ describe('util/merge-confidence/index', () => { }); it('warns and then resolves if base url is invalid', async () => { - process.env.RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL = - 'invalid-url.com'; + GlobalConfig.set({ mergeConfidenceEndpoint: 'invalid-url.com' }); httpMock .scope(defaultApiBaseUrl) .get(`/api/mc/availability`) @@ -349,7 +355,7 @@ describe('util/merge-confidence/index', () => { it('uses a custom base url containing path', async () => { const renovateApi = 'https://domain.com/proxy/renovate-api'; - process.env.RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL = renovateApi; + GlobalConfig.set({ mergeConfidenceEndpoint: renovateApi }); httpMock.scope(renovateApi).get(`/api/mc/availability`).reply(200); await expect(initMergeConfidence()).toResolve(); @@ -423,54 +429,42 @@ describe('util/merge-confidence/index', () => { }); describe('parseSupportedDatasourceList()', () => { - type ParseSupportedDatasourceTestCase = { - name: string; - datasourceListString: string | undefined; - expected: string[] | undefined; - }; - afterEach(() => { - delete process.env.RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES; + GlobalConfig.reset(); }); it.each([ { - name: 'it should do nothing when the input is undefined', - datasourceListString: undefined, - expected: undefined, + name: 'it should do return default value when the input is undefined', + datasources: undefined, + expected: supportedDatasources, }, { name: 'it should successfully parse the given datasource list', - datasourceListString: `["go","npm"]`, + datasources: ['go', 'npm'], expected: ['go', 'npm'], }, { - name: 'it should gracefully handle invalid json', - datasourceListString: `{`, - expected: undefined, + name: 'it should gracefully handle invalid JSON', + datasources: `{`, + expected: supportedDatasources, }, { name: 'it should discard non-array JSON input', - datasourceListString: `{}`, - expected: undefined, + datasources: `{}`, + expected: supportedDatasources, }, { name: 'it should discard non-string array JSON input', - datasourceListString: `[1,2]`, - expected: undefined, + datasources: `[1,2]`, + expected: supportedDatasources, }, - ])( - `$name`, - ({ - datasourceListString, - expected, - }: ParseSupportedDatasourceTestCase) => { - process.env.RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES = - datasourceListString; - - expect(parseSupportedDatasourceString()).toStrictEqual(expected); - }, - ); + ])(`$name`, ({ datasources, expected }) => { + GlobalConfig.set({ + mergeConfidenceDatasources: datasources, + }); + expect(parseSupportedDatasourceString()).toStrictEqual(expected); + }); }); }); }); diff --git a/lib/util/merge-confidence/index.ts b/lib/util/merge-confidence/index.ts index 26be764a898d8c..05c29c8676b223 100644 --- a/lib/util/merge-confidence/index.ts +++ b/lib/util/merge-confidence/index.ts @@ -1,10 +1,10 @@ import is from '@sindresorhus/is'; +import { GlobalConfig } from '../../config/global'; import { supportedDatasources as presetSupportedDatasources } from '../../config/presets/internal/merge-confidence'; import type { UpdateType } from '../../config/types'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../cache/package'; -import { parseJson } from '../common'; import * as hostRules from '../host-rules'; import { Http } from '../http'; import { regEx } from '../regex'; @@ -28,41 +28,28 @@ export const confidenceLevels: Record = { export function initConfig(): void { apiBaseUrl = getApiBaseUrl(); token = getApiToken(); - supportedDatasources = - parseSupportedDatasourceString() ?? presetSupportedDatasources; + supportedDatasources = parseSupportedDatasourceString(); if (!is.nullOrUndefined(token)) { logger.debug(`Merge confidence token found for ${apiBaseUrl}`); } } -export function parseSupportedDatasourceString(): string[] | undefined { - const supportedDatasourceString = - process.env.RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES; - - if (!is.string(supportedDatasourceString)) { - return undefined; - } - - let parsedDatasourceList: unknown; - try { - parsedDatasourceList = parseJson(supportedDatasourceString, '.json5'); - } catch (err) { - logger.error( - { supportedDatasourceString, err }, - 'Failed to parse supported datasources list; Invalid JSON format', - ); - } +export function parseSupportedDatasourceString(): string[] { + const supportedDatasources = GlobalConfig.get( + 'mergeConfidenceDatasources', + presetSupportedDatasources, + ); - if (!is.array(parsedDatasourceList, is.string)) { + if (!is.array(supportedDatasources, is.string)) { logger.warn( - { parsedDatasourceList }, - `Expected a string array but got ${typeof parsedDatasourceList}`, + { supportedDatasources }, + `Expected a string array but got ${typeof supportedDatasources} - using default value instead`, ); - return undefined; + return presetSupportedDatasources; } - return parsedDatasourceList; + return supportedDatasources; } export function resetConfig(): void { @@ -242,12 +229,10 @@ export async function initMergeConfidence(): Promise { function getApiBaseUrl(): string { const defaultBaseUrl = 'https://developer.mend.io/'; - const baseFromEnv = process.env.RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL; - - if (is.nullOrUndefined(baseFromEnv)) { - logger.trace('using default merge confidence API base URL'); - return defaultBaseUrl; - } + const baseFromEnv = GlobalConfig.get( + 'mergeConfidenceEndpoint', + defaultBaseUrl, + ); try { const parsedBaseUrl = new URL(baseFromEnv).toString(); diff --git a/lib/workers/global/config/parse/env.spec.ts b/lib/workers/global/config/parse/env.spec.ts index 1acaa0b5d5d040..4efcc8cc1d526c 100644 --- a/lib/workers/global/config/parse/env.spec.ts +++ b/lib/workers/global/config/parse/env.spec.ts @@ -269,14 +269,19 @@ describe('workers/global/config/parse/env', () => { it('massages converted experimental env vars', async () => { const envParam: NodeJS.ProcessEnv = { + RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL: 'some-url', // converted + RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES: '["docker"]', // converted RENOVATE_X_AUTODISCOVER_REPO_SORT: 'alpha', RENOVATE_X_DOCKER_MAX_PAGES: '10', RENOVATE_AUTODISCOVER_REPO_ORDER: 'desc', }; const config = await env.getConfig(envParam); - expect(config.autodiscoverRepoSort).toBe('alpha'); - expect(config.autodiscoverRepoOrder).toBe('desc'); - expect(config.dockerMaxPages).toBeUndefined(); + expect(config).toMatchObject({ + mergeConfidenceEndpoint: 'some-url', + mergeConfidenceDatasources: ['docker'], + autodiscoverRepoSort: 'alpha', + autodiscoverRepoOrder: 'desc', + }); }); describe('RENOVATE_CONFIG tests', () => { diff --git a/lib/workers/global/config/parse/env.ts b/lib/workers/global/config/parse/env.ts index a37141cff4bba7..b49cee0cbaeb89 100644 --- a/lib/workers/global/config/parse/env.ts +++ b/lib/workers/global/config/parse/env.ts @@ -39,6 +39,8 @@ const renameKeys = { aliases: 'registryAliases', azureAutoComplete: 'platformAutomerge', // migrate: azureAutoComplete gitLabAutomerge: 'platformAutomerge', // migrate: gitLabAutomerge + mergeConfidenceApiBaseUrl: 'mergeConfidenceEndpoint', + mergeConfidenceSupportedDatasources: 'mergeConfidenceDatasources', }; function renameEnvKeys(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { @@ -86,6 +88,8 @@ function massageEnvKeyValues(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const convertedExperimentalEnvVars = [ 'RENOVATE_X_AUTODISCOVER_REPO_SORT', 'RENOVATE_X_AUTODISCOVER_REPO_ORDER', + 'RENOVATE_X_MERGE_CONFIDENCE_API_BASE_URL', + 'RENOVATE_X_MERGE_CONFIDENCE_SUPPORTED_DATASOURCES', ]; /** @@ -112,10 +116,10 @@ export async function getConfig( ): Promise { let env = inputEnv; env = normalizePrefixes(inputEnv, inputEnv.ENV_PREFIX); + env = massageConvertedExperimentalVars(env); env = renameEnvKeys(env); // massage the values of migrated configuration keys env = massageEnvKeyValues(env); - env = massageConvertedExperimentalVars(env); const options = getOptions();