diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 21c10e0cd9551e..5f69e98036fb81 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -1552,12 +1552,13 @@ e.g. { "postUpgradeTasks": { "commands": ["tslint --fix"], - "fileFilters": ["yarn.lock", "**/*.js"] + "fileFilters": ["yarn.lock", "**/*.js"], + "executionMode": "update" } } ``` -The `postUpgradeTasks` configuration consists of two fields: +The `postUpgradeTasks` configuration consists of three fields: ### commands @@ -1567,6 +1568,11 @@ A list of commands that are executed after Renovate has updated a dependency but A list of glob-style matchers that determine which files will be included in the final commit made by Renovate +### executionMode + +Defaults to `update`, but can also be set to `branch`. This sets the level the postUpgradeTask runs on, if set to `update` the postUpgradeTask +will be executed for every dependency on the branch. If set to `branch` the postUpgradeTask is executed for the whole branch. + ## prBodyColumns Use this array to provide a list of column names you wish to include in the PR tables. diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index ee1de0277fd587..84e2d6aefe5704 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -11,7 +11,7 @@ Please also see [Self-Hosted Experimental Options](./self-hosted-experimental.md ## allowPostUpgradeCommandTemplating -Set to true to allow templating of post-upgrade commands. +Set to true to allow templating of dependency level post-upgrade commands. Let's look at an example of configuring packages with existing Angular migrations. diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index 9cffe4473353f3..65dcea16df3888 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -32,6 +32,7 @@ const options: RenovateOptions[] = [ default: { commands: [], fileFilters: [], + executionMode: 'update', }, }, { @@ -54,6 +55,16 @@ const options: RenovateOptions[] = [ default: [], cli: false, }, + { + name: 'executionMode', + description: + 'Controls whether the post upgrade tasks runs for every update or once per upgrade branch', + type: 'string', + parent: 'postUpgradeTasks', + allowedValues: ['update', 'branch'], + default: 'update', + cli: false, + }, { name: 'onboardingBranch', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index 5a1e066b81580d..ba94103c944589 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -114,10 +114,12 @@ export interface LegacyAdminConfig { platform?: string; requireConfig?: boolean; } +export type ExecutionMode = 'branch' | 'update'; export type PostUpgradeTasks = { commands?: string[]; fileFilters?: string[]; + executionMode: ExecutionMode; }; type UpdateConfig< diff --git a/lib/workers/branch/execute-post-upgrade-commands.ts b/lib/workers/branch/execute-post-upgrade-commands.ts new file mode 100644 index 00000000000000..c03dc951635a68 --- /dev/null +++ b/lib/workers/branch/execute-post-upgrade-commands.ts @@ -0,0 +1,196 @@ +import is from '@sindresorhus/is'; +import minimatch from 'minimatch'; +import { getAdminConfig } from '../../config/admin'; +import { addMeta, logger } from '../../logger'; +import type { ArtifactError } from '../../manager/types'; +import { exec } from '../../util/exec'; +import { readLocalFile, writeLocalFile } from '../../util/fs'; +import { File, getRepoStatus } from '../../util/git'; +import { regEx } from '../../util/regex'; +import { sanitize } from '../../util/sanitize'; +import { compile } from '../../util/template'; +import type { BranchConfig, BranchUpgradeConfig } from '../types'; + +export type PostUpgradeCommandsExecutionResult = { + updatedArtifacts: File[]; + artifactErrors: ArtifactError[]; +}; + +export async function postUpgradeCommandsExecutor( + filteredUpgradeCommands: BranchUpgradeConfig[], + config: BranchConfig +): Promise { + let updatedArtifacts = [...(config.updatedArtifacts || [])]; + const artifactErrors = [...(config.artifactErrors || [])]; + const { + allowedPostUpgradeCommands, + allowPostUpgradeCommandTemplating, + } = getAdminConfig(); + + for (const upgrade of filteredUpgradeCommands) { + addMeta({ dep: upgrade.depName }); + logger.trace( + { + tasks: upgrade.postUpgradeTasks, + allowedCommands: allowedPostUpgradeCommands, + }, + `Checking for post-upgrade tasks` + ); + const commands = upgrade.postUpgradeTasks?.commands || []; + const fileFilters = upgrade.postUpgradeTasks?.fileFilters || []; + + if (is.nonEmptyArray(commands)) { + // Persist updated files in file system so any executed commands can see them + for (const file of config.updatedPackageFiles.concat(updatedArtifacts)) { + if (file.name !== '|delete|') { + let contents; + if (typeof file.contents === 'string') { + contents = Buffer.from(file.contents); + } else { + contents = file.contents; + } + await writeLocalFile(file.name, contents); + } + } + + for (const cmd of commands) { + if ( + allowedPostUpgradeCommands.some((pattern) => regEx(pattern).test(cmd)) + ) { + try { + const compiledCmd = allowPostUpgradeCommandTemplating + ? compile(cmd, upgrade) + : cmd; + + logger.debug({ cmd: compiledCmd }, 'Executing post-upgrade task'); + const execResult = await exec(compiledCmd, { + cwd: config.localDir, + }); + + logger.debug( + { cmd: compiledCmd, ...execResult }, + 'Executed post-upgrade task' + ); + } catch (error) { + artifactErrors.push({ + lockFile: upgrade.packageFile, + stderr: sanitize(error.message), + }); + } + } else { + logger.warn( + { + cmd, + allowedPostUpgradeCommands, + }, + 'Post-upgrade task did not match any on allowed list' + ); + artifactErrors.push({ + lockFile: upgrade.packageFile, + stderr: sanitize( + `Post-upgrade command '${cmd}' does not match allowed pattern${ + allowedPostUpgradeCommands.length === 1 ? '' : 's' + } ${allowedPostUpgradeCommands.map((x) => `'${x}'`).join(', ')}` + ), + }); + } + } + + const status = await getRepoStatus(); + + for (const relativePath of status.modified.concat(status.not_added)) { + for (const pattern of fileFilters) { + if (minimatch(relativePath, pattern)) { + logger.debug( + { file: relativePath, pattern }, + 'Post-upgrade file saved' + ); + const existingContent = await readLocalFile(relativePath); + const existingUpdatedArtifacts = updatedArtifacts.find( + (ua) => ua.name === relativePath + ); + if (existingUpdatedArtifacts) { + existingUpdatedArtifacts.contents = existingContent; + } else { + updatedArtifacts.push({ + name: relativePath, + contents: existingContent, + }); + } + // If the file is deleted by a previous post-update command, remove the deletion from updatedArtifacts + updatedArtifacts = updatedArtifacts.filter( + (ua) => ua.name !== '|delete|' || ua.contents !== relativePath + ); + } + } + } + + for (const relativePath of status.deleted || []) { + for (const pattern of fileFilters) { + if (minimatch(relativePath, pattern)) { + logger.debug( + { file: relativePath, pattern }, + 'Post-upgrade file removed' + ); + updatedArtifacts.push({ + name: '|delete|', + contents: relativePath, + }); + // If the file is created or modified by a previous post-update command, remove the modification from updatedArtifacts + updatedArtifacts = updatedArtifacts.filter( + (ua) => ua.name !== relativePath + ); + } + } + } + } + } + return { updatedArtifacts, artifactErrors }; +} + +export default async function executePostUpgradeCommands( + config: BranchConfig +): Promise { + const { allowedPostUpgradeCommands } = getAdminConfig(); + + const hasChangedFiles = + config.updatedPackageFiles?.length > 0 || + config.updatedArtifacts?.length > 0; + + if ( + /* Only run post-upgrade tasks if there are changes to package files... */ + !hasChangedFiles || + is.emptyArray(allowedPostUpgradeCommands) + ) { + return null; + } + + const branchUpgradeCommands: BranchUpgradeConfig[] = [ + { + depName: config.upgrades.map(({ depName }) => depName).join(' '), + branchName: config.branchName, + postUpgradeTasks: + config.postUpgradeTasks.executionMode === 'branch' + ? config.postUpgradeTasks + : undefined, + fileFilters: config.fileFilters, + }, + ]; + + const updateUpgradeCommands: BranchUpgradeConfig[] = config.upgrades.filter( + ({ postUpgradeTasks }) => + !postUpgradeTasks || + !postUpgradeTasks.executionMode || + postUpgradeTasks.executionMode === 'update' + ); + + const { + updatedArtifacts, + artifactErrors, + } = await postUpgradeCommandsExecutor(updateUpgradeCommands, config); + return postUpgradeCommandsExecutor(branchUpgradeCommands, { + ...config, + updatedArtifacts, + artifactErrors, + }); +} diff --git a/lib/workers/branch/index.spec.ts b/lib/workers/branch/index.spec.ts index 51ff0568bcf091..7f78d75eccca14 100644 --- a/lib/workers/branch/index.spec.ts +++ b/lib/workers/branch/index.spec.ts @@ -6,17 +6,22 @@ import { REPOSITORY_CHANGED, } from '../../constants/error-messages'; import * as _npmPostExtract from '../../manager/npm/post-update'; +import type { WriteExistingFilesResult } from '../../manager/npm/post-update'; import { PrState } from '../../types'; import * as _exec from '../../util/exec'; import { File, StatusResult } from '../../util/git'; import * as _sanitize from '../../util/sanitize'; import * as _limits from '../global/limits'; import * as _prWorker from '../pr'; -import { BranchConfig, PrResult, ProcessBranchResult } from '../types'; +import type { EnsurePrResult } from '../pr'; +import type { Pr } from '../repository/onboarding/branch/check'; +import type { BranchConfig, BranchUpgradeConfig } from '../types'; +import { PrResult, ProcessBranchResult } from '../types'; import * as _automerge from './automerge'; import * as _checkExisting from './check-existing'; import * as _commit from './commit'; import * as _getUpdated from './get-updated'; +import type { PackageFilesResult } from './get-updated'; import * as _reuse from './reuse'; import * as _schedule from './schedule'; import * as branchWorker from '.'; @@ -50,7 +55,7 @@ const limits = mocked(_limits); describe('workers/branch', () => { describe('processBranch', () => { - const updatedPackageFiles: _getUpdated.PackageFilesResult = { + const updatedPackageFiles: PackageFilesResult = { updatedPackageFiles: [], artifactErrors: [], updatedArtifacts: [], @@ -64,8 +69,8 @@ describe('workers/branch', () => { branchName: 'renovate/some-branch', errors: [], warnings: [], - upgrades: [{ depName: 'some-dep-name' } as never], - } as never; + upgrades: [{ depName: 'some-dep-name' }], + } as BranchConfig; schedule.isScheduledNow.mockReturnValue(true); commit.commitFilesToBranch.mockResolvedValue('abc123'); @@ -113,7 +118,13 @@ describe('workers/branch', () => { releaseTimestamp: new Date().getTime(), stabilityDays: 1, }, + /* TODO: This test is probably broken and needs to be fixed. + The type definition for "releaseTimestamp" is a string. But when I change it to + one the test starts failing. Once this test has been fixed, the never typing can be removed. + And instead replaced with the pattern used on the other places that have a config.upgrades + */ ] as never; + git.branchExists.mockReturnValueOnce(false); const res = await branchWorker.processBranch(config); expect(res).toEqual(ProcessBranchResult.Pending); @@ -121,11 +132,11 @@ describe('workers/branch', () => { it('skips branch if not stabilityDays not met', async () => { schedule.isScheduledNow.mockReturnValueOnce(true); config.prCreation = 'not-pending'; - config.upgrades = [ + (config.upgrades as Partial[]) = [ { releaseTimestamp: '2099-12-31', stabilityDays: 1, - } as never, + }, ]; const res = await branchWorker.processBranch(config); expect(res).toEqual(ProcessBranchResult.Pending); @@ -136,7 +147,7 @@ describe('workers/branch', () => { git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Open, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(false); await branchWorker.processBranch(config); expect(reuse.shouldReuseExistingBranch).toHaveBeenCalled(); @@ -148,7 +159,7 @@ describe('workers/branch', () => { checkExisting.prAlreadyExisted.mockResolvedValueOnce({ number: 13, state: PrState.Closed, - } as never); + } as Pr); await branchWorker.processBranch(config); expect(reuse.shouldReuseExistingBranch).toHaveBeenCalledTimes(0); }); @@ -159,7 +170,7 @@ describe('workers/branch', () => { checkExisting.prAlreadyExisted.mockResolvedValueOnce({ number: 13, state: PrState.Closed, - } as never); + } as Pr); await branchWorker.processBranch(config); expect(reuse.shouldReuseExistingBranch).toHaveBeenCalledTimes(0); }); @@ -169,7 +180,7 @@ describe('workers/branch', () => { checkExisting.prAlreadyExisted.mockResolvedValueOnce({ number: 13, state: PrState.Closed, - } as never); + } as Pr); await branchWorker.processBranch(config); expect(reuse.shouldReuseExistingBranch).toHaveBeenCalledTimes(0); }); @@ -179,7 +190,7 @@ describe('workers/branch', () => { checkExisting.prAlreadyExisted.mockResolvedValueOnce({ number: 13, state: PrState.Merged, - } as never); + } as Pr); await branchWorker.processBranch(config); expect(reuse.shouldReuseExistingBranch).toHaveBeenCalledTimes(0); }); @@ -188,7 +199,7 @@ describe('workers/branch', () => { git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Merged, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); await expect(branchWorker.processBranch(config)).rejects.toThrow( REPOSITORY_CHANGED @@ -200,7 +211,7 @@ describe('workers/branch', () => { platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Open, labels: ['rebase'], - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); const res = await branchWorker.processBranch(config); expect(res).not.toEqual(ProcessBranchResult.PrEdited); @@ -211,7 +222,7 @@ describe('workers/branch', () => { platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Open, body: '**Rebasing**: something', - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); const res = await branchWorker.processBranch(config); expect(res).toEqual(ProcessBranchResult.PrEdited); @@ -222,7 +233,7 @@ describe('workers/branch', () => { platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Open, targetBranch: 'v6', - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(false); config.baseBranch = 'master'; const res = await branchWorker.processBranch(config); @@ -311,13 +322,13 @@ describe('workers/branch', () => { ); }); it('returns if branch automerged', async () => { - getUpdated.getUpdatedPackageFiles.mockReturnValueOnce({ + getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); - npmPostExtract.getAdditionalFiles.mockReturnValueOnce({ + } as PackageFilesResult); + npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); automerge.tryBranchAutomerge.mockResolvedValueOnce('automerged'); @@ -327,13 +338,13 @@ describe('workers/branch', () => { }); it('returns if branch automerged and no checks', async () => { - getUpdated.getUpdatedPackageFiles.mockReturnValueOnce({ + getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); - npmPostExtract.getAdditionalFiles.mockReturnValueOnce({ + } as PackageFilesResult); + npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(false); automerge.tryBranchAutomerge.mockResolvedValueOnce('automerged'); await branchWorker.processBranch({ @@ -347,11 +358,11 @@ describe('workers/branch', () => { it('returns if branch automerged (dry-run)', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); automerge.tryBranchAutomerge.mockResolvedValueOnce('automerged'); @@ -363,11 +374,11 @@ describe('workers/branch', () => { it('returns if branch exists and prCreation set to approval', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); @@ -382,11 +393,11 @@ describe('workers/branch', () => { expect.assertions(1); getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); @@ -401,11 +412,11 @@ describe('workers/branch', () => { expect.assertions(3); getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); expect( await branchWorker.processBranch({ ...config, @@ -420,17 +431,17 @@ describe('workers/branch', () => { it('ensures PR and tries automerge', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); - npmPostExtract.getAdditionalFiles.mockReturnValueOnce({ + } as PackageFilesResult); + npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); await branchWorker.processBranch(config); @@ -441,17 +452,17 @@ describe('workers/branch', () => { it('ensures PR and adds lock file error comment if no releaseTimestamp', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); await branchWorker.processBranch(config); @@ -462,17 +473,17 @@ describe('workers/branch', () => { it('ensures PR and adds lock file error comment if old releaseTimestamp', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); config.releaseTimestamp = '2018-04-26T05:15:51.877Z'; commit.commitFilesToBranch.mockResolvedValueOnce(null); @@ -484,17 +495,17 @@ describe('workers/branch', () => { it('ensures PR and adds lock file error comment if new releaseTimestamp and branch exists', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); config.releaseTimestamp = new Date().toISOString(); commit.commitFilesToBranch.mockResolvedValueOnce(null); @@ -506,17 +517,17 @@ describe('workers/branch', () => { it('throws error if lock file errors and new releaseTimestamp', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(false); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); config.releaseTimestamp = new Date().toISOString(); await expect(branchWorker.processBranch(config)).rejects.toThrow( @@ -526,18 +537,18 @@ describe('workers/branch', () => { it('ensures PR and adds lock file error comment recreate closed', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); config.recreateClosed = true; git.branchExists.mockReturnValueOnce(true); automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); prWorker.checkAutoMerge.mockResolvedValueOnce(true); commit.commitFilesToBranch.mockResolvedValueOnce(null); await branchWorker.processBranch(config); @@ -555,24 +566,24 @@ describe('workers/branch', () => { it('throws and swallows branch errors', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [{}], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); const processBranchResult = await branchWorker.processBranch(config); expect(processBranchResult).not.toBeNull(); }); it('swallows pr errors', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); - automerge.tryBranchAutomerge.mockResolvedValueOnce(false as never); + automerge.tryBranchAutomerge.mockResolvedValueOnce('failed'); prWorker.ensurePr.mockImplementationOnce(() => { throw new Error('some error'); }); @@ -584,7 +595,7 @@ describe('workers/branch', () => { git.branchExists.mockReturnValueOnce(true); checkExisting.prAlreadyExisted.mockResolvedValueOnce({ state: PrState.Closed, - } as never); + } as Pr); setAdminConfig({ dryRun: true }); expect(await branchWorker.processBranch(config)).toEqual( ProcessBranchResult.AlreadyExisted @@ -595,7 +606,7 @@ describe('workers/branch', () => { git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ state: PrState.Open, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); setAdminConfig({ dryRun: true }); expect(await branchWorker.processBranch(config)).toEqual( @@ -607,17 +618,17 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], artifactErrors: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); schedule.isScheduledNow.mockReturnValueOnce(false); commit.commitFilesToBranch.mockResolvedValueOnce(null); @@ -636,23 +647,23 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], artifactErrors: [{}], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); schedule.isScheduledNow.mockReturnValueOnce(false); prWorker.ensurePr.mockResolvedValueOnce({ - result: PrResult.Created, + prResult: PrResult.Created, pr: {}, - } as never); + } as EnsurePrResult); commit.commitFilesToBranch.mockResolvedValueOnce(null); setAdminConfig({ dryRun: true }); expect( @@ -667,17 +678,17 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], artifactErrors: [], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [{}], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); schedule.isScheduledNow.mockReturnValueOnce(false); commit.commitFilesToBranch.mockResolvedValueOnce(null); @@ -699,7 +710,7 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [updatedPackageFile], artifactErrors: [], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [ @@ -708,13 +719,13 @@ describe('workers/branch', () => { contents: Buffer.from([1, 2, 3]) /* Binary content */, }, ], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); git.getRepoStatus.mockResolvedValueOnce({ modified: ['modified_file'], @@ -738,6 +749,7 @@ describe('workers/branch', () => { const result = await branchWorker.processBranch({ ...config, postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{versioning}}}', 'disallowed task'], fileFilters: ['modified_file', 'deleted_file'], }, @@ -747,10 +759,11 @@ describe('workers/branch', () => { ...defaultConfig, depName: 'some-dep-name', postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{versioning}}}', 'disallowed task'], fileFilters: ['modified_file', 'deleted_file'], }, - } as never, + } as BranchUpgradeConfig, ], }); @@ -847,7 +860,7 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [updatedPackageFile], artifactErrors: [], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [ @@ -856,13 +869,13 @@ describe('workers/branch', () => { contents: Buffer.from([1, 2, 3]) /* Binary content */, }, ], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); git.getRepoStatus.mockResolvedValueOnce({ modified: ['modified_file'], @@ -884,6 +897,7 @@ describe('workers/branch', () => { const result = await branchWorker.processBranch({ ...config, postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{versioning}}}', 'disallowed task'], fileFilters: ['modified_file', 'deleted_file'], }, @@ -893,10 +907,11 @@ describe('workers/branch', () => { ...defaultConfig, depName: 'some-dep-name', postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{versioning}}}', 'disallowed task'], fileFilters: ['modified_file', 'deleted_file'], }, - } as never, + } as BranchUpgradeConfig, ], }); @@ -914,7 +929,7 @@ describe('workers/branch', () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [updatedPackageFile], artifactErrors: [], - } as never); + } as PackageFilesResult); npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ artifactErrors: [], updatedArtifacts: [ @@ -923,13 +938,13 @@ describe('workers/branch', () => { contents: Buffer.from([1, 2, 3]) /* Binary content */, }, ], - } as never); + } as WriteExistingFilesResult); git.branchExists.mockReturnValueOnce(true); platform.getBranchPr.mockResolvedValueOnce({ title: 'rebase!', state: PrState.Open, body: `- [x] `, - } as never); + } as Pr); git.isBranchModified.mockResolvedValueOnce(true); git.getRepoStatus .mockResolvedValueOnce({ @@ -960,9 +975,10 @@ describe('workers/branch', () => { }; setAdminConfig(adminConfig); - const inconfig = { + const inconfig: BranchConfig = { ...config, postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{depName}}}', 'disallowed task'], fileFilters: [ 'modified_file', @@ -977,6 +993,7 @@ describe('workers/branch', () => { ...defaultConfig, depName: 'some-dep-name-1', postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{depName}}}', 'disallowed task'], fileFilters: [ 'modified_file', @@ -985,11 +1002,12 @@ describe('workers/branch', () => { 'modified_then_deleted_file', ], }, - } as never, + } as BranchUpgradeConfig, { ...defaultConfig, depName: 'some-dep-name-2', postUpgradeTasks: { + executionMode: 'update', commands: ['echo {{{depName}}}', 'disallowed task'], fileFilters: [ 'modified_file', @@ -998,7 +1016,7 @@ describe('workers/branch', () => { 'modified_then_deleted_file', ], }, - } as never, + } as BranchUpgradeConfig, ], }; @@ -1040,5 +1058,109 @@ describe('workers/branch', () => { ) ).not.toBeUndefined(); }); + + it('executes post-upgrade tasks once when set to branch mode', async () => { + const updatedPackageFile: File = { + name: 'pom.xml', + contents: 'pom.xml file contents', + }; + getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ + updatedPackageFiles: [updatedPackageFile], + artifactErrors: [], + } as PackageFilesResult); + npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ + artifactErrors: [], + updatedArtifacts: [ + { + name: 'yarn.lock', + contents: Buffer.from([1, 2, 3]) /* Binary content */, + }, + ], + } as WriteExistingFilesResult); + git.branchExists.mockReturnValueOnce(true); + platform.getBranchPr.mockResolvedValueOnce({ + title: 'rebase!', + state: PrState.Open, + body: `- [x] `, + } as Pr); + git.isBranchModified.mockResolvedValueOnce(true); + git.getRepoStatus.mockResolvedValueOnce({ + modified: ['modified_file', 'modified_then_deleted_file'], + not_added: [], + deleted: ['deleted_file', 'deleted_then_created_file'], + } as StatusResult); + + fs.outputFile.mockReturnValue(); + fs.readFile + .mockResolvedValueOnce(Buffer.from('modified file content')) + .mockResolvedValueOnce(Buffer.from('this file will not exists')); + + schedule.isScheduledNow.mockReturnValueOnce(false); + commit.commitFilesToBranch.mockResolvedValueOnce(null); + + const adminConfig = { + allowedPostUpgradeCommands: ['^echo hardcoded-string$'], + allowPostUpgradeCommandTemplating: true, + trustLevel: 'high', + }; + setAdminConfig(adminConfig); + + const inconfig: BranchConfig = { + ...config, + postUpgradeTasks: { + executionMode: 'branch', + commands: ['echo hardcoded-string', 'disallowed task'], + fileFilters: [ + 'modified_file', + 'deleted_file', + 'deleted_then_created_file', + 'modified_then_deleted_file', + ], + }, + localDir: '/localDir', + upgrades: [ + { + ...defaultConfig, + depName: 'some-dep-name-1', + postUpgradeTasks: { + executionMode: 'branch', + commands: ['echo hardcoded-string', 'disallowed task'], + fileFilters: [ + 'modified_file', + 'deleted_file', + 'deleted_then_created_file', + 'modified_then_deleted_file', + ], + }, + } as BranchUpgradeConfig, + { + ...defaultConfig, + depName: 'some-dep-name-2', + postUpgradeTasks: { + executionMode: 'branch', + commands: ['echo hardcoded-string', 'disallowed task'], + fileFilters: [ + 'modified_file', + 'deleted_file', + 'deleted_then_created_file', + 'modified_then_deleted_file', + ], + }, + } as BranchUpgradeConfig, + ], + }; + + const result = await branchWorker.processBranch(inconfig); + expect(result).toEqual(ProcessBranchResult.Done); + expect(exec.exec).toHaveBeenNthCalledWith(1, 'echo hardcoded-string', { + cwd: '/localDir', + }); + expect(exec.exec).toHaveBeenCalledTimes(1); + expect( + (commit.commitFilesToBranch.mock.calls[0][0].updatedArtifacts.find( + (f) => f.name === 'modified_file' + ).contents as Buffer).toString() + ).toBe('modified file content'); + }); }); }); diff --git a/lib/workers/branch/index.ts b/lib/workers/branch/index.ts index ae4eb0ccc9573d..e0bc3eef37119b 100644 --- a/lib/workers/branch/index.ts +++ b/lib/workers/branch/index.ts @@ -1,6 +1,4 @@ -import is from '@sindresorhus/is'; import { DateTime } from 'luxon'; -import minimatch from 'minimatch'; import { RenovateConfig } from '../../config'; import { getAdminConfig } from '../../config/admin'; import { @@ -15,31 +13,26 @@ import { TEMPORARY_ERROR, WORKER_FILE_UPDATE_FAILED, } from '../../constants/error-messages'; -import { addMeta, logger, removeMeta } from '../../logger'; +import { logger, removeMeta } from '../../logger'; import { getAdditionalFiles } from '../../manager/npm/post-update'; import { Pr, platform } from '../../platform'; import { BranchStatus, PrState } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { emojify } from '../../util/emoji'; -import { exec } from '../../util/exec'; -import { readLocalFile, writeLocalFile } from '../../util/fs'; import { checkoutBranch, deleteBranch, getBranchCommit, - getRepoStatus, branchExists as gitBranchExists, isBranchModified, } from '../../util/git'; -import { regEx } from '../../util/regex'; -import { sanitize } from '../../util/sanitize'; -import * as template from '../../util/template'; import { Limit, isLimitReached } from '../global/limits'; import { checkAutoMerge, ensurePr, getPlatformPrOptions } from '../pr'; import { BranchConfig, PrResult, ProcessBranchResult } from '../types'; import { tryBranchAutomerge } from './automerge'; import { prAlreadyExisted } from './check-existing'; import { commitFilesToBranch } from './commit'; +import executePostUpgradeCommands from './execute-post-upgrade-commands'; import { getUpdatedPackageFiles } from './get-updated'; import { shouldReuseExistingBranch } from './reuse'; import { isScheduledNow } from './schedule'; @@ -349,148 +342,14 @@ export async function processBranch( } else { logger.debug('No updated lock files in branch'); } + const postUpgradeCommandResults = await executePostUpgradeCommands(config); - const { - allowedPostUpgradeCommands, - allowPostUpgradeCommandTemplating, - } = getAdminConfig(); - - if ( - /* Only run post-upgrade tasks if there are changes to package files... */ - (config.updatedPackageFiles?.length > 0 || - /* ... or changes to artifacts */ - config.updatedArtifacts?.length > 0) && - getAdminConfig().trustLevel === 'high' && - is.nonEmptyArray(allowedPostUpgradeCommands) - ) { - for (const upgrade of config.upgrades) { - addMeta({ dep: upgrade.depName }); - logger.trace( - { - tasks: upgrade.postUpgradeTasks, - allowedCommands: allowedPostUpgradeCommands, - }, - 'Checking for post-upgrade tasks' - ); - const commands = upgrade.postUpgradeTasks.commands || []; - const fileFilters = upgrade.postUpgradeTasks.fileFilters || []; - - if (is.nonEmptyArray(commands)) { - // Persist updated files in file system so any executed commands can see them - for (const file of config.updatedPackageFiles.concat( - config.updatedArtifacts - )) { - if (file.name !== '|delete|') { - let contents; - if (typeof file.contents === 'string') { - contents = Buffer.from(file.contents); - } else { - contents = file.contents; - } - await writeLocalFile(file.name, contents); - } - } - - for (const cmd of commands) { - if ( - allowedPostUpgradeCommands.some((pattern) => - regEx(pattern).test(cmd) - ) - ) { - try { - const compiledCmd = allowPostUpgradeCommandTemplating - ? template.compile(cmd, upgrade) - : cmd; - - logger.debug( - { cmd: compiledCmd }, - 'Executing post-upgrade task' - ); - const execResult = await exec(compiledCmd, { - cwd: config.localDir, - }); - - logger.debug( - { cmd: compiledCmd, ...execResult }, - 'Executed post-upgrade task' - ); - } catch (error) { - config.artifactErrors.push({ - lockFile: upgrade.packageFile, - stderr: sanitize(error.message), - }); - } - } else { - logger.warn( - { - cmd, - allowedPostUpgradeCommands, - }, - 'Post-upgrade task did not match any on allowed list' - ); - config.artifactErrors.push({ - lockFile: upgrade.packageFile, - stderr: sanitize( - `Post-upgrade command '${cmd}' does not match allowed pattern${ - allowedPostUpgradeCommands.length === 1 ? '' : 's' - } ${allowedPostUpgradeCommands - .map((x) => `'${x}'`) - .join(', ')}` - ), - }); - } - } - - const status = await getRepoStatus(); - - for (const relativePath of status.modified.concat(status.not_added)) { - for (const pattern of fileFilters) { - if (minimatch(relativePath, pattern)) { - logger.debug( - { file: relativePath, pattern }, - 'Post-upgrade file saved' - ); - const existingContent = await readLocalFile(relativePath); - const existingUpdatedArtifacts = config.updatedArtifacts.find( - (ua) => ua.name === relativePath - ); - if (existingUpdatedArtifacts) { - existingUpdatedArtifacts.contents = existingContent; - } else { - config.updatedArtifacts.push({ - name: relativePath, - contents: existingContent, - }); - } - // If the file is deleted by a previous post-update command, remove the deletion from updatedArtifacts - config.updatedArtifacts = config.updatedArtifacts.filter( - (ua) => ua.name !== '|delete|' || ua.contents !== relativePath - ); - } - } - } - - for (const relativePath of status.deleted || []) { - for (const pattern of fileFilters) { - if (minimatch(relativePath, pattern)) { - logger.debug( - { file: relativePath, pattern }, - 'Post-upgrade file removed' - ); - config.updatedArtifacts.push({ - name: '|delete|', - contents: relativePath, - }); - // If the file is created or modified by a previous post-update command, remove the modification from updatedArtifacts - config.updatedArtifacts = config.updatedArtifacts.filter( - (ua) => ua.name !== relativePath - ); - } - } - } - } - } + if (postUpgradeCommandResults !== null) { + const { updatedArtifacts, artifactErrors } = postUpgradeCommandResults; + config.updatedArtifacts = updatedArtifacts; + config.artifactErrors = artifactErrors; } + removeMeta(['dep']); if (config.artifactErrors?.length) { diff --git a/lib/workers/pr/index.ts b/lib/workers/pr/index.ts index af0d395c283d9e..23bbb44b1b13f2 100644 --- a/lib/workers/pr/index.ts +++ b/lib/workers/pr/index.ts @@ -111,14 +111,15 @@ export function getPlatformPrOptions( config.gitLabAutomerge, }; } +export type EnsurePrResult = { + prResult: PrResult; + pr?: Pr; +}; // Ensures that PR exists with matching title/body export async function ensurePr( prConfig: BranchConfig -): Promise<{ - prResult: PrResult; - pr?: Pr; -}> { +): Promise { const config: BranchConfig = { ...prConfig }; logger.trace({ config }, 'ensurePr');