From d5e4e098b395acf42de5f19f51a653ca68fa7113 Mon Sep 17 00:00:00 2001 From: Lucas <16666115+EndzeitBegins@users.noreply.github.com> Date: Mon, 30 Aug 2021 09:06:18 +0200 Subject: [PATCH] feat(gradle-wrapper): add support for multiple version occurrences in distributionUrl (#11454) --- docs/usage/java.md | 57 +++++++++++- .../custom-gradle-wrapper-all.properties | 7 ++ .../custom-gradle-wrapper-bin.properties | 7 ++ ...operties => gradle-wrapper-all.properties} | 5 +- ...operties => gradle-wrapper-bin.properties} | 4 +- ...wrapper-custom-distribution-url.properties | 7 -- ...s => gradle-wrapper-prerelease.properties} | 2 + ... gradle-wrapper-unknown-format.properties} | 2 + .../gradle-wrapper-whitespace.properties | 5 +- .../__snapshots__/extract.spec.ts.snap | 12 --- lib/manager/gradle-wrapper/extract.spec.ts | 90 +++++++++++++++---- lib/manager/gradle-wrapper/extract.ts | 7 +- lib/manager/gradle-wrapper/types.ts | 4 + lib/manager/gradle-wrapper/utils.ts | 19 +++- lib/manager/gradle/deep/utils.ts | 2 +- 15 files changed, 175 insertions(+), 55 deletions(-) create mode 100644 lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-all.properties create mode 100644 lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-bin.properties rename lib/manager/gradle-wrapper/__fixtures__/{gradle-wrapper-2.properties => gradle-wrapper-all.properties} (84%) rename lib/manager/gradle-wrapper/__fixtures__/{gradle-wrapper-1.properties => gradle-wrapper-bin.properties} (73%) delete mode 100644 lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-custom-distribution-url.properties rename lib/manager/gradle-wrapper/__fixtures__/{gradle-wrapper-3.properties => gradle-wrapper-prerelease.properties} (82%) rename lib/manager/gradle-wrapper/__fixtures__/{gradle-wrapper-4.properties => gradle-wrapper-unknown-format.properties} (78%) delete mode 100644 lib/manager/gradle-wrapper/__snapshots__/extract.spec.ts.snap create mode 100644 lib/manager/gradle-wrapper/types.ts diff --git a/docs/usage/java.md b/docs/usage/java.md index 082501663d2d74..e58dbe408117b2 100644 --- a/docs/usage/java.md +++ b/docs/usage/java.md @@ -6,7 +6,7 @@ description: Java versions support in Renovate # Java Dependency Updates Renovate can update Gradle and Maven dependencies. -This includes libraries and plugins. +This includes libraries and plugins as well as the Gradle Wrapper. ## Gradle @@ -23,10 +23,41 @@ Renovate does not support: - Android projects that require extra configuration to run (e.g. setting the Android SDK) - Gradle versions prior to version 5.0. +## Gradle Wrapper + +Renovate can update the [Gradle Wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html) of a project. + +This includes the source declaration inside the `gradle/wrapper/gradle-wrapper.properties` as well as accompanied files such as `gradlew`, `gradlew.bat`, and `gradle/wrapper/gradle-wrapper.jar`. + ### How It Works -Renovate uses a plugin to search and extract versions from projects. -Once the Gradle plugin has detected the dependencies, lookups and updating will be performed like usual with datasources and direct patching of files. +Renovate extracts the Gradle Wrapper version used from the `distributionUrl` inside the `gradle-wrapper.properties`. +Once the version is determined, Renovate will look for newer versions from the `gradle-version` datasource. +Renovate will then invoke the Gradle Wrapper to update itself, [as recommended by Gradle](https://docs.gradle.org/current/userguide/gradle_wrapper.html#sec:upgrading_wrapper). + +For the extraction to work, the `distributionUrl` must point to a file of type `.zip`, which includes the version in its name, and defines one of the official distribution types (bin, all). + +### Support for mirrors and custom distributions + +As Renovate takes the `distributionUrl` defined inside the `gradle-wrapper.properties` as basis for its update, source declarations other than to the official Gradle Wrapper are supported. + +This can be used for hosting the official distributions with a proxy server, an offline mirror or even providing a custom distribution of the Gradle Wrapper, e.g. to provide a company-wide base configuration for all Gradle projects. + +However, the `gradle-version` datasource is used to determine available versions. +In case the available versions at the defined source differ from those available from Gradle or the [default datasource](https://services.gradle.org/versions/all) cannot be reached, e.g. due to network restrictions, the datasource may be reconfigured via a `packageRule`: + +```json +{ + "packageRules": [ + { + "matchDatasources": ["gradle-version"], + "registryUrls": [ + "https://domain.tld/repository/custom-gradle-wrapper/versions.json" + ] + } + ] +} +``` ## Maven @@ -36,7 +67,12 @@ Renovate can update dependency versions found in Maven `pom.xml` files. Renovate will search repositories for all `pom.xml` files and processes them independently. -### Custom registry support, and authentication +## Custom registry support, and authentication + +Unless using `deepExtract`, Renovate does not make use of authentication credentials available to Gradle. + +The manager for Gradle makes use of the `maven` datasource. +Renovate can be configured to access additional repositories and access repositories authenticated. This example shows how you can use a `config.js` file to configure Renovate for use with Artifactory. We're using environment variables to pass the Artifactory username and password to Renovate bot. @@ -53,3 +89,16 @@ module.exports = { ], }; ``` + +You can overwrite the repositories to use for version lookup through configuration. + +```js +module.exports = { + packageRules: [ + { + matchDatasources: ['maven'], + registryUrls: ['https://repo-a.tld/repo', 'https://repo-b.tld/repo'], + }, + ], +}; +``` diff --git a/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-all.properties b/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-all.properties new file mode 100644 index 00000000000000..691f26185e7824 --- /dev/null +++ b/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-all.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists + +# distributionUrl includes version both in file name and path hierarchy, file name has different prefix +distributionUrl=https\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/6.6.6/custom-gradle-wrapper-6.6.6-all.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-bin.properties b/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-bin.properties new file mode 100644 index 00000000000000..909f9745dd90e9 --- /dev/null +++ b/lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-bin.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists + +# distributionUrl includes version both in file name and path hierarchy, file name has different prefix +distributionUrl=https\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/1.3.7/custom-gradle-wrapper-1.3.7-bin.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-2.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-all.properties similarity index 84% rename from lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-2.properties rename to lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-all.properties index aa0b761449f4e0..56482f3365a0b1 100644 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-2.properties +++ b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-all.properties @@ -2,6 +2,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -# @See https://gradle.org/releases/ -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip distributionSha256Sum=336b6898b491f6334502d8074a6b8c2d73ed83b92123106bd4bf837f04111043 + +# distributionUrl includes distribution type "all" +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-1.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-bin.properties similarity index 73% rename from lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-1.properties rename to lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-bin.properties index b206624d48d3ec..83d5d1c8cb5ca3 100644 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-1.properties +++ b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-bin.properties @@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip \ No newline at end of file + +# distributionUrl includes distribution type "bin" +distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-custom-distribution-url.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-custom-distribution-url.properties deleted file mode 100644 index 9953c8e1089608..00000000000000 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-custom-distribution-url.properties +++ /dev/null @@ -1,7 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -# @See https://gradle.org/releases/ -distributionUrl=https\://artifactory/gradle-wrapper-cache/distributions/gradle-4.8-bin.zip -distributionSha256Sum=336b6898b491f6334502d8074a6b8c2d73ed83b92123106bd4bf837f04111043 diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-3.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-prerelease.properties similarity index 82% rename from lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-3.properties rename to lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-prerelease.properties index a78c7b6f53f705..2ce023bdd94c56 100644 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-3.properties +++ b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-prerelease.properties @@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists + +# distributionUrl includes prerelase version distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-milestone-1-bin.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-4.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-unknown-format.properties similarity index 78% rename from lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-4.properties rename to lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-unknown-format.properties index d98eba7fbb009e..4dc14b6b213bbf 100644 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-4.properties +++ b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-unknown-format.properties @@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists + +# distributionUrl includes unsupported file name format distributionUrl=https\://services.gradle.org/distributions/gradle-7-rc-1-bin.zip diff --git a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-whitespace.properties b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-whitespace.properties index 0f14bb607411d6..25fc798d531a94 100644 --- a/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-whitespace.properties +++ b/lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-whitespace.properties @@ -2,6 +2,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -# @See https://gradle.org/releases/ -distributionUrl = https\://services.gradle.org/distributions/gradle-4.10.3-all.zip + +# distributionUrl and distributionSha256Sum include unnecessary whitespace +distributionUrl = https\://services.gradle.org/distributions/gradle-4.10.3-all.zip distributionSha256Sum = 336b6898b491f6334502d8074a6b8c2d73ed83b92123106bd4bf837f04111043 diff --git a/lib/manager/gradle-wrapper/__snapshots__/extract.spec.ts.snap b/lib/manager/gradle-wrapper/__snapshots__/extract.spec.ts.snap deleted file mode 100644 index a94cb18f3e82cc..00000000000000 --- a/lib/manager/gradle-wrapper/__snapshots__/extract.spec.ts.snap +++ /dev/null @@ -1,12 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`manager/gradle-wrapper/extract extractPackageFile() extracts prerelease version line 1`] = ` -Array [ - Object { - "currentValue": "7.0-milestone-1", - "datasource": "gradle-version", - "depName": "gradle", - "versioning": "gradle", - }, -] -`; diff --git a/lib/manager/gradle-wrapper/extract.spec.ts b/lib/manager/gradle-wrapper/extract.spec.ts index 8e2ecacea001f5..36fd0ef4b1d9fd 100644 --- a/lib/manager/gradle-wrapper/extract.spec.ts +++ b/lib/manager/gradle-wrapper/extract.spec.ts @@ -1,25 +1,42 @@ import { loadFixture } from '../../../test/util'; import { extractPackageFile } from './extract'; -const propertiesFile1 = loadFixture('gradle-wrapper-1.properties'); -const propertiesFile2 = loadFixture('gradle-wrapper-2.properties'); -const propertiesFile3 = loadFixture('gradle-wrapper-3.properties'); -const propertiesFile4 = loadFixture('gradle-wrapper-4.properties'); +const typeBinFileContent = loadFixture('gradle-wrapper-bin.properties'); +const typeAllFileContent = loadFixture('gradle-wrapper-all.properties'); +const prereleaseVersionFileContent = loadFixture( + 'gradle-wrapper-prerelease.properties' +); +const unknownFormatFileContent = loadFixture( + 'gradle-wrapper-unknown-format.properties' +); const whitespacePropertiesFile = loadFixture( 'gradle-wrapper-whitespace.properties' ); +const customTypeBinFileContent = loadFixture( + 'custom-gradle-wrapper-bin.properties' +); +const customTypeAllFileContent = loadFixture( + 'custom-gradle-wrapper-all.properties' +); describe('manager/gradle-wrapper/extract', () => { describe('extractPackageFile()', () => { - it('returns null for empty', () => { + it('returns null for property file without distributionUrl', () => { expect(extractPackageFile('nothing here')).toBeNull(); }); - it('extracts bin version line', () => { - const res = extractPackageFile(propertiesFile1); + it('returns null for property file with unsupported distributionUrl format', () => { + const res = extractPackageFile(unknownFormatFileContent); + expect(res).toBeNull(); + }); + + it('extracts version for property file with distribution type "bin" in distributionUrl', () => { + const res = extractPackageFile(typeBinFileContent); expect(res.deps).toEqual([ { currentValue: '4.8', + replaceString: + 'https\\://services.gradle.org/distributions/gradle-4.8-bin.zip', datasource: 'gradle-version', depName: 'gradle', versioning: 'gradle', @@ -27,11 +44,13 @@ describe('manager/gradle-wrapper/extract', () => { ]); }); - it('extracts all version line', () => { - const res = extractPackageFile(propertiesFile2); + it('extracts version for property file with distribution type "all" in distributionUrl', () => { + const res = extractPackageFile(typeAllFileContent); expect(res.deps).toEqual([ { currentValue: '4.10.3', + replaceString: + 'https\\://services.gradle.org/distributions/gradle-4.10.3-all.zip', datasource: 'gradle-version', depName: 'gradle', versioning: 'gradle', @@ -39,22 +58,55 @@ describe('manager/gradle-wrapper/extract', () => { ]); }); - it('extracts prerelease version line', () => { - const res = extractPackageFile(propertiesFile3); - expect(res.deps).toMatchSnapshot(); - expect(res.deps[0].currentValue).toBe('7.0-milestone-1'); - }); - - it('ignores invalid', () => { - const res = extractPackageFile(propertiesFile4); - expect(res).toBeNull(); + it('extracts version for property file with prerelease version in distributionUrl', () => { + const res = extractPackageFile(prereleaseVersionFileContent); + expect(res.deps).toEqual([ + { + currentValue: '7.0-milestone-1', + replaceString: + 'https\\://services.gradle.org/distributions/gradle-7.0-milestone-1-bin.zip', + datasource: 'gradle-version', + depName: 'gradle', + versioning: 'gradle', + }, + ]); }); - it('handles whitespace', () => { + it('extracts version for property file with unnecessary whitespace in distributionUrl', () => { const res = extractPackageFile(whitespacePropertiesFile); expect(res.deps).toEqual([ { currentValue: '4.10.3', + replaceString: + 'https\\://services.gradle.org/distributions/gradle-4.10.3-all.zip', + datasource: 'gradle-version', + depName: 'gradle', + versioning: 'gradle', + }, + ]); + }); + + it('extracts version for property file with custom distribution of type "bin" in distributionUrl', () => { + const res = extractPackageFile(customTypeBinFileContent); + expect(res.deps).toEqual([ + { + currentValue: '1.3.7', + replaceString: + 'https\\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/1.3.7/custom-gradle-wrapper-1.3.7-bin.zip', + datasource: 'gradle-version', + depName: 'gradle', + versioning: 'gradle', + }, + ]); + }); + + it('extracts version for property file with custom distribution of type "all" in distributionUrl', () => { + const res = extractPackageFile(customTypeAllFileContent); + expect(res.deps).toEqual([ + { + currentValue: '6.6.6', + replaceString: + 'https\\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/6.6.6/custom-gradle-wrapper-6.6.6-all.zip', datasource: 'gradle-version', depName: 'gradle', versioning: 'gradle', diff --git a/lib/manager/gradle-wrapper/extract.ts b/lib/manager/gradle-wrapper/extract.ts index 88383865230e98..fbcdf7f848d959 100644 --- a/lib/manager/gradle-wrapper/extract.ts +++ b/lib/manager/gradle-wrapper/extract.ts @@ -6,11 +6,12 @@ import { extractGradleVersion } from './utils'; export function extractPackageFile(fileContent: string): PackageFile | null { logger.trace('gradle-wrapper.extractPackageFile()'); - const currentValue = extractGradleVersion(fileContent); - if (currentValue) { + const extractResult = extractGradleVersion(fileContent); + if (extractResult) { const dependency: PackageDependency = { depName: 'gradle', - currentValue, + currentValue: extractResult.version, + replaceString: extractResult.url, datasource: GradleVersionDatasource.id, versioning, }; diff --git a/lib/manager/gradle-wrapper/types.ts b/lib/manager/gradle-wrapper/types.ts new file mode 100644 index 00000000000000..4d9805ab539e23 --- /dev/null +++ b/lib/manager/gradle-wrapper/types.ts @@ -0,0 +1,4 @@ +export interface GradleVersionExtract { + url: string; + version: string; +} diff --git a/lib/manager/gradle-wrapper/utils.ts b/lib/manager/gradle-wrapper/utils.ts index f328ed53e9a6f6..eb3f75966de5d3 100644 --- a/lib/manager/gradle-wrapper/utils.ts +++ b/lib/manager/gradle-wrapper/utils.ts @@ -2,10 +2,12 @@ import type { Stats } from 'fs'; import os from 'os'; import upath from 'upath'; import { getGlobalConfig } from '../../config/global'; +import { logger } from '../../logger'; import { chmod } from '../../util/fs'; import { regEx } from '../../util/regex'; import gradleVersioning from '../../versioning/gradle'; import { id as npmVersioning } from '../../versioning/npm'; +import { GradleVersionExtract } from './types'; export const extraEnv = { GRADLE_OPTS: @@ -72,20 +74,29 @@ export function getJavaVersioning(): string { return npmVersioning; } -// https://regex101.com/r/1GaQ2X/1 +// https://regex101.com/r/IcOs7P/1 const DISTRIBUTION_URL_REGEX = regEx( - '^(?:distributionUrl\\s*=\\s*)\\S*-(?\\d+\\.\\d+(?:\\.\\d+)?(?:-\\w+)*)-(?bin|all)\\.zip\\s*$' + '^(?:distributionUrl\\s*=\\s*)(?\\S*-(?\\d+\\.\\d+(?:\\.\\d+)?(?:-\\w+)*)-(?bin|all)\\.zip)\\s*$' ); -export function extractGradleVersion(fileContent: string): string | null { +export function extractGradleVersion( + fileContent: string +): GradleVersionExtract | null { const lines = fileContent?.split('\n') ?? []; for (const line of lines) { const distributionUrlMatch = DISTRIBUTION_URL_REGEX.exec(line); + if (distributionUrlMatch) { - return distributionUrlMatch.groups.version; + return { + url: distributionUrlMatch.groups.url, + version: distributionUrlMatch.groups.version, + }; } } + logger.debug( + 'Gradle wrapper version and url could not be extracted from properties - skipping update' + ); return null; } diff --git a/lib/manager/gradle/deep/utils.ts b/lib/manager/gradle/deep/utils.ts index 4b8aa9e1181eec..b984d6c055ab49 100644 --- a/lib/manager/gradle/deep/utils.ts +++ b/lib/manager/gradle/deep/utils.ts @@ -21,7 +21,7 @@ export async function getDockerConstraint( 'utf8' ); - const version = extractGradleVersion(fileContent); + const version = extractGradleVersion(fileContent)?.version; return getJavaContraint(version); }