feat(gradle-wrapper): add support for multiple version occurrences in…

… distributionUrl (#11454)

docs/usage/java.md

@@ -6,7 +6,7 @@ description: Java versions support in Renovate
 # Java Dependency Updates
 
 Renovate can update Gradle and Maven dependencies.
-This includes libraries and plugins.
+This includes libraries and plugins as well as the Gradle Wrapper.
 
 ## Gradle
 
@@ -23,10 +23,41 @@ Renovate does not support:
 - Android projects that require extra configuration to run (e.g. setting the Android SDK)
 - Gradle versions prior to version 5.0.
 
+## Gradle Wrapper
+
+Renovate can update the [Gradle Wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html) of a project.
+
+This includes the source declaration inside the `gradle/wrapper/gradle-wrapper.properties` as well as accompanied files such as `gradlew`, `gradlew.bat`, and `gradle/wrapper/gradle-wrapper.jar`.
+
 ### How It Works
 
-Renovate uses a plugin to search and extract versions from projects.
-Once the Gradle plugin has detected the dependencies, lookups and updating will be performed like usual with datasources and direct patching of files.
+Renovate extracts the Gradle Wrapper version used from the `distributionUrl` inside the `gradle-wrapper.properties`.
+Once the version is determined, Renovate will look for newer versions from the `gradle-version` datasource.
+Renovate will then invoke the Gradle Wrapper to update itself, [as recommended by Gradle](https://docs.gradle.org/current/userguide/gradle_wrapper.html#sec:upgrading_wrapper).
+
+For the extraction to work, the `distributionUrl` must point to a file of type `.zip`, which includes the version in its name, and defines one of the official distribution types (bin, all).
+
+### Support for mirrors and custom distributions
+
+As Renovate takes the `distributionUrl` defined inside the `gradle-wrapper.properties` as basis for its update, source declarations other than to the official Gradle Wrapper are supported.
+
+This can be used for hosting the official distributions with a proxy server, an offline mirror or even providing a custom distribution of the Gradle Wrapper, e.g. to provide a company-wide base configuration for all Gradle projects.
+
+However, the `gradle-version` datasource is used to determine available versions.
+In case the available versions at the defined source differ from those available from Gradle or the [default datasource](https://services.gradle.org/versions/all) cannot be reached, e.g. due to network restrictions, the datasource may be reconfigured via a `packageRule`:
+
+```json
+{
+  "packageRules": [
+    {
+      "matchDatasources": ["gradle-version"],
+      "registryUrls": [
+        "https://domain.tld/repository/custom-gradle-wrapper/versions.json"
+      ]
+    }
+  ]
+}
+```
 
 ## Maven
 
@@ -36,7 +67,12 @@ Renovate can update dependency versions found in Maven `pom.xml` files.
 
 Renovate will search repositories for all `pom.xml` files and processes them independently.
 
-### Custom registry support, and authentication
+## Custom registry support, and authentication
+
+Unless using `deepExtract`, Renovate does not make use of authentication credentials available to Gradle.
+
+The manager for Gradle makes use of the `maven` datasource.
+Renovate can be configured to access additional repositories and access repositories authenticated.
 
 This example shows how you can use a `config.js` file to configure Renovate for use with Artifactory.
 We're using environment variables to pass the Artifactory username and password to Renovate bot.
@@ -53,3 +89,16 @@ module.exports = {
   ],
 };
 ```
+
+You can overwrite the repositories to use for version lookup through configuration.
+
+```js
+module.exports = {
+  packageRules: [
+    {
+      matchDatasources: ['maven'],
+      registryUrls: ['https://repo-a.tld/repo', 'https://repo-b.tld/repo'],
+    },
+  ],
+};
+```

lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-all.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+
+# distributionUrl includes version both in file name and path hierarchy, file name has different prefix
+distributionUrl=https\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/6.6.6/custom-gradle-wrapper-6.6.6-all.zip

lib/manager/gradle-wrapper/__fixtures__/custom-gradle-wrapper-bin.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+
+# distributionUrl includes version both in file name and path hierarchy, file name has different prefix
+distributionUrl=https\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/1.3.7/custom-gradle-wrapper-1.3.7-bin.zip

lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-2.properties → lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-all.properties

@@ -2,6 +2,7 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-# @See https://gradle.org/releases/
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip
 distributionSha256Sum=336b6898b491f6334502d8074a6b8c2d73ed83b92123106bd4bf837f04111043
+
+# distributionUrl includes distribution type "all"
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.3-all.zip

lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-1.properties → lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-bin.properties

@@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip
+
+# distributionUrl includes distribution type "bin"
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip

lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-3.properties → lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-prerelease.properties

@@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+
+# distributionUrl includes prerelase version
 distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-milestone-1-bin.zip

lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-4.properties → lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-unknown-format.properties

@@ -2,4 +2,6 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
+
+# distributionUrl includes unsupported file name format
 distributionUrl=https\://services.gradle.org/distributions/gradle-7-rc-1-bin.zip

lib/manager/gradle-wrapper/__fixtures__/gradle-wrapper-whitespace.properties

@@ -2,6 +2,7 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-# @See https://gradle.org/releases/
-distributionUrl = https\://services.gradle.org/distributions/gradle-4.10.3-all.zip
+
+# distributionUrl and distributionSha256Sum include unnecessary whitespace
+distributionUrl    =      https\://services.gradle.org/distributions/gradle-4.10.3-all.zip
 distributionSha256Sum       =    336b6898b491f6334502d8074a6b8c2d73ed83b92123106bd4bf837f04111043

lib/manager/gradle-wrapper/extract.spec.ts

@@ -1,60 +1,112 @@
 import { loadFixture } from '../../../test/util';
 import { extractPackageFile } from './extract';
 
-const propertiesFile1 = loadFixture('gradle-wrapper-1.properties');
-const propertiesFile2 = loadFixture('gradle-wrapper-2.properties');
-const propertiesFile3 = loadFixture('gradle-wrapper-3.properties');
-const propertiesFile4 = loadFixture('gradle-wrapper-4.properties');
+const typeBinFileContent = loadFixture('gradle-wrapper-bin.properties');
+const typeAllFileContent = loadFixture('gradle-wrapper-all.properties');
+const prereleaseVersionFileContent = loadFixture(
+  'gradle-wrapper-prerelease.properties'
+);
+const unknownFormatFileContent = loadFixture(
+  'gradle-wrapper-unknown-format.properties'
+);
 const whitespacePropertiesFile = loadFixture(
   'gradle-wrapper-whitespace.properties'
 );
+const customTypeBinFileContent = loadFixture(
+  'custom-gradle-wrapper-bin.properties'
+);
+const customTypeAllFileContent = loadFixture(
+  'custom-gradle-wrapper-all.properties'
+);
 
 describe('manager/gradle-wrapper/extract', () => {
   describe('extractPackageFile()', () => {
-    it('returns null for empty', () => {
+    it('returns null for property file without distributionUrl', () => {
       expect(extractPackageFile('nothing here')).toBeNull();
     });
 
-    it('extracts bin version line', () => {
-      const res = extractPackageFile(propertiesFile1);
+    it('returns null for property file with unsupported distributionUrl format', () => {
+      const res = extractPackageFile(unknownFormatFileContent);
+      expect(res).toBeNull();
+    });
+
+    it('extracts version for property file with distribution type "bin" in distributionUrl', () => {
+      const res = extractPackageFile(typeBinFileContent);
       expect(res.deps).toEqual([
         {
           currentValue: '4.8',
+          replaceString:
+            'https\\://services.gradle.org/distributions/gradle-4.8-bin.zip',
           datasource: 'gradle-version',
           depName: 'gradle',
           versioning: 'gradle',
         },
       ]);
     });
 
-    it('extracts all version line', () => {
-      const res = extractPackageFile(propertiesFile2);
+    it('extracts version for property file with distribution type "all" in distributionUrl', () => {
+      const res = extractPackageFile(typeAllFileContent);
       expect(res.deps).toEqual([
         {
           currentValue: '4.10.3',
+          replaceString:
+            'https\\://services.gradle.org/distributions/gradle-4.10.3-all.zip',
           datasource: 'gradle-version',
           depName: 'gradle',
           versioning: 'gradle',
         },
       ]);
     });
 
-    it('extracts prerelease version line', () => {
-      const res = extractPackageFile(propertiesFile3);
-      expect(res.deps).toMatchSnapshot();
-      expect(res.deps[0].currentValue).toBe('7.0-milestone-1');
-    });
-
-    it('ignores invalid', () => {
-      const res = extractPackageFile(propertiesFile4);
-      expect(res).toBeNull();
+    it('extracts version for property file with prerelease version in distributionUrl', () => {
+      const res = extractPackageFile(prereleaseVersionFileContent);
+      expect(res.deps).toEqual([
+        {
+          currentValue: '7.0-milestone-1',
+          replaceString:
+            'https\\://services.gradle.org/distributions/gradle-7.0-milestone-1-bin.zip',
+          datasource: 'gradle-version',
+          depName: 'gradle',
+          versioning: 'gradle',
+        },
+      ]);
     });
 
-    it('handles whitespace', () => {
+    it('extracts version for property file with unnecessary whitespace in distributionUrl', () => {
       const res = extractPackageFile(whitespacePropertiesFile);
       expect(res.deps).toEqual([
         {
           currentValue: '4.10.3',
+          replaceString:
+            'https\\://services.gradle.org/distributions/gradle-4.10.3-all.zip',
+          datasource: 'gradle-version',
+          depName: 'gradle',
+          versioning: 'gradle',
+        },
+      ]);
+    });
+
+    it('extracts version for property file with custom distribution of type "bin" in distributionUrl', () => {
+      const res = extractPackageFile(customTypeBinFileContent);
+      expect(res.deps).toEqual([
+        {
+          currentValue: '1.3.7',
+          replaceString:
+            'https\\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/1.3.7/custom-gradle-wrapper-1.3.7-bin.zip',
+          datasource: 'gradle-version',
+          depName: 'gradle',
+          versioning: 'gradle',
+        },
+      ]);
+    });
+
+    it('extracts version for property file with custom distribution of type "all" in distributionUrl', () => {
+      const res = extractPackageFile(customTypeAllFileContent);
+      expect(res.deps).toEqual([
+        {
+          currentValue: '6.6.6',
+          replaceString:
+            'https\\://domain.tld/repository/maven-releases/tld/domain/gradle-wrapper/custom-gradle-wrapper/6.6.6/custom-gradle-wrapper-6.6.6-all.zip',
           datasource: 'gradle-version',
           depName: 'gradle',
           versioning: 'gradle',

lib/manager/gradle-wrapper/extract.ts

@@ -6,11 +6,12 @@ import { extractGradleVersion } from './utils';
 
 export function extractPackageFile(fileContent: string): PackageFile | null {
   logger.trace('gradle-wrapper.extractPackageFile()');
-  const currentValue = extractGradleVersion(fileContent);
-  if (currentValue) {
+  const extractResult = extractGradleVersion(fileContent);
+  if (extractResult) {
     const dependency: PackageDependency = {
       depName: 'gradle',
-      currentValue,
+      currentValue: extractResult.version,
+      replaceString: extractResult.url,
       datasource: GradleVersionDatasource.id,
       versioning,
     };

lib/manager/gradle-wrapper/types.ts

@@ -0,0 +1,4 @@
+export interface GradleVersionExtract {
+  url: string;
+  version: string;
+}

lib/manager/gradle-wrapper/utils.ts

@@ -2,10 +2,12 @@ import type { Stats } from 'fs';
 import os from 'os';
 import upath from 'upath';
 import { getGlobalConfig } from '../../config/global';
+import { logger } from '../../logger';
 import { chmod } from '../../util/fs';
 import { regEx } from '../../util/regex';
 import gradleVersioning from '../../versioning/gradle';
 import { id as npmVersioning } from '../../versioning/npm';
+import { GradleVersionExtract } from './types';
 
 export const extraEnv = {
   GRADLE_OPTS:
@@ -72,20 +74,29 @@ export function getJavaVersioning(): string {
   return npmVersioning;
 }
 
-// https://regex101.com/r/1GaQ2X/1
+// https://regex101.com/r/IcOs7P/1
 const DISTRIBUTION_URL_REGEX = regEx(
-  '^(?:distributionUrl\\s*=\\s*)\\S*-(?<version>\\d+\\.\\d+(?:\\.\\d+)?(?:-\\w+)*)-(?<type>bin|all)\\.zip\\s*$'
+  '^(?:distributionUrl\\s*=\\s*)(?<url>\\S*-(?<version>\\d+\\.\\d+(?:\\.\\d+)?(?:-\\w+)*)-(?<type>bin|all)\\.zip)\\s*$'
 );
 
-export function extractGradleVersion(fileContent: string): string | null {
+export function extractGradleVersion(
+  fileContent: string
+): GradleVersionExtract | null {
   const lines = fileContent?.split('\n') ?? [];
 
   for (const line of lines) {
     const distributionUrlMatch = DISTRIBUTION_URL_REGEX.exec(line);
+
     if (distributionUrlMatch) {
-      return distributionUrlMatch.groups.version;
+      return {
+        url: distributionUrlMatch.groups.url,
+        version: distributionUrlMatch.groups.version,
+      };
     }
   }
+  logger.debug(
+    'Gradle wrapper version and url could not be extracted from properties - skipping update'
+  );
 
   return null;
 }

lib/manager/gradle/deep/utils.ts

@@ -21,7 +21,7 @@ export async function getDockerConstraint(
     'utf8'
   );
 
-  const version = extractGradleVersion(fileContent);
+  const version = extractGradleVersion(fileContent)?.version;
 
   return getJavaContraint(version);
 }